diff options
Diffstat (limited to 'src')
171 files changed, 0 insertions, 62638 deletions
diff --git a/src/amqqueue.erl b/src/amqqueue.erl deleted file mode 100644 index 3415ebd073..0000000000 --- a/src/amqqueue.erl +++ /dev/null @@ -1,762 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(amqqueue). %% Could become amqqueue_v2 in the future. - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --export([new/8, - new/9, - new_with_version/9, - new_with_version/10, - fields/0, - fields/1, - field_vhost/0, - record_version_to_use/0, - upgrade/1, - upgrade_to/2, - % arguments - get_arguments/1, - set_arguments/2, - % decorators - get_decorators/1, - set_decorators/2, - % exclusive_owner - get_exclusive_owner/1, - % gm_pids - get_gm_pids/1, - set_gm_pids/2, - get_leader/1, - % name (#resource) - get_name/1, - set_name/2, - % operator_policy - get_operator_policy/1, - set_operator_policy/2, - get_options/1, - % pid - get_pid/1, - set_pid/2, - % policy - get_policy/1, - set_policy/2, - % policy_version - get_policy_version/1, - set_policy_version/2, - % type_state - get_type_state/1, - set_type_state/2, - % recoverable_slaves - get_recoverable_slaves/1, - set_recoverable_slaves/2, - % slave_pids - get_slave_pids/1, - set_slave_pids/2, - % slave_pids_pending_shutdown - get_slave_pids_pending_shutdown/1, - set_slave_pids_pending_shutdown/2, - % state - get_state/1, - set_state/2, - % sync_slave_pids - get_sync_slave_pids/1, - set_sync_slave_pids/2, - get_type/1, - get_vhost/1, - is_amqqueue/1, - is_auto_delete/1, - is_durable/1, - is_classic/1, - is_quorum/1, - pattern_match_all/0, - pattern_match_on_name/1, - pattern_match_on_type/1, - reset_mirroring_and_decorators/1, - set_immutable/1, - qnode/1, - macros/0]). - --define(record_version, amqqueue_v2). --define(is_backwards_compat_classic(T), - (T =:= classic orelse T =:= ?amqqueue_v1_type)). - --record(amqqueue, { - name :: rabbit_amqqueue:name() | '_', %% immutable - durable :: boolean() | '_', %% immutable - auto_delete :: boolean() | '_', %% immutable - exclusive_owner = none :: pid() | none | '_', %% immutable - arguments = [] :: rabbit_framing:amqp_table() | '_', %% immutable - pid :: pid() | ra_server_id() | none | '_', %% durable (just so we - %% know home node) - slave_pids = [] :: [pid()] | none | '_', %% transient - sync_slave_pids = [] :: [pid()] | none| '_',%% transient - recoverable_slaves = [] :: [atom()] | none | '_', %% durable - policy :: binary() | none | undefined | '_', %% durable, implicit - %% update as above - operator_policy :: binary() | none | undefined | '_', %% durable, - %% implicit - %% update - %% as above - gm_pids = [] :: [{pid(), pid()}] | none | '_', %% transient - decorators :: [atom()] | none | undefined | '_', %% transient, - %% recalculated - %% as above - state = live :: atom() | none | '_', %% durable (have we crashed?) - policy_version = 0 :: non_neg_integer() | '_', - slave_pids_pending_shutdown = [] :: [pid()] | '_', - vhost :: rabbit_types:vhost() | undefined | '_', %% secondary index - options = #{} :: map() | '_', - type = ?amqqueue_v1_type :: module() | '_', - type_state = #{} :: map() | '_' - }). - --type amqqueue() :: amqqueue_v1:amqqueue_v1() | amqqueue_v2(). --type amqqueue_v2() :: #amqqueue{ - name :: rabbit_amqqueue:name(), - durable :: boolean(), - auto_delete :: boolean(), - exclusive_owner :: pid() | none, - arguments :: rabbit_framing:amqp_table(), - pid :: pid() | ra_server_id() | none, - slave_pids :: [pid()] | none, - sync_slave_pids :: [pid()] | none, - recoverable_slaves :: [atom()] | none, - policy :: binary() | none | undefined, - operator_policy :: binary() | none | undefined, - gm_pids :: [{pid(), pid()}] | none, - decorators :: [atom()] | none | undefined, - state :: atom() | none, - policy_version :: non_neg_integer(), - slave_pids_pending_shutdown :: [pid()], - vhost :: rabbit_types:vhost() | undefined, - options :: map(), - type :: atom(), - type_state :: #{} - }. - --type ra_server_id() :: {Name :: atom(), Node :: node()}. - --type amqqueue_pattern() :: amqqueue_v1:amqqueue_v1_pattern() | - amqqueue_v2_pattern(). --type amqqueue_v2_pattern() :: #amqqueue{ - name :: rabbit_amqqueue:name() | '_', - durable :: '_', - auto_delete :: '_', - exclusive_owner :: '_', - arguments :: '_', - pid :: '_', - slave_pids :: '_', - sync_slave_pids :: '_', - recoverable_slaves :: '_', - policy :: '_', - operator_policy :: '_', - gm_pids :: '_', - decorators :: '_', - state :: '_', - policy_version :: '_', - slave_pids_pending_shutdown :: '_', - vhost :: '_', - options :: '_', - type :: atom() | '_', - type_state :: '_' - }. - --export_type([amqqueue/0, - amqqueue_v2/0, - amqqueue_pattern/0, - amqqueue_v2_pattern/0, - ra_server_id/0]). - --spec new(rabbit_amqqueue:name(), - pid() | ra_server_id() | none, - boolean(), - boolean(), - pid() | none, - rabbit_framing:amqp_table(), - rabbit_types:vhost() | undefined, - map()) -> amqqueue(). - -new(#resource{kind = queue} = Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options) - when (is_pid(Pid) orelse is_tuple(Pid) orelse Pid =:= none) andalso - is_boolean(Durable) andalso - is_boolean(AutoDelete) andalso - (is_pid(Owner) orelse Owner =:= none) andalso - is_list(Args) andalso - (is_binary(VHost) orelse VHost =:= undefined) andalso - is_map(Options) -> - new(Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options, - ?amqqueue_v1_type). - --spec new(rabbit_amqqueue:name(), - pid() | ra_server_id() | none, - boolean(), - boolean(), - pid() | none, - rabbit_framing:amqp_table(), - rabbit_types:vhost() | undefined, - map(), - atom()) -> amqqueue(). - -new(#resource{kind = queue} = Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options, - Type) - when (is_pid(Pid) orelse is_tuple(Pid) orelse Pid =:= none) andalso - is_boolean(Durable) andalso - is_boolean(AutoDelete) andalso - (is_pid(Owner) orelse Owner =:= none) andalso - is_list(Args) andalso - (is_binary(VHost) orelse VHost =:= undefined) andalso - is_map(Options) andalso - is_atom(Type) -> - case record_version_to_use() of - ?record_version -> - new_with_version( - ?record_version, - Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options, - Type); - _ -> - amqqueue_v1:new( - Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options, - Type) - end. - --spec new_with_version -(amqqueue_v1 | amqqueue_v2, - rabbit_amqqueue:name(), - pid() | ra_server_id() | none, - boolean(), - boolean(), - pid() | none, - rabbit_framing:amqp_table(), - rabbit_types:vhost() | undefined, - map()) -> amqqueue(). - -new_with_version(RecordVersion, - #resource{kind = queue} = Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options) - when (is_pid(Pid) orelse is_tuple(Pid) orelse Pid =:= none) andalso - is_boolean(Durable) andalso - is_boolean(AutoDelete) andalso - (is_pid(Owner) orelse Owner =:= none) andalso - is_list(Args) andalso - (is_binary(VHost) orelse VHost =:= undefined) andalso - is_map(Options) -> - new_with_version(RecordVersion, - Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options, - ?amqqueue_v1_type). - --spec new_with_version -(amqqueue_v1 | amqqueue_v2, - rabbit_amqqueue:name(), - pid() | ra_server_id() | none, - boolean(), - boolean(), - pid() | none, - rabbit_framing:amqp_table(), - rabbit_types:vhost() | undefined, - map(), - atom()) -> amqqueue(). - -new_with_version(?record_version, - #resource{kind = queue} = Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options, - Type) - when (is_pid(Pid) orelse is_tuple(Pid) orelse Pid =:= none) andalso - is_boolean(Durable) andalso - is_boolean(AutoDelete) andalso - (is_pid(Owner) orelse Owner =:= none) andalso - is_list(Args) andalso - (is_binary(VHost) orelse VHost =:= undefined) andalso - is_map(Options) andalso - is_atom(Type) -> - #amqqueue{name = Name, - durable = Durable, - auto_delete = AutoDelete, - arguments = Args, - exclusive_owner = Owner, - pid = Pid, - vhost = VHost, - options = Options, - type = ensure_type_compat(Type)}; -new_with_version(Version, - Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options, - Type) - when ?is_backwards_compat_classic(Type) -> - amqqueue_v1:new_with_version( - Version, - Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options). - --spec is_amqqueue(any()) -> boolean(). - -is_amqqueue(#amqqueue{}) -> true; -is_amqqueue(Queue) -> amqqueue_v1:is_amqqueue(Queue). - --spec record_version_to_use() -> amqqueue_v1 | amqqueue_v2. - -record_version_to_use() -> - case rabbit_feature_flags:is_enabled(quorum_queue) of - true -> ?record_version; - false -> amqqueue_v1:record_version_to_use() - end. - --spec upgrade(amqqueue()) -> amqqueue(). - -upgrade(#amqqueue{} = Queue) -> Queue; -upgrade(OldQueue) -> upgrade_to(record_version_to_use(), OldQueue). - --spec upgrade_to -(amqqueue_v2, amqqueue()) -> amqqueue_v2(); -(amqqueue_v1, amqqueue_v1:amqqueue_v1()) -> amqqueue_v1:amqqueue_v1(). - -upgrade_to(?record_version, #amqqueue{} = Queue) -> - Queue; -upgrade_to(?record_version, OldQueue) -> - Fields = erlang:tuple_to_list(OldQueue) ++ [?amqqueue_v1_type, - undefined], - #amqqueue{} = erlang:list_to_tuple(Fields); -upgrade_to(Version, OldQueue) -> - amqqueue_v1:upgrade_to(Version, OldQueue). - -% arguments - --spec get_arguments(amqqueue()) -> rabbit_framing:amqp_table(). - -get_arguments(#amqqueue{arguments = Args}) -> - Args; -get_arguments(Queue) -> - amqqueue_v1:get_arguments(Queue). - --spec set_arguments(amqqueue(), rabbit_framing:amqp_table()) -> amqqueue(). - -set_arguments(#amqqueue{} = Queue, Args) -> - Queue#amqqueue{arguments = Args}; -set_arguments(Queue, Args) -> - amqqueue_v1:set_arguments(Queue, Args). - -% decorators - --spec get_decorators(amqqueue()) -> [atom()] | none | undefined. - -get_decorators(#amqqueue{decorators = Decorators}) -> - Decorators; -get_decorators(Queue) -> - amqqueue_v1:get_decorators(Queue). - --spec set_decorators(amqqueue(), [atom()] | none | undefined) -> amqqueue(). - -set_decorators(#amqqueue{} = Queue, Decorators) -> - Queue#amqqueue{decorators = Decorators}; -set_decorators(Queue, Decorators) -> - amqqueue_v1:set_decorators(Queue, Decorators). - --spec get_exclusive_owner(amqqueue()) -> pid() | none. - -get_exclusive_owner(#amqqueue{exclusive_owner = Owner}) -> - Owner; -get_exclusive_owner(Queue) -> - amqqueue_v1:get_exclusive_owner(Queue). - -% gm_pids - --spec get_gm_pids(amqqueue()) -> [{pid(), pid()}] | none. - -get_gm_pids(#amqqueue{gm_pids = GMPids}) -> - GMPids; -get_gm_pids(Queue) -> - amqqueue_v1:get_gm_pids(Queue). - --spec set_gm_pids(amqqueue(), [{pid(), pid()}] | none) -> amqqueue(). - -set_gm_pids(#amqqueue{} = Queue, GMPids) -> - Queue#amqqueue{gm_pids = GMPids}; -set_gm_pids(Queue, GMPids) -> - amqqueue_v1:set_gm_pids(Queue, GMPids). - --spec get_leader(amqqueue_v2()) -> node(). - -get_leader(#amqqueue{type = rabbit_quorum_queue, pid = {_, Leader}}) -> Leader. - -% operator_policy - --spec get_operator_policy(amqqueue()) -> binary() | none | undefined. - -get_operator_policy(#amqqueue{operator_policy = OpPolicy}) -> OpPolicy; -get_operator_policy(Queue) -> amqqueue_v1:get_operator_policy(Queue). - --spec set_operator_policy(amqqueue(), binary() | none | undefined) -> - amqqueue(). - -set_operator_policy(#amqqueue{} = Queue, Policy) -> - Queue#amqqueue{operator_policy = Policy}; -set_operator_policy(Queue, Policy) -> - amqqueue_v1:set_operator_policy(Queue, Policy). - -% name - --spec get_name(amqqueue()) -> rabbit_amqqueue:name(). - -get_name(#amqqueue{name = Name}) -> Name; -get_name(Queue) -> amqqueue_v1:get_name(Queue). - --spec set_name(amqqueue(), rabbit_amqqueue:name()) -> amqqueue(). - -set_name(#amqqueue{} = Queue, Name) -> - Queue#amqqueue{name = Name}; -set_name(Queue, Name) -> - amqqueue_v1:set_name(Queue, Name). - --spec get_options(amqqueue()) -> map(). - -get_options(#amqqueue{options = Options}) -> Options; -get_options(Queue) -> amqqueue_v1:get_options(Queue). - -% pid - --spec get_pid -(amqqueue_v2()) -> pid() | ra_server_id() | none; -(amqqueue_v1:amqqueue_v1()) -> pid() | none. - -get_pid(#amqqueue{pid = Pid}) -> Pid; -get_pid(Queue) -> amqqueue_v1:get_pid(Queue). - --spec set_pid -(amqqueue_v2(), pid() | ra_server_id() | none) -> amqqueue_v2(); -(amqqueue_v1:amqqueue_v1(), pid() | none) -> amqqueue_v1:amqqueue_v1(). - -set_pid(#amqqueue{} = Queue, Pid) -> - Queue#amqqueue{pid = Pid}; -set_pid(Queue, Pid) -> - amqqueue_v1:set_pid(Queue, Pid). - -% policy - --spec get_policy(amqqueue()) -> proplists:proplist() | none | undefined. - -get_policy(#amqqueue{policy = Policy}) -> Policy; -get_policy(Queue) -> amqqueue_v1:get_policy(Queue). - --spec set_policy(amqqueue(), binary() | none | undefined) -> amqqueue(). - -set_policy(#amqqueue{} = Queue, Policy) -> - Queue#amqqueue{policy = Policy}; -set_policy(Queue, Policy) -> - amqqueue_v1:set_policy(Queue, Policy). - -% policy_version - --spec get_policy_version(amqqueue()) -> non_neg_integer(). - -get_policy_version(#amqqueue{policy_version = PV}) -> - PV; -get_policy_version(Queue) -> - amqqueue_v1:get_policy_version(Queue). - --spec set_policy_version(amqqueue(), non_neg_integer()) -> amqqueue(). - -set_policy_version(#amqqueue{} = Queue, PV) -> - Queue#amqqueue{policy_version = PV}; -set_policy_version(Queue, PV) -> - amqqueue_v1:set_policy_version(Queue, PV). - -% recoverable_slaves - --spec get_recoverable_slaves(amqqueue()) -> [atom()] | none. - -get_recoverable_slaves(#amqqueue{recoverable_slaves = Slaves}) -> - Slaves; -get_recoverable_slaves(Queue) -> - amqqueue_v1:get_recoverable_slaves(Queue). - --spec set_recoverable_slaves(amqqueue(), [atom()] | none) -> amqqueue(). - -set_recoverable_slaves(#amqqueue{} = Queue, Slaves) -> - Queue#amqqueue{recoverable_slaves = Slaves}; -set_recoverable_slaves(Queue, Slaves) -> - amqqueue_v1:set_recoverable_slaves(Queue, Slaves). - -% type_state (new in v2) - --spec get_type_state(amqqueue()) -> map(). -get_type_state(#amqqueue{type_state = TState}) -> - TState; -get_type_state(_) -> - #{}. - --spec set_type_state(amqqueue(), map()) -> amqqueue(). -set_type_state(#amqqueue{} = Queue, TState) -> - Queue#amqqueue{type_state = TState}; -set_type_state(Queue, _TState) -> - Queue. - -% slave_pids - --spec get_slave_pids(amqqueue()) -> [pid()] | none. - -get_slave_pids(#amqqueue{slave_pids = Slaves}) -> - Slaves; -get_slave_pids(Queue) -> - amqqueue_v1:get_slave_pids(Queue). - --spec set_slave_pids(amqqueue(), [pid()] | none) -> amqqueue(). - -set_slave_pids(#amqqueue{} = Queue, SlavePids) -> - Queue#amqqueue{slave_pids = SlavePids}; -set_slave_pids(Queue, SlavePids) -> - amqqueue_v1:set_slave_pids(Queue, SlavePids). - -% slave_pids_pending_shutdown - --spec get_slave_pids_pending_shutdown(amqqueue()) -> [pid()]. - -get_slave_pids_pending_shutdown( - #amqqueue{slave_pids_pending_shutdown = Slaves}) -> - Slaves; -get_slave_pids_pending_shutdown(Queue) -> - amqqueue_v1:get_slave_pids_pending_shutdown(Queue). - --spec set_slave_pids_pending_shutdown(amqqueue(), [pid()]) -> amqqueue(). - -set_slave_pids_pending_shutdown(#amqqueue{} = Queue, SlavePids) -> - Queue#amqqueue{slave_pids_pending_shutdown = SlavePids}; -set_slave_pids_pending_shutdown(Queue, SlavePids) -> - amqqueue_v1:set_slave_pids_pending_shutdown(Queue, SlavePids). - -% state - --spec get_state(amqqueue()) -> atom() | none. - -get_state(#amqqueue{state = State}) -> State; -get_state(Queue) -> amqqueue_v1:get_state(Queue). - --spec set_state(amqqueue(), atom() | none) -> amqqueue(). - -set_state(#amqqueue{} = Queue, State) -> - Queue#amqqueue{state = State}; -set_state(Queue, State) -> - amqqueue_v1:set_state(Queue, State). - -% sync_slave_pids - --spec get_sync_slave_pids(amqqueue()) -> [pid()] | none. - -get_sync_slave_pids(#amqqueue{sync_slave_pids = Pids}) -> - Pids; -get_sync_slave_pids(Queue) -> - amqqueue_v1:get_sync_slave_pids(Queue). - --spec set_sync_slave_pids(amqqueue(), [pid()] | none) -> amqqueue(). - -set_sync_slave_pids(#amqqueue{} = Queue, Pids) -> - Queue#amqqueue{sync_slave_pids = Pids}; -set_sync_slave_pids(Queue, Pids) -> - amqqueue_v1:set_sync_slave_pids(Queue, Pids). - -%% New in v2. - --spec get_type(amqqueue()) -> atom(). - -get_type(#amqqueue{type = Type}) -> Type; -get_type(Queue) when ?is_amqqueue(Queue) -> ?amqqueue_v1_type. - --spec get_vhost(amqqueue()) -> rabbit_types:vhost() | undefined. - -get_vhost(#amqqueue{vhost = VHost}) -> VHost; -get_vhost(Queue) -> amqqueue_v1:get_vhost(Queue). - --spec is_auto_delete(amqqueue()) -> boolean(). - -is_auto_delete(#amqqueue{auto_delete = AutoDelete}) -> - AutoDelete; -is_auto_delete(Queue) -> - amqqueue_v1:is_auto_delete(Queue). - --spec is_durable(amqqueue()) -> boolean(). - -is_durable(#amqqueue{durable = Durable}) -> Durable; -is_durable(Queue) -> amqqueue_v1:is_durable(Queue). - --spec is_classic(amqqueue()) -> boolean(). - -is_classic(Queue) -> - get_type(Queue) =:= ?amqqueue_v1_type. - --spec is_quorum(amqqueue()) -> boolean(). - -is_quorum(Queue) -> - get_type(Queue) =:= rabbit_quorum_queue. - -fields() -> - case record_version_to_use() of - ?record_version -> fields(?record_version); - _ -> amqqueue_v1:fields() - end. - -fields(?record_version) -> record_info(fields, amqqueue); -fields(Version) -> amqqueue_v1:fields(Version). - -field_vhost() -> - case record_version_to_use() of - ?record_version -> #amqqueue.vhost; - _ -> amqqueue_v1:field_vhost() - end. - --spec pattern_match_all() -> amqqueue_pattern(). - -pattern_match_all() -> - case record_version_to_use() of - ?record_version -> #amqqueue{_ = '_'}; - _ -> amqqueue_v1:pattern_match_all() - end. - --spec pattern_match_on_name(rabbit_amqqueue:name()) -> amqqueue_pattern(). - -pattern_match_on_name(Name) -> - case record_version_to_use() of - ?record_version -> #amqqueue{name = Name, _ = '_'}; - _ -> amqqueue_v1:pattern_match_on_name(Name) - end. - --spec pattern_match_on_type(atom()) -> amqqueue_pattern(). - -pattern_match_on_type(Type) -> - case record_version_to_use() of - ?record_version -> - #amqqueue{type = Type, _ = '_'}; - _ when ?is_backwards_compat_classic(Type) -> - amqqueue_v1:pattern_match_all(); - %% FIXME: We try a pattern which should never match when the - %% `quorum_queue` feature flag is not enabled yet. Is there - %% a better solution? - _ -> - amqqueue_v1:pattern_match_on_name( - rabbit_misc:r(<<0>>, queue, <<0>>)) - end. - --spec reset_mirroring_and_decorators(amqqueue()) -> amqqueue(). - -reset_mirroring_and_decorators(#amqqueue{} = Queue) -> - Queue#amqqueue{slave_pids = [], - sync_slave_pids = [], - gm_pids = [], - decorators = undefined}; -reset_mirroring_and_decorators(Queue) -> - amqqueue_v1:reset_mirroring_and_decorators(Queue). - --spec set_immutable(amqqueue()) -> amqqueue(). - -set_immutable(#amqqueue{} = Queue) -> - Queue#amqqueue{pid = none, - slave_pids = [], - sync_slave_pids = none, - recoverable_slaves = none, - gm_pids = none, - policy = none, - decorators = none, - state = none}; -set_immutable(Queue) -> - amqqueue_v1:set_immutable(Queue). - --spec qnode(amqqueue() | pid() | ra_server_id()) -> node(). - -qnode(Queue) when ?is_amqqueue(Queue) -> - QPid = get_pid(Queue), - qnode(QPid); -qnode(QPid) when is_pid(QPid) -> - node(QPid); -qnode({_, Node}) -> - Node. - -% private - -macros() -> - io:format( - "-define(is_~s(Q), is_record(Q, amqqueue, ~b)).~n~n", - [?record_version, record_info(size, amqqueue)]), - %% The field number starts at 2 because the first element is the - %% record name. - macros(record_info(fields, amqqueue), 2). - -macros([Field | Rest], I) -> - io:format( - "-define(~s_field_~s(Q), element(~b, Q)).~n", - [?record_version, Field, I]), - macros(Rest, I + 1); -macros([], _) -> - ok. - -ensure_type_compat(classic) -> - ?amqqueue_v1_type; -ensure_type_compat(Type) -> - Type. diff --git a/src/amqqueue_v1.erl b/src/amqqueue_v1.erl deleted file mode 100644 index dd1de74a4e..0000000000 --- a/src/amqqueue_v1.erl +++ /dev/null @@ -1,584 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(amqqueue_v1). - --include_lib("rabbit_common/include/resource.hrl"). --include("amqqueue.hrl"). - --export([new/8, - new/9, - new_with_version/9, - new_with_version/10, - fields/0, - fields/1, - field_vhost/0, - record_version_to_use/0, - upgrade/1, - upgrade_to/2, - % arguments - get_arguments/1, - set_arguments/2, - % decorators - get_decorators/1, - set_decorators/2, - % exclusive_owner - get_exclusive_owner/1, - % gm_pids - get_gm_pids/1, - set_gm_pids/2, - get_leader/1, - % name (#resource) - get_name/1, - set_name/2, - % operator_policy - get_operator_policy/1, - set_operator_policy/2, - get_options/1, - % pid - get_pid/1, - set_pid/2, - % policy - get_policy/1, - set_policy/2, - % policy_version - get_policy_version/1, - set_policy_version/2, - % type_state - get_type_state/1, - set_type_state/2, - % recoverable_slaves - get_recoverable_slaves/1, - set_recoverable_slaves/2, - % slave_pids - get_slave_pids/1, - set_slave_pids/2, - % slave_pids_pending_shutdown - get_slave_pids_pending_shutdown/1, - set_slave_pids_pending_shutdown/2, - % state - get_state/1, - set_state/2, - % sync_slave_pids - get_sync_slave_pids/1, - set_sync_slave_pids/2, - get_type/1, - get_vhost/1, - is_amqqueue/1, - is_auto_delete/1, - is_durable/1, - is_classic/1, - is_quorum/1, - pattern_match_all/0, - pattern_match_on_name/1, - pattern_match_on_type/1, - reset_mirroring_and_decorators/1, - set_immutable/1, - qnode/1, - macros/0]). - --define(record_version, ?MODULE). --define(is_backwards_compat_classic(T), - (T =:= classic orelse T =:= ?amqqueue_v1_type)). - --record(amqqueue, { - name :: rabbit_amqqueue:name() | '_', %% immutable - durable :: boolean() | '_', %% immutable - auto_delete :: boolean() | '_', %% immutable - exclusive_owner = none :: pid() | none | '_', %% immutable - arguments = [] :: rabbit_framing:amqp_table() | '_', %% immutable - pid :: pid() | none | '_', %% durable (just so we - %% know home node) - slave_pids = [] :: [pid()] | none | '_', %% transient - sync_slave_pids = [] :: [pid()] | none| '_',%% transient - recoverable_slaves = [] :: [atom()] | none | '_', %% durable - policy :: binary() | none | undefined | '_', %% durable, implicit - %% update as above - operator_policy :: binary() | none | undefined | '_', %% durable, - %% implicit - %% update - %% as above - gm_pids = [] :: [{pid(), pid()}] | none | '_', %% transient - decorators :: [atom()] | none | undefined | '_', %% transient, - %% recalculated - %% as above - state = live :: atom() | none | '_', %% durable (have we crashed?) - policy_version = 0 :: non_neg_integer() | '_', - slave_pids_pending_shutdown = [] :: [pid()] | '_', - vhost :: rabbit_types:vhost() | undefined | '_', %% secondary index - options = #{} :: map() | '_' - }). - --type amqqueue() :: amqqueue_v1(). --type amqqueue_v1() :: #amqqueue{ - name :: rabbit_amqqueue:name(), - durable :: boolean(), - auto_delete :: boolean(), - exclusive_owner :: pid() | none, - arguments :: rabbit_framing:amqp_table(), - pid :: pid() | none, - slave_pids :: [pid()] | none, - sync_slave_pids :: [pid()] | none, - recoverable_slaves :: [atom()] | none, - policy :: binary() | none | undefined, - operator_policy :: binary() | none | undefined, - gm_pids :: [{pid(), pid()}] | none, - decorators :: [atom()] | none | undefined, - state :: atom() | none, - policy_version :: non_neg_integer(), - slave_pids_pending_shutdown :: [pid()], - vhost :: rabbit_types:vhost() | undefined, - options :: map() - }. - --type amqqueue_pattern() :: amqqueue_v1_pattern(). --type amqqueue_v1_pattern() :: #amqqueue{ - name :: rabbit_amqqueue:name() | '_', - durable :: '_', - auto_delete :: '_', - exclusive_owner :: '_', - arguments :: '_', - pid :: '_', - slave_pids :: '_', - sync_slave_pids :: '_', - recoverable_slaves :: '_', - policy :: '_', - operator_policy :: '_', - gm_pids :: '_', - decorators :: '_', - state :: '_', - policy_version :: '_', - slave_pids_pending_shutdown :: '_', - vhost :: '_', - options :: '_' - }. - --export_type([amqqueue/0, - amqqueue_v1/0, - amqqueue_pattern/0, - amqqueue_v1_pattern/0]). - --spec new(rabbit_amqqueue:name(), - pid() | none, - boolean(), - boolean(), - pid() | none, - rabbit_framing:amqp_table(), - rabbit_types:vhost() | undefined, - map()) -> amqqueue(). - -new(#resource{kind = queue} = Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options) - when (is_pid(Pid) orelse Pid =:= none) andalso - is_boolean(Durable) andalso - is_boolean(AutoDelete) andalso - (is_pid(Owner) orelse Owner =:= none) andalso - is_list(Args) andalso - (is_binary(VHost) orelse VHost =:= undefined) andalso - is_map(Options) -> - new_with_version( - ?record_version, - Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options). - --spec new(rabbit_amqqueue:name(), - pid() | none, - boolean(), - boolean(), - pid() | none, - rabbit_framing:amqp_table(), - rabbit_types:vhost() | undefined, - map(), - ?amqqueue_v1_type | classic) -> amqqueue(). - -new(#resource{kind = queue} = Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options, - Type) - when (is_pid(Pid) orelse Pid =:= none) andalso - is_boolean(Durable) andalso - is_boolean(AutoDelete) andalso - (is_pid(Owner) orelse Owner =:= none) andalso - is_list(Args) andalso - (is_binary(VHost) orelse VHost =:= undefined) andalso - is_map(Options) andalso - ?is_backwards_compat_classic(Type) -> - new( - Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options). - --spec new_with_version(amqqueue_v1, - rabbit_amqqueue:name(), - pid() | none, - boolean(), - boolean(), - pid() | none, - rabbit_framing:amqp_table(), - rabbit_types:vhost() | undefined, - map()) -> amqqueue(). - -new_with_version(?record_version, - #resource{kind = queue} = Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options) - when (is_pid(Pid) orelse Pid =:= none) andalso - is_boolean(Durable) andalso - is_boolean(AutoDelete) andalso - (is_pid(Owner) orelse Owner =:= none) andalso - is_list(Args) andalso - (is_binary(VHost) orelse VHost =:= undefined) andalso - is_map(Options) -> - #amqqueue{name = Name, - durable = Durable, - auto_delete = AutoDelete, - arguments = Args, - exclusive_owner = Owner, - pid = Pid, - vhost = VHost, - options = Options}. - --spec new_with_version(amqqueue_v1, - rabbit_amqqueue:name(), - pid() | none, - boolean(), - boolean(), - pid() | none, - rabbit_framing:amqp_table(), - rabbit_types:vhost() | undefined, - map(), - ?amqqueue_v1_type | classic) -> amqqueue(). - -new_with_version(?record_version, - #resource{kind = queue} = Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options, - Type) - when (is_pid(Pid) orelse Pid =:= none) andalso - is_boolean(Durable) andalso - is_boolean(AutoDelete) andalso - (is_pid(Owner) orelse Owner =:= none) andalso - is_list(Args) andalso - (is_binary(VHost) orelse VHost =:= undefined) andalso - is_map(Options) andalso - ?is_backwards_compat_classic(Type) -> - new_with_version( - ?record_version, - Name, - Pid, - Durable, - AutoDelete, - Owner, - Args, - VHost, - Options). - --spec is_amqqueue(any()) -> boolean(). - -is_amqqueue(#amqqueue{}) -> true; -is_amqqueue(_) -> false. - --spec record_version_to_use() -> amqqueue_v1. - -record_version_to_use() -> - ?record_version. - --spec upgrade(amqqueue()) -> amqqueue(). - -upgrade(#amqqueue{} = Queue) -> Queue. - --spec upgrade_to(amqqueue_v1, amqqueue()) -> amqqueue(). - -upgrade_to(?record_version, #amqqueue{} = Queue) -> - Queue. - -% arguments - --spec get_arguments(amqqueue()) -> rabbit_framing:amqp_table(). - -get_arguments(#amqqueue{arguments = Args}) -> Args. - --spec set_arguments(amqqueue(), rabbit_framing:amqp_table()) -> amqqueue(). - -set_arguments(#amqqueue{} = Queue, Args) -> - Queue#amqqueue{arguments = Args}. - -% decorators - --spec get_decorators(amqqueue()) -> [atom()] | none | undefined. - -get_decorators(#amqqueue{decorators = Decorators}) -> Decorators. - --spec set_decorators(amqqueue(), [atom()] | none | undefined) -> amqqueue(). - -set_decorators(#amqqueue{} = Queue, Decorators) -> - Queue#amqqueue{decorators = Decorators}. - --spec get_exclusive_owner(amqqueue()) -> pid() | none. - -get_exclusive_owner(#amqqueue{exclusive_owner = Owner}) -> Owner. - -% gm_pids - --spec get_gm_pids(amqqueue()) -> [{pid(), pid()}] | none. - -get_gm_pids(#amqqueue{gm_pids = GMPids}) -> GMPids. - --spec set_gm_pids(amqqueue(), [{pid(), pid()}] | none) -> amqqueue(). - -set_gm_pids(#amqqueue{} = Queue, GMPids) -> - Queue#amqqueue{gm_pids = GMPids}. - --spec get_leader(amqqueue_v1()) -> no_return(). - -get_leader(_) -> throw({unsupported, ?record_version, get_leader}). - -% operator_policy - --spec get_operator_policy(amqqueue()) -> binary() | none | undefined. - -get_operator_policy(#amqqueue{operator_policy = OpPolicy}) -> OpPolicy. - --spec set_operator_policy(amqqueue(), binary() | none | undefined) -> - amqqueue(). - -set_operator_policy(#amqqueue{} = Queue, OpPolicy) -> - Queue#amqqueue{operator_policy = OpPolicy}. - -% name - --spec get_name(amqqueue()) -> rabbit_amqqueue:name(). - -get_name(#amqqueue{name = Name}) -> Name. - --spec set_name(amqqueue(), rabbit_amqqueue:name()) -> amqqueue(). - -set_name(#amqqueue{} = Queue, Name) -> - Queue#amqqueue{name = Name}. - --spec get_options(amqqueue()) -> map(). - -get_options(#amqqueue{options = Options}) -> Options. - -% pid - --spec get_pid -(amqqueue_v1:amqqueue_v1()) -> pid() | none. - -get_pid(#amqqueue{pid = Pid}) -> Pid. - --spec set_pid -(amqqueue_v1:amqqueue_v1(), pid() | none) -> amqqueue_v1:amqqueue_v1(). - -set_pid(#amqqueue{} = Queue, Pid) -> - Queue#amqqueue{pid = Pid}. - -% policy - --spec get_policy(amqqueue()) -> proplists:proplist() | none | undefined. - -get_policy(#amqqueue{policy = Policy}) -> Policy. - --spec set_policy(amqqueue(), binary() | none | undefined) -> amqqueue(). - -set_policy(#amqqueue{} = Queue, Policy) -> - Queue#amqqueue{policy = Policy}. - -% policy_version - --spec get_policy_version(amqqueue()) -> non_neg_integer(). - -get_policy_version(#amqqueue{policy_version = PV}) -> - PV. - --spec set_policy_version(amqqueue(), non_neg_integer()) -> amqqueue(). - -set_policy_version(#amqqueue{} = Queue, PV) -> - Queue#amqqueue{policy_version = PV}. - -% recoverable_slaves - --spec get_recoverable_slaves(amqqueue()) -> [atom()] | none. - -get_recoverable_slaves(#amqqueue{recoverable_slaves = Slaves}) -> - Slaves. - --spec set_recoverable_slaves(amqqueue(), [atom()] | none) -> amqqueue(). - -set_recoverable_slaves(#amqqueue{} = Queue, Slaves) -> - Queue#amqqueue{recoverable_slaves = Slaves}. - -% type_state (new in v2) - --spec get_type_state(amqqueue()) -> no_return(). - -get_type_state(_) -> throw({unsupported, ?record_version, get_type_state}). - --spec set_type_state(amqqueue(), [node()]) -> no_return(). - -set_type_state(_, _) -> - throw({unsupported, ?record_version, set_type_state}). - -% slave_pids - -get_slave_pids(#amqqueue{slave_pids = Slaves}) -> - Slaves. - -set_slave_pids(#amqqueue{} = Queue, SlavePids) -> - Queue#amqqueue{slave_pids = SlavePids}. - -% slave_pids_pending_shutdown - -get_slave_pids_pending_shutdown( - #amqqueue{slave_pids_pending_shutdown = Slaves}) -> - Slaves. - -set_slave_pids_pending_shutdown(#amqqueue{} = Queue, SlavePids) -> - Queue#amqqueue{slave_pids_pending_shutdown = SlavePids}. - -% state - --spec get_state(amqqueue()) -> atom() | none. - -get_state(#amqqueue{state = State}) -> State. - --spec set_state(amqqueue(), atom() | none) -> amqqueue(). - -set_state(#amqqueue{} = Queue, State) -> - Queue#amqqueue{state = State}. - -% sync_slave_pids - --spec get_sync_slave_pids(amqqueue()) -> [pid()] | none. - -get_sync_slave_pids(#amqqueue{sync_slave_pids = Pids}) -> - Pids. - --spec set_sync_slave_pids(amqqueue(), [pid()] | none) -> amqqueue(). - -set_sync_slave_pids(#amqqueue{} = Queue, Pids) -> - Queue#amqqueue{sync_slave_pids = Pids}. - -%% New in v2. - --spec get_type(amqqueue()) -> atom(). - -get_type(Queue) when ?is_amqqueue(Queue) -> ?amqqueue_v1_type. - --spec get_vhost(amqqueue()) -> rabbit_types:vhost() | undefined. - -get_vhost(#amqqueue{vhost = VHost}) -> VHost. - --spec is_auto_delete(amqqueue()) -> boolean(). - -is_auto_delete(#amqqueue{auto_delete = AutoDelete}) -> AutoDelete. - --spec is_durable(amqqueue()) -> boolean(). - -is_durable(#amqqueue{durable = Durable}) -> Durable. - --spec is_classic(amqqueue()) -> boolean(). - -is_classic(Queue) -> - get_type(Queue) =:= ?amqqueue_v1_type. - --spec is_quorum(amqqueue()) -> boolean(). - -is_quorum(Queue) when ?is_amqqueue(Queue) -> - false. - -fields() -> fields(?record_version). - -fields(?record_version) -> record_info(fields, amqqueue). - -field_vhost() -> #amqqueue.vhost. - --spec pattern_match_all() -> amqqueue_pattern(). - -pattern_match_all() -> #amqqueue{_ = '_'}. - --spec pattern_match_on_name(rabbit_amqqueue:name()) -> - amqqueue_pattern(). - -pattern_match_on_name(Name) -> #amqqueue{name = Name, _ = '_'}. - --spec pattern_match_on_type(atom()) -> no_return(). - -pattern_match_on_type(_) -> - throw({unsupported, ?record_version, pattern_match_on_type}). - -reset_mirroring_and_decorators(#amqqueue{} = Queue) -> - Queue#amqqueue{slave_pids = [], - sync_slave_pids = [], - gm_pids = [], - decorators = undefined}. - -set_immutable(#amqqueue{} = Queue) -> - Queue#amqqueue{pid = none, - slave_pids = none, - sync_slave_pids = none, - recoverable_slaves = none, - gm_pids = none, - policy = none, - decorators = none, - state = none}. - --spec qnode(amqqueue() | pid()) -> node(). - -qnode(Queue) when ?is_amqqueue(Queue) -> - QPid = get_pid(Queue), - qnode(QPid); -qnode(QPid) when is_pid(QPid) -> - node(QPid). - -macros() -> - io:format( - "-define(is_~s(Q), is_record(Q, amqqueue, ~b)).~n~n", - [?record_version, record_info(size, amqqueue)]), - %% The field number starts at 2 because the first element is the - %% record name. - macros(record_info(fields, amqqueue), 2). - -macros([Field | Rest], I) -> - io:format( - "-define(~s_field_~s(Q), element(~b, Q)).~n", - [?record_version, Field, I]), - macros(Rest, I + 1); -macros([], _) -> - ok. diff --git a/src/background_gc.erl b/src/background_gc.erl deleted file mode 100644 index be5bf0c995..0000000000 --- a/src/background_gc.erl +++ /dev/null @@ -1,78 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(background_gc). - --behaviour(gen_server2). - --export([start_link/0, run/0]). --export([gc/0]). %% For run_interval only - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --define(MAX_RATIO, 0.01). --define(MAX_INTERVAL, 240000). - --record(state, {last_interval}). - -%%---------------------------------------------------------------------------- - --spec start_link() -> {'ok', pid()} | {'error', any()}. - -start_link() -> gen_server2:start_link({local, ?MODULE}, ?MODULE, [], - [{timeout, infinity}]). - --spec run() -> 'ok'. - -run() -> gen_server2:cast(?MODULE, run). - -%%---------------------------------------------------------------------------- - -init([]) -> - {ok, IdealInterval} = application:get_env(rabbit, background_gc_target_interval), - {ok, interval_gc(#state{last_interval = IdealInterval})}. - -handle_call(Msg, _From, State) -> - {stop, {unexpected_call, Msg}, {unexpected_call, Msg}, State}. - -handle_cast(run, State) -> gc(), {noreply, State}; - -handle_cast(Msg, State) -> {stop, {unexpected_cast, Msg}, State}. - -handle_info(run, State) -> {noreply, interval_gc(State)}; - -handle_info(Msg, State) -> {stop, {unexpected_info, Msg}, State}. - -code_change(_OldVsn, State, _Extra) -> {ok, State}. - -terminate(_Reason, State) -> State. - -%%---------------------------------------------------------------------------- - -interval_gc(State = #state{last_interval = LastInterval}) -> - {ok, IdealInterval} = application:get_env(rabbit, background_gc_target_interval), - {ok, Interval} = rabbit_misc:interval_operation( - {?MODULE, gc, []}, - ?MAX_RATIO, ?MAX_INTERVAL, IdealInterval, LastInterval), - erlang:send_after(Interval, self(), run), - State#state{last_interval = Interval}. - --spec gc() -> 'ok'. - -gc() -> - Enabled = rabbit_misc:get_env(rabbit, background_gc_enabled, false), - case Enabled of - true -> - [garbage_collect(P) || P <- processes(), - {status, waiting} == process_info(P, status)], - %% since we will never be waiting... - garbage_collect(); - false -> - ok - end, - ok. diff --git a/src/code_server_cache.erl b/src/code_server_cache.erl deleted file mode 100644 index b53f5dcee9..0000000000 --- a/src/code_server_cache.erl +++ /dev/null @@ -1,81 +0,0 @@ -%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(code_server_cache). - --behaviour(gen_server). - -%% API --export([start_link/0, - maybe_call_mfa/4]). - -%% gen_server callbacks --export([init/1, - handle_call/3, - handle_cast/2, - handle_info/2, - terminate/2, - code_change/3]). - --record(state, { - modules = #{} :: #{atom() => boolean()} -}). - -%% API -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -maybe_call_mfa(Module, Function, Args, Default) -> - gen_server:call(?MODULE, {maybe_call_mfa, {Module, Function, Args, Default}}). - -%% gen_server callbacks - -init([]) -> - {ok, #state{}}. - -handle_call({maybe_call_mfa, {Mod, _F, _A, _D} = MFA}, _From, #state{modules = ModuleMap} = State0) -> - Value = maps:get(Mod, ModuleMap, true), - {ok, Reply, State1} = handle_maybe_call_mfa(Value, MFA, State0), - {reply, Reply, State1}; -handle_call(_Request, _From, State) -> - {reply, ignored, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%% Internal functions - -handle_maybe_call_mfa(false, {_M, _F, _A, Default}, State) -> - {ok, Default, State}; -handle_maybe_call_mfa(true, {Module, Function, Args, Default}, State) -> - try - Reply = erlang:apply(Module, Function, Args), - {ok, Reply, State} - catch - error:undef -> - handle_maybe_call_mfa_error(Module, Default, State); - Err:Reason -> - rabbit_log:error("Calling ~p:~p failed: ~p:~p~n", - [Module, Function, Err, Reason]), - handle_maybe_call_mfa_error(Module, Default, State) - end. - -handle_maybe_call_mfa_error(Module, Default, #state{modules = ModuleMap0} = State0) -> - ModuleMap1 = maps:put(Module, false, ModuleMap0), - State1 = State0#state{modules = ModuleMap1}, - {ok, Default, State1}. diff --git a/src/gatherer.erl b/src/gatherer.erl deleted file mode 100644 index 2b46ec02b1..0000000000 --- a/src/gatherer.erl +++ /dev/null @@ -1,151 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(gatherer). - -%% Gatherer is a queue which has producer and consumer processes. Before producers -%% push items to the queue using gatherer:in/2 they need to declare their intent -%% to do so with gatherer:fork/1. When a publisher's work is done, it states so -%% using gatherer:finish/1. -%% -%% Consumers pop messages off queues with gatherer:out/1. If a queue is empty -%% and there are producers that haven't finished working, the caller is blocked -%% until an item is available. If there are no active producers, gatherer:out/1 -%% immediately returns 'empty'. -%% -%% This module is primarily used to collect results from asynchronous tasks -%% running in a worker pool, e.g. when recovering bindings or rebuilding -%% message store indices. - --behaviour(gen_server2). - --export([start_link/0, stop/1, fork/1, finish/1, in/2, sync_in/2, out/1]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - -%%---------------------------------------------------------------------------- - --define(HIBERNATE_AFTER_MIN, 1000). --define(DESIRED_HIBERNATE, 10000). - -%%---------------------------------------------------------------------------- - --record(gstate, { forks, values, blocked }). - -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - gen_server2:start_link(?MODULE, [], [{timeout, infinity}]). - --spec stop(pid()) -> 'ok'. - -stop(Pid) -> - unlink(Pid), - gen_server2:call(Pid, stop, infinity). - --spec fork(pid()) -> 'ok'. - -fork(Pid) -> - gen_server2:call(Pid, fork, infinity). - --spec finish(pid()) -> 'ok'. - -finish(Pid) -> - gen_server2:cast(Pid, finish). - --spec in(pid(), any()) -> 'ok'. - -in(Pid, Value) -> - gen_server2:cast(Pid, {in, Value}). - --spec sync_in(pid(), any()) -> 'ok'. - -sync_in(Pid, Value) -> - gen_server2:call(Pid, {in, Value}, infinity). - --spec out(pid()) -> {'value', any()} | 'empty'. - -out(Pid) -> - gen_server2:call(Pid, out, infinity). - -%%---------------------------------------------------------------------------- - -init([]) -> - {ok, #gstate { forks = 0, values = queue:new(), blocked = queue:new() }, - hibernate, - {backoff, ?HIBERNATE_AFTER_MIN, ?HIBERNATE_AFTER_MIN, ?DESIRED_HIBERNATE}}. - -handle_call(stop, _From, State) -> - {stop, normal, ok, State}; - -handle_call(fork, _From, State = #gstate { forks = Forks }) -> - {reply, ok, State #gstate { forks = Forks + 1 }, hibernate}; - -handle_call({in, Value}, From, State) -> - {noreply, in(Value, From, State), hibernate}; - -handle_call(out, From, State = #gstate { forks = Forks, - values = Values, - blocked = Blocked }) -> - case queue:out(Values) of - {empty, _} when Forks == 0 -> - {reply, empty, State, hibernate}; - {empty, _} -> - {noreply, State #gstate { blocked = queue:in(From, Blocked) }, - hibernate}; - {{value, {PendingIn, Value}}, NewValues} -> - reply(PendingIn, ok), - {reply, {value, Value}, State #gstate { values = NewValues }, - hibernate} - end; - -handle_call(Msg, _From, State) -> - {stop, {unexpected_call, Msg}, State}. - -handle_cast(finish, State = #gstate { forks = Forks, blocked = Blocked }) -> - NewForks = Forks - 1, - NewBlocked = case NewForks of - 0 -> _ = [gen_server2:reply(From, empty) || - From <- queue:to_list(Blocked)], - queue:new(); - _ -> Blocked - end, - {noreply, State #gstate { forks = NewForks, blocked = NewBlocked }, - hibernate}; - -handle_cast({in, Value}, State) -> - {noreply, in(Value, undefined, State), hibernate}; - -handle_cast(Msg, State) -> - {stop, {unexpected_cast, Msg}, State}. - -handle_info(Msg, State) -> - {stop, {unexpected_info, Msg}, State}. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -terminate(_Reason, State) -> - State. - -%%---------------------------------------------------------------------------- - -in(Value, From, State = #gstate { values = Values, blocked = Blocked }) -> - case queue:out(Blocked) of - {empty, _} -> - State #gstate { values = queue:in({From, Value}, Values) }; - {{value, PendingOut}, NewBlocked} -> - reply(From, ok), - gen_server2:reply(PendingOut, {value, Value}), - State #gstate { blocked = NewBlocked } - end. - -reply(undefined, _Reply) -> ok; -reply(From, Reply) -> gen_server2:reply(From, Reply). diff --git a/src/gm.erl b/src/gm.erl deleted file mode 100644 index af24a2958a..0000000000 --- a/src/gm.erl +++ /dev/null @@ -1,1650 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(gm). - -%% Guaranteed Multicast -%% ==================== -%% -%% This module provides the ability to create named groups of -%% processes to which members can be dynamically added and removed, -%% and for messages to be broadcast within the group that are -%% guaranteed to reach all members of the group during the lifetime of -%% the message. The lifetime of a message is defined as being, at a -%% minimum, the time from which the message is first sent to any -%% member of the group, up until the time at which it is known by the -%% member who published the message that the message has reached all -%% group members. -%% -%% The guarantee given is that provided a message, once sent, makes it -%% to members who do not all leave the group, the message will -%% continue to propagate to all group members. -%% -%% Another way of stating the guarantee is that if member P publishes -%% messages m and m', then for all members P', if P' is a member of -%% the group prior to the publication of m, and P' receives m', then -%% P' will receive m. -%% -%% Note that only local-ordering is enforced: i.e. if member P sends -%% message m and then message m', then for-all members P', if P' -%% receives m and m', then they will receive m' after m. Causality -%% ordering is _not_ enforced. I.e. if member P receives message m -%% and as a result publishes message m', there is no guarantee that -%% other members P' will receive m before m'. -%% -%% -%% API Use -%% ------- -%% -%% Mnesia must be started. Use the idempotent create_tables/0 function -%% to create the tables required. -%% -%% start_link/3 -%% Provide the group name, the callback module name, and any arguments -%% you wish to be passed into the callback module's functions. The -%% joined/2 function will be called when we have joined the group, -%% with the arguments passed to start_link and a list of the current -%% members of the group. See the callbacks specs and the comments -%% below for further details of the callback functions. -%% -%% leave/1 -%% Provide the Pid. Removes the Pid from the group. The callback -%% handle_terminate/2 function will be called. -%% -%% broadcast/2 -%% Provide the Pid and a Message. The message will be sent to all -%% members of the group as per the guarantees given above. This is a -%% cast and the function call will return immediately. There is no -%% guarantee that the message will reach any member of the group. -%% -%% confirmed_broadcast/2 -%% Provide the Pid and a Message. As per broadcast/2 except that this -%% is a call, not a cast, and only returns 'ok' once the Message has -%% reached every member of the group. Do not call -%% confirmed_broadcast/2 directly from the callback module otherwise -%% you will deadlock the entire group. -%% -%% info/1 -%% Provide the Pid. Returns a proplist with various facts, including -%% the group name and the current group members. -%% -%% validate_members/2 -%% Check whether a given member list agrees with the chosen member's -%% view. Any differences will be communicated via the members_changed -%% callback. If there are no differences then there will be no reply. -%% Note that members will not necessarily share the same view. -%% -%% forget_group/1 -%% Provide the group name. Removes its mnesia record. Makes no attempt -%% to ensure the group is empty. -%% -%% Implementation Overview -%% ----------------------- -%% -%% One possible means of implementation would be a fan-out from the -%% sender to every member of the group. This would require that the -%% group is fully connected, and, in the event that the original -%% sender of the message disappears from the group before the message -%% has made it to every member of the group, raises questions as to -%% who is responsible for sending on the message to new group members. -%% In particular, the issue is with [ Pid ! Msg || Pid <- Members ] - -%% if the sender dies part way through, who is responsible for -%% ensuring that the remaining Members receive the Msg? In the event -%% that within the group, messages sent are broadcast from a subset of -%% the members, the fan-out arrangement has the potential to -%% substantially impact the CPU and network workload of such members, -%% as such members would have to accommodate the cost of sending each -%% message to every group member. -%% -%% Instead, if the members of the group are arranged in a chain, then -%% it becomes easier to reason about who within the group has received -%% each message and who has not. It eases issues of responsibility: in -%% the event of a group member disappearing, the nearest upstream -%% member of the chain is responsible for ensuring that messages -%% continue to propagate down the chain. It also results in equal -%% distribution of sending and receiving workload, even if all -%% messages are being sent from just a single group member. This -%% configuration has the further advantage that it is not necessary -%% for every group member to know of every other group member, and -%% even that a group member does not have to be accessible from all -%% other group members. -%% -%% Performance is kept high by permitting pipelining and all -%% communication between joined group members is asynchronous. In the -%% chain A -> B -> C -> D, if A sends a message to the group, it will -%% not directly contact C or D. However, it must know that D receives -%% the message (in addition to B and C) before it can consider the -%% message fully sent. A simplistic implementation would require that -%% D replies to C, C replies to B and B then replies to A. This would -%% result in a propagation delay of twice the length of the chain. It -%% would also require, in the event of the failure of C, that D knows -%% to directly contact B and issue the necessary replies. Instead, the -%% chain forms a ring: D sends the message on to A: D does not -%% distinguish A as the sender, merely as the next member (downstream) -%% within the chain (which has now become a ring). When A receives -%% from D messages that A sent, it knows that all members have -%% received the message. However, the message is not dead yet: if C -%% died as B was sending to C, then B would need to detect the death -%% of C and forward the message on to D instead: thus every node has -%% to remember every message published until it is told that it can -%% forget about the message. This is essential not just for dealing -%% with failure of members, but also for the addition of new members. -%% -%% Thus once A receives the message back again, it then sends to B an -%% acknowledgement for the message, indicating that B can now forget -%% about the message. B does so, and forwards the ack to C. C forgets -%% the message, and forwards the ack to D, which forgets the message -%% and finally forwards the ack back to A. At this point, A takes no -%% further action: the message and its acknowledgement have made it to -%% every member of the group. The message is now dead, and any new -%% member joining the group at this point will not receive the -%% message. -%% -%% We therefore have two roles: -%% -%% 1. The sender, who upon receiving their own messages back, must -%% then send out acknowledgements, and upon receiving their own -%% acknowledgements back perform no further action. -%% -%% 2. The other group members who upon receiving messages and -%% acknowledgements must update their own internal state accordingly -%% (the sending member must also do this in order to be able to -%% accommodate failures), and forwards messages on to their downstream -%% neighbours. -%% -%% -%% Implementation: It gets trickier -%% -------------------------------- -%% -%% Chain A -> B -> C -> D -%% -%% A publishes a message which B receives. A now dies. B and D will -%% detect the death of A, and will link up, thus the chain is now B -> -%% C -> D. B forwards A's message on to C, who forwards it to D, who -%% forwards it to B. Thus B is now responsible for A's messages - both -%% publications and acknowledgements that were in flight at the point -%% at which A died. Even worse is that this is transitive: after B -%% forwards A's message to C, B dies as well. Now C is not only -%% responsible for B's in-flight messages, but is also responsible for -%% A's in-flight messages. -%% -%% Lemma 1: A member can only determine which dead members they have -%% inherited responsibility for if there is a total ordering on the -%% conflicting additions and subtractions of members from the group. -%% -%% Consider the simultaneous death of B and addition of B' that -%% transitions a chain from A -> B -> C to A -> B' -> C. Either B' or -%% C is responsible for in-flight messages from B. It is easy to -%% ensure that at least one of them thinks they have inherited B, but -%% if we do not ensure that exactly one of them inherits B, then we -%% could have B' converting publishes to acks, which then will crash C -%% as C does not believe it has issued acks for those messages. -%% -%% More complex scenarios are easy to concoct: A -> B -> C -> D -> E -%% becoming A -> C' -> E. Who has inherited which of B, C and D? -%% -%% However, for non-conflicting membership changes, only a partial -%% ordering is required. For example, A -> B -> C becoming A -> A' -> -%% B. The addition of A', between A and B can have no conflicts with -%% the death of C: it is clear that A has inherited C's messages. -%% -%% For ease of implementation, we adopt the simple solution, of -%% imposing a total order on all membership changes. -%% -%% On the death of a member, it is ensured the dead member's -%% neighbours become aware of the death, and the upstream neighbour -%% now sends to its new downstream neighbour its state, including the -%% messages pending acknowledgement. The downstream neighbour can then -%% use this to calculate which publishes and acknowledgements it has -%% missed out on, due to the death of its old upstream. Thus the -%% downstream can catch up, and continues the propagation of messages -%% through the group. -%% -%% Lemma 2: When a member is joining, it must synchronously -%% communicate with its upstream member in order to receive its -%% starting state atomically with its addition to the group. -%% -%% New members must start with the same state as their nearest -%% upstream neighbour. This ensures that it is not surprised by -%% acknowledgements they are sent, and that should their downstream -%% neighbour die, they are able to send the correct state to their new -%% downstream neighbour to ensure it can catch up. Thus in the -%% transition A -> B -> C becomes A -> A' -> B -> C becomes A -> A' -> -%% C, A' must start with the state of A, so that it can send C the -%% correct state when B dies, allowing C to detect any missed -%% messages. -%% -%% If A' starts by adding itself to the group membership, A could then -%% die, without A' having received the necessary state from A. This -%% would leave A' responsible for in-flight messages from A, but -%% having the least knowledge of all, of those messages. Thus A' must -%% start by synchronously calling A, which then immediately sends A' -%% back its state. A then adds A' to the group. If A dies at this -%% point then A' will be able to see this (as A' will fail to appear -%% in the group membership), and thus A' will ignore the state it -%% receives from A, and will simply repeat the process, trying to now -%% join downstream from some other member. This ensures that should -%% the upstream die as soon as the new member has been joined, the new -%% member is guaranteed to receive the correct state, allowing it to -%% correctly process messages inherited due to the death of its -%% upstream neighbour. -%% -%% The canonical definition of the group membership is held by a -%% distributed database. Whilst this allows the total ordering of -%% changes to be achieved, it is nevertheless undesirable to have to -%% query this database for the current view, upon receiving each -%% message. Instead, we wish for members to be able to cache a view of -%% the group membership, which then requires a cache invalidation -%% mechanism. Each member maintains its own view of the group -%% membership. Thus when the group's membership changes, members may -%% need to become aware of such changes in order to be able to -%% accurately process messages they receive. Because of the -%% requirement of a total ordering of conflicting membership changes, -%% it is not possible to use the guaranteed broadcast mechanism to -%% communicate these changes: to achieve the necessary ordering, it -%% would be necessary for such messages to be published by exactly one -%% member, which can not be guaranteed given that such a member could -%% die. -%% -%% The total ordering we enforce on membership changes gives rise to a -%% view version number: every change to the membership creates a -%% different view, and the total ordering permits a simple -%% monotonically increasing view version number. -%% -%% Lemma 3: If a message is sent from a member that holds view version -%% N, it can be correctly processed by any member receiving the -%% message with a view version >= N. -%% -%% Initially, let us suppose that each view contains the ordering of -%% every member that was ever part of the group. Dead members are -%% marked as such. Thus we have a ring of members, some of which are -%% dead, and are thus inherited by the nearest alive downstream -%% member. -%% -%% In the chain A -> B -> C, all three members initially have view -%% version 1, which reflects reality. B publishes a message, which is -%% forward by C to A. B now dies, which A notices very quickly. Thus A -%% updates the view, creating version 2. It now forwards B's -%% publication, sending that message to its new downstream neighbour, -%% C. This happens before C is aware of the death of B. C must become -%% aware of the view change before it interprets the message its -%% received, otherwise it will fail to learn of the death of B, and -%% thus will not realise it has inherited B's messages (and will -%% likely crash). -%% -%% Thus very simply, we have that each subsequent view contains more -%% information than the preceding view. -%% -%% However, to avoid the views growing indefinitely, we need to be -%% able to delete members which have died _and_ for which no messages -%% are in-flight. This requires that upon inheriting a dead member, we -%% know the last publication sent by the dead member (this is easy: we -%% inherit a member because we are the nearest downstream member which -%% implies that we know at least as much than everyone else about the -%% publications of the dead member), and we know the earliest message -%% for which the acknowledgement is still in flight. -%% -%% In the chain A -> B -> C, when B dies, A will send to C its state -%% (as C is the new downstream from A), allowing C to calculate which -%% messages it has missed out on (described above). At this point, C -%% also inherits B's messages. If that state from A also includes the -%% last message published by B for which an acknowledgement has been -%% seen, then C knows exactly which further acknowledgements it must -%% receive (also including issuing acknowledgements for publications -%% still in-flight that it receives), after which it is known there -%% are no more messages in flight for B, thus all evidence that B was -%% ever part of the group can be safely removed from the canonical -%% group membership. -%% -%% Thus, for every message that a member sends, it includes with that -%% message its view version. When a member receives a message it will -%% update its view from the canonical copy, should its view be older -%% than the view version included in the message it has received. -%% -%% The state held by each member therefore includes the messages from -%% each publisher pending acknowledgement, the last publication seen -%% from that publisher, and the last acknowledgement from that -%% publisher. In the case of the member's own publications or -%% inherited members, this last acknowledgement seen state indicates -%% the last acknowledgement retired, rather than sent. -%% -%% -%% Proof sketch -%% ------------ -%% -%% We need to prove that with the provided operational semantics, we -%% can never reach a state that is not well formed from a well-formed -%% starting state. -%% -%% Operational semantics (small step): straight-forward message -%% sending, process monitoring, state updates. -%% -%% Well formed state: dead members inherited by exactly one non-dead -%% member; for every entry in anyone's pending-acks, either (the -%% publication of the message is in-flight downstream from the member -%% and upstream from the publisher) or (the acknowledgement of the -%% message is in-flight downstream from the publisher and upstream -%% from the member). -%% -%% Proof by induction on the applicable operational semantics. -%% -%% -%% Related work -%% ------------ -%% -%% The ring configuration and double traversal of messages around the -%% ring is similar (though developed independently) to the LCR -%% protocol by [Levy 2008]. However, LCR differs in several -%% ways. Firstly, by using vector clocks, it enforces a total order of -%% message delivery, which is unnecessary for our purposes. More -%% significantly, it is built on top of a "group communication system" -%% which performs the group management functions, taking -%% responsibility away from the protocol as to how to cope with safely -%% adding and removing members. When membership changes do occur, the -%% protocol stipulates that every member must perform communication -%% with every other member of the group, to ensure all outstanding -%% deliveries complete, before the entire group transitions to the new -%% view. This, in total, requires two sets of all-to-all synchronous -%% communications. -%% -%% This is not only rather inefficient, but also does not explain what -%% happens upon the failure of a member during this process. It does -%% though entirely avoid the need for inheritance of responsibility of -%% dead members that our protocol incorporates. -%% -%% In [Marandi et al 2010], a Paxos-based protocol is described. This -%% work explicitly focuses on the efficiency of communication. LCR -%% (and our protocol too) are more efficient, but at the cost of -%% higher latency. The Ring-Paxos protocol is itself built on top of -%% IP-multicast, which rules it out for many applications where -%% point-to-point communication is all that can be required. They also -%% have an excellent related work section which I really ought to -%% read... -%% -%% -%% [Levy 2008] The Complexity of Reliable Distributed Storage, 2008. -%% [Marandi et al 2010] Ring Paxos: A High-Throughput Atomic Broadcast -%% Protocol - - --behaviour(gen_server2). - --export([create_tables/0, start_link/4, leave/1, broadcast/2, broadcast/3, - confirmed_broadcast/2, info/1, validate_members/2, forget_group/1]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3, prioritise_info/3]). - -%% For INSTR_MOD callbacks --export([call/3, cast/2, monitor/1, demonitor/1]). - --export([table_definitions/0]). - --define(GROUP_TABLE, gm_group). --define(MAX_BUFFER_SIZE, 100000000). %% 100MB --define(BROADCAST_TIMER, 25). --define(FORCE_GC_TIMER, 250). --define(VERSION_START, 0). --define(SETS, ordsets). - --record(state, - { self, - left, - right, - group_name, - module, - view, - pub_count, - members_state, - callback_args, - confirms, - broadcast_buffer, - broadcast_buffer_sz, - broadcast_timer, - force_gc_timer, - txn_executor, - shutting_down - }). - --record(gm_group, { name, version, members }). - --record(view_member, { id, aliases, left, right }). - --record(member, { pending_ack, last_pub, last_ack }). - --define(TABLE, {?GROUP_TABLE, [{record_name, gm_group}, - {attributes, record_info(fields, gm_group)}]}). --define(TABLE_MATCH, {match, #gm_group { _ = '_' }}). - --define(TAG, '$gm'). - --export_type([group_name/0]). - --type group_name() :: any(). --type txn_fun() :: fun((fun(() -> any())) -> any()). - -%% The joined, members_changed and handle_msg callbacks can all return -%% any of the following terms: -%% -%% 'ok' - the callback function returns normally -%% -%% {'stop', Reason} - the callback indicates the member should stop -%% with reason Reason and should leave the group. -%% -%% {'become', Module, Args} - the callback indicates that the callback -%% module should be changed to Module and that the callback functions -%% should now be passed the arguments Args. This allows the callback -%% module to be dynamically changed. - -%% Called when we've successfully joined the group. Supplied with Args -%% provided in start_link, plus current group members. --callback joined(Args :: term(), Members :: [pid()]) -> - ok | {stop, Reason :: term()} | {become, Module :: atom(), Args :: any()}. - -%% Supplied with Args provided in start_link, the list of new members -%% and the list of members previously known to us that have since -%% died. Note that if a member joins and dies very quickly, it's -%% possible that we will never see that member appear in either births -%% or deaths. However we are guaranteed that (1) we will see a member -%% joining either in the births here, or in the members passed to -%% joined/2 before receiving any messages from it; and (2) we will not -%% see members die that we have not seen born (or supplied in the -%% members to joined/2). --callback members_changed(Args :: term(), - Births :: [pid()], Deaths :: [pid()]) -> - ok | {stop, Reason :: term()} | {become, Module :: atom(), Args :: any()}. - -%% Supplied with Args provided in start_link, the sender, and the -%% message. This does get called for messages injected by this member, -%% however, in such cases, there is no special significance of this -%% invocation: it does not indicate that the message has made it to -%% any other members, let alone all other members. --callback handle_msg(Args :: term(), From :: pid(), Message :: term()) -> - ok | {stop, Reason :: term()} | {become, Module :: atom(), Args :: any()}. - -%% Called on gm member termination as per rules in gen_server, with -%% the Args provided in start_link plus the termination Reason. --callback handle_terminate(Args :: term(), Reason :: term()) -> - ok | term(). - --spec create_tables() -> 'ok' | {'aborted', any()}. - -create_tables() -> - create_tables([?TABLE]). - -create_tables([]) -> - ok; -create_tables([{Table, Attributes} | Tables]) -> - case mnesia:create_table(Table, Attributes) of - {atomic, ok} -> create_tables(Tables); - {aborted, {already_exists, Table}} -> create_tables(Tables); - Err -> Err - end. - -table_definitions() -> - {Name, Attributes} = ?TABLE, - [{Name, [?TABLE_MATCH | Attributes]}]. - --spec start_link(group_name(), atom(), any(), txn_fun()) -> - rabbit_types:ok_pid_or_error(). - -start_link(GroupName, Module, Args, TxnFun) -> - gen_server2:start_link(?MODULE, [GroupName, Module, Args, TxnFun], - [{spawn_opt, [{fullsweep_after, 0}]}]). - --spec leave(pid()) -> 'ok'. - -leave(Server) -> - gen_server2:cast(Server, leave). - --spec broadcast(pid(), any()) -> 'ok'. - -broadcast(Server, Msg) -> broadcast(Server, Msg, 0). - -broadcast(Server, Msg, SizeHint) -> - gen_server2:cast(Server, {broadcast, Msg, SizeHint}). - --spec confirmed_broadcast(pid(), any()) -> 'ok'. - -confirmed_broadcast(Server, Msg) -> - gen_server2:call(Server, {confirmed_broadcast, Msg}, infinity). - --spec info(pid()) -> rabbit_types:infos(). - -info(Server) -> - gen_server2:call(Server, info, infinity). - --spec validate_members(pid(), [pid()]) -> 'ok'. - -validate_members(Server, Members) -> - gen_server2:cast(Server, {validate_members, Members}). - --spec forget_group(group_name()) -> 'ok'. - -forget_group(GroupName) -> - {atomic, ok} = mnesia:sync_transaction( - fun () -> - mnesia:delete({?GROUP_TABLE, GroupName}) - end), - ok. - -init([GroupName, Module, Args, TxnFun]) -> - put(process_name, {?MODULE, GroupName}), - Self = make_member(GroupName), - gen_server2:cast(self(), join), - {ok, #state { self = Self, - left = {Self, undefined}, - right = {Self, undefined}, - group_name = GroupName, - module = Module, - view = undefined, - pub_count = -1, - members_state = undefined, - callback_args = Args, - confirms = queue:new(), - broadcast_buffer = [], - broadcast_buffer_sz = 0, - broadcast_timer = undefined, - force_gc_timer = undefined, - txn_executor = TxnFun, - shutting_down = false }}. - - -handle_call({confirmed_broadcast, _Msg}, _From, - State = #state { shutting_down = {true, _} }) -> - reply(shutting_down, State); - -handle_call({confirmed_broadcast, _Msg}, _From, - State = #state { members_state = undefined }) -> - reply(not_joined, State); - -handle_call({confirmed_broadcast, Msg}, _From, - State = #state { self = Self, - right = {Self, undefined}, - module = Module, - callback_args = Args }) -> - handle_callback_result({Module:handle_msg(Args, get_pid(Self), Msg), - ok, State}); - -handle_call({confirmed_broadcast, Msg}, From, State) -> - {Result, State1 = #state { pub_count = PubCount, confirms = Confirms }} = - internal_broadcast(Msg, 0, State), - Confirms1 = queue:in({PubCount, From}, Confirms), - handle_callback_result({Result, flush_broadcast_buffer( - State1 #state { confirms = Confirms1 })}); - -handle_call(info, _From, - State = #state { members_state = undefined }) -> - reply(not_joined, State); - -handle_call(info, _From, State = #state { group_name = GroupName, - module = Module, - view = View }) -> - reply([{group_name, GroupName}, - {module, Module}, - {group_members, get_pids(alive_view_members(View))}], State); - -handle_call({add_on_right, _NewMember}, _From, - State = #state { members_state = undefined }) -> - reply(not_ready, State); - -handle_call({add_on_right, NewMember}, _From, - State = #state { self = Self, - group_name = GroupName, - members_state = MembersState, - txn_executor = TxnFun }) -> - try - Group = record_new_member_in_group( - NewMember, Self, GroupName, TxnFun), - View1 = group_to_view(check_membership(Self, Group)), - MembersState1 = remove_erased_members(MembersState, View1), - ok = send_right(NewMember, View1, - {catchup, Self, prepare_members_state(MembersState1)}), - {Result, State1} = change_view(View1, State #state { - members_state = MembersState1 }), - handle_callback_result({Result, {ok, Group}, State1}) - catch - lost_membership -> - {stop, shutdown, State} - end. - -%% add_on_right causes a catchup to be sent immediately from the left, -%% so we can never see this from the left neighbour. However, it's -%% possible for the right neighbour to send us a check_neighbours -%% immediately before that. We can't possibly handle it, but if we're -%% in this state we know a catchup is coming imminently anyway. So -%% just ignore it. -handle_cast({?TAG, _ReqVer, check_neighbours}, - State = #state { members_state = undefined }) -> - noreply(State); - -handle_cast({?TAG, ReqVer, Msg}, - State = #state { view = View, - self = Self, - members_state = MembersState, - group_name = GroupName }) -> - try - {Result, State1} = - case needs_view_update(ReqVer, View) of - true -> - View1 = group_to_view( - check_membership(Self, - dirty_read_group(GroupName))), - MemberState1 = remove_erased_members(MembersState, View1), - change_view(View1, State #state { - members_state = MemberState1 }); - false -> {ok, State} - end, - handle_callback_result( - if_callback_success( - Result, fun handle_msg_true/3, fun handle_msg_false/3, Msg, State1)) - catch - lost_membership -> - {stop, shutdown, State} - end; - -handle_cast({broadcast, _Msg, _SizeHint}, - State = #state { shutting_down = {true, _} }) -> - noreply(State); - -handle_cast({broadcast, _Msg, _SizeHint}, - State = #state { members_state = undefined }) -> - noreply(State); - -handle_cast({broadcast, Msg, _SizeHint}, - State = #state { self = Self, - right = {Self, undefined}, - module = Module, - callback_args = Args }) -> - handle_callback_result({Module:handle_msg(Args, get_pid(Self), Msg), - State}); - -handle_cast({broadcast, Msg, SizeHint}, State) -> - {Result, State1} = internal_broadcast(Msg, SizeHint, State), - handle_callback_result({Result, maybe_flush_broadcast_buffer(State1)}); - -handle_cast(join, State = #state { self = Self, - group_name = GroupName, - members_state = undefined, - module = Module, - callback_args = Args, - txn_executor = TxnFun }) -> - try - View = join_group(Self, GroupName, TxnFun), - MembersState = - case alive_view_members(View) of - [Self] -> blank_member_state(); - _ -> undefined - end, - State1 = check_neighbours(State #state { view = View, - members_state = MembersState }), - handle_callback_result( - {Module:joined(Args, get_pids(all_known_members(View))), State1}) - catch - lost_membership -> - {stop, shutdown, State} - end; - -handle_cast({validate_members, OldMembers}, - State = #state { view = View, - module = Module, - callback_args = Args }) -> - NewMembers = get_pids(all_known_members(View)), - Births = NewMembers -- OldMembers, - Deaths = OldMembers -- NewMembers, - case {Births, Deaths} of - {[], []} -> noreply(State); - _ -> Result = Module:members_changed(Args, Births, Deaths), - handle_callback_result({Result, State}) - end; - -handle_cast(leave, State) -> - {stop, normal, State}. - - -handle_info(force_gc, State) -> - garbage_collect(), - noreply(State #state { force_gc_timer = undefined }); - -handle_info(flush, State) -> - noreply( - flush_broadcast_buffer(State #state { broadcast_timer = undefined })); - -handle_info(timeout, State) -> - noreply(flush_broadcast_buffer(State)); - -handle_info({'DOWN', _MRef, process, _Pid, _Reason}, - State = #state { shutting_down = - {true, {shutdown, ring_shutdown}} }) -> - noreply(State); -handle_info({'DOWN', MRef, process, _Pid, Reason}, - State = #state { self = Self, - left = Left, - right = Right, - group_name = GroupName, - confirms = Confirms, - txn_executor = TxnFun }) -> - try - check_membership(GroupName), - Member = case {Left, Right} of - {{Member1, MRef}, _} -> Member1; - {_, {Member1, MRef}} -> Member1; - _ -> undefined - end, - case {Member, Reason} of - {undefined, _} -> - noreply(State); - {_, {shutdown, ring_shutdown}} -> - noreply(State); - _ -> - %% In the event of a partial partition we could see another member - %% go down and then remove them from Mnesia. While they can - %% recover from this they'd have to restart the queue - not - %% ideal. So let's sleep here briefly just in case this was caused - %% by a partial partition; in which case by the time we record the - %% member death in Mnesia we will probably be in a full - %% partition and will not be assassinating another member. - timer:sleep(100), - View1 = group_to_view(record_dead_member_in_group(Self, - Member, GroupName, TxnFun, true)), - handle_callback_result( - case alive_view_members(View1) of - [Self] -> maybe_erase_aliases( - State #state { - members_state = blank_member_state(), - confirms = purge_confirms(Confirms) }, - View1); - _ -> change_view(View1, State) - end) - end - catch - lost_membership -> - {stop, shutdown, State} - end; -handle_info(_, State) -> - %% Discard any unexpected messages, such as late replies from neighbour_call/2 - %% TODO: For #gm_group{} related info messages, it could be worthwhile to - %% change_view/2, as this might reflect an alteration in the gm group, meaning - %% we now need to update our state. see rabbitmq-server#914. - noreply(State). - -terminate(Reason, #state { module = Module, callback_args = Args }) -> - Module:handle_terminate(Args, Reason). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -prioritise_info(flush, _Len, _State) -> - 1; -%% DOWN messages should not overtake initial catchups; if they do we -%% will receive a DOWN we do not know what to do with. -prioritise_info({'DOWN', _MRef, process, _Pid, _Reason}, _Len, - #state { members_state = undefined }) -> - 0; -%% We should not prioritise DOWN messages from our left since -%% otherwise the DOWN can overtake any last activity from the left, -%% causing that activity to be lost. -prioritise_info({'DOWN', _MRef, process, LeftPid, _Reason}, _Len, - #state { left = {{_LeftVer, LeftPid}, _MRef2} }) -> - 0; -%% But prioritise all other DOWNs - we want to make sure we are not -%% sending activity into the void for too long because our right is -%% down but we don't know it. -prioritise_info({'DOWN', _MRef, process, _Pid, _Reason}, _Len, _State) -> - 1; -prioritise_info(_, _Len, _State) -> - 0. - - -handle_msg(check_neighbours, State) -> - %% no-op - it's already been done by the calling handle_cast - {ok, State}; - -handle_msg({catchup, Left, MembersStateLeft}, - State = #state { self = Self, - left = {Left, _MRefL}, - right = {Right, _MRefR}, - view = View, - members_state = undefined }) -> - ok = send_right(Right, View, {catchup, Self, MembersStateLeft}), - MembersStateLeft1 = build_members_state(MembersStateLeft), - {ok, State #state { members_state = MembersStateLeft1 }}; - -handle_msg({catchup, Left, MembersStateLeft}, - State = #state { self = Self, - left = {Left, _MRefL}, - view = View, - members_state = MembersState }) - when MembersState =/= undefined -> - MembersStateLeft1 = build_members_state(MembersStateLeft), - AllMembers = lists:usort(maps:keys(MembersState) ++ - maps:keys(MembersStateLeft1)), - {MembersState1, Activity} = - lists:foldl( - fun (Id, MembersStateActivity) -> - #member { pending_ack = PALeft, last_ack = LA } = - find_member_or_blank(Id, MembersStateLeft1), - with_member_acc( - fun (#member { pending_ack = PA } = Member, Activity1) -> - case is_member_alias(Id, Self, View) of - true -> - {_AcksInFlight, Pubs, _PA1} = - find_prefix_common_suffix(PALeft, PA), - {Member #member { last_ack = LA }, - activity_cons(Id, pubs_from_queue(Pubs), - [], Activity1)}; - false -> - {Acks, _Common, Pubs} = - find_prefix_common_suffix(PA, PALeft), - {Member, - activity_cons(Id, pubs_from_queue(Pubs), - acks_from_queue(Acks), - Activity1)} - end - end, Id, MembersStateActivity) - end, {MembersState, activity_nil()}, AllMembers), - handle_msg({activity, Left, activity_finalise(Activity)}, - State #state { members_state = MembersState1 }); - -handle_msg({catchup, _NotLeft, _MembersState}, State) -> - {ok, State}; - -handle_msg({activity, Left, Activity}, - State = #state { self = Self, - group_name = GroupName, - left = {Left, _MRefL}, - view = View, - members_state = MembersState, - confirms = Confirms }) - when MembersState =/= undefined -> - try - %% If we have to stop, do it asap so we avoid any ack confirmation - %% Membership must be checked again by erase_members_in_group, as the - %% node can be marked as dead on the meanwhile - check_membership(GroupName), - {MembersState1, {Confirms1, Activity1}} = - calculate_activity(MembersState, Confirms, Activity, Self, View), - State1 = State #state { members_state = MembersState1, - confirms = Confirms1 }, - Activity3 = activity_finalise(Activity1), - ok = maybe_send_activity(Activity3, State1), - {Result, State2} = maybe_erase_aliases(State1, View), - if_callback_success( - Result, fun activity_true/3, fun activity_false/3, Activity3, State2) - catch - lost_membership -> - {{stop, shutdown}, State} - end; - -handle_msg({activity, _NotLeft, _Activity}, State) -> - {ok, State}. - - -noreply(State) -> - {noreply, ensure_timers(State), flush_timeout(State)}. - -reply(Reply, State) -> - {reply, Reply, ensure_timers(State), flush_timeout(State)}. - -ensure_timers(State) -> - ensure_force_gc_timer(ensure_broadcast_timer(State)). - -flush_timeout(#state{broadcast_buffer = []}) -> infinity; -flush_timeout(_) -> 0. - -ensure_force_gc_timer(State = #state { force_gc_timer = TRef }) - when is_reference(TRef) -> - State; -ensure_force_gc_timer(State = #state { force_gc_timer = undefined }) -> - TRef = erlang:send_after(?FORCE_GC_TIMER, self(), force_gc), - State #state { force_gc_timer = TRef }. - -ensure_broadcast_timer(State = #state { broadcast_buffer = [], - broadcast_timer = undefined }) -> - State; -ensure_broadcast_timer(State = #state { broadcast_buffer = [], - broadcast_timer = TRef }) -> - _ = erlang:cancel_timer(TRef), - State #state { broadcast_timer = undefined }; -ensure_broadcast_timer(State = #state { broadcast_timer = undefined }) -> - TRef = erlang:send_after(?BROADCAST_TIMER, self(), flush), - State #state { broadcast_timer = TRef }; -ensure_broadcast_timer(State) -> - State. - -internal_broadcast(Msg, SizeHint, - State = #state { self = Self, - pub_count = PubCount, - module = Module, - callback_args = Args, - broadcast_buffer = Buffer, - broadcast_buffer_sz = BufferSize }) -> - PubCount1 = PubCount + 1, - {Module:handle_msg(Args, get_pid(Self), Msg), - State #state { pub_count = PubCount1, - broadcast_buffer = [{PubCount1, Msg} | Buffer], - broadcast_buffer_sz = BufferSize + SizeHint}}. - -%% The Erlang distribution mechanism has an interesting quirk - it -%% will kill the VM cold with "Absurdly large distribution output data -%% buffer" if you attempt to send a message which serialises out to -%% more than 2^31 bytes in size. It's therefore a very good idea to -%% make sure that we don't exceed that size! -%% -%% Now, we could figure out the size of messages as they come in using -%% size(term_to_binary(Msg)) or similar. The trouble is, that requires -%% us to serialise the message only to throw the serialised form -%% away. Hard to believe that's a sensible thing to do. So instead we -%% accept a size hint from the application, via broadcast/3. This size -%% hint can be the size of anything in the message which we expect -%% could be large, and we just ignore the size of any small bits of -%% the message term. Therefore MAX_BUFFER_SIZE is set somewhat -%% conservatively at 100MB - but the buffer is only to allow us to -%% buffer tiny messages anyway, so 100MB is plenty. - -maybe_flush_broadcast_buffer(State = #state{broadcast_buffer_sz = Size}) -> - case Size > ?MAX_BUFFER_SIZE of - true -> flush_broadcast_buffer(State); - false -> State - end. - -flush_broadcast_buffer(State = #state { broadcast_buffer = [] }) -> - State; -flush_broadcast_buffer(State = #state { self = Self, - members_state = MembersState, - broadcast_buffer = Buffer, - pub_count = PubCount }) -> - [{PubCount, _Msg}|_] = Buffer, %% ASSERTION match on PubCount - Pubs = lists:reverse(Buffer), - Activity = activity_cons(Self, Pubs, [], activity_nil()), - ok = maybe_send_activity(activity_finalise(Activity), State), - MembersState1 = with_member( - fun (Member = #member { pending_ack = PA }) -> - PA1 = queue:join(PA, queue:from_list(Pubs)), - Member #member { pending_ack = PA1, - last_pub = PubCount } - end, Self, MembersState), - State #state { members_state = MembersState1, - broadcast_buffer = [], - broadcast_buffer_sz = 0 }. - -%% --------------------------------------------------------------------------- -%% View construction and inspection -%% --------------------------------------------------------------------------- - -needs_view_update(ReqVer, {Ver, _View}) -> Ver < ReqVer. - -view_version({Ver, _View}) -> Ver. - -is_member_alive({dead, _Member}) -> false; -is_member_alive(_) -> true. - -is_member_alias(Self, Self, _View) -> - true; -is_member_alias(Member, Self, View) -> - ?SETS:is_element(Member, - ((fetch_view_member(Self, View)) #view_member.aliases)). - -dead_member_id({dead, Member}) -> Member. - -store_view_member(VMember = #view_member { id = Id }, {Ver, View}) -> - {Ver, maps:put(Id, VMember, View)}. - -with_view_member(Fun, View, Id) -> - store_view_member(Fun(fetch_view_member(Id, View)), View). - -fetch_view_member(Id, {_Ver, View}) -> maps:get(Id, View). - -find_view_member(Id, {_Ver, View}) -> maps:find(Id, View). - -blank_view(Ver) -> {Ver, maps:new()}. - -alive_view_members({_Ver, View}) -> maps:keys(View). - -all_known_members({_Ver, View}) -> - maps:fold( - fun (Member, #view_member { aliases = Aliases }, Acc) -> - ?SETS:to_list(Aliases) ++ [Member | Acc] - end, [], View). - -group_to_view(#gm_group { members = Members, version = Ver }) -> - Alive = lists:filter(fun is_member_alive/1, Members), - [_|_] = Alive, %% ASSERTION - can't have all dead members - add_aliases(link_view(Alive ++ Alive ++ Alive, blank_view(Ver)), Members). - -link_view([Left, Middle, Right | Rest], View) -> - case find_view_member(Middle, View) of - error -> - link_view( - [Middle, Right | Rest], - store_view_member(#view_member { id = Middle, - aliases = ?SETS:new(), - left = Left, - right = Right }, View)); - {ok, _} -> - View - end; -link_view(_, View) -> - View. - -add_aliases(View, Members) -> - Members1 = ensure_alive_suffix(Members), - {EmptyDeadSet, View1} = - lists:foldl( - fun (Member, {DeadAcc, ViewAcc}) -> - case is_member_alive(Member) of - true -> - {?SETS:new(), - with_view_member( - fun (VMember = - #view_member { aliases = Aliases }) -> - VMember #view_member { - aliases = ?SETS:union(Aliases, DeadAcc) } - end, ViewAcc, Member)}; - false -> - {?SETS:add_element(dead_member_id(Member), DeadAcc), - ViewAcc} - end - end, {?SETS:new(), View}, Members1), - 0 = ?SETS:size(EmptyDeadSet), %% ASSERTION - View1. - -ensure_alive_suffix(Members) -> - queue:to_list(ensure_alive_suffix1(queue:from_list(Members))). - -ensure_alive_suffix1(MembersQ) -> - {{value, Member}, MembersQ1} = queue:out_r(MembersQ), - case is_member_alive(Member) of - true -> MembersQ; - false -> ensure_alive_suffix1(queue:in_r(Member, MembersQ1)) - end. - - -%% --------------------------------------------------------------------------- -%% View modification -%% --------------------------------------------------------------------------- - -join_group(Self, GroupName, TxnFun) -> - join_group(Self, GroupName, dirty_read_group(GroupName), TxnFun). - -join_group(Self, GroupName, {error, not_found}, TxnFun) -> - join_group(Self, GroupName, - prune_or_create_group(Self, GroupName, TxnFun), TxnFun); -join_group(Self, _GroupName, #gm_group { members = [Self] } = Group, _TxnFun) -> - group_to_view(Group); -join_group(Self, GroupName, #gm_group { members = Members } = Group, TxnFun) -> - case lists:member(Self, Members) of - true -> - group_to_view(Group); - false -> - case lists:filter(fun is_member_alive/1, Members) of - [] -> - join_group(Self, GroupName, - prune_or_create_group(Self, GroupName, TxnFun), - TxnFun); - Alive -> - Left = lists:nth(rand:uniform(length(Alive)), Alive), - Handler = - fun () -> - join_group( - Self, GroupName, - record_dead_member_in_group(Self, - Left, GroupName, TxnFun, false), - TxnFun) - end, - try - case neighbour_call(Left, {add_on_right, Self}) of - {ok, Group1} -> group_to_view(Group1); - not_ready -> join_group(Self, GroupName, TxnFun) - end - catch - exit:{R, _} - when R =:= noproc; R =:= normal; R =:= shutdown -> - Handler(); - exit:{{R, _}, _} - when R =:= nodedown; R =:= shutdown -> - Handler() - end - end - end. - -dirty_read_group(GroupName) -> - case mnesia:dirty_read(?GROUP_TABLE, GroupName) of - [] -> {error, not_found}; - [Group] -> Group - end. - -read_group(GroupName) -> - case mnesia:read({?GROUP_TABLE, GroupName}) of - [] -> {error, not_found}; - [Group] -> Group - end. - -write_group(Group) -> mnesia:write(?GROUP_TABLE, Group, write), Group. - -prune_or_create_group(Self, GroupName, TxnFun) -> - TxnFun( - fun () -> - GroupNew = #gm_group { name = GroupName, - members = [Self], - version = get_version(Self) }, - case read_group(GroupName) of - {error, not_found} -> - write_group(GroupNew); - Group = #gm_group { members = Members } -> - case lists:any(fun is_member_alive/1, Members) of - true -> Group; - false -> write_group(GroupNew) - end - end - end). - -record_dead_member_in_group(Self, Member, GroupName, TxnFun, Verify) -> - Fun = - fun () -> - try - Group = #gm_group { members = Members, version = Ver } = - case Verify of - true -> - check_membership(Self, read_group(GroupName)); - false -> - check_group(read_group(GroupName)) - end, - case lists:splitwith( - fun (Member1) -> Member1 =/= Member end, Members) of - {_Members1, []} -> %% not found - already recorded dead - Group; - {Members1, [Member | Members2]} -> - Members3 = Members1 ++ [{dead, Member} | Members2], - write_group(Group #gm_group { members = Members3, - version = Ver + 1 }) - end - catch - lost_membership -> - %% The transaction must not be abruptly crashed, but - %% leave the gen_server to stop normally - {error, lost_membership} - end - end, - handle_lost_membership_in_txn(TxnFun, Fun). - -handle_lost_membership_in_txn(TxnFun, Fun) -> - case TxnFun(Fun) of - {error, lost_membership = T} -> - throw(T); - Any -> - Any - end. - -record_new_member_in_group(NewMember, Left, GroupName, TxnFun) -> - Fun = - fun () -> - try - Group = #gm_group { members = Members, version = Ver } = - check_membership(Left, read_group(GroupName)), - case lists:member(NewMember, Members) of - true -> - %% This avois duplicates during partial partitions, - %% as inconsistent views might happen during them - rabbit_log:warning("(~p) GM avoiding duplicate of ~p", - [self(), NewMember]), - Group; - false -> - {Prefix, [Left | Suffix]} = - lists:splitwith(fun (M) -> M =/= Left end, Members), - write_group(Group #gm_group { - members = Prefix ++ [Left, NewMember | Suffix], - version = Ver + 1 }) - end - catch - lost_membership -> - %% The transaction must not be abruptly crashed, but - %% leave the gen_server to stop normally - {error, lost_membership} - end - end, - handle_lost_membership_in_txn(TxnFun, Fun). - -erase_members_in_group(Self, Members, GroupName, TxnFun) -> - DeadMembers = [{dead, Id} || Id <- Members], - Fun = - fun () -> - try - Group = #gm_group { members = [_|_] = Members1, version = Ver } = - check_membership(Self, read_group(GroupName)), - case Members1 -- DeadMembers of - Members1 -> Group; - Members2 -> write_group( - Group #gm_group { members = Members2, - version = Ver + 1 }) - end - catch - lost_membership -> - %% The transaction must not be abruptly crashed, but - %% leave the gen_server to stop normally - {error, lost_membership} - end - end, - handle_lost_membership_in_txn(TxnFun, Fun). - -maybe_erase_aliases(State = #state { self = Self, - group_name = GroupName, - members_state = MembersState, - txn_executor = TxnFun }, View) -> - #view_member { aliases = Aliases } = fetch_view_member(Self, View), - {Erasable, MembersState1} - = ?SETS:fold( - fun (Id, {ErasableAcc, MembersStateAcc} = Acc) -> - #member { last_pub = LP, last_ack = LA } = - find_member_or_blank(Id, MembersState), - case can_erase_view_member(Self, Id, LA, LP) of - true -> {[Id | ErasableAcc], - erase_member(Id, MembersStateAcc)}; - false -> Acc - end - end, {[], MembersState}, Aliases), - View1 = case Erasable of - [] -> View; - _ -> group_to_view( - erase_members_in_group(Self, Erasable, GroupName, TxnFun)) - end, - change_view(View1, State #state { members_state = MembersState1 }). - -can_erase_view_member(Self, Self, _LA, _LP) -> false; -can_erase_view_member(_Self, _Id, N, N) -> true; -can_erase_view_member(_Self, _Id, _LA, _LP) -> false. - -neighbour_cast(N, Msg) -> ?INSTR_MOD:cast(get_pid(N), Msg). -neighbour_call(N, Msg) -> ?INSTR_MOD:call(get_pid(N), Msg, infinity). - -%% --------------------------------------------------------------------------- -%% View monitoring and maintenance -%% --------------------------------------------------------------------------- - -ensure_neighbour(_Ver, Self, {Self, undefined}, Self) -> - {Self, undefined}; -ensure_neighbour(Ver, Self, {Self, undefined}, RealNeighbour) -> - ok = neighbour_cast(RealNeighbour, {?TAG, Ver, check_neighbours}), - {RealNeighbour, maybe_monitor(RealNeighbour, Self)}; -ensure_neighbour(_Ver, _Self, {RealNeighbour, MRef}, RealNeighbour) -> - {RealNeighbour, MRef}; -ensure_neighbour(Ver, Self, {RealNeighbour, MRef}, Neighbour) -> - true = ?INSTR_MOD:demonitor(MRef), - Msg = {?TAG, Ver, check_neighbours}, - ok = neighbour_cast(RealNeighbour, Msg), - ok = case Neighbour of - Self -> ok; - _ -> neighbour_cast(Neighbour, Msg) - end, - {Neighbour, maybe_monitor(Neighbour, Self)}. - -maybe_monitor( Self, Self) -> undefined; -maybe_monitor(Other, _Self) -> ?INSTR_MOD:monitor(get_pid(Other)). - -check_neighbours(State = #state { self = Self, - left = Left, - right = Right, - view = View, - broadcast_buffer = Buffer }) -> - #view_member { left = VLeft, right = VRight } - = fetch_view_member(Self, View), - Ver = view_version(View), - Left1 = ensure_neighbour(Ver, Self, Left, VLeft), - Right1 = ensure_neighbour(Ver, Self, Right, VRight), - Buffer1 = case Right1 of - {Self, undefined} -> []; - _ -> Buffer - end, - State1 = State #state { left = Left1, right = Right1, - broadcast_buffer = Buffer1 }, - ok = maybe_send_catchup(Right, State1), - State1. - -maybe_send_catchup(Right, #state { right = Right }) -> - ok; -maybe_send_catchup(_Right, #state { self = Self, - right = {Self, undefined} }) -> - ok; -maybe_send_catchup(_Right, #state { members_state = undefined }) -> - ok; -maybe_send_catchup(_Right, #state { self = Self, - right = {Right, _MRef}, - view = View, - members_state = MembersState }) -> - send_right(Right, View, - {catchup, Self, prepare_members_state(MembersState)}). - - -%% --------------------------------------------------------------------------- -%% Catch_up delta detection -%% --------------------------------------------------------------------------- - -find_prefix_common_suffix(A, B) -> - {Prefix, A1} = find_prefix(A, B, queue:new()), - {Common, Suffix} = find_common(A1, B, queue:new()), - {Prefix, Common, Suffix}. - -%% Returns the elements of A that occur before the first element of B, -%% plus the remainder of A. -find_prefix(A, B, Prefix) -> - case {queue:out(A), queue:out(B)} of - {{{value, Val}, _A1}, {{value, Val}, _B1}} -> - {Prefix, A}; - {{empty, A1}, {{value, _A}, _B1}} -> - {Prefix, A1}; - {{{value, {NumA, _MsgA} = Val}, A1}, - {{value, {NumB, _MsgB}}, _B1}} when NumA < NumB -> - find_prefix(A1, B, queue:in(Val, Prefix)); - {_, {empty, _B1}} -> - {A, Prefix} %% Prefix well be empty here - end. - -%% A should be a prefix of B. Returns the commonality plus the -%% remainder of B. -find_common(A, B, Common) -> - case {queue:out(A), queue:out(B)} of - {{{value, Val}, A1}, {{value, Val}, B1}} -> - find_common(A1, B1, queue:in(Val, Common)); - {{empty, _A}, _} -> - {Common, B}; - %% Drop value from B. - %% Match value to avoid infinite loop, since {empty, B} = queue:out(B). - {_, {{value, _}, B1}} -> - find_common(A, B1, Common); - %% Drop value from A. Empty A should be matched by second close. - {{{value, _}, A1}, _} -> - find_common(A1, B, Common) - end. - - -%% --------------------------------------------------------------------------- -%% Members helpers -%% --------------------------------------------------------------------------- - -with_member(Fun, Id, MembersState) -> - store_member( - Id, Fun(find_member_or_blank(Id, MembersState)), MembersState). - -with_member_acc(Fun, Id, {MembersState, Acc}) -> - {MemberState, Acc1} = Fun(find_member_or_blank(Id, MembersState), Acc), - {store_member(Id, MemberState, MembersState), Acc1}. - -find_member_or_blank(Id, MembersState) -> - case maps:find(Id, MembersState) of - {ok, Result} -> Result; - error -> blank_member() - end. - -erase_member(Id, MembersState) -> maps:remove(Id, MembersState). - -blank_member() -> - #member { pending_ack = queue:new(), last_pub = -1, last_ack = -1 }. - -blank_member_state() -> maps:new(). - -store_member(Id, MemberState, MembersState) -> - maps:put(Id, MemberState, MembersState). - -prepare_members_state(MembersState) -> maps:to_list(MembersState). - -build_members_state(MembersStateList) -> maps:from_list(MembersStateList). - -make_member(GroupName) -> - {case dirty_read_group(GroupName) of - #gm_group { version = Version } -> Version; - {error, not_found} -> ?VERSION_START - end, self()}. - -remove_erased_members(MembersState, View) -> - lists:foldl(fun (Id, MembersState1) -> - store_member(Id, find_member_or_blank(Id, MembersState), - MembersState1) - end, blank_member_state(), all_known_members(View)). - -get_version({Version, _Pid}) -> Version. - -get_pid({_Version, Pid}) -> Pid. - -get_pids(Ids) -> [Pid || {_Version, Pid} <- Ids]. - -%% --------------------------------------------------------------------------- -%% Activity assembly -%% --------------------------------------------------------------------------- - -activity_nil() -> queue:new(). - -activity_cons( _Id, [], [], Tail) -> Tail; -activity_cons(Sender, Pubs, Acks, Tail) -> queue:in({Sender, Pubs, Acks}, Tail). - -activity_finalise(Activity) -> queue:to_list(Activity). - -maybe_send_activity([], _State) -> - ok; -maybe_send_activity(Activity, #state { self = Self, - right = {Right, _MRefR}, - view = View }) -> - send_right(Right, View, {activity, Self, Activity}). - -send_right(Right, View, Msg) -> - ok = neighbour_cast(Right, {?TAG, view_version(View), Msg}). - -calculate_activity(MembersState, Confirms, Activity, Self, View) -> - lists:foldl( - fun ({Id, Pubs, Acks}, MembersStateConfirmsActivity) -> - with_member_acc( - fun (Member = #member { pending_ack = PA, - last_pub = LP, - last_ack = LA }, - {Confirms2, Activity2}) -> - case is_member_alias(Id, Self, View) of - true -> - {ToAck, PA1} = - find_common(queue_from_pubs(Pubs), PA, - queue:new()), - LA1 = last_ack(Acks, LA), - AckNums = acks_from_queue(ToAck), - Confirms3 = maybe_confirm( - Self, Id, Confirms2, AckNums), - {Member #member { pending_ack = PA1, - last_ack = LA1 }, - {Confirms3, - activity_cons( - Id, [], AckNums, Activity2)}}; - false -> - PA1 = apply_acks(Acks, join_pubs(PA, Pubs)), - LA1 = last_ack(Acks, LA), - LP1 = last_pub(Pubs, LP), - {Member #member { pending_ack = PA1, - last_pub = LP1, - last_ack = LA1 }, - {Confirms2, - activity_cons(Id, Pubs, Acks, Activity2)}} - end - end, Id, MembersStateConfirmsActivity) - end, {MembersState, {Confirms, activity_nil()}}, Activity). - -callback(Args, Module, Activity) -> - Result = - lists:foldl( - fun ({Id, Pubs, _Acks}, {Args1, Module1, ok}) -> - lists:foldl(fun ({_PubNum, Pub}, Acc = {Args2, Module2, ok}) -> - case Module2:handle_msg( - Args2, get_pid(Id), Pub) of - ok -> - Acc; - {become, Module3, Args3} -> - {Args3, Module3, ok}; - {stop, _Reason} = Error -> - Error - end; - (_, Error = {stop, _Reason}) -> - Error - end, {Args1, Module1, ok}, Pubs); - (_, Error = {stop, _Reason}) -> - Error - end, {Args, Module, ok}, Activity), - case Result of - {Args, Module, ok} -> ok; - {Args1, Module1, ok} -> {become, Module1, Args1}; - {stop, _Reason} = Error -> Error - end. - -change_view(View, State = #state { view = View0, - module = Module, - callback_args = Args }) -> - OldMembers = all_known_members(View0), - NewMembers = all_known_members(View), - Births = NewMembers -- OldMembers, - Deaths = OldMembers -- NewMembers, - Result = case {Births, Deaths} of - {[], []} -> ok; - _ -> Module:members_changed( - Args, get_pids(Births), get_pids(Deaths)) - end, - {Result, check_neighbours(State #state { view = View })}. - -handle_callback_result({Result, State}) -> - if_callback_success( - Result, fun no_reply_true/3, fun no_reply_false/3, undefined, State); -handle_callback_result({Result, Reply, State}) -> - if_callback_success( - Result, fun reply_true/3, fun reply_false/3, Reply, State). - -no_reply_true (_Result, _Undefined, State) -> noreply(State). -no_reply_false({stop, Reason}, _Undefined, State) -> {stop, Reason, State}. - -reply_true (_Result, Reply, State) -> reply(Reply, State). -reply_false({stop, Reason}, Reply, State) -> {stop, Reason, Reply, State}. - -handle_msg_true (_Result, Msg, State) -> handle_msg(Msg, State). -handle_msg_false(Result, _Msg, State) -> {Result, State}. - -activity_true(_Result, Activity, State = #state { module = Module, - callback_args = Args }) -> - {callback(Args, Module, Activity), State}. -activity_false(Result, _Activity, State) -> - {Result, State}. - -if_callback_success(Result, True, False, Arg, State) -> - {NewResult, NewState} = maybe_stop(Result, State), - if_callback_success1(NewResult, True, False, Arg, NewState). - -if_callback_success1(ok, True, _False, Arg, State) -> - True(ok, Arg, State); -if_callback_success1( - {become, Module, Args} = Result, True, _False, Arg, State) -> - True(Result, Arg, State #state { module = Module, - callback_args = Args }); -if_callback_success1({stop, _Reason} = Result, _True, False, Arg, State) -> - False(Result, Arg, State). - -maybe_stop({stop, Reason}, #state{ shutting_down = false } = State) -> - ShuttingDown = {true, Reason}, - case has_pending_messages(State) of - true -> {ok, State #state{ shutting_down = ShuttingDown }}; - false -> {{stop, Reason}, State #state{ shutting_down = ShuttingDown }} - end; -maybe_stop(Result, #state{ shutting_down = false } = State) -> - {Result, State}; -maybe_stop(Result, #state{ shutting_down = {true, Reason} } = State) -> - case has_pending_messages(State) of - true -> {Result, State}; - false -> {{stop, Reason}, State} - end. - -has_pending_messages(#state{ broadcast_buffer = Buffer }) - when Buffer =/= [] -> - true; -has_pending_messages(#state{ members_state = MembersState }) -> - MembersWithPubAckMismatches = maps:filter(fun(_Id, #member{last_pub = LP, last_ack = LA}) -> - LP =/= LA - end, MembersState), - 0 =/= maps:size(MembersWithPubAckMismatches). - -maybe_confirm(_Self, _Id, Confirms, []) -> - Confirms; -maybe_confirm(Self, Self, Confirms, [PubNum | PubNums]) -> - case queue:out(Confirms) of - {empty, _Confirms} -> - Confirms; - {{value, {PubNum, From}}, Confirms1} -> - gen_server2:reply(From, ok), - maybe_confirm(Self, Self, Confirms1, PubNums); - {{value, {PubNum1, _From}}, _Confirms} when PubNum1 > PubNum -> - maybe_confirm(Self, Self, Confirms, PubNums) - end; -maybe_confirm(_Self, _Id, Confirms, _PubNums) -> - Confirms. - -purge_confirms(Confirms) -> - _ = [gen_server2:reply(From, ok) || {_PubNum, From} <- queue:to_list(Confirms)], - queue:new(). - - -%% --------------------------------------------------------------------------- -%% Msg transformation -%% --------------------------------------------------------------------------- - -acks_from_queue(Q) -> [PubNum || {PubNum, _Msg} <- queue:to_list(Q)]. - -pubs_from_queue(Q) -> queue:to_list(Q). - -queue_from_pubs(Pubs) -> queue:from_list(Pubs). - -apply_acks( [], Pubs) -> Pubs; -apply_acks(List, Pubs) -> {_, Pubs1} = queue:split(length(List), Pubs), - Pubs1. - -join_pubs(Q, []) -> Q; -join_pubs(Q, Pubs) -> queue:join(Q, queue_from_pubs(Pubs)). - -last_ack( [], LA) -> LA; -last_ack(List, LA) -> LA1 = lists:last(List), - true = LA1 > LA, %% ASSERTION - LA1. - -last_pub( [], LP) -> LP; -last_pub(List, LP) -> {PubNum, _Msg} = lists:last(List), - true = PubNum > LP, %% ASSERTION - PubNum. - -%% --------------------------------------------------------------------------- - -%% Uninstrumented versions - -call(Pid, Msg, Timeout) -> gen_server2:call(Pid, Msg, Timeout). -cast(Pid, Msg) -> gen_server2:cast(Pid, Msg). -monitor(Pid) -> erlang:monitor(process, Pid). -demonitor(MRef) -> erlang:demonitor(MRef). - -check_membership(Self, #gm_group{members = M} = Group) -> - case lists:member(Self, M) of - true -> - Group; - false -> - throw(lost_membership) - end; -check_membership(_Self, {error, not_found}) -> - throw(lost_membership). - -check_membership(GroupName) -> - case dirty_read_group(GroupName) of - #gm_group{members = M} -> - case lists:keymember(self(), 2, M) of - true -> - ok; - false -> - throw(lost_membership) - end; - {error, not_found} -> - throw(lost_membership) - end. - -check_group({error, not_found}) -> - throw(lost_membership); -check_group(Any) -> - Any. diff --git a/src/internal_user.erl b/src/internal_user.erl deleted file mode 100644 index b2bdcb6785..0000000000 --- a/src/internal_user.erl +++ /dev/null @@ -1,216 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(internal_user). - --include_lib("rabbit_common/include/rabbit.hrl"). - --export([ - new/0, - new/1, - record_version_to_use/0, - fields/0, - fields/1, - upgrade/1, - upgrade_to/2, - pattern_match_all/0, - get_username/1, - get_password_hash/1, - get_tags/1, - get_hashing_algorithm/1, - get_limits/1, - create_user/3, - set_password_hash/3, - set_tags/2, - update_limits/3, - clear_limits/1 -]). - --define(record_version, internal_user_v2). - --type(username() :: binary()). - --type(password_hash() :: binary()). - --type internal_user() :: internal_user_v1:internal_user_v1() | internal_user_v2(). - --record(internal_user, { - username :: username() | '_', - password_hash :: password_hash() | '_', - tags :: [atom()] | '_', - %% password hashing implementation module, - %% typically rabbit_password_hashing_* but can - %% come from a plugin - hashing_algorithm :: atom() | '_', - limits = #{} :: map() | '_'}). - --type(internal_user_v2() :: - #internal_user{username :: username() | '_', - password_hash :: password_hash() | '_', - tags :: [atom()] | '_', - hashing_algorithm :: atom() | '_', - limits :: map()}). - --type internal_user_pattern() :: internal_user_v1:internal_user_v1_pattern() | - internal_user_v2_pattern(). - --type internal_user_v2_pattern() :: #internal_user{ - username :: username() | '_', - password_hash :: '_', - tags :: '_', - hashing_algorithm :: '_', - limits :: '_' - }. - --export_type([username/0, - password_hash/0, - internal_user/0, - internal_user_v2/0, - internal_user_pattern/0, - internal_user_v2_pattern/0]). - --spec new() -> internal_user(). -new() -> - case record_version_to_use() of - ?record_version -> - #internal_user{ - username = <<"">>, - password_hash = <<"">>, - tags = [] - }; - _ -> - internal_user_v1:new() - end. - --spec new(tuple()) -> internal_user(). -new({hashing_algorithm, HashingAlgorithm}) -> - case record_version_to_use() of - ?record_version -> - #internal_user{ - username = <<"">>, - password_hash = <<"">>, - tags = [], - hashing_algorithm = HashingAlgorithm - }; - _ -> - internal_user_v1:new({hashing_algorithm, HashingAlgorithm}) - end; -new({tags, Tags}) -> - case record_version_to_use() of - ?record_version -> - #internal_user{ - username = <<"">>, - password_hash = <<"">>, - tags = Tags - }; - _ -> - internal_user_v1:new({tags, Tags}) - end. - --spec record_version_to_use() -> internal_user_v1 | internal_user_v2. -record_version_to_use() -> - case rabbit_feature_flags:is_enabled(user_limits) of - true -> ?record_version; - false -> internal_user_v1:record_version_to_use() - end. - --spec fields() -> list(). -fields() -> - case record_version_to_use() of - ?record_version -> fields(?record_version); - _ -> internal_user_v1:fields() - end. - --spec fields(atom()) -> list(). -fields(?record_version) -> record_info(fields, internal_user); -fields(Version) -> internal_user_v1:fields(Version). - --spec upgrade(internal_user()) -> internal_user(). -upgrade(#internal_user{} = User) -> User; -upgrade(OldUser) -> upgrade_to(record_version_to_use(), OldUser). - --spec upgrade_to -(internal_user_v2, internal_user()) -> internal_user_v2(); -(internal_user_v1, internal_user_v1:internal_user_v1()) -> internal_user_v1:internal_user_v1(). - -upgrade_to(?record_version, #internal_user{} = User) -> - User; -upgrade_to(?record_version, OldUser) -> - Fields = erlang:tuple_to_list(OldUser) ++ [#{}], - #internal_user{} = erlang:list_to_tuple(Fields); -upgrade_to(Version, OldUser) -> - internal_user_v1:upgrade_to(Version, OldUser). - --spec pattern_match_all() -> internal_user_pattern(). -pattern_match_all() -> - case record_version_to_use() of - ?record_version -> #internal_user{_ = '_'}; - _ -> internal_user_v1:pattern_match_all() - end. - --spec get_username(internal_user()) -> username(). -get_username(#internal_user{username = Value}) -> Value; -get_username(User) -> internal_user_v1:get_username(User). - --spec get_password_hash(internal_user()) -> password_hash(). -get_password_hash(#internal_user{password_hash = Value}) -> Value; -get_password_hash(User) -> internal_user_v1:get_password_hash(User). - --spec get_tags(internal_user()) -> [atom()]. -get_tags(#internal_user{tags = Value}) -> Value; -get_tags(User) -> internal_user_v1:get_tags(User). - --spec get_hashing_algorithm(internal_user()) -> atom(). -get_hashing_algorithm(#internal_user{hashing_algorithm = Value}) -> Value; -get_hashing_algorithm(User) -> internal_user_v1:get_hashing_algorithm(User). - --spec get_limits(internal_user()) -> map(). -get_limits(#internal_user{limits = Value}) -> Value; -get_limits(User) -> internal_user_v1:get_limits(User). - --spec create_user(username(), password_hash(), atom()) -> internal_user(). -create_user(Username, PasswordHash, HashingMod) -> - case record_version_to_use() of - ?record_version -> - #internal_user{username = Username, - password_hash = PasswordHash, - tags = [], - hashing_algorithm = HashingMod, - limits = #{} - }; - _ -> - internal_user_v1:create_user(Username, PasswordHash, HashingMod) - end. - --spec set_password_hash(internal_user(), password_hash(), atom()) -> internal_user(). -set_password_hash(#internal_user{} = User, PasswordHash, HashingAlgorithm) -> - User#internal_user{password_hash = PasswordHash, - hashing_algorithm = HashingAlgorithm}; -set_password_hash(User, PasswordHash, HashingAlgorithm) -> - internal_user_v1:set_password_hash(User, PasswordHash, HashingAlgorithm). - --spec set_tags(internal_user(), [atom()]) -> internal_user(). -set_tags(#internal_user{} = User, Tags) -> - User#internal_user{tags = Tags}; -set_tags(User, Tags) -> - internal_user_v1:set_tags(User, Tags). - --spec update_limits -(add, internal_user(), map()) -> internal_user(); -(remove, internal_user(), term()) -> internal_user(). -update_limits(add, #internal_user{limits = Limits} = User, Term) -> - User#internal_user{limits = maps:merge(Limits, Term)}; -update_limits(remove, #internal_user{limits = Limits} = User, LimitType) -> - User#internal_user{limits = maps:remove(LimitType, Limits)}; -update_limits(Action, User, Term) -> - internal_user_v1:update_limits(Action, User, Term). - --spec clear_limits(internal_user()) -> internal_user(). -clear_limits(#internal_user{} = User) -> - User#internal_user{limits = #{}}; -clear_limits(User) -> - internal_user_v1:clear_limits(User). diff --git a/src/internal_user_v1.erl b/src/internal_user_v1.erl deleted file mode 100644 index edb956436f..0000000000 --- a/src/internal_user_v1.erl +++ /dev/null @@ -1,151 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(internal_user_v1). - --include_lib("rabbit_common/include/rabbit.hrl"). - --export([ - new/0, - new/1, - record_version_to_use/0, - fields/0, - fields/1, - upgrade/1, - upgrade_to/2, - pattern_match_all/0, - get_username/1, - get_password_hash/1, - get_tags/1, - get_hashing_algorithm/1, - get_limits/1, - create_user/3, - set_password_hash/3, - set_tags/2, - update_limits/3, - clear_limits/1 -]). - --define(record_version, ?MODULE). - --record(internal_user, { - username :: internal_user:username() | '_', - password_hash :: internal_user:password_hash() | '_', - tags :: [atom()] | '_', - %% password hashing implementation module, - %% typically rabbit_password_hashing_* but can - %% come from a plugin - hashing_algorithm :: atom() | '_'}). - --type internal_user() :: internal_user_v1(). - --type(internal_user_v1() :: - #internal_user{username :: internal_user:username(), - password_hash :: internal_user:password_hash(), - tags :: [atom()], - hashing_algorithm :: atom()}). - --type internal_user_pattern() :: internal_user_v1_pattern(). - --type internal_user_v1_pattern() :: #internal_user{ - username :: internal_user:username() | '_', - password_hash :: '_', - tags :: '_', - hashing_algorithm :: '_' - }. - --export_type([internal_user/0, - internal_user_v1/0, - internal_user_pattern/0, - internal_user_v1_pattern/0]). - --spec record_version_to_use() -> internal_user_v1. -record_version_to_use() -> - ?record_version. - --spec new() -> internal_user(). -new() -> - #internal_user{ - username = <<"">>, - password_hash = <<"">>, - tags = [] - }. - --spec new(tuple()) -> internal_user(). -new({hashing_algorithm, HashingAlgorithm}) -> - #internal_user{ - username = <<"">>, - password_hash = <<"">>, - hashing_algorithm = HashingAlgorithm, - tags = [] - }; -new({tags, Tags}) -> - #internal_user{ - username = <<"">>, - password_hash = <<"">>, - tags = Tags - }. - --spec fields() -> list(). -fields() -> fields(?record_version). - --spec fields(atom()) -> list(). -fields(?record_version) -> record_info(fields, internal_user). - --spec upgrade(internal_user()) -> internal_user(). -upgrade(#internal_user{} = User) -> User. - --spec upgrade_to(internal_user_v1, internal_user()) -> internal_user(). -upgrade_to(?record_version, #internal_user{} = User) -> - User. - --spec pattern_match_all() -> internal_user_pattern(). -pattern_match_all() -> #internal_user{_ = '_'}. - --spec get_username(internal_user()) -> internal_user:username(). -get_username(#internal_user{username = Value}) -> Value. - --spec get_password_hash(internal_user()) -> internal_user:password_hash(). -get_password_hash(#internal_user{password_hash = Value}) -> Value. - --spec get_tags(internal_user()) -> [atom()]. -get_tags(#internal_user{tags = Value}) -> Value. - --spec get_hashing_algorithm(internal_user()) -> atom(). -get_hashing_algorithm(#internal_user{hashing_algorithm = Value}) -> Value. - --spec get_limits(internal_user()) -> map(). -get_limits(_User) -> #{}. - --spec create_user(internal_user:username(), internal_user:password_hash(), - atom()) -> internal_user(). -create_user(Username, PasswordHash, HashingMod) -> - #internal_user{username = Username, - password_hash = PasswordHash, - tags = [], - hashing_algorithm = HashingMod - }. - --spec set_password_hash(internal_user:internal_user(), - internal_user:password_hash(), atom()) -> internal_user(). -set_password_hash(#internal_user{} = User, PasswordHash, HashingAlgorithm) -> - User#internal_user{password_hash = PasswordHash, - hashing_algorithm = HashingAlgorithm}. - --spec set_tags(internal_user(), [atom()]) -> internal_user(). -set_tags(#internal_user{} = User, Tags) -> - User#internal_user{tags = Tags}. - --spec update_limits -(add, internal_user(), map()) -> internal_user(); -(remove, internal_user(), term()) -> internal_user(). -update_limits(_, User, _) -> - User. - --spec clear_limits(internal_user()) -> internal_user(). -clear_limits(User) -> - User. diff --git a/src/lager_exchange_backend.erl b/src/lager_exchange_backend.erl deleted file mode 100644 index cd96f2230e..0000000000 --- a/src/lager_exchange_backend.erl +++ /dev/null @@ -1,233 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% @doc RabbitMQ backend for lager. -%% Configuration is a proplist with the following keys: -%% <ul> -%% <li>`level' - log level to use</li> -%% <li>`formatter' - the module to use when formatting log messages. Defaults to -%% `lager_default_formatter'</li> -%% <li>`formatter_config' - the format configuration string. Defaults to -%% `time [ severity ] message'</li> -%% </ul> - --module(lager_exchange_backend). - --behaviour(gen_event). - --export([init/1, terminate/2, code_change/3, - handle_call/2, handle_event/2, handle_info/2]). - --export([maybe_init_exchange/0]). - --include("rabbit.hrl"). --include("rabbit_framing.hrl"). - --include_lib("lager/include/lager.hrl"). - --record(state, {level :: {'mask', integer()}, - formatter :: atom(), - format_config :: any(), - init_exchange_ts = undefined :: integer() | undefined, - exchange = undefined :: #resource{} | undefined}). - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --compile([{parse_transform, lager_transform}]). --endif. - --define(INIT_EXCHANGE_INTERVAL_SECS, 5). --define(TERSE_FORMAT, [time, " [", severity, "] ", message]). --define(DEFAULT_FORMAT_CONFIG, ?TERSE_FORMAT). --define(FORMAT_CONFIG_OFF, []). - --ifdef(TEST). --define(DEPRECATED(_Msg), ok). --else. --define(DEPRECATED(Msg), - io:format(user, "WARNING: This is a deprecated lager_exchange_backend configuration. Please use \"~w\" instead.~n", [Msg])). --endif. - --define(LOG_EXCH_NAME, <<"amq.rabbitmq.log">>). - -init([Level]) when is_atom(Level) -> - ?DEPRECATED([{level, Level}]), - init([{level, Level}]); -init([Level, true]) when is_atom(Level) -> % for backwards compatibility - ?DEPRECATED([{level, Level}, {formatter_config, [{eol, "\\r\\n\\"}]}]), - init([{level, Level}, {formatter_config, ?FORMAT_CONFIG_OFF}]); -init([Level, false]) when is_atom(Level) -> % for backwards compatibility - ?DEPRECATED([{level, Level}]), - init([{level, Level}]); - -init(Options) when is_list(Options) -> - true = validate_options(Options), - Level = get_option(level, Options, undefined), - try lager_util:config_to_mask(Level) of - L -> - DefaultOptions = [{formatter, lager_default_formatter}, - {formatter_config, ?DEFAULT_FORMAT_CONFIG}], - [Formatter, Config] = [get_option(K, Options, Default) || {K, Default} <- DefaultOptions], - State0 = #state{level=L, - formatter=Formatter, - format_config=Config}, - % NB: this will probably always fail since the / vhost isn't available - State1 = maybe_init_exchange(State0), - {ok, State1} - catch - _:_ -> - {error, {fatal, bad_log_level}} - end; -init(Level) when is_atom(Level) -> - ?DEPRECATED([{level, Level}]), - init([{level, Level}]); -init(Other) -> - {error, {fatal, {bad_lager_exchange_backend_config, Other}}}. - -% rabbitmq/rabbitmq-server#1973 -% This is called immediatly after the / vhost is created -% or recovered -maybe_init_exchange() -> - case lists:member(?MODULE, gen_event:which_handlers(lager_event)) of - true -> - _ = init_exchange(true), - ok; - _ -> - ok - end. - -validate_options([]) -> true; -validate_options([{level, L}|T]) when is_atom(L) -> - case lists:member(L, ?LEVELS) of - false -> - throw({error, {fatal, {bad_level, L}}}); - true -> - validate_options(T) - end; -validate_options([{formatter, M}|T]) when is_atom(M) -> - validate_options(T); -validate_options([{formatter_config, C}|T]) when is_list(C) -> - validate_options(T); -validate_options([H|_]) -> - throw({error, {fatal, {bad_lager_exchange_backend_config, H}}}). - -get_option(K, Options, Default) -> - case lists:keyfind(K, 1, Options) of - {K, V} -> V; - false -> Default - end. - -handle_call(get_loglevel, #state{level=Level} = State) -> - {ok, Level, State}; -handle_call({set_loglevel, Level}, State) -> - try lager_util:config_to_mask(Level) of - Levels -> - {ok, ok, State#state{level=Levels}} - catch - _:_ -> - {ok, {error, bad_log_level}, State} - end; -handle_call(_Request, State) -> - {ok, ok, State}. - -handle_event({log, _Message} = Event, State0) -> - State1 = maybe_init_exchange(State0), - handle_log_event(Event, State1); -handle_event(_Event, State) -> - {ok, State}. - -handle_info(_Info, State) -> - {ok, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%% @private -handle_log_event({log, _Message}, #state{exchange=undefined} = State) -> - % NB: tried to define the exchange but still undefined, - % so not logging this message. Note: we can't log this dropped - % message because it will start an infinite loop - {ok, State}; -handle_log_event({log, Message}, - #state{level=L, exchange=LogExch, - formatter=Formatter, format_config=FormatConfig} = State) -> - case lager_util:is_loggable(Message, L, ?MODULE) of - true -> - %% 0-9-1 says the timestamp is a "64 bit POSIX timestamp". That's - %% second resolution, not millisecond. - RoutingKey = rabbit_data_coercion:to_binary(lager_msg:severity(Message)), - Timestamp = os:system_time(seconds), - Node = rabbit_data_coercion:to_binary(node()), - Headers = [{<<"node">>, longstr, Node}], - AmqpMsg = #'P_basic'{content_type = <<"text/plain">>, - timestamp = Timestamp, - headers = Headers}, - Body = rabbit_data_coercion:to_binary(Formatter:format(Message, FormatConfig)), - case rabbit_basic:publish(LogExch, RoutingKey, AmqpMsg, Body) of - ok -> ok; - {error, not_found} -> ok - end, - {ok, State}; - false -> - {ok, State} - end. - -%% @private -maybe_init_exchange(#state{exchange=undefined, init_exchange_ts=undefined} = State) -> - Now = erlang:monotonic_time(second), - handle_init_exchange(init_exchange(true), Now, State); -maybe_init_exchange(#state{exchange=undefined, init_exchange_ts=Timestamp} = State) -> - Now = erlang:monotonic_time(second), - % NB: since we may try to declare the exchange on every log message, this ensures - % that we only try once every 5 seconds - HasEnoughTimeElapsed = Now - Timestamp > ?INIT_EXCHANGE_INTERVAL_SECS, - Result = init_exchange(HasEnoughTimeElapsed), - handle_init_exchange(Result, Now, State); -maybe_init_exchange(State) -> - State. - -%% @private -init_exchange(true) -> - {ok, DefaultVHost} = application:get_env(rabbit, default_vhost), - Exchange = rabbit_misc:r(DefaultVHost, exchange, ?LOG_EXCH_NAME), - try - %% durable - #exchange{} = rabbit_exchange:declare(Exchange, topic, true, false, true, [], ?INTERNAL_USER), - rabbit_log:info("Declared exchange '~s' in vhost '~s'", [?LOG_EXCH_NAME, DefaultVHost]), - {ok, Exchange} - catch - ErrType:Err -> - rabbit_log:error("Could not declare exchange '~s' in vhost '~s', reason: ~p:~p", - [?LOG_EXCH_NAME, DefaultVHost, ErrType, Err]), - {ok, undefined} - end; -init_exchange(_) -> - {ok, undefined}. - -%% @private -handle_init_exchange({ok, undefined}, Now, State) -> - State#state{init_exchange_ts=Now}; -handle_init_exchange({ok, Exchange}, Now, State) -> - State#state{exchange=Exchange, init_exchange_ts=Now}. - --ifdef(TEST). -console_config_validation_test_() -> - Good = [{level, info}], - Bad1 = [{level, foo}], - Bad2 = [{larval, info}], - AllGood = [{level, info}, {formatter, my_formatter}, - {formatter_config, ["blort", "garbage"]}], - [ - ?_assertEqual(true, validate_options(Good)), - ?_assertThrow({error, {fatal, {bad_level, foo}}}, validate_options(Bad1)), - ?_assertThrow({error, {fatal, {bad_lager_exchange_backend_config, {larval, info}}}}, validate_options(Bad2)), - ?_assertEqual(true, validate_options(AllGood)) - ]. --endif. diff --git a/src/lqueue.erl b/src/lqueue.erl deleted file mode 100644 index 1e267210d9..0000000000 --- a/src/lqueue.erl +++ /dev/null @@ -1,102 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(lqueue). - -%% lqueue implements a subset of Erlang's queue module. lqueues -%% maintain their own length, so lqueue:len/1 -%% is an O(1) operation, in contrast with queue:len/1 which is O(n). - --export([new/0, is_empty/1, len/1, in/2, in_r/2, out/1, out_r/1, join/2, - foldl/3, foldr/3, from_list/1, drop/1, to_list/1, peek/1, peek_r/1]). - --define(QUEUE, queue). - --export_type([ - ?MODULE/0, - ?MODULE/1 - ]). - --opaque ?MODULE() :: ?MODULE(_). --opaque ?MODULE(T) :: {non_neg_integer(), queue:queue(T)}. --type value() :: any(). --type result(T) :: 'empty' | {'value', T}. - --spec new() -> ?MODULE(_). - -new() -> {0, ?QUEUE:new()}. - --spec drop(?MODULE(T)) -> ?MODULE(T). - -drop({L, Q}) -> {L - 1, ?QUEUE:drop(Q)}. - --spec is_empty(?MODULE(_)) -> boolean(). - -is_empty({0, _Q}) -> true; -is_empty(_) -> false. - --spec in(T, ?MODULE(T)) -> ?MODULE(T). - -in(V, {L, Q}) -> {L+1, ?QUEUE:in(V, Q)}. - --spec in_r(value(), ?MODULE(T)) -> ?MODULE(T). - -in_r(V, {L, Q}) -> {L+1, ?QUEUE:in_r(V, Q)}. - --spec out(?MODULE(T)) -> {result(T), ?MODULE(T)}. - -out({0, _Q} = Q) -> {empty, Q}; -out({L, Q}) -> {Result, Q1} = ?QUEUE:out(Q), - {Result, {L-1, Q1}}. - --spec out_r(?MODULE(T)) -> {result(T), ?MODULE(T)}. - -out_r({0, _Q} = Q) -> {empty, Q}; -out_r({L, Q}) -> {Result, Q1} = ?QUEUE:out_r(Q), - {Result, {L-1, Q1}}. - --spec join(?MODULE(A), ?MODULE(B)) -> ?MODULE(A | B). - -join({L1, Q1}, {L2, Q2}) -> {L1 + L2, ?QUEUE:join(Q1, Q2)}. - --spec to_list(?MODULE(T)) -> [T]. - -to_list({_L, Q}) -> ?QUEUE:to_list(Q). - --spec from_list([T]) -> ?MODULE(T). - -from_list(L) -> {length(L), ?QUEUE:from_list(L)}. - --spec foldl(fun ((T, B) -> B), B, ?MODULE(T)) -> B. - -foldl(Fun, Init, Q) -> - case out(Q) of - {empty, _Q} -> Init; - {{value, V}, Q1} -> foldl(Fun, Fun(V, Init), Q1) - end. - --spec foldr(fun ((T, B) -> B), B, ?MODULE(T)) -> B. - -foldr(Fun, Init, Q) -> - case out_r(Q) of - {empty, _Q} -> Init; - {{value, V}, Q1} -> foldr(Fun, Fun(V, Init), Q1) - end. - --spec len(?MODULE(_)) -> non_neg_integer(). - -len({L, _}) -> L. - --spec peek(?MODULE(T)) -> result(T). - -peek({ 0, _Q}) -> empty; -peek({_L, Q}) -> ?QUEUE:peek(Q). - --spec peek_r(?MODULE(T)) -> result(T). - -peek_r({ 0, _Q}) -> empty; -peek_r({_L, Q}) -> ?QUEUE:peek_r(Q). diff --git a/src/mirrored_supervisor_sups.erl b/src/mirrored_supervisor_sups.erl deleted file mode 100644 index b29d4d48e6..0000000000 --- a/src/mirrored_supervisor_sups.erl +++ /dev/null @@ -1,34 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(mirrored_supervisor_sups). - --define(SUPERVISOR, supervisor2). --define(GS_MODULE, mirrored_supervisor). - --behaviour(?SUPERVISOR). - --export([init/1]). - -%%---------------------------------------------------------------------------- - -init({overall, _Group, _TxFun, ignore}) -> ignore; -init({overall, Group, TxFun, {ok, {Restart, ChildSpecs}}}) -> - %% Important: Delegate MUST start before Mirroring so that when we - %% shut down from above it shuts down last, so Mirroring does not - %% see it die. - %% - %% See comment in handle_info('DOWN', ...) in mirrored_supervisor - {ok, {{one_for_all, 0, 1}, - [{delegate, {?SUPERVISOR, start_link, [?MODULE, {delegate, Restart}]}, - temporary, 16#ffffffff, supervisor, [?SUPERVISOR]}, - {mirroring, {?GS_MODULE, start_internal, [Group, TxFun, ChildSpecs]}, - permanent, 16#ffffffff, worker, [?MODULE]}]}}; - - -init({delegate, Restart}) -> - {ok, {Restart, []}}. diff --git a/src/pg_local.erl b/src/pg_local.erl deleted file mode 100644 index 263e743d1f..0000000000 --- a/src/pg_local.erl +++ /dev/null @@ -1,249 +0,0 @@ -%% This file is a copy of pg2.erl from the R13B-3 Erlang/OTP -%% distribution, with the following modifications: -%% -%% 1) Process groups are node-local only. -%% -%% 2) Groups are created/deleted implicitly. -%% -%% 3) 'join' and 'leave' are asynchronous. -%% -%% 4) the type specs of the exported non-callback functions have been -%% extracted into a separate, guarded section, and rewritten in -%% old-style spec syntax, for better compatibility with older -%% versions of Erlang/OTP. The remaining type specs have been -%% removed. - -%% All modifications are (C) 2010-2020 VMware, Inc. or its affiliates. - -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at https://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% --module(pg_local). - --export([join/2, leave/2, get_members/1, in_group/2]). -%% intended for testing only; not part of official API --export([sync/0, clear/0]). --export([start/0, start_link/0, init/1, handle_call/3, handle_cast/2, - handle_info/2, terminate/2]). - -%%---------------------------------------------------------------------------- - --type name() :: term(). - -%%---------------------------------------------------------------------------- - --define(TABLE, pg_local_table). - -%%% -%%% Exported functions -%%% - --spec start_link() -> {'ok', pid()} | {'error', any()}. - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - --spec start() -> {'ok', pid()} | {'error', any()}. - -start() -> - ensure_started(). - --spec join(name(), pid()) -> 'ok'. - -join(Name, Pid) when is_pid(Pid) -> - _ = ensure_started(), - gen_server:cast(?MODULE, {join, Name, Pid}). - --spec leave(name(), pid()) -> 'ok'. - -leave(Name, Pid) when is_pid(Pid) -> - _ = ensure_started(), - gen_server:cast(?MODULE, {leave, Name, Pid}). - --spec get_members(name()) -> [pid()]. - -get_members(Name) -> - _ = ensure_started(), - group_members(Name). - --spec in_group(name(), pid()) -> boolean(). - -in_group(Name, Pid) -> - _ = ensure_started(), - %% The join message is a cast and thus can race, but we want to - %% keep it that way to be fast in the common case. - case member_present(Name, Pid) of - true -> true; - false -> sync(), - member_present(Name, Pid) - end. - --spec sync() -> 'ok'. - -sync() -> - _ = ensure_started(), - gen_server:call(?MODULE, sync, infinity). - -clear() -> - _ = ensure_started(), - gen_server:call(?MODULE, clear, infinity). - -%%% -%%% Callback functions from gen_server -%%% - --record(state, {}). - -init([]) -> - ?TABLE = ets:new(?TABLE, [ordered_set, protected, named_table]), - {ok, #state{}}. - -handle_call(sync, _From, S) -> - {reply, ok, S}; - -handle_call(clear, _From, S) -> - ets:delete_all_objects(?TABLE), - {reply, ok, S}; - -handle_call(Request, From, S) -> - error_logger:warning_msg("The pg_local server received an unexpected message:\n" - "handle_call(~p, ~p, _)\n", - [Request, From]), - {noreply, S}. - -handle_cast({join, Name, Pid}, S) -> - _ = join_group(Name, Pid), - {noreply, S}; -handle_cast({leave, Name, Pid}, S) -> - leave_group(Name, Pid), - {noreply, S}; -handle_cast(_, S) -> - {noreply, S}. - -handle_info({'DOWN', MonitorRef, process, Pid, _Info}, S) -> - member_died(MonitorRef, Pid), - {noreply, S}; -handle_info(_, S) -> - {noreply, S}. - -terminate(_Reason, _S) -> - true = ets:delete(?TABLE), - ok. - -%%% -%%% Local functions -%%% - -%%% One ETS table, pg_local_table, is used for bookkeeping. The type of the -%%% table is ordered_set, and the fast matching of partially -%%% instantiated keys is used extensively. -%%% -%%% {{ref, Pid}, MonitorRef, Counter} -%%% {{ref, MonitorRef}, Pid} -%%% Each process has one monitor. Counter is incremented when the -%%% Pid joins some group. -%%% {{member, Name, Pid}, _} -%%% Pid is a member of group Name, GroupCounter is incremented when the -%%% Pid joins the group Name. -%%% {{pid, Pid, Name}} -%%% Pid is a member of group Name. - -member_died(Ref, Pid) -> - case ets:lookup(?TABLE, {ref, Ref}) of - [{{ref, Ref}, Pid}] -> - leave_all_groups(Pid); - %% in case the key has already been removed - %% we can clean up using the value from the DOWN message - _ -> - leave_all_groups(Pid) - end, - ok. - -leave_all_groups(Pid) -> - Names = member_groups(Pid), - _ = [leave_group(Name, P) || - Name <- Names, - P <- member_in_group(Pid, Name)]. - -join_group(Name, Pid) -> - Ref_Pid = {ref, Pid}, - try _ = ets:update_counter(?TABLE, Ref_Pid, {3, +1}) - catch _:_ -> - Ref = erlang:monitor(process, Pid), - true = ets:insert(?TABLE, {Ref_Pid, Ref, 1}), - true = ets:insert(?TABLE, {{ref, Ref}, Pid}) - end, - Member_Name_Pid = {member, Name, Pid}, - try _ = ets:update_counter(?TABLE, Member_Name_Pid, {2, +1}) - catch _:_ -> - true = ets:insert(?TABLE, {Member_Name_Pid, 1}), - true = ets:insert(?TABLE, {{pid, Pid, Name}}) - end. - -leave_group(Name, Pid) -> - Member_Name_Pid = {member, Name, Pid}, - try ets:update_counter(?TABLE, Member_Name_Pid, {2, -1}) of - N -> - if - N =:= 0 -> - true = ets:delete(?TABLE, {pid, Pid, Name}), - true = ets:delete(?TABLE, Member_Name_Pid); - true -> - ok - end, - Ref_Pid = {ref, Pid}, - case ets:update_counter(?TABLE, Ref_Pid, {3, -1}) of - 0 -> - [{Ref_Pid,Ref,0}] = ets:lookup(?TABLE, Ref_Pid), - true = ets:delete(?TABLE, {ref, Ref}), - true = ets:delete(?TABLE, Ref_Pid), - true = erlang:demonitor(Ref, [flush]), - ok; - _ -> - ok - end - catch _:_ -> - ok - end. - -group_members(Name) -> - [P || - [P, N] <- ets:match(?TABLE, {{member, Name, '$1'},'$2'}), - _ <- lists:seq(1, N)]. - -member_in_group(Pid, Name) -> - [{{member, Name, Pid}, N}] = ets:lookup(?TABLE, {member, Name, Pid}), - lists:duplicate(N, Pid). - -member_present(Name, Pid) -> - case ets:lookup(?TABLE, {member, Name, Pid}) of - [_] -> true; - [] -> false - end. - -member_groups(Pid) -> - [Name || [Name] <- ets:match(?TABLE, {{pid, Pid, '$1'}})]. - -ensure_started() -> - case whereis(?MODULE) of - undefined -> - C = {pg_local, {?MODULE, start_link, []}, permanent, - 16#ffffffff, worker, [?MODULE]}, - supervisor:start_child(kernel_safe_sup, C); - PgLocalPid -> - {ok, PgLocalPid} - end. diff --git a/src/rabbit.erl b/src/rabbit.erl deleted file mode 100644 index 9248c945dc..0000000000 --- a/src/rabbit.erl +++ /dev/null @@ -1,1511 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit). - -%% Transitional step until we can require Erlang/OTP 21 and -%% use the now recommended try/catch syntax for obtaining the stack trace. --compile(nowarn_deprecated_function). - --behaviour(application). - --export([start/0, boot/0, stop/0, - stop_and_halt/0, await_startup/0, await_startup/1, await_startup/3, - status/0, is_running/0, alarms/0, - is_running/1, environment/0, rotate_logs/0, force_event_refresh/1, - start_fhc/0]). - --export([start/2, stop/1, prep_stop/1]). --export([start_apps/1, start_apps/2, stop_apps/1]). --export([product_info/0, - product_name/0, - product_version/0, - base_product_name/0, - base_product_version/0, - motd_file/0, - motd/0]). --export([log_locations/0, config_files/0]). %% for testing and mgmt-agent --export([is_booted/1, is_booted/0, is_booting/1, is_booting/0]). - -%%--------------------------------------------------------------------------- -%% Boot steps. --export([maybe_insert_default_data/0, boot_delegate/0, recover/0]). - -%% for tests --export([validate_msg_store_io_batch_size_and_credit_disc_bound/2]). - --rabbit_boot_step({pre_boot, [{description, "rabbit boot start"}]}). - --rabbit_boot_step({codec_correctness_check, - [{description, "codec correctness check"}, - {mfa, {rabbit_binary_generator, - check_empty_frame_size, - []}}, - {requires, pre_boot}, - {enables, external_infrastructure}]}). - -%% rabbit_alarm currently starts memory and disk space monitors --rabbit_boot_step({rabbit_alarm, - [{description, "alarm handler"}, - {mfa, {rabbit_alarm, start, []}}, - {requires, pre_boot}, - {enables, external_infrastructure}]}). - --rabbit_boot_step({feature_flags, - [{description, "feature flags registry and initial state"}, - {mfa, {rabbit_feature_flags, init, []}}, - {requires, pre_boot}, - {enables, external_infrastructure}]}). - --rabbit_boot_step({database, - [{mfa, {rabbit_mnesia, init, []}}, - {requires, file_handle_cache}, - {enables, external_infrastructure}]}). - --rabbit_boot_step({database_sync, - [{description, "database sync"}, - {mfa, {rabbit_sup, start_child, [mnesia_sync]}}, - {requires, database}, - {enables, external_infrastructure}]}). - --rabbit_boot_step({code_server_cache, - [{description, "code_server cache server"}, - {mfa, {rabbit_sup, start_child, [code_server_cache]}}, - {requires, rabbit_alarm}, - {enables, file_handle_cache}]}). - --rabbit_boot_step({file_handle_cache, - [{description, "file handle cache server"}, - {mfa, {rabbit, start_fhc, []}}, - %% FHC needs memory monitor to be running - {requires, code_server_cache}, - {enables, worker_pool}]}). - --rabbit_boot_step({worker_pool, - [{description, "default worker pool"}, - {mfa, {rabbit_sup, start_supervisor_child, - [worker_pool_sup]}}, - {requires, pre_boot}, - {enables, external_infrastructure}]}). - --rabbit_boot_step({definition_import_worker_pool, - [{description, "dedicated worker pool for definition import"}, - {mfa, {rabbit_definitions, boot, []}}, - {requires, external_infrastructure}]}). - --rabbit_boot_step({external_infrastructure, - [{description, "external infrastructure ready"}]}). - --rabbit_boot_step({rabbit_registry, - [{description, "plugin registry"}, - {mfa, {rabbit_sup, start_child, - [rabbit_registry]}}, - {requires, external_infrastructure}, - {enables, kernel_ready}]}). - --rabbit_boot_step({rabbit_core_metrics, - [{description, "core metrics storage"}, - {mfa, {rabbit_sup, start_child, - [rabbit_metrics]}}, - {requires, pre_boot}, - {enables, external_infrastructure}]}). - --rabbit_boot_step({rabbit_osiris_metrics, - [{description, "osiris metrics scraper"}, - {mfa, {rabbit_sup, start_child, - [rabbit_osiris_metrics]}}, - {requires, pre_boot}, - {enables, external_infrastructure}]}). - -%% -rabbit_boot_step({rabbit_stream_coordinator, -%% [{description, "stream queues coordinator"}, -%% {mfa, {rabbit_stream_coordinator, start, -%% []}}, -%% {requires, pre_boot}, -%% {enables, external_infrastructure}]}). - --rabbit_boot_step({rabbit_event, - [{description, "statistics event manager"}, - {mfa, {rabbit_sup, start_restartable_child, - [rabbit_event]}}, - {requires, external_infrastructure}, - {enables, kernel_ready}]}). - --rabbit_boot_step({kernel_ready, - [{description, "kernel ready"}, - {requires, external_infrastructure}]}). - --rabbit_boot_step({rabbit_memory_monitor, - [{description, "memory monitor"}, - {mfa, {rabbit_sup, start_restartable_child, - [rabbit_memory_monitor]}}, - {requires, rabbit_alarm}, - {enables, core_initialized}]}). - --rabbit_boot_step({guid_generator, - [{description, "guid generator"}, - {mfa, {rabbit_sup, start_restartable_child, - [rabbit_guid]}}, - {requires, kernel_ready}, - {enables, core_initialized}]}). - --rabbit_boot_step({delegate_sup, - [{description, "cluster delegate"}, - {mfa, {rabbit, boot_delegate, []}}, - {requires, kernel_ready}, - {enables, core_initialized}]}). - --rabbit_boot_step({rabbit_node_monitor, - [{description, "node monitor"}, - {mfa, {rabbit_sup, start_restartable_child, - [rabbit_node_monitor]}}, - {requires, [rabbit_alarm, guid_generator]}, - {enables, core_initialized}]}). - --rabbit_boot_step({rabbit_epmd_monitor, - [{description, "epmd monitor"}, - {mfa, {rabbit_sup, start_restartable_child, - [rabbit_epmd_monitor]}}, - {requires, kernel_ready}, - {enables, core_initialized}]}). - --rabbit_boot_step({rabbit_sysmon_minder, - [{description, "sysmon_handler supervisor"}, - {mfa, {rabbit_sup, start_restartable_child, - [rabbit_sysmon_minder]}}, - {requires, kernel_ready}, - {enables, core_initialized}]}). - --rabbit_boot_step({core_initialized, - [{description, "core initialized"}, - {requires, kernel_ready}]}). - --rabbit_boot_step({upgrade_queues, - [{description, "per-vhost message store migration"}, - {mfa, {rabbit_upgrade, - maybe_migrate_queues_to_per_vhost_storage, - []}}, - {requires, [core_initialized]}, - {enables, recovery}]}). - --rabbit_boot_step({recovery, - [{description, "exchange, queue and binding recovery"}, - {mfa, {rabbit, recover, []}}, - {requires, [core_initialized]}, - {enables, routing_ready}]}). - --rabbit_boot_step({empty_db_check, - [{description, "empty DB check"}, - {mfa, {?MODULE, maybe_insert_default_data, []}}, - {requires, recovery}, - {enables, routing_ready}]}). - --rabbit_boot_step({routing_ready, - [{description, "message delivery logic ready"}, - {requires, [core_initialized, recovery]}]}). - --rabbit_boot_step({connection_tracking, - [{description, "connection tracking infrastructure"}, - {mfa, {rabbit_connection_tracking, boot, []}}, - {enables, routing_ready}]}). - --rabbit_boot_step({channel_tracking, - [{description, "channel tracking infrastructure"}, - {mfa, {rabbit_channel_tracking, boot, []}}, - {enables, routing_ready}]}). - --rabbit_boot_step({background_gc, - [{description, "background garbage collection"}, - {mfa, {rabbit_sup, start_restartable_child, - [background_gc]}}, - {requires, [core_initialized, recovery]}, - {enables, routing_ready}]}). - --rabbit_boot_step({rabbit_core_metrics_gc, - [{description, "background core metrics garbage collection"}, - {mfa, {rabbit_sup, start_restartable_child, - [rabbit_core_metrics_gc]}}, - {requires, [core_initialized, recovery]}, - {enables, routing_ready}]}). - --rabbit_boot_step({rabbit_looking_glass, - [{description, "Looking Glass tracer and profiler"}, - {mfa, {rabbit_looking_glass, boot, []}}, - {requires, [core_initialized, recovery]}, - {enables, routing_ready}]}). - --rabbit_boot_step({pre_flight, - [{description, "ready to communicate with peers and clients"}, - {requires, [core_initialized, recovery, routing_ready]}]}). - --rabbit_boot_step({cluster_name, - [{description, "sets cluster name if configured"}, - {mfa, {rabbit_nodes, boot, []}}, - {requires, pre_flight} - ]}). - --rabbit_boot_step({direct_client, - [{description, "direct client"}, - {mfa, {rabbit_direct, boot, []}}, - {requires, pre_flight} - ]}). - --rabbit_boot_step({notify_cluster, - [{description, "notifies cluster peers of our presence"}, - {mfa, {rabbit_node_monitor, notify_node_up, []}}, - {requires, pre_flight}]}). - --rabbit_boot_step({networking, - [{description, "TCP and TLS listeners (backwards compatibility)"}, - {mfa, {rabbit_log, debug, ["'networking' boot step skipped and moved to end of startup", []]}}, - {requires, notify_cluster}]}). - -%%--------------------------------------------------------------------------- - --include("rabbit_framing.hrl"). --include("rabbit.hrl"). - --define(APPS, [os_mon, mnesia, rabbit_common, rabbitmq_prelaunch, ra, sysmon_handler, rabbit, osiris]). - --define(ASYNC_THREADS_WARNING_THRESHOLD, 8). - -%% 1 minute --define(BOOT_START_TIMEOUT, 1 * 60 * 1000). -%% 12 hours --define(BOOT_FINISH_TIMEOUT, 12 * 60 * 60 * 1000). -%% 100 ms --define(BOOT_STATUS_CHECK_INTERVAL, 100). - -%%---------------------------------------------------------------------------- - --type restart_type() :: 'permanent' | 'transient' | 'temporary'. - --type param() :: atom(). --type app_name() :: atom(). - -%%---------------------------------------------------------------------------- - --spec start() -> 'ok'. - -start() -> - %% start() vs. boot(): we want to throw an error in start(). - start_it(temporary). - --spec boot() -> 'ok'. - -boot() -> - %% start() vs. boot(): we want the node to exit in boot(). Because - %% applications are started with `transient`, any error during their - %% startup will abort the node. - start_it(transient). - -run_prelaunch_second_phase() -> - %% Finish the prelaunch phase started by the `rabbitmq_prelaunch` - %% application. - %% - %% The first phase was handled by the `rabbitmq_prelaunch` - %% application. It was started in one of the following way: - %% - from an Erlang release boot script; - %% - from the rabbit:boot/0 or rabbit:start/0 functions. - %% - %% The `rabbitmq_prelaunch` application creates the context map from - %% the environment and the configuration files early during Erlang - %% VM startup. Once it is done, all application environments are - %% configured (in particular `mnesia` and `ra`). - %% - %% This second phase depends on other modules & facilities of - %% RabbitMQ core. That's why we need to run it now, from the - %% `rabbit` application start function. - - %% We assert Mnesia is stopped before we run the prelaunch - %% phases. See `rabbit_prelaunch` for an explanation. - %% - %% This is the second assertion, just in case Mnesia is started - %% between the two prelaunch phases. - rabbit_prelaunch:assert_mnesia_is_stopped(), - - %% Get the context created by `rabbitmq_prelaunch` then proceed - %% with all steps in this phase. - #{initial_pass := IsInitialPass} = - Context = rabbit_prelaunch:get_context(), - - case IsInitialPass of - true -> - rabbit_log_prelaunch:debug(""), - rabbit_log_prelaunch:debug( - "== Prelaunch phase [2/2] (initial pass) =="); - false -> - rabbit_log_prelaunch:debug(""), - rabbit_log_prelaunch:debug("== Prelaunch phase [2/2] =="), - ok - end, - - %% 1. Enabled plugins file. - ok = rabbit_prelaunch_enabled_plugins_file:setup(Context), - - %% 2. Feature flags registry. - ok = rabbit_prelaunch_feature_flags:setup(Context), - - %% 3. Logging. - ok = rabbit_prelaunch_logging:setup(Context), - - %% 4. Clustering. - ok = rabbit_prelaunch_cluster:setup(Context), - - %% Start Mnesia now that everything is ready. - rabbit_log_prelaunch:debug("Starting Mnesia"), - ok = mnesia:start(), - - rabbit_log_prelaunch:debug(""), - rabbit_log_prelaunch:debug("== Prelaunch DONE =="), - - case IsInitialPass of - true -> rabbit_prelaunch:initial_pass_finished(); - false -> ok - end, - ok. - -start_it(StartType) -> - case spawn_boot_marker() of - {ok, Marker} -> - T0 = erlang:timestamp(), - rabbit_log:info("RabbitMQ is asked to start...", []), - try - {ok, _} = application:ensure_all_started(rabbitmq_prelaunch, - StartType), - {ok, _} = application:ensure_all_started(rabbit, - StartType), - ok = wait_for_ready_or_stopped(), - - T1 = erlang:timestamp(), - rabbit_log_prelaunch:debug( - "Time to start RabbitMQ: ~p µs", - [timer:now_diff(T1, T0)]), - stop_boot_marker(Marker), - ok - catch - error:{badmatch, Error}:_ -> - stop_boot_marker(Marker), - case StartType of - temporary -> throw(Error); - _ -> exit(Error) - end - end; - {already_booting, Marker} -> - stop_boot_marker(Marker), - ok - end. - -wait_for_ready_or_stopped() -> - ok = rabbit_boot_state:wait_for(ready, ?BOOT_FINISH_TIMEOUT), - case rabbit_boot_state:get() of - ready -> - ok; - _ -> - ok = rabbit_boot_state:wait_for(stopped, ?BOOT_FINISH_TIMEOUT), - rabbit_prelaunch:get_stop_reason() - end. - -spawn_boot_marker() -> - %% Compatibility with older RabbitMQ versions: - %% We register a process doing nothing to indicate that RabbitMQ is - %% booting. This is checked by `is_booting(Node)` on a remote node. - Marker = spawn_link(fun() -> receive stop -> ok end end), - case catch register(rabbit_boot, Marker) of - true -> {ok, Marker}; - _ -> {already_booting, Marker} - end. - -stop_boot_marker(Marker) -> - unlink(Marker), - Marker ! stop, - ok. - --spec stop() -> 'ok'. - -stop() -> - case wait_for_ready_or_stopped() of - ok -> - case rabbit_boot_state:get() of - ready -> - Product = product_name(), - rabbit_log:info("~s is asked to stop...", [Product]), - do_stop(), - rabbit_log:info( - "Successfully stopped ~s and its dependencies", - [Product]), - ok; - stopped -> - ok - end; - _ -> - ok - end. - -do_stop() -> - Apps0 = ?APPS ++ rabbit_plugins:active(), - %% We ensure that Mnesia is stopped last (or more exactly, after rabbit). - Apps1 = app_utils:app_dependency_order(Apps0, true) -- [mnesia], - Apps = [mnesia | Apps1], - %% this will also perform unregistration with the peer discovery backend - %% as needed - stop_apps(Apps). - --spec stop_and_halt() -> no_return(). - -stop_and_halt() -> - try - stop() - catch Type:Reason -> - rabbit_log:error( - "Error trying to stop ~s: ~p:~p", - [product_name(), Type, Reason]), - error({Type, Reason}) - after - %% Enclose all the logging in the try block. - %% init:stop() will be called regardless of any errors. - try - AppsLeft = [ A || {A, _, _} <- application:which_applications() ], - rabbit_log:info( - lists:flatten(["Halting Erlang VM with the following applications:~n", - [" ~p~n" || _ <- AppsLeft]]), - AppsLeft), - %% Also duplicate this information to stderr, so console where - %% foreground broker was running (or systemd journal) will - %% contain information about graceful termination. - io:format(standard_error, "Gracefully halting Erlang VM~n", []) - after - init:stop() - end - end, - ok. - --spec start_apps([app_name()]) -> 'ok'. - -start_apps(Apps) -> - start_apps(Apps, #{}). - --spec start_apps([app_name()], - #{app_name() => restart_type()}) -> 'ok'. - -%% TODO: start_apps/2 and is now specific to plugins. This function -%% should be moved over `rabbit_plugins`, along with stop_apps/1, once -%% the latter stops using app_utils as well. - -start_apps(Apps, RestartTypes) -> - false = lists:member(rabbit, Apps), %% Assertion. - %% We need to load all applications involved in order to be able to - %% find new feature flags. - app_utils:load_applications(Apps), - ok = rabbit_feature_flags:refresh_feature_flags_after_app_load(Apps), - rabbit_prelaunch_conf:decrypt_config(Apps), - lists:foreach( - fun(App) -> - RestartType = maps:get(App, RestartTypes, temporary), - ok = rabbit_boot_steps:run_boot_steps([App]), - case application:ensure_all_started(App, RestartType) of - {ok, _} -> ok; - {error, Reason} -> throw({could_not_start, App, Reason}) - end - end, Apps). - --spec stop_apps([app_name()]) -> 'ok'. - -stop_apps([]) -> - ok; -stop_apps(Apps) -> - rabbit_log:info( - lists:flatten(["Stopping ~s applications and their dependencies in the following order:~n", - [" ~p~n" || _ <- Apps]]), - [product_name() | lists:reverse(Apps)]), - ok = app_utils:stop_applications( - Apps, handle_app_error(error_during_shutdown)), - case lists:member(rabbit, Apps) of - %% plugin deactivation - false -> rabbit_boot_steps:run_cleanup_steps(Apps); - true -> ok %% it's all going anyway - end, - ok. - --spec handle_app_error(_) -> fun((_, _) -> no_return()). -handle_app_error(Term) -> - fun(App, {bad_return, {_MFA, {'EXIT', ExitReason}}}) -> - throw({Term, App, ExitReason}); - (App, Reason) -> - throw({Term, App, Reason}) - end. - -is_booting() -> is_booting(node()). - -is_booting(Node) when Node =:= node() -> - case rabbit_boot_state:get() of - booting -> true; - _ -> false - end; -is_booting(Node) -> - case rpc:call(Node, rabbit, is_booting, []) of - {badrpc, _} = Err -> Err; - Ret -> Ret - end. - - --spec await_startup() -> 'ok' | {'error', 'timeout'}. - -await_startup() -> - await_startup(node(), false). - --spec await_startup(node() | non_neg_integer()) -> 'ok' | {'error', 'timeout'}. - -await_startup(Node) when is_atom(Node) -> - await_startup(Node, false); - await_startup(Timeout) when is_integer(Timeout) -> - await_startup(node(), false, Timeout). - --spec await_startup(node(), boolean()) -> 'ok' | {'error', 'timeout'}. - -await_startup(Node, PrintProgressReports) -> - case is_booting(Node) of - true -> wait_for_boot_to_finish(Node, PrintProgressReports); - false -> - case is_running(Node) of - true -> ok; - false -> wait_for_boot_to_start(Node), - wait_for_boot_to_finish(Node, PrintProgressReports) - end - end. - --spec await_startup(node(), boolean(), non_neg_integer()) -> 'ok' | {'error', 'timeout'}. - -await_startup(Node, PrintProgressReports, Timeout) -> - case is_booting(Node) of - true -> wait_for_boot_to_finish(Node, PrintProgressReports, Timeout); - false -> - case is_running(Node) of - true -> ok; - false -> wait_for_boot_to_start(Node, Timeout), - wait_for_boot_to_finish(Node, PrintProgressReports, Timeout) - end - end. - -wait_for_boot_to_start(Node) -> - wait_for_boot_to_start(Node, ?BOOT_START_TIMEOUT). - -wait_for_boot_to_start(Node, infinity) -> - %% This assumes that 100K iterations is close enough to "infinity". - %% Now that's deep. - do_wait_for_boot_to_start(Node, 100000); -wait_for_boot_to_start(Node, Timeout) -> - Iterations = Timeout div ?BOOT_STATUS_CHECK_INTERVAL, - do_wait_for_boot_to_start(Node, Iterations). - -do_wait_for_boot_to_start(_Node, IterationsLeft) when IterationsLeft =< 0 -> - {error, timeout}; -do_wait_for_boot_to_start(Node, IterationsLeft) -> - case is_booting(Node) of - false -> - timer:sleep(?BOOT_STATUS_CHECK_INTERVAL), - do_wait_for_boot_to_start(Node, IterationsLeft - 1); - {badrpc, _} = Err -> - Err; - true -> - ok - end. - -wait_for_boot_to_finish(Node, PrintProgressReports) -> - wait_for_boot_to_finish(Node, PrintProgressReports, ?BOOT_FINISH_TIMEOUT). - -wait_for_boot_to_finish(Node, PrintProgressReports, infinity) -> - %% This assumes that 100K iterations is close enough to "infinity". - %% Now that's deep. - do_wait_for_boot_to_finish(Node, PrintProgressReports, 100000); -wait_for_boot_to_finish(Node, PrintProgressReports, Timeout) -> - Iterations = Timeout div ?BOOT_STATUS_CHECK_INTERVAL, - do_wait_for_boot_to_finish(Node, PrintProgressReports, Iterations). - -do_wait_for_boot_to_finish(_Node, _PrintProgressReports, IterationsLeft) when IterationsLeft =< 0 -> - {error, timeout}; -do_wait_for_boot_to_finish(Node, PrintProgressReports, IterationsLeft) -> - case is_booting(Node) of - false -> - %% We don't want badrpc error to be interpreted as false, - %% so we don't call rabbit:is_running(Node) - case rpc:call(Node, rabbit, is_running, []) of - true -> ok; - false -> {error, rabbit_is_not_running}; - {badrpc, _} = Err -> Err - end; - {badrpc, _} = Err -> - Err; - true -> - maybe_print_boot_progress(PrintProgressReports, IterationsLeft), - timer:sleep(?BOOT_STATUS_CHECK_INTERVAL), - do_wait_for_boot_to_finish(Node, PrintProgressReports, IterationsLeft - 1) - end. - -maybe_print_boot_progress(false = _PrintProgressReports, _IterationsLeft) -> - ok; -maybe_print_boot_progress(true, IterationsLeft) -> - case IterationsLeft rem 100 of - %% This will be printed on the CLI command end to illustrate some - %% progress. - 0 -> io:format("Still booting, will check again in 10 seconds...~n"); - _ -> ok - end. - --spec status - () -> [{pid, integer()} | - {running_applications, [{atom(), string(), string()}]} | - {os, {atom(), atom()}} | - {erlang_version, string()} | - {memory, any()}]. - -status() -> - Version = base_product_version(), - S1 = [{pid, list_to_integer(os:getpid())}, - %% The timeout value used is twice that of gen_server:call/2. - {running_applications, rabbit_misc:which_applications()}, - {os, os:type()}, - {rabbitmq_version, Version}, - {erlang_version, erlang:system_info(system_version)}, - {memory, rabbit_vm:memory()}, - {alarms, alarms()}, - {is_under_maintenance, rabbit_maintenance:is_being_drained_local_read(node())}, - {listeners, listeners()}, - {vm_memory_calculation_strategy, vm_memory_monitor:get_memory_calculation_strategy()}], - S2 = rabbit_misc:filter_exit_map( - fun ({Key, {M, F, A}}) -> {Key, erlang:apply(M, F, A)} end, - [{vm_memory_high_watermark, {vm_memory_monitor, - get_vm_memory_high_watermark, []}}, - {vm_memory_limit, {vm_memory_monitor, - get_memory_limit, []}}, - {disk_free_limit, {rabbit_disk_monitor, - get_disk_free_limit, []}}, - {disk_free, {rabbit_disk_monitor, - get_disk_free, []}}]), - S3 = rabbit_misc:with_exit_handler( - fun () -> [] end, - fun () -> [{file_descriptors, file_handle_cache:info()}] end), - S4 = [{processes, [{limit, erlang:system_info(process_limit)}, - {used, erlang:system_info(process_count)}]}, - {run_queue, erlang:statistics(run_queue)}, - {uptime, begin - {T,_} = erlang:statistics(wall_clock), - T div 1000 - end}, - {kernel, {net_ticktime, net_kernel:get_net_ticktime()}}], - S5 = [{active_plugins, rabbit_plugins:active()}, - {enabled_plugin_file, rabbit_plugins:enabled_plugins_file()}], - S6 = [{config_files, config_files()}, - {log_files, log_locations()}, - {data_directory, rabbit_mnesia:dir()}, - {raft_data_directory, ra_env:data_dir()}], - Totals = case is_running() of - true -> - [{virtual_host_count, rabbit_vhost:count()}, - {connection_count, - length(rabbit_networking:connections_local())}, - {queue_count, total_queue_count()}]; - false -> - [] - end, - S7 = [{totals, Totals}], - S8 = lists:filter( - fun - ({product_base_name, _}) -> true; - ({product_base_version, _}) -> true; - ({product_name, _}) -> true; - ({product_version, _}) -> true; - (_) -> false - end, - maps:to_list(product_info())), - S1 ++ S2 ++ S3 ++ S4 ++ S5 ++ S6 ++ S7 ++ S8. - -alarms() -> - Alarms = rabbit_misc:with_exit_handler(rabbit_misc:const([]), - fun rabbit_alarm:get_alarms/0), - N = node(), - %% [{{resource_limit,memory,rabbit@mercurio},[]}] - [{resource_limit, Limit, Node} || {{resource_limit, Limit, Node}, _} <- Alarms, Node =:= N]. - -listeners() -> - Listeners = try - rabbit_networking:active_listeners() - catch - exit:{aborted, _} -> [] - end, - [L || L = #listener{node = Node} <- Listeners, Node =:= node()]. - -total_queue_count() -> - lists:foldl(fun (VirtualHost, Acc) -> - Acc + rabbit_amqqueue:count(VirtualHost) - end, - 0, rabbit_vhost:list_names()). - --spec is_running() -> boolean(). - -is_running() -> is_running(node()). - --spec is_running(node()) -> boolean(). - -is_running(Node) when Node =:= node() -> - case rabbit_boot_state:get() of - ready -> true; - _ -> false - end; -is_running(Node) -> - case rpc:call(Node, rabbit, is_running, []) of - true -> true; - _ -> false - end. - -is_booted() -> is_booted(node()). - -is_booted(Node) -> - case is_booting(Node) of - false -> - is_running(Node); - _ -> false - end. - --spec environment() -> [{param(), term()}]. - -environment() -> - %% The timeout value is twice that of gen_server:call/2. - [{A, environment(A)} || - {A, _, _} <- lists:keysort(1, application:which_applications(10000))]. - -environment(App) -> - Ignore = [default_pass, included_applications], - lists:keysort(1, [P || P = {K, _} <- application:get_all_env(App), - not lists:member(K, Ignore)]). - --spec rotate_logs() -> rabbit_types:ok_or_error(any()). - -rotate_logs() -> - rabbit_lager:fold_sinks( - fun - (_, [], Acc) -> - Acc; - (SinkName, FileNames, Acc) -> - lager:log(SinkName, info, self(), - "Log file rotation forced", []), - %% FIXME: We use an internal message, understood by - %% lager_file_backend. We should use a proper API, when - %% it's added to Lager. - %% - %% FIXME: This call is effectively asynchronous: at the - %% end of this function, we can't guaranty the rotation - %% is completed. - [ok = gen_event:call(SinkName, - {lager_file_backend, FileName}, - rotate, - infinity) || FileName <- FileNames], - lager:log(SinkName, info, self(), - "Log file re-opened after forced rotation", []), - Acc - end, ok). - -%%-------------------------------------------------------------------- - --spec start('normal',[]) -> - {'error', - {'erlang_version_too_old', - {'found',string(),string()}, - {'required',string(),string()}}} | - {'ok',pid()}. - -start(normal, []) -> - %% Reset boot state and clear the stop reason again (it was already - %% made in rabbitmq_prelaunch). - %% - %% This is important if the previous startup attempt failed after - %% rabbitmq_prelaunch was started and the application is still - %% running. - rabbit_boot_state:set(booting), - rabbit_prelaunch:clear_stop_reason(), - - try - run_prelaunch_second_phase(), - - ProductInfo = product_info(), - case ProductInfo of - #{product_overridden := true, - product_base_name := BaseName, - product_base_version := BaseVersion} -> - rabbit_log:info("~n Starting ~s ~s on Erlang ~s~n Based on ~s ~s~n ~s~n ~s~n", - [product_name(), product_version(), rabbit_misc:otp_release(), - BaseName, BaseVersion, - ?COPYRIGHT_MESSAGE, ?INFORMATION_MESSAGE]); - _ -> - rabbit_log:info("~n Starting ~s ~s on Erlang ~s~n ~s~n ~s~n", - [product_name(), product_version(), rabbit_misc:otp_release(), - ?COPYRIGHT_MESSAGE, ?INFORMATION_MESSAGE]) - end, - log_motd(), - {ok, SupPid} = rabbit_sup:start_link(), - - %% Compatibility with older RabbitMQ versions + required by - %% rabbit_node_monitor:notify_node_up/0: - %% - %% We register the app process under the name `rabbit`. This is - %% checked by `is_running(Node)` on a remote node. The process - %% is also monitord by rabbit_node_monitor. - %% - %% The process name must be registered *before* running the boot - %% steps: that's when rabbit_node_monitor will set the process - %% monitor up. - %% - %% Note that plugins were not taken care of at this point - %% either. - rabbit_log_prelaunch:debug( - "Register `rabbit` process (~p) for rabbit_node_monitor", - [self()]), - true = register(rabbit, self()), - - print_banner(), - log_banner(), - warn_if_kernel_config_dubious(), - warn_if_disc_io_options_dubious(), - %% We run `rabbit` boot steps only for now. Plugins boot steps - %% will be executed as part of the postlaunch phase after they - %% are started. - rabbit_boot_steps:run_boot_steps([rabbit]), - run_postlaunch_phase(), - {ok, SupPid} - catch - throw:{error, _} = Error -> - mnesia:stop(), - rabbit_prelaunch_errors:log_error(Error), - rabbit_prelaunch:set_stop_reason(Error), - rabbit_boot_state:set(stopped), - Error; - Class:Exception:Stacktrace -> - mnesia:stop(), - rabbit_prelaunch_errors:log_exception( - Class, Exception, Stacktrace), - Error = {error, Exception}, - rabbit_prelaunch:set_stop_reason(Error), - rabbit_boot_state:set(stopped), - Error - end. - -run_postlaunch_phase() -> - spawn(fun() -> do_run_postlaunch_phase() end). - -do_run_postlaunch_phase() -> - %% Once RabbitMQ itself is started, we need to run a few more steps, - %% in particular start plugins. - rabbit_log_prelaunch:debug(""), - rabbit_log_prelaunch:debug("== Postlaunch phase =="), - - try - rabbit_log_prelaunch:debug(""), - rabbit_log_prelaunch:debug("== Plugins =="), - - rabbit_log_prelaunch:debug("Setting plugins up"), - %% `Plugins` contains all the enabled plugins, plus their - %% dependencies. The order is important: dependencies appear - %% before plugin which depend on them. - Plugins = rabbit_plugins:setup(), - rabbit_log_prelaunch:debug( - "Starting the following plugins: ~p", [Plugins]), - %% We can load all plugins and refresh their feature flags at - %% once, because it does not involve running code from the - %% plugins. - app_utils:load_applications(Plugins), - ok = rabbit_feature_flags:refresh_feature_flags_after_app_load( - Plugins), - %% However, we want to run their boot steps and actually start - %% them one by one, to ensure a dependency is fully started - %% before a plugin which depends on it gets a chance to start. - lists:foreach( - fun(Plugin) -> - ok = rabbit_boot_steps:run_boot_steps([Plugin]), - case application:ensure_all_started(Plugin) of - {ok, _} -> ok; - Error -> throw(Error) - end - end, Plugins), - - %% Successful boot resets node maintenance state. - rabbit_log_prelaunch:info("Resetting node maintenance status"), - _ = rabbit_maintenance:unmark_as_being_drained(), - - %% Export definitions after all plugins have been enabled, - %% see rabbitmq/rabbitmq-server#2384 - case rabbit_definitions:maybe_load_definitions() of - ok -> ok; - DefLoadError -> throw(DefLoadError) - end, - - %% Start listeners after all plugins have been enabled, - %% see rabbitmq/rabbitmq-server#2405. - rabbit_log_prelaunch:info( - "Ready to start client connection listeners"), - ok = rabbit_networking:boot(), - - %% The node is ready: mark it as such and log it. - %% NOTE: PLEASE DO NOT ADD CRITICAL NODE STARTUP CODE AFTER THIS. - ok = rabbit_lager:broker_is_started(), - ok = log_broker_started( - rabbit_plugins:strictly_plugins(rabbit_plugins:active())), - - rabbit_log_prelaunch:debug("Marking ~s as running", [product_name()]), - rabbit_boot_state:set(ready) - catch - throw:{error, _} = Error -> - rabbit_prelaunch_errors:log_error(Error), - rabbit_prelaunch:set_stop_reason(Error), - do_stop(); - Class:Exception:Stacktrace -> - rabbit_prelaunch_errors:log_exception( - Class, Exception, Stacktrace), - Error = {error, Exception}, - rabbit_prelaunch:set_stop_reason(Error), - do_stop() - end. - -prep_stop(State) -> - rabbit_boot_state:set(stopping), - rabbit_peer_discovery:maybe_unregister(), - State. - --spec stop(_) -> 'ok'. - -stop(State) -> - ok = rabbit_alarm:stop(), - ok = case rabbit_mnesia:is_clustered() of - true -> ok; - false -> rabbit_table:clear_ram_only_tables() - end, - case State of - [] -> rabbit_prelaunch:set_stop_reason(normal); - _ -> rabbit_prelaunch:set_stop_reason(State) - end, - rabbit_boot_state:set(stopped), - ok. - -%%--------------------------------------------------------------------------- -%% boot step functions - --spec boot_delegate() -> 'ok'. - -boot_delegate() -> - {ok, Count} = application:get_env(rabbit, delegate_count), - rabbit_sup:start_supervisor_child(delegate_sup, [Count]). - --spec recover() -> 'ok'. - -recover() -> - ok = rabbit_policy:recover(), - ok = rabbit_vhost:recover(), - ok = lager_exchange_backend:maybe_init_exchange(). - --spec maybe_insert_default_data() -> 'ok'. - -maybe_insert_default_data() -> - NoDefsToImport = not rabbit_definitions:has_configured_definitions_to_load(), - case rabbit_table:needs_default_data() andalso NoDefsToImport of - true -> - rabbit_log:info("Will seed default virtual host and user..."), - insert_default_data(); - false -> - rabbit_log:info("Will not seed default virtual host and user: have definitions to load..."), - ok - end. - -insert_default_data() -> - {ok, DefaultUser} = application:get_env(default_user), - {ok, DefaultPass} = application:get_env(default_pass), - {ok, DefaultTags} = application:get_env(default_user_tags), - {ok, DefaultVHost} = application:get_env(default_vhost), - {ok, [DefaultConfigurePerm, DefaultWritePerm, DefaultReadPerm]} = - application:get_env(default_permissions), - - DefaultUserBin = rabbit_data_coercion:to_binary(DefaultUser), - DefaultPassBin = rabbit_data_coercion:to_binary(DefaultPass), - DefaultVHostBin = rabbit_data_coercion:to_binary(DefaultVHost), - DefaultConfigurePermBin = rabbit_data_coercion:to_binary(DefaultConfigurePerm), - DefaultWritePermBin = rabbit_data_coercion:to_binary(DefaultWritePerm), - DefaultReadPermBin = rabbit_data_coercion:to_binary(DefaultReadPerm), - - ok = rabbit_vhost:add(DefaultVHostBin, <<"Default virtual host">>, [], ?INTERNAL_USER), - ok = lager_exchange_backend:maybe_init_exchange(), - ok = rabbit_auth_backend_internal:add_user( - DefaultUserBin, - DefaultPassBin, - ?INTERNAL_USER - ), - ok = rabbit_auth_backend_internal:set_tags(DefaultUserBin, DefaultTags, - ?INTERNAL_USER), - ok = rabbit_auth_backend_internal:set_permissions(DefaultUserBin, - DefaultVHostBin, - DefaultConfigurePermBin, - DefaultWritePermBin, - DefaultReadPermBin, - ?INTERNAL_USER), - ok. - -%%--------------------------------------------------------------------------- -%% logging - --spec log_locations() -> [rabbit_lager:log_location()]. -log_locations() -> - rabbit_lager:log_locations(). - --spec config_locations() -> [rabbit_config:config_location()]. -config_locations() -> - rabbit_config:config_files(). - --spec force_event_refresh(reference()) -> 'ok'. - -% Note: https://www.pivotaltracker.com/story/show/166962656 -% This event is necessary for the stats timer to be initialized with -% the correct values once the management agent has started -force_event_refresh(Ref) -> - % direct connections, e.g. MQTT, STOMP - ok = rabbit_direct:force_event_refresh(Ref), - % AMQP connections - ok = rabbit_networking:force_connection_event_refresh(Ref), - % "external" connections, which are not handled by the "AMQP core", - % e.g. connections to the stream plugin - ok = rabbit_networking:force_non_amqp_connection_event_refresh(Ref), - ok = rabbit_channel:force_event_refresh(Ref), - ok = rabbit_amqqueue:force_event_refresh(Ref). - -%%--------------------------------------------------------------------------- -%% misc - -log_broker_started(Plugins) -> - PluginList = iolist_to_binary([rabbit_misc:format(" * ~s~n", [P]) - || P <- Plugins]), - Message = string:strip(rabbit_misc:format( - "Server startup complete; ~b plugins started.~n~s", - [length(Plugins), PluginList]), right, $\n), - rabbit_log:info(Message), - io:format(" completed with ~p plugins.~n", [length(Plugins)]). - --define(RABBIT_TEXT_LOGO, - "~n ## ## ~s ~s" - "~n ## ##" - "~n ########## ~s" - "~n ###### ##" - "~n ########## ~s"). --define(FG8_START, "\033[38;5;202m"). --define(BG8_START, "\033[48;5;202m"). --define(FG32_START, "\033[38;2;255;102;0m"). --define(BG32_START, "\033[48;2;255;102;0m"). --define(C_END, "\033[0m"). --define(RABBIT_8BITCOLOR_LOGO, - "~n " ?BG8_START " " ?C_END " " ?BG8_START " " ?C_END " \033[1m" ?FG8_START "~s" ?C_END " ~s" - "~n " ?BG8_START " " ?C_END " " ?BG8_START " " ?C_END - "~n " ?BG8_START " " ?C_END " ~s" - "~n " ?BG8_START " " ?C_END " " ?BG8_START " " ?C_END - "~n " ?BG8_START " " ?C_END " ~s"). --define(RABBIT_32BITCOLOR_LOGO, - "~n " ?BG32_START " " ?C_END " " ?BG32_START " " ?C_END " \033[1m" ?FG32_START "~s" ?C_END " ~s" - "~n " ?BG32_START " " ?C_END " " ?BG32_START " " ?C_END - "~n " ?BG32_START " " ?C_END " ~s" - "~n " ?BG32_START " " ?C_END " " ?BG32_START " " ?C_END - "~n " ?BG32_START " " ?C_END " ~s"). - -print_banner() -> - Product = product_name(), - Version = product_version(), - LineListFormatter = fun (Placeholder, [_ | Tail] = LL) -> - LF = lists:flatten([Placeholder || _ <- lists:seq(1, length(Tail))]), - {LF, LL}; - (_, []) -> - {"", ["(none)"]} - end, - Logo = case rabbit_prelaunch:get_context() of - %% We use the colored logo only when running the - %% interactive shell and when colors are supported. - %% - %% Basically it means it will be used on Unix when - %% running "make run-broker" and that's about it. - #{os_type := {unix, darwin}, - interactive_shell := true, - output_supports_colors := true} -> ?RABBIT_8BITCOLOR_LOGO; - #{interactive_shell := true, - output_supports_colors := true} -> ?RABBIT_32BITCOLOR_LOGO; - _ -> ?RABBIT_TEXT_LOGO - end, - %% padded list lines - {LogFmt, LogLocations} = LineListFormatter("~n ~ts", log_locations()), - {CfgFmt, CfgLocations} = LineListFormatter("~n ~ts", config_locations()), - {MOTDFormat, MOTDArgs} = case motd() of - undefined -> - {"", []}; - MOTD -> - Lines = string:split(MOTD, "\n", all), - Padded = [case Line of - <<>> -> "\n"; - _ -> [" ", Line, "\n"] - end - || Line <- Lines], - {"~n~ts", [Padded]} - end, - io:format(Logo ++ - "~n" ++ - MOTDFormat ++ - "~n Doc guides: https://rabbitmq.com/documentation.html" - "~n Support: https://rabbitmq.com/contact.html" - "~n Tutorials: https://rabbitmq.com/getstarted.html" - "~n Monitoring: https://rabbitmq.com/monitoring.html" - "~n" - "~n Logs: ~ts" ++ LogFmt ++ "~n" - "~n Config file(s): ~ts" ++ CfgFmt ++ "~n" - "~n Starting broker...", - [Product, Version, ?COPYRIGHT_MESSAGE, ?INFORMATION_MESSAGE] ++ - MOTDArgs ++ - LogLocations ++ - CfgLocations). - -log_motd() -> - case motd() of - undefined -> - ok; - MOTD -> - Lines = string:split(MOTD, "\n", all), - Padded = [case Line of - <<>> -> "\n"; - _ -> [" ", Line, "\n"] - end - || Line <- Lines], - rabbit_log:info("~n~ts", [string:trim(Padded, trailing, [$\r, $\n])]) - end. - -log_banner() -> - {FirstLog, OtherLogs} = case log_locations() of - [Head | Tail] -> - {Head, [{"", F} || F <- Tail]}; - [] -> - {"(none)", []} - end, - Settings = [{"node", node()}, - {"home dir", home_dir()}, - {"config file(s)", config_files()}, - {"cookie hash", rabbit_nodes:cookie_hash()}, - {"log(s)", FirstLog}] ++ - OtherLogs ++ - [{"database dir", rabbit_mnesia:dir()}], - DescrLen = 1 + lists:max([length(K) || {K, _V} <- Settings]), - Format = fun (K, V) -> - rabbit_misc:format( - " ~-" ++ integer_to_list(DescrLen) ++ "s: ~ts~n", [K, V]) - end, - Banner = string:strip(lists:flatten( - [case S of - {"config file(s)" = K, []} -> - Format(K, "(none)"); - {"config file(s)" = K, [V0 | Vs]} -> - [Format(K, V0) | [Format("", V) || V <- Vs]]; - {K, V} -> - Format(K, V) - end || S <- Settings]), right, $\n), - rabbit_log:info("~n~ts", [Banner]). - -warn_if_kernel_config_dubious() -> - case os:type() of - {win32, _} -> - ok; - _ -> - case erlang:system_info(kernel_poll) of - true -> ok; - false -> rabbit_log:warning( - "Kernel poll (epoll, kqueue, etc) is disabled. Throughput " - "and CPU utilization may worsen.~n") - end - end, - AsyncThreads = erlang:system_info(thread_pool_size), - case AsyncThreads < ?ASYNC_THREADS_WARNING_THRESHOLD of - true -> rabbit_log:warning( - "Erlang VM is running with ~b I/O threads, " - "file I/O performance may worsen~n", [AsyncThreads]); - false -> ok - end, - IDCOpts = case application:get_env(kernel, inet_default_connect_options) of - undefined -> []; - {ok, Val} -> Val - end, - case proplists:get_value(nodelay, IDCOpts, false) of - false -> rabbit_log:warning("Nagle's algorithm is enabled for sockets, " - "network I/O latency will be higher~n"); - true -> ok - end. - -warn_if_disc_io_options_dubious() -> - %% if these values are not set, it doesn't matter since - %% rabbit_variable_queue will pick up the values defined in the - %% IO_BATCH_SIZE and CREDIT_DISC_BOUND constants. - CreditDiscBound = rabbit_misc:get_env(rabbit, msg_store_credit_disc_bound, - undefined), - IoBatchSize = rabbit_misc:get_env(rabbit, msg_store_io_batch_size, - undefined), - case catch validate_msg_store_io_batch_size_and_credit_disc_bound( - CreditDiscBound, IoBatchSize) of - ok -> ok; - {error, {Reason, Vars}} -> - rabbit_log:warning(Reason, Vars) - end. - -validate_msg_store_io_batch_size_and_credit_disc_bound(CreditDiscBound, - IoBatchSize) -> - case IoBatchSize of - undefined -> - ok; - IoBatchSize when is_integer(IoBatchSize) -> - if IoBatchSize < ?IO_BATCH_SIZE -> - throw({error, - {"io_batch_size of ~b lower than recommended value ~b, " - "paging performance may worsen~n", - [IoBatchSize, ?IO_BATCH_SIZE]}}); - true -> - ok - end; - IoBatchSize -> - throw({error, - {"io_batch_size should be an integer, but ~b given", - [IoBatchSize]}}) - end, - - %% CreditDiscBound = {InitialCredit, MoreCreditAfter} - {RIC, RMCA} = ?CREDIT_DISC_BOUND, - case CreditDiscBound of - undefined -> - ok; - {IC, MCA} when is_integer(IC), is_integer(MCA) -> - if IC < RIC; MCA < RMCA -> - throw({error, - {"msg_store_credit_disc_bound {~b, ~b} lower than" - "recommended value {~b, ~b}," - " paging performance may worsen~n", - [IC, MCA, RIC, RMCA]}}); - true -> - ok - end; - {IC, MCA} -> - throw({error, - {"both msg_store_credit_disc_bound values should be integers, but ~p given", - [{IC, MCA}]}}); - CreditDiscBound -> - throw({error, - {"invalid msg_store_credit_disc_bound value given: ~p", - [CreditDiscBound]}}) - end, - - case {CreditDiscBound, IoBatchSize} of - {undefined, undefined} -> - ok; - {_CDB, undefined} -> - ok; - {undefined, _IBS} -> - ok; - {{InitialCredit, _MCA}, IoBatchSize} -> - if IoBatchSize < InitialCredit -> - throw( - {error, - {"msg_store_io_batch_size ~b should be bigger than the initial " - "credit value from msg_store_credit_disc_bound ~b," - " paging performance may worsen~n", - [IoBatchSize, InitialCredit]}}); - true -> - ok - end - end. - --spec product_name() -> string(). - -product_name() -> - case product_info() of - #{product_name := ProductName} -> ProductName; - #{product_base_name := BaseName} -> BaseName - end. - --spec product_version() -> string(). - -product_version() -> - case product_info() of - #{product_version := ProductVersion} -> ProductVersion; - #{product_base_version := BaseVersion} -> BaseVersion - end. - --spec product_info() -> #{product_base_name := string(), - product_base_version := string(), - product_overridden := boolean(), - product_name => string(), - product_version => string(), - otp_release := string()}. - -product_info() -> - PTKey = {?MODULE, product}, - try - %% The value is cached the first time to avoid calling the - %% application master many times just for that. - persistent_term:get(PTKey) - catch - error:badarg -> - BaseName = base_product_name(), - BaseVersion = base_product_version(), - Info0 = #{product_base_name => BaseName, - product_base_version => BaseVersion, - otp_release => rabbit_misc:otp_release()}, - - {NameFromEnv, VersionFromEnv} = - case rabbit_prelaunch:get_context() of - #{product_name := NFE, - product_version := VFE} -> {NFE, VFE}; - _ -> {undefined, undefined} - end, - - Info1 = case NameFromEnv of - undefined -> - NameFromApp = string_from_app_env( - product_name, - undefined), - case NameFromApp of - undefined -> - Info0; - _ -> - Info0#{product_name => NameFromApp, - product_overridden => true} - end; - _ -> - Info0#{product_name => NameFromEnv, - product_overridden => true} - end, - - Info2 = case VersionFromEnv of - undefined -> - VersionFromApp = string_from_app_env( - product_version, - undefined), - case VersionFromApp of - undefined -> - Info1; - _ -> - Info1#{product_version => VersionFromApp, - product_overridden => true} - end; - _ -> - Info1#{product_version => VersionFromEnv, - product_overridden => true} - end, - persistent_term:put(PTKey, Info2), - Info2 - end. - -string_from_app_env(Key, Default) -> - case application:get_env(rabbit, Key) of - {ok, Val} -> - case io_lib:deep_char_list(Val) of - true -> - case lists:flatten(Val) of - "" -> Default; - String -> String - end; - false -> - Default - end; - undefined -> - Default - end. - -base_product_name() -> - %% This function assumes the `rabbit` application was loaded in - %% product_info(). - {ok, Product} = application:get_key(rabbit, description), - Product. - -base_product_version() -> - %% This function assumes the `rabbit` application was loaded in - %% product_info(). - rabbit_misc:version(). - -motd_file() -> - %% Precendence is: - %% 1. The environment variable; - %% 2. The `motd_file` configuration parameter; - %% 3. The default value. - Context = rabbit_prelaunch:get_context(), - case Context of - #{motd_file := File, - var_origins := #{motd_file := environment}} - when File =/= undefined -> - File; - _ -> - Default = case Context of - #{motd_file := File} -> File; - _ -> undefined - end, - string_from_app_env(motd_file, Default) - end. - -motd() -> - case motd_file() of - undefined -> - undefined; - File -> - case file:read_file(File) of - {ok, MOTD} -> string:trim(MOTD, trailing, [$\r,$\n]); - {error, _} -> undefined - end - end. - -home_dir() -> - case init:get_argument(home) of - {ok, [[Home]]} -> Home; - Other -> Other - end. - -config_files() -> - rabbit_config:config_files(). - -%% We don't want this in fhc since it references rabbit stuff. And we can't put -%% this in the bootstep directly. -start_fhc() -> - ok = rabbit_sup:start_restartable_child( - file_handle_cache, - [fun rabbit_alarm:set_alarm/1, fun rabbit_alarm:clear_alarm/1]), - ensure_working_fhc(). - -ensure_working_fhc() -> - %% To test the file handle cache, we simply read a file we know it - %% exists (Erlang kernel's .app file). - %% - %% To avoid any pollution of the application process' dictionary by - %% file_handle_cache, we spawn a separate process. - Parent = self(), - TestFun = fun() -> - ReadBuf = case application:get_env(rabbit, fhc_read_buffering) of - {ok, true} -> "ON"; - {ok, false} -> "OFF" - end, - WriteBuf = case application:get_env(rabbit, fhc_write_buffering) of - {ok, true} -> "ON"; - {ok, false} -> "OFF" - end, - rabbit_log:info("FHC read buffering: ~s~n", [ReadBuf]), - rabbit_log:info("FHC write buffering: ~s~n", [WriteBuf]), - Filename = filename:join(code:lib_dir(kernel, ebin), "kernel.app"), - {ok, Fd} = file_handle_cache:open(Filename, [raw, binary, read], []), - {ok, _} = file_handle_cache:read(Fd, 1), - ok = file_handle_cache:close(Fd), - Parent ! fhc_ok - end, - TestPid = spawn_link(TestFun), - %% Because we are waiting for the test fun, abuse the - %% 'mnesia_table_loading_retry_timeout' parameter to find a sane timeout - %% value. - Timeout = rabbit_table:retry_timeout(), - receive - fhc_ok -> ok; - {'EXIT', TestPid, Exception} -> throw({ensure_working_fhc, Exception}) - after Timeout -> - throw({ensure_working_fhc, {timeout, TestPid}}) - end. diff --git a/src/rabbit_access_control.erl b/src/rabbit_access_control.erl deleted file mode 100644 index 72260d5723..0000000000 --- a/src/rabbit_access_control.erl +++ /dev/null @@ -1,257 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_access_control). - --include("rabbit.hrl"). - --export([check_user_pass_login/2, check_user_login/2, check_user_loopback/2, - check_vhost_access/4, check_resource_access/4, check_topic_access/4]). - --export([permission_cache_can_expire/1, update_state/2]). - -%%---------------------------------------------------------------------------- - --export_type([permission_atom/0]). - --type permission_atom() :: 'configure' | 'read' | 'write'. - -%%---------------------------------------------------------------------------- - --spec check_user_pass_login - (rabbit_types:username(), rabbit_types:password()) -> - {'ok', rabbit_types:user()} | - {'refused', rabbit_types:username(), string(), [any()]}. - -check_user_pass_login(Username, Password) -> - check_user_login(Username, [{password, Password}]). - --spec check_user_login - (rabbit_types:username(), [{atom(), any()}]) -> - {'ok', rabbit_types:user()} | - {'refused', rabbit_types:username(), string(), [any()]}. - -check_user_login(Username, AuthProps) -> - %% extra auth properties like MQTT client id are in AuthProps - {ok, Modules} = application:get_env(rabbit, auth_backends), - R = lists:foldl( - fun (rabbit_auth_backend_cache=ModN, {refused, _, _, _}) -> - %% It is possible to specify authn/authz within the cache module settings, - %% so we have to do both auth steps here - %% See this rabbitmq-users discussion: - %% https://groups.google.com/d/topic/rabbitmq-users/ObqM7MQdA3I/discussion - try_authenticate_and_try_authorize(ModN, ModN, Username, AuthProps); - ({ModN, ModZs}, {refused, _, _, _}) -> - %% Different modules for authN vs authZ. So authenticate - %% with authN module, then if that succeeds do - %% passwordless (i.e pre-authenticated) login with authZ. - try_authenticate_and_try_authorize(ModN, ModZs, Username, AuthProps); - (Mod, {refused, _, _, _}) -> - %% Same module for authN and authZ. Just take the result - %% it gives us - case try_authenticate(Mod, Username, AuthProps) of - {ok, ModNUser = #auth_user{username = Username2, impl = Impl}} -> - rabbit_log:debug("User '~s' authenticated successfully by backend ~s", [Username2, Mod]), - user(ModNUser, {ok, [{Mod, Impl}], []}); - Else -> - rabbit_log:debug("User '~s' failed authenticatation by backend ~s", [Username, Mod]), - Else - end; - (_, {ok, User}) -> - %% We've successfully authenticated. Skip to the end... - {ok, User} - end, - {refused, Username, "No modules checked '~s'", [Username]}, Modules), - R. - -try_authenticate_and_try_authorize(ModN, ModZs0, Username, AuthProps) -> - ModZs = case ModZs0 of - A when is_atom(A) -> [A]; - L when is_list(L) -> L - end, - case try_authenticate(ModN, Username, AuthProps) of - {ok, ModNUser = #auth_user{username = Username2}} -> - rabbit_log:debug("User '~s' authenticated successfully by backend ~s", [Username2, ModN]), - user(ModNUser, try_authorize(ModZs, Username2, AuthProps)); - Else -> - Else - end. - -try_authenticate(Module, Username, AuthProps) -> - case Module:user_login_authentication(Username, AuthProps) of - {ok, AuthUser} -> {ok, AuthUser}; - {error, E} -> {refused, Username, - "~s failed authenticating ~s: ~p~n", - [Module, Username, E]}; - {refused, F, A} -> {refused, Username, F, A} - end. - -try_authorize(Modules, Username, AuthProps) -> - lists:foldr( - fun (Module, {ok, ModsImpls, ModsTags}) -> - case Module:user_login_authorization(Username, AuthProps) of - {ok, Impl, Tags}-> {ok, [{Module, Impl} | ModsImpls], ModsTags ++ Tags}; - {ok, Impl} -> {ok, [{Module, Impl} | ModsImpls], ModsTags}; - {error, E} -> {refused, Username, - "~s failed authorizing ~s: ~p~n", - [Module, Username, E]}; - {refused, F, A} -> {refused, Username, F, A} - end; - (_, {refused, F, A}) -> - {refused, Username, F, A} - end, {ok, [], []}, Modules). - -user(#auth_user{username = Username, tags = Tags}, {ok, ModZImpls, ModZTags}) -> - {ok, #user{username = Username, - tags = Tags ++ ModZTags, - authz_backends = ModZImpls}}; -user(_AuthUser, Error) -> - Error. - -auth_user(#user{username = Username, tags = Tags}, Impl) -> - #auth_user{username = Username, - tags = Tags, - impl = Impl}. - --spec check_user_loopback - (rabbit_types:username(), rabbit_net:socket() | inet:ip_address()) -> - 'ok' | 'not_allowed'. - -check_user_loopback(Username, SockOrAddr) -> - {ok, Users} = application:get_env(rabbit, loopback_users), - case rabbit_net:is_loopback(SockOrAddr) - orelse not lists:member(Username, Users) of - true -> ok; - false -> not_allowed - end. - -get_authz_data_from({ip, Address}) -> - #{peeraddr => Address}; -get_authz_data_from({socket, Sock}) -> - {ok, {Address, _Port}} = rabbit_net:peername(Sock), - #{peeraddr => Address}; -get_authz_data_from(undefined) -> - undefined. - -% Note: ip can be either a tuple or, a binary if reverse_dns_lookups -% is enabled and it's a direct connection. --spec check_vhost_access(User :: rabbit_types:user(), - VHostPath :: rabbit_types:vhost(), - AuthzRawData :: {socket, rabbit_net:socket()} | {ip, inet:ip_address() | binary()} | undefined, - AuthzContext :: map()) -> - 'ok' | rabbit_types:channel_exit(). -check_vhost_access(User = #user{username = Username, - authz_backends = Modules}, VHostPath, AuthzRawData, AuthzContext) -> - AuthzData = get_authz_data_from(AuthzRawData), - FullAuthzContext = create_vhost_access_authz_data(AuthzData, AuthzContext), - lists:foldl( - fun({Mod, Impl}, ok) -> - check_access( - fun() -> - rabbit_vhost:exists(VHostPath) andalso - Mod:check_vhost_access( - auth_user(User, Impl), VHostPath, FullAuthzContext) - end, - Mod, "access to vhost '~s' refused for user '~s'", - [VHostPath, Username], not_allowed); - (_, Else) -> - Else - end, ok, Modules). - -create_vhost_access_authz_data(undefined, Context) when map_size(Context) == 0 -> - undefined; -create_vhost_access_authz_data(undefined, Context) -> - Context; -create_vhost_access_authz_data(PeerAddr, Context) when map_size(Context) == 0 -> - PeerAddr; -create_vhost_access_authz_data(PeerAddr, Context) -> - maps:merge(PeerAddr, Context). - --spec check_resource_access - (rabbit_types:user(), rabbit_types:r(atom()), permission_atom(), rabbit_types:authz_context()) -> - 'ok' | rabbit_types:channel_exit(). - -check_resource_access(User, R = #resource{kind = exchange, name = <<"">>}, - Permission, Context) -> - check_resource_access(User, R#resource{name = <<"amq.default">>}, - Permission, Context); -check_resource_access(User = #user{username = Username, - authz_backends = Modules}, - Resource, Permission, Context) -> - lists:foldl( - fun({Module, Impl}, ok) -> - check_access( - fun() -> Module:check_resource_access( - auth_user(User, Impl), Resource, Permission, Context) end, - Module, "access to ~s refused for user '~s'", - [rabbit_misc:rs(Resource), Username]); - (_, Else) -> Else - end, ok, Modules). - -check_topic_access(User = #user{username = Username, - authz_backends = Modules}, - Resource, Permission, Context) -> - lists:foldl( - fun({Module, Impl}, ok) -> - check_access( - fun() -> Module:check_topic_access( - auth_user(User, Impl), Resource, Permission, Context) end, - Module, "access to topic '~s' in exchange ~s refused for user '~s'", - [maps:get(routing_key, Context), rabbit_misc:rs(Resource), Username]); - (_, Else) -> Else - end, ok, Modules). - -check_access(Fun, Module, ErrStr, ErrArgs) -> - check_access(Fun, Module, ErrStr, ErrArgs, access_refused). - -check_access(Fun, Module, ErrStr, ErrArgs, ErrName) -> - case Fun() of - true -> - ok; - false -> - rabbit_misc:protocol_error(ErrName, ErrStr, ErrArgs); - {error, E} -> - FullErrStr = ErrStr ++ ", backend ~s returned an error: ~p~n", - FullErrArgs = ErrArgs ++ [Module, E], - rabbit_log:error(FullErrStr, FullErrArgs), - rabbit_misc:protocol_error(ErrName, FullErrStr, FullErrArgs) - end. - --spec update_state(User :: rabbit_types:user(), NewState :: term()) -> - {'ok', rabbit_types:auth_user()} | - {'refused', string()} | - {'error', any()}. - -update_state(User = #user{authz_backends = Backends0}, NewState) -> - %% N.B.: we use foldl/3 and prepending, so the final list of - %% backends is in reverse order from the original list. - Backends = lists:foldl( - fun({Module, Impl}, {ok, Acc}) -> - case Module:state_can_expire() of - true -> - case Module:update_state(auth_user(User, Impl), NewState) of - {ok, #auth_user{impl = Impl1}} -> - {ok, [{Module, Impl1} | Acc]}; - Else -> Else - end; - false -> - {ok, [{Module, Impl} | Acc]} - end; - (_, {error, _} = Err) -> Err; - (_, {refused, _, _} = Err) -> Err - end, {ok, []}, Backends0), - case Backends of - {ok, Pairs} -> {ok, User#user{authz_backends = lists:reverse(Pairs)}}; - Else -> Else - end. - --spec permission_cache_can_expire(User :: rabbit_types:user()) -> boolean(). - -%% Returns true if any of the backends support credential expiration, -%% otherwise returns false. -permission_cache_can_expire(#user{authz_backends = Backends}) -> - lists:any(fun ({Module, _State}) -> Module:state_can_expire() end, Backends). diff --git a/src/rabbit_alarm.erl b/src/rabbit_alarm.erl deleted file mode 100644 index 3f1ab7ae62..0000000000 --- a/src/rabbit_alarm.erl +++ /dev/null @@ -1,365 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% -%% There are two types of alarms handled by this module: -%% -%% * per-node resource (disk, memory) alarms for the whole cluster. If any node -%% has an alarm, then all publishing should be disabled across the -%% cluster until all alarms clear. When a node sets such an alarm, -%% this information is automatically propagated throughout the cluster. -%% `#alarms.alarmed_nodes' is being used to track this type of alarms. -%% * limits local to this node (file_descriptor_limit). Used for information -%% purposes only: logging and getting node status. This information is not propagated -%% throughout the cluster. `#alarms.alarms' is being used to track this type of alarms. -%% @end - --module(rabbit_alarm). - --behaviour(gen_event). - --export([start_link/0, start/0, stop/0, register/2, set_alarm/1, - clear_alarm/1, get_alarms/0, get_alarms/1, get_local_alarms/0, get_local_alarms/1, on_node_up/1, on_node_down/1, - format_as_map/1, format_as_maps/1, is_local/1]). - --export([init/1, handle_call/2, handle_event/2, handle_info/2, - terminate/2, code_change/3]). - --export([remote_conserve_resources/3]). %% Internal use only - --define(SERVER, ?MODULE). - --define(FILE_DESCRIPTOR_RESOURCE, <<"file descriptors">>). --define(MEMORY_RESOURCE, <<"memory">>). --define(DISK_SPACE_RESOURCE, <<"disk space">>). - -%%---------------------------------------------------------------------------- - --record(alarms, {alertees :: dict:dict(pid(), rabbit_types:mfargs()), - alarmed_nodes :: dict:dict(node(), [resource_alarm_source()]), - alarms :: [alarm()]}). - --type local_alarm() :: 'file_descriptor_limit'. --type resource_alarm_source() :: 'disk' | 'memory'. --type resource_alarm() :: {resource_limit, resource_alarm_source(), node()}. --type alarm() :: local_alarm() | resource_alarm(). - -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - gen_event:start_link({local, ?SERVER}). - --spec start() -> 'ok'. - -start() -> - ok = rabbit_sup:start_restartable_child(?MODULE), - ok = gen_event:add_handler(?SERVER, ?MODULE, []), - {ok, MemoryWatermark} = application:get_env(vm_memory_high_watermark), - - rabbit_sup:start_restartable_child( - vm_memory_monitor, [MemoryWatermark, - fun (Alarm) -> - background_gc:run(), - set_alarm(Alarm) - end, - fun clear_alarm/1]), - {ok, DiskLimit} = application:get_env(disk_free_limit), - rabbit_sup:start_delayed_restartable_child( - rabbit_disk_monitor, [DiskLimit]), - ok. - --spec stop() -> 'ok'. - -stop() -> ok. - -%% Registers a handler that should be called on every resource alarm change. -%% Given a call rabbit_alarm:register(Pid, {M, F, A}), the handler would be -%% called like this: `apply(M, F, A ++ [Pid, Source, Alert])', where `Source' -%% has the type of resource_alarm_source() and `Alert' has the type of resource_alert(). - --spec register(pid(), rabbit_types:mfargs()) -> [atom()]. - -register(Pid, AlertMFA) -> - gen_event:call(?SERVER, ?MODULE, {register, Pid, AlertMFA}, infinity). - --spec set_alarm({alarm(), []}) -> 'ok'. - -set_alarm(Alarm) -> gen_event:notify(?SERVER, {set_alarm, Alarm}). - --spec clear_alarm(alarm()) -> 'ok'. - -clear_alarm(Alarm) -> gen_event:notify(?SERVER, {clear_alarm, Alarm}). - --spec get_alarms() -> [{alarm(), []}]. -get_alarms() -> gen_event:call(?SERVER, ?MODULE, get_alarms, infinity). - --spec get_alarms(timeout()) -> [{alarm(), []}]. -get_alarms(Timeout) -> gen_event:call(?SERVER, ?MODULE, get_alarms, Timeout). - --spec get_local_alarms() -> [alarm()]. -get_local_alarms() -> gen_event:call(?SERVER, ?MODULE, get_local_alarms, infinity). - --spec get_local_alarms(timeout()) -> [alarm()]. -get_local_alarms(Timeout) -> gen_event:call(?SERVER, ?MODULE, get_local_alarms, Timeout). - --spec filter_local_alarms([alarm()]) -> [alarm()]. -filter_local_alarms(Alarms) -> - lists:filter(fun is_local/1, Alarms). - --spec is_local({alarm(), any()}) -> boolean(). -is_local({file_descriptor_limit, _}) -> true; -is_local({{resource_limit, _Resource, Node}, _}) when Node =:= node() -> true; -is_local({{resource_limit, _Resource, Node}, _}) when Node =/= node() -> false. - --spec format_as_map(alarm()) -> #{binary() => term()}. -format_as_map(file_descriptor_limit) -> - #{ - <<"resource">> => ?FILE_DESCRIPTOR_RESOURCE, - <<"node">> => node() - }; -format_as_map({resource_limit, disk, Node}) -> - #{ - <<"resource">> => ?DISK_SPACE_RESOURCE, - <<"node">> => Node - }; -format_as_map({resource_limit, memory, Node}) -> - #{ - <<"resource">> => ?MEMORY_RESOURCE, - <<"node">> => Node - }; -format_as_map({resource_limit, Limit, Node}) -> - #{ - <<"resource">> => rabbit_data_coercion:to_binary(Limit), - <<"node">> => Node - }. - --spec format_as_maps([{alarm(), []}]) -> [#{any() => term()}]. -format_as_maps(Alarms) when is_list(Alarms) -> - %% get_alarms/0 returns - %% - %% [ - %% {file_descriptor_limit, []}, - %% {{resource_limit, disk, rabbit@warp10}, []}, - %% {{resource_limit, memory, rabbit@warp10}, []} - %% ] - lists:map(fun({Resource, _}) -> format_as_map(Resource); - (Resource) -> format_as_map(Resource) - end, Alarms). - - --spec on_node_up(node()) -> 'ok'. -on_node_up(Node) -> gen_event:notify(?SERVER, {node_up, Node}). - --spec on_node_down(node()) -> 'ok'. -on_node_down(Node) -> gen_event:notify(?SERVER, {node_down, Node}). - -remote_conserve_resources(Pid, Source, {true, _, _}) -> - gen_event:notify({?SERVER, node(Pid)}, - {set_alarm, {{resource_limit, Source, node()}, []}}); -remote_conserve_resources(Pid, Source, {false, _, _}) -> - gen_event:notify({?SERVER, node(Pid)}, - {clear_alarm, {resource_limit, Source, node()}}). - - -%%---------------------------------------------------------------------------- - -init([]) -> - {ok, #alarms{alertees = dict:new(), - alarmed_nodes = dict:new(), - alarms = []}}. - -handle_call({register, Pid, AlertMFA}, State = #alarms{alarmed_nodes = AN}) -> - {ok, lists:usort(lists:append([V || {_, V} <- dict:to_list(AN)])), - internal_register(Pid, AlertMFA, State)}; - -handle_call(get_alarms, State) -> - {ok, compute_alarms(State), State}; - -handle_call(get_local_alarms, State) -> - {ok, filter_local_alarms(compute_alarms(State)), State}; - -handle_call(_Request, State) -> - {ok, not_understood, State}. - -handle_event({set_alarm, {{resource_limit, Source, Node}, []}}, State) -> - case is_node_alarmed(Source, Node, State) of - true -> - {ok, State}; - false -> - rabbit_event:notify(alarm_set, [{source, Source}, - {node, Node}]), - handle_set_resource_alarm(Source, Node, State) - end; -handle_event({set_alarm, Alarm}, State = #alarms{alarms = Alarms}) -> - case lists:member(Alarm, Alarms) of - true -> {ok, State}; - false -> UpdatedAlarms = lists:usort([Alarm|Alarms]), - handle_set_alarm(Alarm, State#alarms{alarms = UpdatedAlarms}) - end; - -handle_event({clear_alarm, {resource_limit, Source, Node}}, State) -> - case is_node_alarmed(Source, Node, State) of - true -> - rabbit_event:notify(alarm_cleared, [{source, Source}, - {node, Node}]), - handle_clear_resource_alarm(Source, Node, State); - false -> - {ok, State} - end; -handle_event({clear_alarm, Alarm}, State = #alarms{alarms = Alarms}) -> - case lists:keymember(Alarm, 1, Alarms) of - true -> handle_clear_alarm( - Alarm, State#alarms{alarms = lists:keydelete( - Alarm, 1, Alarms)}); - false -> {ok, State} - - end; - -handle_event({node_up, Node}, State) -> - %% Must do this via notify and not call to avoid possible deadlock. - ok = gen_event:notify( - {?SERVER, Node}, - {register, self(), {?MODULE, remote_conserve_resources, []}}), - {ok, State}; - -handle_event({node_down, Node}, #alarms{alarmed_nodes = AN} = State) -> - AlarmsForDeadNode = case dict:find(Node, AN) of - {ok, V} -> V; - error -> [] - end, - {ok, lists:foldr(fun(Source, AccState) -> - rabbit_log:warning("~s resource limit alarm cleared for dead node ~p~n", - [Source, Node]), - maybe_alert(fun dict_unappend/3, Node, Source, false, AccState) - end, State, AlarmsForDeadNode)}; - -handle_event({register, Pid, AlertMFA}, State) -> - {ok, internal_register(Pid, AlertMFA, State)}; - -handle_event(_Event, State) -> - {ok, State}. - -handle_info({'DOWN', _MRef, process, Pid, _Reason}, - State = #alarms{alertees = Alertees}) -> - {ok, State#alarms{alertees = dict:erase(Pid, Alertees)}}; - -handle_info(_Info, State) -> - {ok, State}. - -terminate(_Arg, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%---------------------------------------------------------------------------- - -dict_append(Key, Val, Dict) -> - L = case dict:find(Key, Dict) of - {ok, V} -> V; - error -> [] - end, - dict:store(Key, lists:usort([Val|L]), Dict). - -dict_unappend(Key, Val, Dict) -> - L = case dict:find(Key, Dict) of - {ok, V} -> V; - error -> [] - end, - - case lists:delete(Val, L) of - [] -> dict:erase(Key, Dict); - X -> dict:store(Key, X, Dict) - end. - -maybe_alert(UpdateFun, Node, Source, WasAlertAdded, - State = #alarms{alarmed_nodes = AN, - alertees = Alertees}) -> - AN1 = UpdateFun(Node, Source, AN), - %% Is alarm for Source still set on any node? - StillHasAlerts = lists:any(fun ({_Node, NodeAlerts}) -> lists:member(Source, NodeAlerts) end, dict:to_list(AN1)), - case StillHasAlerts of - true -> ok; - false -> rabbit_log:warning("~s resource limit alarm cleared across the cluster~n", [Source]) - end, - Alert = {WasAlertAdded, StillHasAlerts, Node}, - case node() of - Node -> ok = alert_remote(Alert, Alertees, Source); - _ -> ok - end, - ok = alert_local(Alert, Alertees, Source), - State#alarms{alarmed_nodes = AN1}. - -alert_local(Alert, Alertees, Source) -> - alert(Alertees, Source, Alert, fun erlang:'=:='/2). - -alert_remote(Alert, Alertees, Source) -> - alert(Alertees, Source, Alert, fun erlang:'=/='/2). - -alert(Alertees, Source, Alert, NodeComparator) -> - Node = node(), - dict:fold(fun (Pid, {M, F, A}, ok) -> - case NodeComparator(Node, node(Pid)) of - true -> apply(M, F, A ++ [Pid, Source, Alert]); - false -> ok - end - end, ok, Alertees). - -internal_register(Pid, {M, F, A} = AlertMFA, - State = #alarms{alertees = Alertees}) -> - _MRef = erlang:monitor(process, Pid), - case dict:find(node(), State#alarms.alarmed_nodes) of - {ok, Sources} -> [apply(M, F, A ++ [Pid, R, {true, true, node()}]) || R <- Sources]; - error -> ok - end, - NewAlertees = dict:store(Pid, AlertMFA, Alertees), - State#alarms{alertees = NewAlertees}. - -handle_set_resource_alarm(Source, Node, State) -> - rabbit_log:warning( - "~s resource limit alarm set on node ~p.~n~n" - "**********************************************************~n" - "*** Publishers will be blocked until this alarm clears ***~n" - "**********************************************************~n", - [Source, Node]), - {ok, maybe_alert(fun dict_append/3, Node, Source, true, State)}. - -handle_set_alarm({file_descriptor_limit, []}, State) -> - rabbit_log:warning( - "file descriptor limit alarm set.~n~n" - "********************************************************************~n" - "*** New connections will not be accepted until this alarm clears ***~n" - "********************************************************************~n"), - {ok, State}; -handle_set_alarm(Alarm, State) -> - rabbit_log:warning("alarm '~p' set~n", [Alarm]), - {ok, State}. - -handle_clear_resource_alarm(Source, Node, State) -> - rabbit_log:warning("~s resource limit alarm cleared on node ~p~n", - [Source, Node]), - {ok, maybe_alert(fun dict_unappend/3, Node, Source, false, State)}. - -handle_clear_alarm(file_descriptor_limit, State) -> - rabbit_log:warning("file descriptor limit alarm cleared~n"), - {ok, State}; -handle_clear_alarm(Alarm, State) -> - rabbit_log:warning("alarm '~p' cleared~n", [Alarm]), - {ok, State}. - -is_node_alarmed(Source, Node, #alarms{alarmed_nodes = AN}) -> - case dict:find(Node, AN) of - {ok, Sources} -> - lists:member(Source, Sources); - error -> - false - end. - -compute_alarms(#alarms{alarms = Alarms, - alarmed_nodes = AN}) -> - Alarms ++ [ {{resource_limit, Source, Node}, []} - || {Node, Sources} <- dict:to_list(AN), Source <- Sources ]. diff --git a/src/rabbit_amqqueue.erl b/src/rabbit_amqqueue.erl deleted file mode 100644 index cd5f894680..0000000000 --- a/src/rabbit_amqqueue.erl +++ /dev/null @@ -1,1889 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_amqqueue). - --export([warn_file_limit/0]). --export([recover/1, stop/1, start/1, declare/6, declare/7, - delete_immediately/1, delete_exclusive/2, delete/4, purge/1, - forget_all_durable/1]). --export([pseudo_queue/2, pseudo_queue/3, immutable/1]). --export([lookup/1, lookup_many/1, not_found_or_absent/1, not_found_or_absent_dirty/1, - with/2, with/3, with_or_die/2, - assert_equivalence/5, - check_exclusive_access/2, with_exclusive_access_or_die/3, - stat/1, deliver/2, - requeue/3, ack/3, reject/4]). --export([not_found/1, absent/2]). --export([list/0, list/1, info_keys/0, info/1, info/2, info_all/1, info_all/2, - emit_info_all/5, list_local/1, info_local/1, - emit_info_local/4, emit_info_down/4]). --export([count/0]). --export([list_down/1, count/1, list_names/0, list_names/1, list_local_names/0, - list_local_names_down/0, list_with_possible_retry/1]). --export([list_by_type/1, sample_local_queues/0, sample_n_by_name/2, sample_n/2]). --export([force_event_refresh/1, notify_policy_changed/1]). --export([consumers/1, consumers_all/1, emit_consumers_all/4, consumer_info_keys/0]). --export([basic_get/5, basic_consume/12, basic_cancel/5, notify_decorators/1]). --export([notify_sent/2, notify_sent_queue_down/1, resume/2]). --export([notify_down_all/2, notify_down_all/3, activate_limit_all/2, credit/5]). --export([on_node_up/1, on_node_down/1]). --export([update/2, store_queue/1, update_decorators/1, policy_changed/2]). --export([update_mirroring/1, sync_mirrors/1, cancel_sync_mirrors/1]). --export([emit_unresponsive/6, emit_unresponsive_local/5, is_unresponsive/2]). --export([has_synchronised_mirrors_online/1]). --export([is_replicated/1, is_exclusive/1, is_not_exclusive/1, is_dead_exclusive/1]). --export([list_local_quorum_queues/0, list_local_quorum_queue_names/0, - list_local_mirrored_classic_queues/0, list_local_mirrored_classic_names/0, - list_local_leaders/0, list_local_followers/0, get_quorum_nodes/1, - list_local_mirrored_classic_without_synchronised_mirrors/0, - list_local_mirrored_classic_without_synchronised_mirrors_for_cli/0]). --export([ensure_rabbit_queue_record_is_initialized/1]). --export([format/1]). --export([delete_immediately_by_resource/1]). --export([delete_crashed/1, - delete_crashed/2, - delete_crashed_internal/2]). - --export([pid_of/1, pid_of/2]). --export([mark_local_durable_queues_stopped/1]). - --export([rebalance/3]). --export([collect_info_all/2]). - --export([is_policy_applicable/2]). --export([is_server_named_allowed/1]). - --export([check_max_age/1]). --export([get_queue_type/1]). - -%% internal --export([internal_declare/2, internal_delete/2, run_backing_queue/3, - set_ram_duration_target/2, set_maximum_since_use/2, - emit_consumers_local/3, internal_delete/3]). - --include_lib("rabbit_common/include/rabbit.hrl"). --include_lib("stdlib/include/qlc.hrl"). --include("amqqueue.hrl"). - --define(INTEGER_ARG_TYPES, [byte, short, signedint, long, - unsignedbyte, unsignedshort, unsignedint]). - --define(MORE_CONSUMER_CREDIT_AFTER, 50). - --define(IS_CLASSIC(QPid), is_pid(QPid)). --define(IS_QUORUM(QPid), is_tuple(QPid)). -%%---------------------------------------------------------------------------- - --export_type([name/0, qmsg/0, absent_reason/0]). - --type name() :: rabbit_types:r('queue'). - --type qpids() :: [pid()]. --type qlen() :: rabbit_types:ok(non_neg_integer()). --type qfun(A) :: fun ((amqqueue:amqqueue()) -> A | no_return()). --type qmsg() :: {name(), pid() | {atom(), pid()}, msg_id(), - boolean(), rabbit_types:message()}. --type msg_id() :: non_neg_integer(). --type ok_or_errors() :: - 'ok' | {'error', [{'error' | 'exit' | 'throw', any()}]}. --type absent_reason() :: 'nodedown' | 'crashed' | stopped | timeout. --type queue_not_found() :: not_found. --type queue_absent() :: {'absent', amqqueue:amqqueue(), absent_reason()}. --type not_found_or_absent() :: queue_not_found() | queue_absent(). - -%%---------------------------------------------------------------------------- - --define(CONSUMER_INFO_KEYS, - [queue_name, channel_pid, consumer_tag, ack_required, prefetch_count, - active, activity_status, arguments]). - -warn_file_limit() -> - DurableQueues = find_recoverable_queues(), - L = length(DurableQueues), - - %% if there are not enough file handles, the server might hang - %% when trying to recover queues, warn the user: - case file_handle_cache:get_limit() < L of - true -> - rabbit_log:warning( - "Recovering ~p queues, available file handles: ~p. Please increase max open file handles limit to at least ~p!~n", - [L, file_handle_cache:get_limit(), L]); - false -> - ok - end. - --spec recover(rabbit_types:vhost()) -> - {Recovered :: [amqqueue:amqqueue()], - Failed :: [amqqueue:amqqueue()]}. -recover(VHost) -> - AllDurable = find_local_durable_queues(VHost), - rabbit_queue_type:recover(VHost, AllDurable). - -filter_pid_per_type(QPids) -> - lists:partition(fun(QPid) -> ?IS_CLASSIC(QPid) end, QPids). - -filter_resource_per_type(Resources) -> - Queues = [begin - {ok, Q} = lookup(Resource), - QPid = amqqueue:get_pid(Q), - {Resource, QPid} - end || Resource <- Resources], - lists:partition(fun({_Resource, QPid}) -> ?IS_CLASSIC(QPid) end, Queues). - --spec stop(rabbit_types:vhost()) -> 'ok'. -stop(VHost) -> - %% Classic queues - ok = rabbit_amqqueue_sup_sup:stop_for_vhost(VHost), - {ok, BQ} = application:get_env(rabbit, backing_queue_module), - ok = BQ:stop(VHost), - rabbit_quorum_queue:stop(VHost). - --spec start([amqqueue:amqqueue()]) -> 'ok'. - -start(Qs) -> - %% At this point all recovered queues and their bindings are - %% visible to routing, so now it is safe for them to complete - %% their initialisation (which may involve interacting with other - %% queues). - _ = [amqqueue:get_pid(Q) ! {self(), go} - || Q <- Qs, - %% All queues are supposed to be classic here. - amqqueue:is_classic(Q)], - ok. - -mark_local_durable_queues_stopped(VHost) -> - ?try_mnesia_tx_or_upgrade_amqqueue_and_retry( - do_mark_local_durable_queues_stopped(VHost), - do_mark_local_durable_queues_stopped(VHost)). - -do_mark_local_durable_queues_stopped(VHost) -> - Qs = find_local_durable_queues(VHost), - rabbit_misc:execute_mnesia_transaction( - fun() -> - [ store_queue(amqqueue:set_state(Q, stopped)) - || Q <- Qs, amqqueue:get_type(Q) =:= rabbit_classic_queue, - amqqueue:get_state(Q) =/= stopped ] - end). - -find_local_durable_queues(VHost) -> - mnesia:async_dirty( - fun () -> - qlc:e( - qlc:q( - [Q || Q <- mnesia:table(rabbit_durable_queue), - amqqueue:get_vhost(Q) =:= VHost andalso - rabbit_queue_type:is_recoverable(Q) - ])) - end). - -find_recoverable_queues() -> - mnesia:async_dirty( - fun () -> - qlc:e(qlc:q([Q || Q <- mnesia:table(rabbit_durable_queue), - rabbit_queue_type:is_recoverable(Q)])) - end). - --spec declare(name(), - boolean(), - boolean(), - rabbit_framing:amqp_table(), - rabbit_types:maybe(pid()), - rabbit_types:username()) -> - {'new' | 'existing' | 'owner_died', amqqueue:amqqueue()} | - {'new', amqqueue:amqqueue(), rabbit_fifo_client:state()} | - {'absent', amqqueue:amqqueue(), absent_reason()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -declare(QueueName, Durable, AutoDelete, Args, Owner, ActingUser) -> - declare(QueueName, Durable, AutoDelete, Args, Owner, ActingUser, node()). - - -%% The Node argument suggests where the queue (master if mirrored) -%% should be. Note that in some cases (e.g. with "nodes" policy in -%% effect) this might not be possible to satisfy. - --spec declare(name(), - boolean(), - boolean(), - rabbit_framing:amqp_table(), - rabbit_types:maybe(pid()), - rabbit_types:username(), - node()) -> - {'new' | 'existing' | 'owner_died', amqqueue:amqqueue()} | - {'absent', amqqueue:amqqueue(), absent_reason()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -declare(QueueName = #resource{virtual_host = VHost}, Durable, AutoDelete, Args, - Owner, ActingUser, Node) -> - ok = check_declare_arguments(QueueName, Args), - Type = get_queue_type(Args), - case rabbit_queue_type:is_enabled(Type) of - true -> - Q0 = amqqueue:new(QueueName, - none, - Durable, - AutoDelete, - Owner, - Args, - VHost, - #{user => ActingUser}, - Type), - Q = rabbit_queue_decorator:set( - rabbit_policy:set(Q0)), - rabbit_queue_type:declare(Q, Node); - false -> - {protocol_error, internal_error, - "Cannot declare a queue '~s' of type '~s' on node '~s': " - "the corresponding feature flag is disabled", - [rabbit_misc:rs(QueueName), Type, Node]} - end. - -get_queue_type(Args) -> - case rabbit_misc:table_lookup(Args, <<"x-queue-type">>) of - undefined -> - rabbit_queue_type:default(); - {_, V} -> - rabbit_queue_type:discover(V) - end. - --spec internal_declare(amqqueue:amqqueue(), boolean()) -> - {created | existing, amqqueue:amqqueue()} | queue_absent(). - -internal_declare(Q, Recover) -> - ?try_mnesia_tx_or_upgrade_amqqueue_and_retry( - do_internal_declare(Q, Recover), - begin - Q1 = amqqueue:upgrade(Q), - do_internal_declare(Q1, Recover) - end). - -do_internal_declare(Q, true) -> - rabbit_misc:execute_mnesia_tx_with_tail( - fun () -> - ok = store_queue(amqqueue:set_state(Q, live)), - rabbit_misc:const({created, Q}) - end); -do_internal_declare(Q, false) -> - QueueName = amqqueue:get_name(Q), - rabbit_misc:execute_mnesia_tx_with_tail( - fun () -> - case mnesia:wread({rabbit_queue, QueueName}) of - [] -> - case not_found_or_absent(QueueName) of - not_found -> Q1 = rabbit_policy:set(Q), - Q2 = amqqueue:set_state(Q1, live), - ok = store_queue(Q2), - fun () -> {created, Q2} end; - {absent, _Q, _} = R -> rabbit_misc:const(R) - end; - [ExistingQ] -> - rabbit_misc:const({existing, ExistingQ}) - end - end). - --spec update - (name(), fun((amqqueue:amqqueue()) -> amqqueue:amqqueue())) -> - 'not_found' | amqqueue:amqqueue(). - -update(Name, Fun) -> - case mnesia:wread({rabbit_queue, Name}) of - [Q] -> - Durable = amqqueue:is_durable(Q), - Q1 = Fun(Q), - ok = mnesia:write(rabbit_queue, Q1, write), - case Durable of - true -> ok = mnesia:write(rabbit_durable_queue, Q1, write); - _ -> ok - end, - Q1; - [] -> - not_found - end. - -%% only really used for quorum queues to ensure the rabbit_queue record -%% is initialised -ensure_rabbit_queue_record_is_initialized(Q) -> - ?try_mnesia_tx_or_upgrade_amqqueue_and_retry( - do_ensure_rabbit_queue_record_is_initialized(Q), - begin - Q1 = amqqueue:upgrade(Q), - do_ensure_rabbit_queue_record_is_initialized(Q1) - end). - -do_ensure_rabbit_queue_record_is_initialized(Q) -> - rabbit_misc:execute_mnesia_tx_with_tail( - fun () -> - ok = store_queue(Q), - rabbit_misc:const({ok, Q}) - end). - --spec store_queue(amqqueue:amqqueue()) -> 'ok'. - -store_queue(Q) when ?amqqueue_is_durable(Q) -> - Q1 = amqqueue:reset_mirroring_and_decorators(Q), - ok = mnesia:write(rabbit_durable_queue, Q1, write), - store_queue_ram(Q); -store_queue(Q) when not ?amqqueue_is_durable(Q) -> - store_queue_ram(Q). - -store_queue_ram(Q) -> - ok = mnesia:write(rabbit_queue, rabbit_queue_decorator:set(Q), write). - --spec update_decorators(name()) -> 'ok'. - -update_decorators(Name) -> - rabbit_misc:execute_mnesia_transaction( - fun() -> - case mnesia:wread({rabbit_queue, Name}) of - [Q] -> store_queue_ram(Q), - ok; - [] -> ok - end - end). - --spec policy_changed(amqqueue:amqqueue(), amqqueue:amqqueue()) -> - 'ok'. - -policy_changed(Q1, Q2) -> - Decorators1 = amqqueue:get_decorators(Q1), - Decorators2 = amqqueue:get_decorators(Q2), - rabbit_mirror_queue_misc:update_mirrors(Q1, Q2), - D1 = rabbit_queue_decorator:select(Decorators1), - D2 = rabbit_queue_decorator:select(Decorators2), - [ok = M:policy_changed(Q1, Q2) || M <- lists:usort(D1 ++ D2)], - %% Make sure we emit a stats event even if nothing - %% mirroring-related has changed - the policy may have changed anyway. - notify_policy_changed(Q2). - -is_policy_applicable(QName, Policy) -> - case lookup(QName) of - {ok, Q} -> - rabbit_queue_type:is_policy_applicable(Q, Policy); - _ -> - %% Defaults to previous behaviour. Apply always - true - end. - -is_server_named_allowed(Args) -> - Type = get_queue_type(Args), - rabbit_queue_type:is_server_named_allowed(Type). - --spec lookup - (name()) -> - rabbit_types:ok(amqqueue:amqqueue()) | - rabbit_types:error('not_found'); - ([name()]) -> - [amqqueue:amqqueue()]. - -lookup([]) -> []; %% optimisation -lookup([Name]) -> ets:lookup(rabbit_queue, Name); %% optimisation -lookup(Names) when is_list(Names) -> - %% Normally we'd call mnesia:dirty_read/1 here, but that is quite - %% expensive for reasons explained in rabbit_misc:dirty_read/1. - lists:append([ets:lookup(rabbit_queue, Name) || Name <- Names]); -lookup(Name) -> - rabbit_misc:dirty_read({rabbit_queue, Name}). - --spec lookup_many ([name()]) -> [amqqueue:amqqueue()]. - -lookup_many(Names) when is_list(Names) -> - lookup(Names). - --spec not_found_or_absent(name()) -> not_found_or_absent(). - -not_found_or_absent(Name) -> - %% NB: we assume that the caller has already performed a lookup on - %% rabbit_queue and not found anything - case mnesia:read({rabbit_durable_queue, Name}) of - [] -> not_found; - [Q] -> {absent, Q, nodedown} %% Q exists on stopped node - end. - --spec not_found_or_absent_dirty(name()) -> not_found_or_absent(). - -not_found_or_absent_dirty(Name) -> - %% We should read from both tables inside a tx, to get a - %% consistent view. But the chances of an inconsistency are small, - %% and only affect the error kind. - case rabbit_misc:dirty_read({rabbit_durable_queue, Name}) of - {error, not_found} -> not_found; - {ok, Q} -> {absent, Q, nodedown} - end. - --spec get_rebalance_lock(pid()) -> - {true, {rebalance_queues, pid()}} | false. -get_rebalance_lock(Pid) when is_pid(Pid) -> - Id = {rebalance_queues, Pid}, - Nodes = [node()|nodes()], - %% Note that we're not re-trying. We want to immediately know - %% if a re-balance is taking place and stop accordingly. - case global:set_lock(Id, Nodes, 0) of - true -> - {true, Id}; - false -> - false - end. - --spec rebalance('all' | 'quorum' | 'classic', binary(), binary()) -> - {ok, [{node(), pos_integer()}]} | {error, term()}. -rebalance(Type, VhostSpec, QueueSpec) -> - %% We have not yet acquired the rebalance_queues global lock. - maybe_rebalance(get_rebalance_lock(self()), Type, VhostSpec, QueueSpec). - -maybe_rebalance({true, Id}, Type, VhostSpec, QueueSpec) -> - rabbit_log:info("Starting queue rebalance operation: '~s' for vhosts matching '~s' and queues matching '~s'", - [Type, VhostSpec, QueueSpec]), - Running = rabbit_nodes:all_running(), - NumRunning = length(Running), - ToRebalance = [Q || Q <- rabbit_amqqueue:list(), - filter_per_type(Type, Q), - is_replicated(Q), - is_match(amqqueue:get_vhost(Q), VhostSpec) andalso - is_match(get_resource_name(amqqueue:get_name(Q)), QueueSpec)], - NumToRebalance = length(ToRebalance), - ByNode = group_by_node(ToRebalance), - Rem = case (NumToRebalance rem NumRunning) of - 0 -> 0; - _ -> 1 - end, - MaxQueuesDesired = (NumToRebalance div NumRunning) + Rem, - Result = iterative_rebalance(ByNode, MaxQueuesDesired), - global:del_lock(Id), - rabbit_log:info("Finished queue rebalance operation"), - Result; -maybe_rebalance(false, _Type, _VhostSpec, _QueueSpec) -> - rabbit_log:warning("Queue rebalance operation is in progress, please wait."), - {error, rebalance_in_progress}. - -filter_per_type(all, _) -> - true; -filter_per_type(quorum, Q) -> - ?amqqueue_is_quorum(Q); -filter_per_type(classic, Q) -> - ?amqqueue_is_classic(Q). - -rebalance_module(Q) when ?amqqueue_is_quorum(Q) -> - rabbit_quorum_queue; -rebalance_module(Q) when ?amqqueue_is_classic(Q) -> - rabbit_mirror_queue_misc. - -get_resource_name(#resource{name = Name}) -> - Name. - -is_match(Subj, E) -> - nomatch /= re:run(Subj, E). - -iterative_rebalance(ByNode, MaxQueuesDesired) -> - case maybe_migrate(ByNode, MaxQueuesDesired) of - {ok, Summary} -> - rabbit_log:info("All queue masters are balanced"), - {ok, Summary}; - {migrated, Other} -> - iterative_rebalance(Other, MaxQueuesDesired); - {not_migrated, Other} -> - iterative_rebalance(Other, MaxQueuesDesired) - end. - -maybe_migrate(ByNode, MaxQueuesDesired) -> - maybe_migrate(ByNode, MaxQueuesDesired, maps:keys(ByNode)). - -maybe_migrate(ByNode, _, []) -> - {ok, maps:fold(fun(K, V, Acc) -> - {CQs, QQs} = lists:partition(fun({_, Q, _}) -> - ?amqqueue_is_classic(Q) - end, V), - [[{<<"Node name">>, K}, {<<"Number of quorum queues">>, length(QQs)}, - {<<"Number of classic queues">>, length(CQs)}] | Acc] - end, [], ByNode)}; -maybe_migrate(ByNode, MaxQueuesDesired, [N | Nodes]) -> - case maps:get(N, ByNode, []) of - [{_, Q, false} = Queue | Queues] = All when length(All) > MaxQueuesDesired -> - Name = amqqueue:get_name(Q), - Module = rebalance_module(Q), - OtherNodes = Module:get_replicas(Q) -- [N], - case OtherNodes of - [] -> - {not_migrated, update_not_migrated_queue(N, Queue, Queues, ByNode)}; - _ -> - [{Length, Destination} | _] = sort_by_number_of_queues(OtherNodes, ByNode), - rabbit_log:warning("Migrating queue ~p from node ~p with ~p queues to node ~p with ~p queues", - [Name, N, length(All), Destination, Length]), - case Module:transfer_leadership(Q, Destination) of - {migrated, NewNode} -> - rabbit_log:warning("Queue ~p migrated to ~p", [Name, NewNode]), - {migrated, update_migrated_queue(Destination, N, Queue, Queues, ByNode)}; - {not_migrated, Reason} -> - rabbit_log:warning("Error migrating queue ~p: ~p", [Name, Reason]), - {not_migrated, update_not_migrated_queue(N, Queue, Queues, ByNode)} - end - end; - [{_, _, true} | _] = All when length(All) > MaxQueuesDesired -> - rabbit_log:warning("Node ~p contains ~p queues, but all have already migrated. " - "Do nothing", [N, length(All)]), - maybe_migrate(ByNode, MaxQueuesDesired, Nodes); - All -> - rabbit_log:warning("Node ~p only contains ~p queues, do nothing", - [N, length(All)]), - maybe_migrate(ByNode, MaxQueuesDesired, Nodes) - end. - -update_not_migrated_queue(N, {Entries, Q, _}, Queues, ByNode) -> - maps:update(N, Queues ++ [{Entries, Q, true}], ByNode). - -update_migrated_queue(NewNode, OldNode, {Entries, Q, _}, Queues, ByNode) -> - maps:update_with(NewNode, - fun(L) -> L ++ [{Entries, Q, true}] end, - [{Entries, Q, true}], maps:update(OldNode, Queues, ByNode)). - -sort_by_number_of_queues(Nodes, ByNode) -> - lists:keysort(1, - lists:map(fun(Node) -> - {num_queues(Node, ByNode), Node} - end, Nodes)). - -num_queues(Node, ByNode) -> - length(maps:get(Node, ByNode, [])). - -group_by_node(Queues) -> - ByNode = lists:foldl(fun(Q, Acc) -> - Module = rebalance_module(Q), - Length = Module:queue_length(Q), - maps:update_with(amqqueue:qnode(Q), - fun(L) -> [{Length, Q, false} | L] end, - [{Length, Q, false}], Acc) - end, #{}, Queues), - maps:map(fun(_K, V) -> lists:keysort(1, V) end, ByNode). - --spec with(name(), - qfun(A), - fun((not_found_or_absent()) -> rabbit_types:channel_exit())) -> - A | rabbit_types:channel_exit(). - -with(Name, F, E) -> - with(Name, F, E, 2000). - -with(#resource{} = Name, F, E, RetriesLeft) -> - case lookup(Name) of - {ok, Q} when ?amqqueue_state_is(Q, live) andalso RetriesLeft =:= 0 -> - %% Something bad happened to that queue, we are bailing out - %% on processing current request. - E({absent, Q, timeout}); - {ok, Q} when ?amqqueue_state_is(Q, stopped) andalso RetriesLeft =:= 0 -> - %% The queue was stopped and not migrated - E({absent, Q, stopped}); - %% The queue process has crashed with unknown error - {ok, Q} when ?amqqueue_state_is(Q, crashed) -> - E({absent, Q, crashed}); - %% The queue process has been stopped by a supervisor. - %% In that case a synchronised mirror can take over - %% so we should retry. - {ok, Q} when ?amqqueue_state_is(Q, stopped) -> - %% The queue process was stopped by the supervisor - rabbit_misc:with_exit_handler( - fun () -> retry_wait(Q, F, E, RetriesLeft) end, - fun () -> F(Q) end); - %% The queue is supposed to be active. - %% The master node can go away or queue can be killed - %% so we retry, waiting for a mirror to take over. - {ok, Q} when ?amqqueue_state_is(Q, live) -> - %% We check is_process_alive(QPid) in case we receive a - %% nodedown (for example) in F() that has nothing to do - %% with the QPid. F() should be written s.t. that this - %% cannot happen, so we bail if it does since that - %% indicates a code bug and we don't want to get stuck in - %% the retry loop. - rabbit_misc:with_exit_handler( - fun () -> retry_wait(Q, F, E, RetriesLeft) end, - fun () -> F(Q) end); - {error, not_found} -> - E(not_found_or_absent_dirty(Name)) - end. - --spec retry_wait(amqqueue:amqqueue(), - qfun(A), - fun((not_found_or_absent()) -> rabbit_types:channel_exit()), - non_neg_integer()) -> - A | rabbit_types:channel_exit(). - -retry_wait(Q, F, E, RetriesLeft) -> - Name = amqqueue:get_name(Q), - QPid = amqqueue:get_pid(Q), - QState = amqqueue:get_state(Q), - case {QState, is_replicated(Q)} of - %% We don't want to repeat an operation if - %% there are no mirrors to migrate to - {stopped, false} -> - E({absent, Q, stopped}); - _ -> - case rabbit_mnesia:is_process_alive(QPid) of - true -> - % rabbitmq-server#1682 - % The old check would have crashed here, - % instead, log it and run the exit fun. absent & alive is weird, - % but better than crashing with badmatch,true - rabbit_log:debug("Unexpected alive queue process ~p~n", [QPid]), - E({absent, Q, alive}); - false -> - ok % Expected result - end, - timer:sleep(30), - with(Name, F, E, RetriesLeft - 1) - end. - --spec with(name(), qfun(A)) -> - A | rabbit_types:error(not_found_or_absent()). - -with(Name, F) -> with(Name, F, fun (E) -> {error, E} end). - --spec with_or_die(name(), qfun(A)) -> A | rabbit_types:channel_exit(). - -with_or_die(Name, F) -> - with(Name, F, die_fun(Name)). - --spec die_fun(name()) -> - fun((not_found_or_absent()) -> rabbit_types:channel_exit()). - -die_fun(Name) -> - fun (not_found) -> not_found(Name); - ({absent, Q, Reason}) -> absent(Q, Reason) - end. - --spec not_found(name()) -> rabbit_types:channel_exit(). - -not_found(R) -> rabbit_misc:protocol_error(not_found, "no ~s", [rabbit_misc:rs(R)]). - --spec absent(amqqueue:amqqueue(), absent_reason()) -> - rabbit_types:channel_exit(). - -absent(Q, AbsentReason) -> - QueueName = amqqueue:get_name(Q), - QPid = amqqueue:get_pid(Q), - IsDurable = amqqueue:is_durable(Q), - priv_absent(QueueName, QPid, IsDurable, AbsentReason). - --spec priv_absent(name(), pid(), boolean(), absent_reason()) -> - rabbit_types:channel_exit(). - -priv_absent(QueueName, QPid, true, nodedown) -> - %% The assertion of durability is mainly there because we mention - %% durability in the error message. That way we will hopefully - %% notice if at some future point our logic changes s.t. we get - %% here with non-durable queues. - rabbit_misc:protocol_error( - not_found, - "home node '~s' of durable ~s is down or inaccessible", - [node(QPid), rabbit_misc:rs(QueueName)]); - -priv_absent(QueueName, _QPid, _IsDurable, stopped) -> - rabbit_misc:protocol_error( - not_found, - "~s process is stopped by supervisor", [rabbit_misc:rs(QueueName)]); - -priv_absent(QueueName, _QPid, _IsDurable, crashed) -> - rabbit_misc:protocol_error( - not_found, - "~s has crashed and failed to restart", [rabbit_misc:rs(QueueName)]); - -priv_absent(QueueName, _QPid, _IsDurable, timeout) -> - rabbit_misc:protocol_error( - not_found, - "failed to perform operation on ~s due to timeout", [rabbit_misc:rs(QueueName)]); - -priv_absent(QueueName, QPid, _IsDurable, alive) -> - rabbit_misc:protocol_error( - not_found, - "failed to perform operation on ~s: its master replica ~w may be stopping or being demoted", - [rabbit_misc:rs(QueueName), QPid]). - --spec assert_equivalence - (amqqueue:amqqueue(), boolean(), boolean(), - rabbit_framing:amqp_table(), rabbit_types:maybe(pid())) -> - 'ok' | rabbit_types:channel_exit() | rabbit_types:connection_exit(). - -assert_equivalence(Q, DurableDeclare, AutoDeleteDeclare, Args1, Owner) -> - QName = amqqueue:get_name(Q), - DurableQ = amqqueue:is_durable(Q), - AutoDeleteQ = amqqueue:is_auto_delete(Q), - ok = check_exclusive_access(Q, Owner, strict), - ok = rabbit_misc:assert_field_equivalence(DurableQ, DurableDeclare, QName, durable), - ok = rabbit_misc:assert_field_equivalence(AutoDeleteQ, AutoDeleteDeclare, QName, auto_delete), - ok = assert_args_equivalence(Q, Args1). - --spec check_exclusive_access(amqqueue:amqqueue(), pid()) -> - 'ok' | rabbit_types:channel_exit(). - -check_exclusive_access(Q, Owner) -> check_exclusive_access(Q, Owner, lax). - -check_exclusive_access(Q, Owner, _MatchType) - when ?amqqueue_exclusive_owner_is(Q, Owner) -> - ok; -check_exclusive_access(Q, _ReaderPid, lax) - when ?amqqueue_exclusive_owner_is(Q, none) -> - ok; -check_exclusive_access(Q, _ReaderPid, _MatchType) -> - QueueName = amqqueue:get_name(Q), - rabbit_misc:protocol_error( - resource_locked, - "cannot obtain exclusive access to locked ~s. It could be originally " - "declared on another connection or the exclusive property value does not " - "match that of the original declaration.", - [rabbit_misc:rs(QueueName)]). - --spec with_exclusive_access_or_die(name(), pid(), qfun(A)) -> - A | rabbit_types:channel_exit(). - -with_exclusive_access_or_die(Name, ReaderPid, F) -> - with_or_die(Name, - fun (Q) -> check_exclusive_access(Q, ReaderPid), F(Q) end). - -assert_args_equivalence(Q, RequiredArgs) -> - QueueName = amqqueue:get_name(Q), - Args = amqqueue:get_arguments(Q), - rabbit_misc:assert_args_equivalence(Args, RequiredArgs, QueueName, - [Key || {Key, _Fun} <- declare_args()]). - -check_declare_arguments(QueueName, Args) -> - check_arguments(QueueName, Args, declare_args()). - -check_consume_arguments(QueueName, Args) -> - check_arguments(QueueName, Args, consume_args()). - -check_arguments(QueueName, Args, Validators) -> - [case rabbit_misc:table_lookup(Args, Key) of - undefined -> ok; - TypeVal -> case Fun(TypeVal, Args) of - ok -> ok; - {error, Error} -> rabbit_misc:protocol_error( - precondition_failed, - "invalid arg '~s' for ~s: ~255p", - [Key, rabbit_misc:rs(QueueName), - Error]) - end - end || {Key, Fun} <- Validators], - ok. - -declare_args() -> - [{<<"x-expires">>, fun check_expires_arg/2}, - {<<"x-message-ttl">>, fun check_message_ttl_arg/2}, - {<<"x-dead-letter-exchange">>, fun check_dlxname_arg/2}, - {<<"x-dead-letter-routing-key">>, fun check_dlxrk_arg/2}, - {<<"x-max-length">>, fun check_non_neg_int_arg/2}, - {<<"x-max-length-bytes">>, fun check_non_neg_int_arg/2}, - {<<"x-max-in-memory-length">>, fun check_non_neg_int_arg/2}, - {<<"x-max-in-memory-bytes">>, fun check_non_neg_int_arg/2}, - {<<"x-max-priority">>, fun check_max_priority_arg/2}, - {<<"x-overflow">>, fun check_overflow/2}, - {<<"x-queue-mode">>, fun check_queue_mode/2}, - {<<"x-single-active-consumer">>, fun check_single_active_consumer_arg/2}, - {<<"x-queue-type">>, fun check_queue_type/2}, - {<<"x-quorum-initial-group-size">>, fun check_initial_cluster_size_arg/2}, - {<<"x-max-age">>, fun check_max_age_arg/2}, - {<<"x-max-segment-size">>, fun check_non_neg_int_arg/2}, - {<<"x-initial-cluster-size">>, fun check_initial_cluster_size_arg/2}, - {<<"x-queue-leader-locator">>, fun check_queue_leader_locator_arg/2}]. - -consume_args() -> [{<<"x-priority">>, fun check_int_arg/2}, - {<<"x-cancel-on-ha-failover">>, fun check_bool_arg/2}]. - -check_int_arg({Type, _}, _) -> - case lists:member(Type, ?INTEGER_ARG_TYPES) of - true -> ok; - false -> {error, {unacceptable_type, Type}} - end. - -check_bool_arg({bool, _}, _) -> ok; -check_bool_arg({Type, _}, _) -> {error, {unacceptable_type, Type}}. - -check_non_neg_int_arg({Type, Val}, Args) -> - case check_int_arg({Type, Val}, Args) of - ok when Val >= 0 -> ok; - ok -> {error, {value_negative, Val}}; - Error -> Error - end. - -check_expires_arg({Type, Val}, Args) -> - case check_int_arg({Type, Val}, Args) of - ok when Val == 0 -> {error, {value_zero, Val}}; - ok -> rabbit_misc:check_expiry(Val); - Error -> Error - end. - -check_message_ttl_arg({Type, Val}, Args) -> - case check_int_arg({Type, Val}, Args) of - ok -> rabbit_misc:check_expiry(Val); - Error -> Error - end. - -check_max_priority_arg({Type, Val}, Args) -> - case check_non_neg_int_arg({Type, Val}, Args) of - ok when Val =< ?MAX_SUPPORTED_PRIORITY -> ok; - ok -> {error, {max_value_exceeded, Val}}; - Error -> Error - end. - -check_single_active_consumer_arg({Type, Val}, Args) -> - case check_bool_arg({Type, Val}, Args) of - ok -> ok; - Error -> Error - end. - -check_initial_cluster_size_arg({Type, Val}, Args) -> - case check_non_neg_int_arg({Type, Val}, Args) of - ok when Val == 0 -> {error, {value_zero, Val}}; - ok -> ok; - Error -> Error - end. - -check_max_age_arg({longstr, Val}, _Args) -> - case check_max_age(Val) of - {error, _} = E -> - E; - _ -> - ok - end; -check_max_age_arg({Type, _}, _Args) -> - {error, {unacceptable_type, Type}}. - -check_max_age(MaxAge) -> - case re:run(MaxAge, "(^[0-9]*)(.*)", [{capture, all_but_first, list}]) of - {match, [Value, Unit]} -> - case list_to_integer(Value) of - I when I > 0 -> - case lists:member(Unit, ["Y", "M", "D", "h", "m", "s"]) of - true -> - Int = list_to_integer(Value), - Int * unit_value_in_ms(Unit); - false -> - {error, invalid_max_age} - end; - _ -> - {error, invalid_max_age} - end; - _ -> - {error, invalid_max_age} - end. - -unit_value_in_ms("Y") -> - 365 * unit_value_in_ms("D"); -unit_value_in_ms("M") -> - 30 * unit_value_in_ms("D"); -unit_value_in_ms("D") -> - 24 * unit_value_in_ms("h"); -unit_value_in_ms("h") -> - 3600 * unit_value_in_ms("s"); -unit_value_in_ms("m") -> - 60 * unit_value_in_ms("s"); -unit_value_in_ms("s") -> - 1000. - -%% Note that the validity of x-dead-letter-exchange is already verified -%% by rabbit_channel's queue.declare handler. -check_dlxname_arg({longstr, _}, _) -> ok; -check_dlxname_arg({Type, _}, _) -> {error, {unacceptable_type, Type}}. - -check_dlxrk_arg({longstr, _}, Args) -> - case rabbit_misc:table_lookup(Args, <<"x-dead-letter-exchange">>) of - undefined -> {error, routing_key_but_no_dlx_defined}; - _ -> ok - end; -check_dlxrk_arg({Type, _}, _Args) -> - {error, {unacceptable_type, Type}}. - -check_overflow({longstr, Val}, _Args) -> - case lists:member(Val, [<<"drop-head">>, - <<"reject-publish">>, - <<"reject-publish-dlx">>]) of - true -> ok; - false -> {error, invalid_overflow} - end; -check_overflow({Type, _}, _Args) -> - {error, {unacceptable_type, Type}}. - -check_queue_leader_locator_arg({longstr, Val}, _Args) -> - case lists:member(Val, [<<"client-local">>, - <<"random">>, - <<"least-leaders">>]) of - true -> ok; - false -> {error, invalid_queue_locator_arg} - end; -check_queue_leader_locator_arg({Type, _}, _Args) -> - {error, {unacceptable_type, Type}}. - -check_queue_mode({longstr, Val}, _Args) -> - case lists:member(Val, [<<"default">>, <<"lazy">>]) of - true -> ok; - false -> {error, invalid_queue_mode} - end; -check_queue_mode({Type, _}, _Args) -> - {error, {unacceptable_type, Type}}. - -check_queue_type({longstr, Val}, _Args) -> - case lists:member(Val, [<<"classic">>, <<"quorum">>, <<"stream">>]) of - true -> ok; - false -> {error, invalid_queue_type} - end; -check_queue_type({Type, _}, _Args) -> - {error, {unacceptable_type, Type}}. - --spec list() -> [amqqueue:amqqueue()]. - -list() -> - list_with_possible_retry(fun do_list/0). - -do_list() -> - mnesia:dirty_match_object(rabbit_queue, amqqueue:pattern_match_all()). - --spec count() -> non_neg_integer(). - -count() -> - mnesia:table_info(rabbit_queue, size). - --spec list_names() -> [rabbit_amqqueue:name()]. - -list_names() -> mnesia:dirty_all_keys(rabbit_queue). - -list_names(VHost) -> [amqqueue:get_name(Q) || Q <- list(VHost)]. - -list_local_names() -> - [ amqqueue:get_name(Q) || Q <- list(), - amqqueue:get_state(Q) =/= crashed, is_local_to_node(amqqueue:get_pid(Q), node())]. - -list_local_names_down() -> - [ amqqueue:get_name(Q) || Q <- list(), - is_down(Q), - is_local_to_node(amqqueue:get_pid(Q), node())]. - -is_down(Q) -> - try - info(Q, [state]) == [{state, down}] - catch - _:_ -> - true - end. - - --spec sample_local_queues() -> [amqqueue:amqqueue()]. -sample_local_queues() -> sample_n_by_name(list_local_names(), 300). - --spec sample_n_by_name([rabbit_amqqueue:name()], pos_integer()) -> [amqqueue:amqqueue()]. -sample_n_by_name([], _N) -> - []; -sample_n_by_name(Names, N) when is_list(Names) andalso is_integer(N) andalso N > 0 -> - %% lists:nth/2 throws when position is > list length - M = erlang:min(N, length(Names)), - Ids = lists:foldl(fun( _, Acc) when length(Acc) >= 100 -> - Acc; - (_, Acc) -> - Pick = lists:nth(rand:uniform(M), Names), - [Pick | Acc] - end, - [], lists:seq(1, M)), - lists:map(fun (Id) -> - {ok, Q} = rabbit_amqqueue:lookup(Id), - Q - end, - lists:usort(Ids)). - --spec sample_n([amqqueue:amqqueue()], pos_integer()) -> [amqqueue:amqqueue()]. -sample_n([], _N) -> - []; -sample_n(Queues, N) when is_list(Queues) andalso is_integer(N) andalso N > 0 -> - Names = [amqqueue:get_name(Q) || Q <- Queues], - sample_n_by_name(Names, N). - - --spec list_by_type(atom()) -> [amqqueue:amqqueue()]. - -list_by_type(classic) -> list_by_type(rabbit_classic_queue); -list_by_type(quorum) -> list_by_type(rabbit_quorum_queue); -list_by_type(Type) -> - {atomic, Qs} = - mnesia:sync_transaction( - fun () -> - mnesia:match_object(rabbit_durable_queue, - amqqueue:pattern_match_on_type(Type), - read) - end), - Qs. - --spec list_local_quorum_queue_names() -> [rabbit_amqqueue:name()]. - -list_local_quorum_queue_names() -> - [ amqqueue:get_name(Q) || Q <- list_by_type(quorum), - amqqueue:get_state(Q) =/= crashed, - lists:member(node(), get_quorum_nodes(Q))]. - --spec list_local_quorum_queues() -> [amqqueue:amqqueue()]. -list_local_quorum_queues() -> - [ Q || Q <- list_by_type(quorum), - amqqueue:get_state(Q) =/= crashed, - lists:member(node(), get_quorum_nodes(Q))]. - --spec list_local_leaders() -> [amqqueue:amqqueue()]. -list_local_leaders() -> - [ Q || Q <- list(), - amqqueue:is_quorum(Q), - amqqueue:get_state(Q) =/= crashed, amqqueue:get_leader(Q) =:= node()]. - --spec list_local_followers() -> [amqqueue:amqqueue()]. -list_local_followers() -> - [Q - || Q <- list(), - amqqueue:is_quorum(Q), - amqqueue:get_state(Q) =/= crashed, - amqqueue:get_leader(Q) =/= node(), - rabbit_quorum_queue:is_recoverable(Q) - ]. - --spec list_local_mirrored_classic_queues() -> [amqqueue:amqqueue()]. -list_local_mirrored_classic_queues() -> - [ Q || Q <- list(), - amqqueue:get_state(Q) =/= crashed, - amqqueue:is_classic(Q), - is_local_to_node(amqqueue:get_pid(Q), node()), - is_replicated(Q)]. - --spec list_local_mirrored_classic_names() -> [rabbit_amqqueue:name()]. -list_local_mirrored_classic_names() -> - [ amqqueue:get_name(Q) || Q <- list(), - amqqueue:get_state(Q) =/= crashed, - amqqueue:is_classic(Q), - is_local_to_node(amqqueue:get_pid(Q), node()), - is_replicated(Q)]. - --spec list_local_mirrored_classic_without_synchronised_mirrors() -> - [amqqueue:amqqueue()]. -list_local_mirrored_classic_without_synchronised_mirrors() -> - [ Q || Q <- list(), - amqqueue:get_state(Q) =/= crashed, - amqqueue:is_classic(Q), - %% filter out exclusive queues as they won't actually be mirrored - is_not_exclusive(Q), - is_local_to_node(amqqueue:get_pid(Q), node()), - is_replicated(Q), - not has_synchronised_mirrors_online(Q)]. - --spec list_local_mirrored_classic_without_synchronised_mirrors_for_cli() -> - [#{binary => any()}]. -list_local_mirrored_classic_without_synchronised_mirrors_for_cli() -> - ClassicQs = list_local_mirrored_classic_without_synchronised_mirrors(), - [begin - #resource{name = Name} = amqqueue:get_name(Q), - #{ - <<"readable_name">> => rabbit_data_coercion:to_binary(rabbit_misc:rs(amqqueue:get_name(Q))), - <<"name">> => Name, - <<"virtual_host">> => amqqueue:get_vhost(Q), - <<"type">> => <<"classic">> - } - end || Q <- ClassicQs]. - -is_local_to_node(QPid, Node) when ?IS_CLASSIC(QPid) -> - Node =:= node(QPid); -is_local_to_node({_, Leader} = QPid, Node) when ?IS_QUORUM(QPid) -> - Node =:= Leader. - --spec list(rabbit_types:vhost()) -> [amqqueue:amqqueue()]. - -list(VHostPath) -> - list(VHostPath, rabbit_queue). - -list(VHostPath, TableName) -> - list_with_possible_retry(fun() -> do_list(VHostPath, TableName) end). - -%% Not dirty_match_object since that would not be transactional when used in a -%% tx context -do_list(VHostPath, TableName) -> - mnesia:async_dirty( - fun () -> - mnesia:match_object( - TableName, - amqqueue:pattern_match_on_name(rabbit_misc:r(VHostPath, queue)), - read) - end). - -list_with_possible_retry(Fun) -> - %% amqqueue migration: - %% The `rabbit_queue` or `rabbit_durable_queue` tables - %% might be migrated between the time we query the pattern - %% (with the `amqqueue` module) and the time we call - %% `mnesia:dirty_match_object()`. This would lead to an empty list - %% (no object matching the now incorrect pattern), not a Mnesia - %% error. - %% - %% So if the result is an empty list and the version of the - %% `amqqueue` record changed in between, we retry the operation. - %% - %% However, we don't do this if inside a Mnesia transaction: we - %% could end up with a live lock between this started transaction - %% and the Mnesia table migration which is blocked (but the - %% rabbit_feature_flags lock is held). - AmqqueueRecordVersion = amqqueue:record_version_to_use(), - case Fun() of - [] -> - case mnesia:is_transaction() of - true -> - []; - false -> - case amqqueue:record_version_to_use() of - AmqqueueRecordVersion -> []; - _ -> Fun() - end - end; - Ret -> - Ret - end. - --spec list_down(rabbit_types:vhost()) -> [amqqueue:amqqueue()]. - -list_down(VHostPath) -> - case rabbit_vhost:exists(VHostPath) of - false -> []; - true -> - Present = list(VHostPath), - Durable = list(VHostPath, rabbit_durable_queue), - PresentS = sets:from_list([amqqueue:get_name(Q) || Q <- Present]), - sets:to_list(sets:filter(fun (Q) -> - N = amqqueue:get_name(Q), - not sets:is_element(N, PresentS) - end, sets:from_list(Durable))) - end. - -count(VHost) -> - try - %% this is certainly suboptimal but there is no way to count - %% things using a secondary index in Mnesia. Our counter-table-per-node - %% won't work here because with master migration of mirrored queues - %% the "ownership" of queues by nodes becomes a non-trivial problem - %% that requires a proper consensus algorithm. - length(list_for_count(VHost)) - catch _:Err -> - rabbit_log:error("Failed to fetch number of queues in vhost ~p:~n~p~n", - [VHost, Err]), - 0 - end. - -list_for_count(VHost) -> - list_with_possible_retry( - fun() -> - mnesia:dirty_index_read(rabbit_queue, - VHost, - amqqueue:field_vhost()) - end). - --spec info_keys() -> rabbit_types:info_keys(). - -%% It should no default to classic queue keys, but a subset of those that must be shared -%% by all queue types. Not sure this is even being used, so will leave it here for backwards -%% compatibility. Each queue type handles now info(Q, all_keys) with the keys it supports. -info_keys() -> rabbit_amqqueue_process:info_keys(). - -map(Qs, F) -> rabbit_misc:filter_exit_map(F, Qs). - -is_unresponsive(Q, _Timeout) when ?amqqueue_state_is(Q, crashed) -> - false; -is_unresponsive(Q, Timeout) when ?amqqueue_is_classic(Q) -> - QPid = amqqueue:get_pid(Q), - try - delegate:invoke(QPid, {gen_server2, call, [{info, [name]}, Timeout]}), - false - catch - %% TODO catch any exit?? - exit:{timeout, _} -> - true - end; -is_unresponsive(Q, Timeout) when ?amqqueue_is_quorum(Q) -> - try - Leader = amqqueue:get_pid(Q), - case rabbit_fifo_client:stat(Leader, Timeout) of - {ok, _, _} -> false; - {timeout, _} -> true; - {error, _} -> true - end - catch - exit:{timeout, _} -> - true - end. - -format(Q) when ?amqqueue_is_quorum(Q) -> rabbit_quorum_queue:format(Q); -format(Q) -> rabbit_amqqueue_process:format(Q). - --spec info(amqqueue:amqqueue()) -> rabbit_types:infos(). - -info(Q) when ?is_amqqueue(Q) -> rabbit_queue_type:info(Q, all_keys). - - --spec info(amqqueue:amqqueue(), rabbit_types:info_keys()) -> - rabbit_types:infos(). - -info(Q, Items) when ?is_amqqueue(Q) -> - rabbit_queue_type:info(Q, Items). - -info_down(Q, DownReason) -> - rabbit_queue_type:info_down(Q, DownReason). - -info_down(Q, Items, DownReason) -> - rabbit_queue_type:info_down(Q, Items, DownReason). - --spec info_all(rabbit_types:vhost()) -> [rabbit_types:infos()]. - -info_all(VHostPath) -> - map(list(VHostPath), fun (Q) -> info(Q) end) ++ - map(list_down(VHostPath), fun (Q) -> info_down(Q, down) end). - --spec info_all(rabbit_types:vhost(), rabbit_types:info_keys()) -> - [rabbit_types:infos()]. - -info_all(VHostPath, Items) -> - map(list(VHostPath), fun (Q) -> info(Q, Items) end) ++ - map(list_down(VHostPath), fun (Q) -> info_down(Q, Items, down) end). - -emit_info_local(VHostPath, Items, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map_with_exit_handler( - AggregatorPid, Ref, fun(Q) -> info(Q, Items) end, list_local(VHostPath)). - -emit_info_all(Nodes, VHostPath, Items, Ref, AggregatorPid) -> - Pids = [ spawn_link(Node, rabbit_amqqueue, emit_info_local, [VHostPath, Items, Ref, AggregatorPid]) || Node <- Nodes ], - rabbit_control_misc:await_emitters_termination(Pids). - -collect_info_all(VHostPath, Items) -> - Nodes = rabbit_nodes:all_running(), - Ref = make_ref(), - Pids = [ spawn_link(Node, rabbit_amqqueue, emit_info_local, [VHostPath, Items, Ref, self()]) || Node <- Nodes ], - rabbit_control_misc:await_emitters_termination(Pids), - wait_for_queues(Ref, length(Pids), []). - -wait_for_queues(Ref, N, Acc) -> - receive - {Ref, finished} when N == 1 -> - Acc; - {Ref, finished} -> - wait_for_queues(Ref, N - 1, Acc); - {Ref, Items, continue} -> - wait_for_queues(Ref, N, [Items | Acc]) - after - 1000 -> - Acc - end. - -emit_info_down(VHostPath, Items, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map_with_exit_handler( - AggregatorPid, Ref, fun(Q) -> info_down(Q, Items, down) end, - list_down(VHostPath)). - -emit_unresponsive_local(VHostPath, Items, Timeout, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map_with_exit_handler( - AggregatorPid, Ref, fun(Q) -> case is_unresponsive(Q, Timeout) of - true -> info_down(Q, Items, unresponsive); - false -> [] - end - end, list_local(VHostPath) - ). - -emit_unresponsive(Nodes, VHostPath, Items, Timeout, Ref, AggregatorPid) -> - Pids = [ spawn_link(Node, rabbit_amqqueue, emit_unresponsive_local, - [VHostPath, Items, Timeout, Ref, AggregatorPid]) || Node <- Nodes ], - rabbit_control_misc:await_emitters_termination(Pids). - -info_local(VHostPath) -> - map(list_local(VHostPath), fun (Q) -> info(Q, [name]) end). - -list_local(VHostPath) -> - [Q || Q <- list(VHostPath), - amqqueue:get_state(Q) =/= crashed, is_local_to_node(amqqueue:get_pid(Q), node())]. - --spec force_event_refresh(reference()) -> 'ok'. - -% Note: https://www.pivotaltracker.com/story/show/166962656 -% This event is necessary for the stats timer to be initialized with -% the correct values once the management agent has started -force_event_refresh(Ref) -> - %% note: quorum queuse emit stats on periodic ticks that run unconditionally, - %% so force_event_refresh is unnecessary (and, in fact, would only produce log noise) for QQs. - ClassicQs = list_by_type(rabbit_classic_queue), - [gen_server2:cast(amqqueue:get_pid(Q), - {force_event_refresh, Ref}) || Q <- ClassicQs], - ok. - --spec notify_policy_changed(amqqueue:amqqueue()) -> 'ok'. -notify_policy_changed(Q) when ?is_amqqueue(Q) -> - rabbit_queue_type:policy_changed(Q). - --spec consumers(amqqueue:amqqueue()) -> - [{pid(), rabbit_types:ctag(), boolean(), non_neg_integer(), - boolean(), atom(), - rabbit_framing:amqp_table(), rabbit_types:username()}]. - -consumers(Q) when ?amqqueue_is_classic(Q) -> - QPid = amqqueue:get_pid(Q), - delegate:invoke(QPid, {gen_server2, call, [consumers, infinity]}); -consumers(Q) when ?amqqueue_is_quorum(Q) -> - QPid = amqqueue:get_pid(Q), - case ra:local_query(QPid, fun rabbit_fifo:query_consumers/1) of - {ok, {_, Result}, _} -> maps:values(Result); - _ -> [] - end; -consumers(Q) when ?amqqueue_is_stream(Q) -> - %% TODO how??? they only exist on the channel - %% we could list the offset listener on the writer but we don't even have a consumer tag, - %% only a (channel) pid and offset - []. - --spec consumer_info_keys() -> rabbit_types:info_keys(). - -consumer_info_keys() -> ?CONSUMER_INFO_KEYS. - --spec consumers_all(rabbit_types:vhost()) -> - [{name(), pid(), rabbit_types:ctag(), boolean(), - non_neg_integer(), rabbit_framing:amqp_table()}]. - -consumers_all(VHostPath) -> - ConsumerInfoKeys = consumer_info_keys(), - lists:append( - map(list(VHostPath), - fun(Q) -> get_queue_consumer_info(Q, ConsumerInfoKeys) end)). - -emit_consumers_all(Nodes, VHostPath, Ref, AggregatorPid) -> - Pids = [ spawn_link(Node, rabbit_amqqueue, emit_consumers_local, [VHostPath, Ref, AggregatorPid]) || Node <- Nodes ], - rabbit_control_misc:await_emitters_termination(Pids), - ok. - -emit_consumers_local(VHostPath, Ref, AggregatorPid) -> - ConsumerInfoKeys = consumer_info_keys(), - rabbit_control_misc:emitting_map( - AggregatorPid, Ref, - fun(Q) -> get_queue_consumer_info(Q, ConsumerInfoKeys) end, - list_local(VHostPath)). - -get_queue_consumer_info(Q, ConsumerInfoKeys) -> - [lists:zip(ConsumerInfoKeys, - [amqqueue:get_name(Q), ChPid, CTag, - AckRequired, Prefetch, Active, ActivityStatus, Args]) || - {ChPid, CTag, AckRequired, Prefetch, Active, ActivityStatus, Args, _} <- consumers(Q)]. - --spec stat(amqqueue:amqqueue()) -> - {'ok', non_neg_integer(), non_neg_integer()}. -stat(Q) -> - rabbit_queue_type:stat(Q). - --spec pid_of(amqqueue:amqqueue()) -> - pid(). - -pid_of(Q) -> amqqueue:get_pid(Q). - --spec pid_of(rabbit_types:vhost(), rabbit_misc:resource_name()) -> - pid() | rabbit_types:error('not_found'). - -pid_of(VHost, QueueName) -> - case lookup(rabbit_misc:r(VHost, queue, QueueName)) of - {ok, Q} -> pid_of(Q); - {error, not_found} = E -> E - end. - --spec delete_exclusive(qpids(), pid()) -> 'ok'. - -delete_exclusive(QPids, ConnId) -> - rabbit_amqqueue_common:delete_exclusive(QPids, ConnId). - --spec delete_immediately(qpids()) -> 'ok'. - -delete_immediately(QPids) -> - {Classic, Quorum} = filter_pid_per_type(QPids), - [gen_server2:cast(QPid, delete_immediately) || QPid <- Classic], - case Quorum of - [] -> ok; - _ -> {error, cannot_delete_quorum_queues, Quorum} - end. - -delete_immediately_by_resource(Resources) -> - {Classic, Quorum} = filter_resource_per_type(Resources), - [gen_server2:cast(QPid, delete_immediately) || {_, QPid} <- Classic], - [rabbit_quorum_queue:delete_immediately(Resource, QPid) - || {Resource, QPid} <- Quorum], - ok. - --spec delete - (amqqueue:amqqueue(), 'false', 'false', rabbit_types:username()) -> - qlen() | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}; - (amqqueue:amqqueue(), 'true' , 'false', rabbit_types:username()) -> - qlen() | rabbit_types:error('in_use') | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}; - (amqqueue:amqqueue(), 'false', 'true', rabbit_types:username()) -> - qlen() | rabbit_types:error('not_empty') | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}; - (amqqueue:amqqueue(), 'true' , 'true', rabbit_types:username()) -> - qlen() | - rabbit_types:error('in_use') | - rabbit_types:error('not_empty') | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -delete(Q, IfUnused, IfEmpty, ActingUser) -> - rabbit_queue_type:delete(Q, IfUnused, IfEmpty, ActingUser). - -%% delete_crashed* INCLUDED FOR BACKWARDS COMPATBILITY REASONS -delete_crashed(Q) when ?amqqueue_is_classic(Q) -> - rabbit_classic_queue:delete_crashed(Q). - -delete_crashed(Q, ActingUser) when ?amqqueue_is_classic(Q) -> - rabbit_classic_queue:delete_crashed(Q, ActingUser). - --spec delete_crashed_internal(amqqueue:amqqueue(), rabbit_types:username()) -> 'ok'. -delete_crashed_internal(Q, ActingUser) when ?amqqueue_is_classic(Q) -> - rabbit_classic_queue:delete_crashed_internal(Q, ActingUser). - --spec purge(amqqueue:amqqueue()) -> qlen(). -purge(Q) when ?is_amqqueue(Q) -> - rabbit_queue_type:purge(Q). - --spec requeue(name(), - {rabbit_fifo:consumer_tag(), [msg_id()]}, - rabbit_queue_type:state()) -> - {ok, rabbit_queue_type:state(), rabbit_queue_type:actions()}. -requeue(QRef, {CTag, MsgIds}, QStates) -> - reject(QRef, true, {CTag, MsgIds}, QStates). - --spec ack(name(), - {rabbit_fifo:consumer_tag(), [msg_id()]}, - rabbit_queue_type:state()) -> - {ok, rabbit_queue_type:state(), rabbit_queue_type:actions()}. -ack(QPid, {CTag, MsgIds}, QueueStates) -> - rabbit_queue_type:settle(QPid, complete, CTag, MsgIds, QueueStates). - - --spec reject(name(), - boolean(), - {rabbit_fifo:consumer_tag(), [msg_id()]}, - rabbit_queue_type:state()) -> - {ok, rabbit_queue_type:state(), rabbit_queue_type:actions()}. -reject(QRef, Requeue, {CTag, MsgIds}, QStates) -> - Op = case Requeue of - true -> requeue; - false -> discard - end, - rabbit_queue_type:settle(QRef, Op, CTag, MsgIds, QStates). - --spec notify_down_all(qpids(), pid()) -> ok_or_errors(). -notify_down_all(QPids, ChPid) -> - notify_down_all(QPids, ChPid, ?CHANNEL_OPERATION_TIMEOUT). - --spec notify_down_all(qpids(), pid(), non_neg_integer()) -> - ok_or_errors(). -notify_down_all(QPids, ChPid, Timeout) -> - case rpc:call(node(), delegate, invoke, - [QPids, {gen_server2, call, [{notify_down, ChPid}, infinity]}], Timeout) of - {badrpc, timeout} -> {error, {channel_operation_timeout, Timeout}}; - {badrpc, Reason} -> {error, Reason}; - {_, Bads} -> - case lists:filter( - fun ({_Pid, {exit, {R, _}, _}}) -> - rabbit_misc:is_abnormal_exit(R); - ({_Pid, _}) -> false - end, Bads) of - [] -> ok; - Bads1 -> {error, Bads1} - end; - Error -> {error, Error} - end. - --spec activate_limit_all(qpids(), pid()) -> ok. - -activate_limit_all(QRefs, ChPid) -> - QPids = [P || P <- QRefs, ?IS_CLASSIC(P)], - delegate:invoke_no_result(QPids, {gen_server2, cast, - [{activate_limit, ChPid}]}). - --spec credit(amqqueue:amqqueue(), - rabbit_types:ctag(), - non_neg_integer(), - boolean(), - rabbit_queue_type:state()) -> - {ok, rabbit_queue_type:state(), rabbit_queue_type:actions()}. -credit(Q, CTag, Credit, Drain, QStates) -> - rabbit_queue_type:credit(Q, CTag, Credit, Drain, QStates). - --spec basic_get(amqqueue:amqqueue(), boolean(), pid(), rabbit_types:ctag(), - rabbit_queue_type:state()) -> - {'ok', non_neg_integer(), qmsg(), rabbit_queue_type:state()} | - {'empty', rabbit_queue_type:state()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -basic_get(Q, NoAck, LimiterPid, CTag, QStates0) -> - rabbit_queue_type:dequeue(Q, NoAck, LimiterPid, CTag, QStates0). - - --spec basic_consume(amqqueue:amqqueue(), boolean(), pid(), pid(), boolean(), - non_neg_integer(), rabbit_types:ctag(), boolean(), - rabbit_framing:amqp_table(), any(), rabbit_types:username(), - rabbit_queue_type:state()) -> - {ok, rabbit_queue_type:state(), rabbit_queue_type:actions()} | - {error, term()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -basic_consume(Q, NoAck, ChPid, LimiterPid, - LimiterActive, ConsumerPrefetchCount, ConsumerTag, - ExclusiveConsume, Args, OkMsg, ActingUser, Contexts) -> - - QName = amqqueue:get_name(Q), - %% first phase argument validation - %% each queue type may do further validations - ok = check_consume_arguments(QName, Args), - Spec = #{no_ack => NoAck, - channel_pid => ChPid, - limiter_pid => LimiterPid, - limiter_active => LimiterActive, - prefetch_count => ConsumerPrefetchCount, - consumer_tag => ConsumerTag, - exclusive_consume => ExclusiveConsume, - args => Args, - ok_msg => OkMsg, - acting_user => ActingUser}, - rabbit_queue_type:consume(Q, Spec, Contexts). - --spec basic_cancel(amqqueue:amqqueue(), rabbit_types:ctag(), any(), - rabbit_types:username(), - rabbit_queue_type:state()) -> - {ok, rabbit_queue_type:state()} | {error, term()}. -basic_cancel(Q, ConsumerTag, OkMsg, ActingUser, QStates) -> - rabbit_queue_type:cancel(Q, ConsumerTag, - OkMsg, ActingUser, QStates). - --spec notify_decorators(amqqueue:amqqueue()) -> 'ok'. - -notify_decorators(Q) -> - QPid = amqqueue:get_pid(Q), - delegate:invoke_no_result(QPid, {gen_server2, cast, [notify_decorators]}). - -notify_sent(QPid, ChPid) -> - rabbit_amqqueue_common:notify_sent(QPid, ChPid). - -notify_sent_queue_down(QPid) -> - rabbit_amqqueue_common:notify_sent_queue_down(QPid). - --spec resume(pid(), pid()) -> 'ok'. - -resume(QPid, ChPid) -> delegate:invoke_no_result(QPid, {gen_server2, cast, - [{resume, ChPid}]}). - -internal_delete1(QueueName, OnlyDurable) -> - internal_delete1(QueueName, OnlyDurable, normal). - -internal_delete1(QueueName, OnlyDurable, Reason) -> - ok = mnesia:delete({rabbit_queue, QueueName}), - case Reason of - auto_delete -> - case mnesia:wread({rabbit_durable_queue, QueueName}) of - [] -> ok; - [_] -> ok = mnesia:delete({rabbit_durable_queue, QueueName}) - end; - _ -> - mnesia:delete({rabbit_durable_queue, QueueName}) - end, - %% we want to execute some things, as decided by rabbit_exchange, - %% after the transaction. - rabbit_binding:remove_for_destination(QueueName, OnlyDurable). - --spec internal_delete(name(), rabbit_types:username()) -> 'ok'. - -internal_delete(QueueName, ActingUser) -> - internal_delete(QueueName, ActingUser, normal). - -internal_delete(QueueName, ActingUser, Reason) -> - rabbit_misc:execute_mnesia_tx_with_tail( - fun () -> - case {mnesia:wread({rabbit_queue, QueueName}), - mnesia:wread({rabbit_durable_queue, QueueName})} of - {[], []} -> - rabbit_misc:const(ok); - _ -> - Deletions = internal_delete1(QueueName, false, Reason), - T = rabbit_binding:process_deletions(Deletions, - ?INTERNAL_USER), - fun() -> - ok = T(), - rabbit_core_metrics:queue_deleted(QueueName), - ok = rabbit_event:notify(queue_deleted, - [{name, QueueName}, - {user_who_performed_action, ActingUser}]) - end - end - end). - --spec forget_all_durable(node()) -> 'ok'. - -forget_all_durable(Node) -> - %% Note rabbit is not running so we avoid e.g. the worker pool. Also why - %% we don't invoke the return from rabbit_binding:process_deletions/1. - {atomic, ok} = - mnesia:sync_transaction( - fun () -> - Qs = mnesia:match_object(rabbit_durable_queue, - amqqueue:pattern_match_all(), write), - [forget_node_for_queue(Node, Q) || - Q <- Qs, - is_local_to_node(amqqueue:get_pid(Q), Node)], - ok - end), - ok. - -%% Try to promote a mirror while down - it should recover as a -%% master. We try to take the oldest mirror here for best chance of -%% recovery. -forget_node_for_queue(_DeadNode, Q) - when ?amqqueue_is_quorum(Q) -> - ok; -forget_node_for_queue(DeadNode, Q) -> - RS = amqqueue:get_recoverable_slaves(Q), - forget_node_for_queue(DeadNode, RS, Q). - -forget_node_for_queue(_DeadNode, [], Q) -> - %% No mirrors to recover from, queue is gone. - %% Don't process_deletions since that just calls callbacks and we - %% are not really up. - Name = amqqueue:get_name(Q), - internal_delete1(Name, true); - -%% Should not happen, but let's be conservative. -forget_node_for_queue(DeadNode, [DeadNode | T], Q) -> - forget_node_for_queue(DeadNode, T, Q); - -forget_node_for_queue(DeadNode, [H|T], Q) when ?is_amqqueue(Q) -> - Type = amqqueue:get_type(Q), - case {node_permits_offline_promotion(H), Type} of - {false, _} -> forget_node_for_queue(DeadNode, T, Q); - {true, rabbit_classic_queue} -> - Q1 = amqqueue:set_pid(Q, rabbit_misc:node_to_fake_pid(H)), - ok = mnesia:write(rabbit_durable_queue, Q1, write); - {true, rabbit_quorum_queue} -> - ok - end. - -node_permits_offline_promotion(Node) -> - case node() of - Node -> not rabbit:is_running(); %% [1] - _ -> All = rabbit_mnesia:cluster_nodes(all), - Running = rabbit_nodes:all_running(), - lists:member(Node, All) andalso - not lists:member(Node, Running) %% [2] - end. -%% [1] In this case if we are a real running node (i.e. rabbitmqctl -%% has RPCed into us) then we cannot allow promotion. If on the other -%% hand we *are* rabbitmqctl impersonating the node for offline -%% node-forgetting then we can. -%% -%% [2] This is simpler; as long as it's down that's OK - --spec run_backing_queue - (pid(), atom(), (fun ((atom(), A) -> {[rabbit_types:msg_id()], A}))) -> - 'ok'. - -run_backing_queue(QPid, Mod, Fun) -> - gen_server2:cast(QPid, {run_backing_queue, Mod, Fun}). - --spec set_ram_duration_target(pid(), number() | 'infinity') -> 'ok'. - -set_ram_duration_target(QPid, Duration) -> - gen_server2:cast(QPid, {set_ram_duration_target, Duration}). - --spec set_maximum_since_use(pid(), non_neg_integer()) -> 'ok'. - -set_maximum_since_use(QPid, Age) -> - gen_server2:cast(QPid, {set_maximum_since_use, Age}). - --spec update_mirroring(pid()) -> 'ok'. - -update_mirroring(QPid) -> - ok = delegate:invoke_no_result(QPid, {gen_server2, cast, [update_mirroring]}). - --spec sync_mirrors(amqqueue:amqqueue() | pid()) -> - 'ok' | rabbit_types:error('not_mirrored'). - -sync_mirrors(Q) when ?is_amqqueue(Q) -> - QPid = amqqueue:get_pid(Q), - delegate:invoke(QPid, {gen_server2, call, [sync_mirrors, infinity]}); -sync_mirrors(QPid) -> - delegate:invoke(QPid, {gen_server2, call, [sync_mirrors, infinity]}). - --spec cancel_sync_mirrors(amqqueue:amqqueue() | pid()) -> - 'ok' | {'ok', 'not_syncing'}. - -cancel_sync_mirrors(Q) when ?is_amqqueue(Q) -> - QPid = amqqueue:get_pid(Q), - delegate:invoke(QPid, {gen_server2, call, [cancel_sync_mirrors, infinity]}); -cancel_sync_mirrors(QPid) -> - delegate:invoke(QPid, {gen_server2, call, [cancel_sync_mirrors, infinity]}). - --spec is_replicated(amqqueue:amqqueue()) -> boolean(). - -is_replicated(Q) when ?amqqueue_is_quorum(Q) -> - true; -is_replicated(Q) -> - rabbit_mirror_queue_misc:is_mirrored(Q). - -is_exclusive(Q) when ?amqqueue_exclusive_owner_is(Q, none) -> - false; -is_exclusive(Q) when ?amqqueue_exclusive_owner_is_pid(Q) -> - true. - -is_not_exclusive(Q) -> - not is_exclusive(Q). - -is_dead_exclusive(Q) when ?amqqueue_exclusive_owner_is(Q, none) -> - false; -is_dead_exclusive(Q) when ?amqqueue_exclusive_owner_is_pid(Q) -> - Pid = amqqueue:get_pid(Q), - not rabbit_mnesia:is_process_alive(Pid). - --spec has_synchronised_mirrors_online(amqqueue:amqqueue()) -> boolean(). -has_synchronised_mirrors_online(Q) -> - %% a queue with all mirrors down would have no mirror pids. - %% We treat these as in sync intentionally to avoid false positives. - MirrorPids = amqqueue:get_sync_slave_pids(Q), - MirrorPids =/= [] andalso lists:any(fun rabbit_misc:is_process_alive/1, MirrorPids). - --spec on_node_up(node()) -> 'ok'. - -on_node_up(Node) -> - ok = rabbit_misc:execute_mnesia_transaction( - fun () -> - Qs = mnesia:match_object(rabbit_queue, - amqqueue:pattern_match_all(), write), - [maybe_clear_recoverable_node(Node, Q) || Q <- Qs], - ok - end). - -maybe_clear_recoverable_node(Node, Q) -> - SPids = amqqueue:get_sync_slave_pids(Q), - RSs = amqqueue:get_recoverable_slaves(Q), - case lists:member(Node, RSs) of - true -> - %% There is a race with - %% rabbit_mirror_queue_slave:record_synchronised/1 called - %% by the incoming mirror node and this function, called - %% by the master node. If this function is executed after - %% record_synchronised/1, the node is erroneously removed - %% from the recoverable mirrors list. - %% - %% We check if the mirror node's queue PID is alive. If it is - %% the case, then this function is executed after. In this - %% situation, we don't touch the queue record, it is already - %% correct. - DoClearNode = - case [SP || SP <- SPids, node(SP) =:= Node] of - [SPid] -> not rabbit_misc:is_process_alive(SPid); - _ -> true - end, - if - DoClearNode -> RSs1 = RSs -- [Node], - store_queue( - amqqueue:set_recoverable_slaves(Q, RSs1)); - true -> ok - end; - false -> - ok - end. - --spec on_node_down(node()) -> 'ok'. - -on_node_down(Node) -> - {QueueNames, QueueDeletions} = delete_queues_on_node_down(Node), - notify_queue_binding_deletions(QueueDeletions), - rabbit_core_metrics:queues_deleted(QueueNames), - notify_queues_deleted(QueueNames), - ok. - -delete_queues_on_node_down(Node) -> - lists:unzip(lists:flatten([ - rabbit_misc:execute_mnesia_transaction( - fun () -> [{Queue, delete_queue(Queue)} || Queue <- Queues] end - ) || Queues <- partition_queues(queues_to_delete_when_node_down(Node)) - ])). - -delete_queue(QueueName) -> - ok = mnesia:delete({rabbit_queue, QueueName}), - rabbit_binding:remove_transient_for_destination(QueueName). - -% If there are many queues and we delete them all in a single Mnesia transaction, -% this can block all other Mnesia operations for a really long time. -% In situations where a node wants to (re-)join a cluster, -% Mnesia won't be able to sync on the new node until this operation finishes. -% As a result, we want to have multiple Mnesia transactions so that other -% operations can make progress in between these queue delete transactions. -% -% 10 queues per Mnesia transaction is an arbitrary number, but it seems to work OK with 50k queues per node. -partition_queues([Q0,Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9 | T]) -> - [[Q0,Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9] | partition_queues(T)]; -partition_queues(T) -> - [T]. - -queues_to_delete_when_node_down(NodeDown) -> - rabbit_misc:execute_mnesia_transaction(fun () -> - qlc:e(qlc:q([amqqueue:get_name(Q) || - Q <- mnesia:table(rabbit_queue), - amqqueue:qnode(Q) == NodeDown andalso - not rabbit_mnesia:is_process_alive(amqqueue:get_pid(Q)) andalso - (not rabbit_amqqueue:is_replicated(Q) orelse - rabbit_amqqueue:is_dead_exclusive(Q))] - )) - end). - -notify_queue_binding_deletions(QueueDeletions) -> - rabbit_misc:execute_mnesia_tx_with_tail( - fun() -> - rabbit_binding:process_deletions( - lists:foldl( - fun rabbit_binding:combine_deletions/2, - rabbit_binding:new_deletions(), - QueueDeletions - ), - ?INTERNAL_USER - ) - end - ). - -notify_queues_deleted(QueueDeletions) -> - lists:foreach( - fun(Queue) -> - ok = rabbit_event:notify(queue_deleted, - [{name, Queue}, - {user, ?INTERNAL_USER}]) - end, - QueueDeletions). - --spec pseudo_queue(name(), pid()) -> amqqueue:amqqueue(). - -pseudo_queue(QueueName, Pid) -> - pseudo_queue(QueueName, Pid, false). - --spec pseudo_queue(name(), pid(), boolean()) -> amqqueue:amqqueue(). - -pseudo_queue(#resource{kind = queue} = QueueName, Pid, Durable) - when is_pid(Pid) andalso - is_boolean(Durable) -> - amqqueue:new(QueueName, - Pid, - Durable, - false, - none, % Owner, - [], - undefined, % VHost, - #{user => undefined}, % ActingUser - rabbit_classic_queue % Type - ). - --spec immutable(amqqueue:amqqueue()) -> amqqueue:amqqueue(). - -immutable(Q) -> amqqueue:set_immutable(Q). - --spec deliver([amqqueue:amqqueue()], rabbit_types:delivery()) -> 'ok'. - -deliver(Qs, Delivery) -> - _ = rabbit_queue_type:deliver(Qs, Delivery, stateless), - ok. - -get_quorum_nodes(Q) -> - case amqqueue:get_type_state(Q) of - #{nodes := Nodes} -> - Nodes; - _ -> - [] - end. diff --git a/src/rabbit_amqqueue_process.erl b/src/rabbit_amqqueue_process.erl deleted file mode 100644 index abad3b5ad4..0000000000 --- a/src/rabbit_amqqueue_process.erl +++ /dev/null @@ -1,1849 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_amqqueue_process). --include_lib("rabbit_common/include/rabbit.hrl"). --include_lib("rabbit_common/include/rabbit_framing.hrl"). --include("amqqueue.hrl"). - --behaviour(gen_server2). - --define(SYNC_INTERVAL, 200). %% milliseconds --define(RAM_DURATION_UPDATE_INTERVAL, 5000). --define(CONSUMER_BIAS_RATIO, 2.0). %% i.e. consume 100% faster - --export([info_keys/0]). - --export([init_with_backing_queue_state/7]). - --export([init/1, terminate/2, code_change/3, handle_call/3, handle_cast/2, - handle_info/2, handle_pre_hibernate/1, prioritise_call/4, - prioritise_cast/3, prioritise_info/3, format_message_queue/2]). --export([format/1]). --export([is_policy_applicable/2]). - -%% Queue's state --record(q, { - %% an #amqqueue record - q :: amqqueue:amqqueue(), - %% none | {exclusive consumer channel PID, consumer tag} | {single active consumer channel PID, consumer} - active_consumer, - %% Set to true if a queue has ever had a consumer. - %% This is used to determine when to delete auto-delete queues. - has_had_consumers, - %% backing queue module. - %% for mirrored queues, this will be rabbit_mirror_queue_master. - %% for non-priority and non-mirrored queues, rabbit_variable_queue. - %% see rabbit_backing_queue. - backing_queue, - %% backing queue state. - %% see rabbit_backing_queue, rabbit_variable_queue. - backing_queue_state, - %% consumers state, see rabbit_queue_consumers - consumers, - %% queue expiration value - expires, - %% timer used to periodically sync (flush) queue index - sync_timer_ref, - %% timer used to update ingress/egress rates and queue RAM duration target - rate_timer_ref, - %% timer used to clean up this queue due to TTL (on when unused) - expiry_timer_ref, - %% stats emission timer - stats_timer, - %% maps message IDs to {channel pid, MsgSeqNo} - %% pairs - msg_id_to_channel, - %% message TTL value - ttl, - %% timer used to delete expired messages - ttl_timer_ref, - ttl_timer_expiry, - %% Keeps track of channels that publish to this queue. - %% When channel process goes down, queues have to perform - %% certain cleanup. - senders, - %% dead letter exchange as a #resource record, if any - dlx, - dlx_routing_key, - %% max length in messages, if configured - max_length, - %% max length in bytes, if configured - max_bytes, - %% an action to perform if queue is to be over a limit, - %% can be either drop-head (default), reject-publish or reject-publish-dlx - overflow, - %% when policies change, this version helps queue - %% determine what previously scheduled/set up state to ignore, - %% e.g. message expiration messages from previously set up timers - %% that may or may not be still valid - args_policy_version, - %% used to discard outdated/superseded policy updates, - %% e.g. when policies are applied concurrently. See - %% https://github.com/rabbitmq/rabbitmq-server/issues/803 for one - %% example. - mirroring_policy_version = 0, - %% running | flow | idle - status, - %% true | false - single_active_consumer_on - }). - -%%---------------------------------------------------------------------------- - --define(STATISTICS_KEYS, - [messages_ready, - messages_unacknowledged, - messages, - reductions, - name, - policy, - operator_policy, - effective_policy_definition, - exclusive_consumer_pid, - exclusive_consumer_tag, - single_active_consumer_pid, - single_active_consumer_tag, - consumers, - consumer_utilisation, - memory, - slave_pids, - synchronised_slave_pids, - recoverable_slaves, - state, - garbage_collection - ]). - --define(CREATION_EVENT_KEYS, - [name, - durable, - auto_delete, - arguments, - owner_pid, - exclusive, - user_who_performed_action - ]). - --define(INFO_KEYS, [pid | ?CREATION_EVENT_KEYS ++ ?STATISTICS_KEYS -- [name]]). - -%%---------------------------------------------------------------------------- - --spec info_keys() -> rabbit_types:info_keys(). - -info_keys() -> ?INFO_KEYS ++ rabbit_backing_queue:info_keys(). -statistics_keys() -> ?STATISTICS_KEYS ++ rabbit_backing_queue:info_keys(). - -%%---------------------------------------------------------------------------- - -init(Q) -> - process_flag(trap_exit, true), - ?store_proc_name(amqqueue:get_name(Q)), - {ok, init_state(amqqueue:set_pid(Q, self())), hibernate, - {backoff, ?HIBERNATE_AFTER_MIN, ?HIBERNATE_AFTER_MIN, ?DESIRED_HIBERNATE}, - ?MODULE}. - -init_state(Q) -> - SingleActiveConsumerOn = case rabbit_misc:table_lookup(amqqueue:get_arguments(Q), <<"x-single-active-consumer">>) of - {bool, true} -> true; - _ -> false - end, - State = #q{q = Q, - active_consumer = none, - has_had_consumers = false, - consumers = rabbit_queue_consumers:new(), - senders = pmon:new(delegate), - msg_id_to_channel = #{}, - status = running, - args_policy_version = 0, - overflow = 'drop-head', - single_active_consumer_on = SingleActiveConsumerOn}, - rabbit_event:init_stats_timer(State, #q.stats_timer). - -init_it(Recover, From, State = #q{q = Q}) - when ?amqqueue_exclusive_owner_is(Q, none) -> - init_it2(Recover, From, State); - -%% You used to be able to declare an exclusive durable queue. Sadly we -%% need to still tidy up after that case, there could be the remnants -%% of one left over from an upgrade. So that's why we don't enforce -%% Recover = new here. -init_it(Recover, From, State = #q{q = Q0}) -> - Owner = amqqueue:get_exclusive_owner(Q0), - case rabbit_misc:is_process_alive(Owner) of - true -> erlang:monitor(process, Owner), - init_it2(Recover, From, State); - false -> #q{backing_queue = undefined, - backing_queue_state = undefined, - q = Q} = State, - send_reply(From, {owner_died, Q}), - BQ = backing_queue_module(Q), - {_, Terms} = recovery_status(Recover), - BQS = bq_init(BQ, Q, Terms), - %% Rely on terminate to delete the queue. - log_delete_exclusive(Owner, State), - {stop, {shutdown, missing_owner}, - State#q{backing_queue = BQ, backing_queue_state = BQS}} - end. - -init_it2(Recover, From, State = #q{q = Q, - backing_queue = undefined, - backing_queue_state = undefined}) -> - {Barrier, TermsOrNew} = recovery_status(Recover), - case rabbit_amqqueue:internal_declare(Q, Recover /= new) of - {Res, Q1} - when ?is_amqqueue(Q1) andalso - (Res == created orelse Res == existing) -> - case matches(Recover, Q, Q1) of - true -> - ok = file_handle_cache:register_callback( - rabbit_amqqueue, set_maximum_since_use, [self()]), - ok = rabbit_memory_monitor:register( - self(), {rabbit_amqqueue, - set_ram_duration_target, [self()]}), - BQ = backing_queue_module(Q1), - BQS = bq_init(BQ, Q, TermsOrNew), - send_reply(From, {new, Q}), - recovery_barrier(Barrier), - State1 = process_args_policy( - State#q{backing_queue = BQ, - backing_queue_state = BQS}), - notify_decorators(startup, State), - rabbit_event:notify(queue_created, - infos(?CREATION_EVENT_KEYS, State1)), - rabbit_event:if_enabled(State1, #q.stats_timer, - fun() -> emit_stats(State1) end), - noreply(State1); - false -> - {stop, normal, {existing, Q1}, State} - end; - Err -> - {stop, normal, Err, State} - end. - -recovery_status(new) -> {no_barrier, new}; -recovery_status({Recover, Terms}) -> {Recover, Terms}. - -send_reply(none, _Q) -> ok; -send_reply(From, Q) -> gen_server2:reply(From, Q). - -matches(new, Q1, Q2) -> - %% i.e. not policy - amqqueue:get_name(Q1) =:= amqqueue:get_name(Q2) andalso - amqqueue:is_durable(Q1) =:= amqqueue:is_durable(Q2) andalso - amqqueue:is_auto_delete(Q1) =:= amqqueue:is_auto_delete(Q2) andalso - amqqueue:get_exclusive_owner(Q1) =:= amqqueue:get_exclusive_owner(Q2) andalso - amqqueue:get_arguments(Q1) =:= amqqueue:get_arguments(Q2) andalso - amqqueue:get_pid(Q1) =:= amqqueue:get_pid(Q2) andalso - amqqueue:get_slave_pids(Q1) =:= amqqueue:get_slave_pids(Q2); -%% FIXME: Should v1 vs. v2 of the same record match? -matches(_, Q, Q) -> true; -matches(_, _Q, _Q1) -> false. - -recovery_barrier(no_barrier) -> - ok; -recovery_barrier(BarrierPid) -> - MRef = erlang:monitor(process, BarrierPid), - receive - {BarrierPid, go} -> erlang:demonitor(MRef, [flush]); - {'DOWN', MRef, process, _, _} -> ok - end. - --spec init_with_backing_queue_state - (amqqueue:amqqueue(), atom(), tuple(), any(), - [rabbit_types:delivery()], pmon:pmon(), map()) -> - #q{}. - -init_with_backing_queue_state(Q, BQ, BQS, - RateTRef, Deliveries, Senders, MTC) -> - Owner = amqqueue:get_exclusive_owner(Q), - case Owner of - none -> ok; - _ -> erlang:monitor(process, Owner) - end, - State = init_state(Q), - State1 = State#q{backing_queue = BQ, - backing_queue_state = BQS, - rate_timer_ref = RateTRef, - senders = Senders, - msg_id_to_channel = MTC}, - State2 = process_args_policy(State1), - State3 = lists:foldl(fun (Delivery, StateN) -> - maybe_deliver_or_enqueue(Delivery, true, StateN) - end, State2, Deliveries), - notify_decorators(startup, State3), - State3. - -terminate(shutdown = R, State = #q{backing_queue = BQ, q = Q0}) -> - QName = amqqueue:get_name(Q0), - rabbit_core_metrics:queue_deleted(qname(State)), - terminate_shutdown( - fun (BQS) -> - rabbit_misc:execute_mnesia_transaction( - fun() -> - [Q] = mnesia:read({rabbit_queue, QName}), - Q2 = amqqueue:set_state(Q, stopped), - %% amqqueue migration: - %% The amqqueue was read from this transaction, no need - %% to handle migration. - rabbit_amqqueue:store_queue(Q2) - end), - BQ:terminate(R, BQS) - end, State); -terminate({shutdown, missing_owner} = Reason, State) -> - %% if the owner was missing then there will be no queue, so don't emit stats - terminate_shutdown(terminate_delete(false, Reason, State), State); -terminate({shutdown, _} = R, State = #q{backing_queue = BQ}) -> - rabbit_core_metrics:queue_deleted(qname(State)), - terminate_shutdown(fun (BQS) -> BQ:terminate(R, BQS) end, State); -terminate(normal, State = #q{status = {terminated_by, auto_delete}}) -> - %% auto_delete case - %% To increase performance we want to avoid a mnesia_sync:sync call - %% after every transaction, as we could be deleting simultaneously - %% thousands of queues. A optimisation introduced by server#1513 - %% needs to be reverted by this case, avoiding to guard the delete - %% operation on `rabbit_durable_queue` - terminate_shutdown(terminate_delete(true, auto_delete, State), State); -terminate(normal, State) -> %% delete case - terminate_shutdown(terminate_delete(true, normal, State), State); -%% If we crashed don't try to clean up the BQS, probably best to leave it. -terminate(_Reason, State = #q{q = Q}) -> - terminate_shutdown(fun (BQS) -> - Q2 = amqqueue:set_state(Q, crashed), - rabbit_misc:execute_mnesia_transaction( - fun() -> - ?try_mnesia_tx_or_upgrade_amqqueue_and_retry( - rabbit_amqqueue:store_queue(Q2), - begin - Q3 = amqqueue:upgrade(Q2), - rabbit_amqqueue:store_queue(Q3) - end) - end), - BQS - end, State). - -terminate_delete(EmitStats, Reason0, - State = #q{q = Q, - backing_queue = BQ, - status = Status}) -> - QName = amqqueue:get_name(Q), - ActingUser = terminated_by(Status), - fun (BQS) -> - Reason = case Reason0 of - auto_delete -> normal; - Any -> Any - end, - BQS1 = BQ:delete_and_terminate(Reason, BQS), - if EmitStats -> rabbit_event:if_enabled(State, #q.stats_timer, - fun() -> emit_stats(State) end); - true -> ok - end, - %% This try-catch block transforms throws to errors since throws are not - %% logged. - try - %% don't care if the internal delete doesn't return 'ok'. - rabbit_amqqueue:internal_delete(QName, ActingUser, Reason0) - catch - {error, ReasonE} -> error(ReasonE) - end, - BQS1 - end. - -terminated_by({terminated_by, auto_delete}) -> - ?INTERNAL_USER; -terminated_by({terminated_by, ActingUser}) -> - ActingUser; -terminated_by(_) -> - ?INTERNAL_USER. - -terminate_shutdown(Fun, #q{status = Status} = State) -> - ActingUser = terminated_by(Status), - State1 = #q{backing_queue_state = BQS, consumers = Consumers} = - lists:foldl(fun (F, S) -> F(S) end, State, - [fun stop_sync_timer/1, - fun stop_rate_timer/1, - fun stop_expiry_timer/1, - fun stop_ttl_timer/1]), - case BQS of - undefined -> State1; - _ -> ok = rabbit_memory_monitor:deregister(self()), - QName = qname(State), - notify_decorators(shutdown, State), - [emit_consumer_deleted(Ch, CTag, QName, ActingUser) || - {Ch, CTag, _, _, _, _, _, _} <- - rabbit_queue_consumers:all(Consumers)], - State1#q{backing_queue_state = Fun(BQS)} - end. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%---------------------------------------------------------------------------- - -maybe_notify_decorators(false, State) -> State; -maybe_notify_decorators(true, State) -> notify_decorators(State), State. - -notify_decorators(Event, State) -> decorator_callback(qname(State), Event, []). - -notify_decorators(State = #q{consumers = Consumers, - backing_queue = BQ, - backing_queue_state = BQS}) -> - P = rabbit_queue_consumers:max_active_priority(Consumers), - decorator_callback(qname(State), consumer_state_changed, - [P, BQ:is_empty(BQS)]). - -decorator_callback(QName, F, A) -> - %% Look up again in case policy and hence decorators have changed - case rabbit_amqqueue:lookup(QName) of - {ok, Q} -> - Ds = amqqueue:get_decorators(Q), - [ok = apply(M, F, [Q|A]) || M <- rabbit_queue_decorator:select(Ds)]; - {error, not_found} -> - ok - end. - -bq_init(BQ, Q, Recover) -> - Self = self(), - BQ:init(Q, Recover, - fun (Mod, Fun) -> - rabbit_amqqueue:run_backing_queue(Self, Mod, Fun) - end). - -process_args_policy(State = #q{q = Q, - args_policy_version = N}) -> - ArgsTable = - [{<<"expires">>, fun res_min/2, fun init_exp/2}, - {<<"dead-letter-exchange">>, fun res_arg/2, fun init_dlx/2}, - {<<"dead-letter-routing-key">>, fun res_arg/2, fun init_dlx_rkey/2}, - {<<"message-ttl">>, fun res_min/2, fun init_ttl/2}, - {<<"max-length">>, fun res_min/2, fun init_max_length/2}, - {<<"max-length-bytes">>, fun res_min/2, fun init_max_bytes/2}, - {<<"overflow">>, fun res_arg/2, fun init_overflow/2}, - {<<"queue-mode">>, fun res_arg/2, fun init_queue_mode/2}], - drop_expired_msgs( - lists:foldl(fun({Name, Resolve, Fun}, StateN) -> - Fun(rabbit_queue_type_util:args_policy_lookup(Name, Resolve, Q), StateN) - end, State#q{args_policy_version = N + 1}, ArgsTable)). - -res_arg(_PolVal, ArgVal) -> ArgVal. -res_min(PolVal, ArgVal) -> erlang:min(PolVal, ArgVal). - -%% In both these we init with the undefined variant first to stop any -%% existing timer, then start a new one which may fire after a -%% different time. -init_exp(undefined, State) -> stop_expiry_timer(State#q{expires = undefined}); -init_exp(Expires, State) -> State1 = init_exp(undefined, State), - ensure_expiry_timer(State1#q{expires = Expires}). - -init_ttl(undefined, State) -> stop_ttl_timer(State#q{ttl = undefined}); -init_ttl(TTL, State) -> (init_ttl(undefined, State))#q{ttl = TTL}. - -init_dlx(undefined, State) -> - State#q{dlx = undefined}; -init_dlx(DLX, State = #q{q = Q}) -> - QName = amqqueue:get_name(Q), - State#q{dlx = rabbit_misc:r(QName, exchange, DLX)}. - -init_dlx_rkey(RoutingKey, State) -> State#q{dlx_routing_key = RoutingKey}. - -init_max_length(MaxLen, State) -> - {_Dropped, State1} = maybe_drop_head(State#q{max_length = MaxLen}), - State1. - -init_max_bytes(MaxBytes, State) -> - {_Dropped, State1} = maybe_drop_head(State#q{max_bytes = MaxBytes}), - State1. - -%% Reset overflow to default 'drop-head' value if it's undefined. -init_overflow(undefined, #q{overflow = 'drop-head'} = State) -> - State; -init_overflow(undefined, State) -> - {_Dropped, State1} = maybe_drop_head(State#q{overflow = 'drop-head'}), - State1; -init_overflow(Overflow, State) -> - OverflowVal = binary_to_existing_atom(Overflow, utf8), - case OverflowVal of - 'drop-head' -> - {_Dropped, State1} = maybe_drop_head(State#q{overflow = OverflowVal}), - State1; - _ -> - State#q{overflow = OverflowVal} - end. - -init_queue_mode(undefined, State) -> - State; -init_queue_mode(Mode, State = #q {backing_queue = BQ, - backing_queue_state = BQS}) -> - BQS1 = BQ:set_queue_mode(binary_to_existing_atom(Mode, utf8), BQS), - State#q{backing_queue_state = BQS1}. - -reply(Reply, NewState) -> - {NewState1, Timeout} = next_state(NewState), - {reply, Reply, ensure_stats_timer(ensure_rate_timer(NewState1)), Timeout}. - -noreply(NewState) -> - {NewState1, Timeout} = next_state(NewState), - {noreply, ensure_stats_timer(ensure_rate_timer(NewState1)), Timeout}. - -next_state(State = #q{q = Q, - backing_queue = BQ, - backing_queue_state = BQS, - msg_id_to_channel = MTC}) -> - assert_invariant(State), - {MsgIds, BQS1} = BQ:drain_confirmed(BQS), - MTC1 = confirm_messages(MsgIds, MTC, amqqueue:get_name(Q)), - State1 = State#q{backing_queue_state = BQS1, msg_id_to_channel = MTC1}, - case BQ:needs_timeout(BQS1) of - false -> {stop_sync_timer(State1), hibernate }; - idle -> {stop_sync_timer(State1), ?SYNC_INTERVAL}; - timed -> {ensure_sync_timer(State1), 0 } - end. - -backing_queue_module(Q) -> - case rabbit_mirror_queue_misc:is_mirrored(Q) of - false -> {ok, BQM} = application:get_env(backing_queue_module), - BQM; - true -> rabbit_mirror_queue_master - end. - -ensure_sync_timer(State) -> - rabbit_misc:ensure_timer(State, #q.sync_timer_ref, - ?SYNC_INTERVAL, sync_timeout). - -stop_sync_timer(State) -> rabbit_misc:stop_timer(State, #q.sync_timer_ref). - -ensure_rate_timer(State) -> - rabbit_misc:ensure_timer(State, #q.rate_timer_ref, - ?RAM_DURATION_UPDATE_INTERVAL, - update_ram_duration). - -stop_rate_timer(State) -> rabbit_misc:stop_timer(State, #q.rate_timer_ref). - -%% We wish to expire only when there are no consumers *and* the expiry -%% hasn't been refreshed (by queue.declare or basic.get) for the -%% configured period. -ensure_expiry_timer(State = #q{expires = undefined}) -> - State; -ensure_expiry_timer(State = #q{expires = Expires, - args_policy_version = Version}) -> - case is_unused(State) of - true -> NewState = stop_expiry_timer(State), - rabbit_misc:ensure_timer(NewState, #q.expiry_timer_ref, - Expires, {maybe_expire, Version}); - false -> State - end. - -stop_expiry_timer(State) -> rabbit_misc:stop_timer(State, #q.expiry_timer_ref). - -ensure_ttl_timer(undefined, State) -> - State; -ensure_ttl_timer(Expiry, State = #q{ttl_timer_ref = undefined, - args_policy_version = Version}) -> - After = (case Expiry - os:system_time(micro_seconds) of - V when V > 0 -> V + 999; %% always fire later - _ -> 0 - end) div 1000, - TRef = rabbit_misc:send_after(After, self(), {drop_expired, Version}), - State#q{ttl_timer_ref = TRef, ttl_timer_expiry = Expiry}; -ensure_ttl_timer(Expiry, State = #q{ttl_timer_ref = TRef, - ttl_timer_expiry = TExpiry}) - when Expiry + 1000 < TExpiry -> - rabbit_misc:cancel_timer(TRef), - ensure_ttl_timer(Expiry, State#q{ttl_timer_ref = undefined}); -ensure_ttl_timer(_Expiry, State) -> - State. - -stop_ttl_timer(State) -> rabbit_misc:stop_timer(State, #q.ttl_timer_ref). - -ensure_stats_timer(State) -> - rabbit_event:ensure_stats_timer(State, #q.stats_timer, emit_stats). - -assert_invariant(#q{single_active_consumer_on = true}) -> - %% queue may contain messages and have available consumers with exclusive consumer - ok; -assert_invariant(State = #q{consumers = Consumers, single_active_consumer_on = false}) -> - true = (rabbit_queue_consumers:inactive(Consumers) orelse is_empty(State)). - -is_empty(#q{backing_queue = BQ, backing_queue_state = BQS}) -> BQ:is_empty(BQS). - -maybe_send_drained(WasEmpty, State) -> - case (not WasEmpty) andalso is_empty(State) of - true -> notify_decorators(State), - rabbit_queue_consumers:send_drained(); - false -> ok - end, - State. - -confirm_messages([], MTC, _QName) -> - MTC; -confirm_messages(MsgIds, MTC, QName) -> - {CMs, MTC1} = - lists:foldl( - fun(MsgId, {CMs, MTC0}) -> - case maps:get(MsgId, MTC0, none) of - none -> - {CMs, MTC0}; - {SenderPid, MsgSeqNo} -> - {maps:update_with(SenderPid, - fun(MsgSeqNos) -> - [MsgSeqNo | MsgSeqNos] - end, - [MsgSeqNo], - CMs), - maps:remove(MsgId, MTC0)} - - end - end, {#{}, MTC}, MsgIds), - maps:fold( - fun(Pid, MsgSeqNos, _) -> - confirm_to_sender(Pid, QName, MsgSeqNos) - end, - ok, - CMs), - MTC1. - -send_or_record_confirm(#delivery{confirm = false}, State) -> - {never, State}; -send_or_record_confirm(#delivery{confirm = true, - sender = SenderPid, - msg_seq_no = MsgSeqNo, - message = #basic_message { - is_persistent = true, - id = MsgId}}, - State = #q{q = Q, - msg_id_to_channel = MTC}) - when ?amqqueue_is_durable(Q) -> - MTC1 = maps:put(MsgId, {SenderPid, MsgSeqNo}, MTC), - {eventually, State#q{msg_id_to_channel = MTC1}}; -send_or_record_confirm(#delivery{confirm = true, - sender = SenderPid, - msg_seq_no = MsgSeqNo}, - #q{q = Q} = State) -> - confirm_to_sender(SenderPid, amqqueue:get_name(Q), [MsgSeqNo]), - {immediately, State}. - -%% This feature was used by `rabbit_amqqueue_process` and -%% `rabbit_mirror_queue_slave` up-to and including RabbitMQ 3.7.x. It is -%% unused in 3.8.x and thus deprecated. We keep it to support in-place -%% upgrades to 3.8.x (i.e. mixed-version clusters), but it is a no-op -%% starting with that version. -send_mandatory(#delivery{mandatory = false}) -> - ok; -send_mandatory(#delivery{mandatory = true, - sender = SenderPid, - msg_seq_no = MsgSeqNo}) -> - gen_server2:cast(SenderPid, {mandatory_received, MsgSeqNo}). - -discard(#delivery{confirm = Confirm, - sender = SenderPid, - flow = Flow, - message = #basic_message{id = MsgId}}, BQ, BQS, MTC, QName) -> - MTC1 = case Confirm of - true -> confirm_messages([MsgId], MTC, QName); - false -> MTC - end, - BQS1 = BQ:discard(MsgId, SenderPid, Flow, BQS), - {BQS1, MTC1}. - -run_message_queue(State) -> run_message_queue(false, State). - -run_message_queue(ActiveConsumersChanged, State) -> - case is_empty(State) of - true -> maybe_notify_decorators(ActiveConsumersChanged, State); - false -> case rabbit_queue_consumers:deliver( - fun(AckRequired) -> fetch(AckRequired, State) end, - qname(State), State#q.consumers, - State#q.single_active_consumer_on, State#q.active_consumer) of - {delivered, ActiveConsumersChanged1, State1, Consumers} -> - run_message_queue( - ActiveConsumersChanged or ActiveConsumersChanged1, - State1#q{consumers = Consumers}); - {undelivered, ActiveConsumersChanged1, Consumers} -> - maybe_notify_decorators( - ActiveConsumersChanged or ActiveConsumersChanged1, - State#q{consumers = Consumers}) - end - end. - -attempt_delivery(Delivery = #delivery{sender = SenderPid, - flow = Flow, - message = Message}, - Props, Delivered, State = #q{q = Q, - backing_queue = BQ, - backing_queue_state = BQS, - msg_id_to_channel = MTC}) -> - case rabbit_queue_consumers:deliver( - fun (true) -> true = BQ:is_empty(BQS), - {AckTag, BQS1} = - BQ:publish_delivered( - Message, Props, SenderPid, Flow, BQS), - {{Message, Delivered, AckTag}, {BQS1, MTC}}; - (false) -> {{Message, Delivered, undefined}, - discard(Delivery, BQ, BQS, MTC, amqqueue:get_name(Q))} - end, qname(State), State#q.consumers, State#q.single_active_consumer_on, State#q.active_consumer) of - {delivered, ActiveConsumersChanged, {BQS1, MTC1}, Consumers} -> - {delivered, maybe_notify_decorators( - ActiveConsumersChanged, - State#q{backing_queue_state = BQS1, - msg_id_to_channel = MTC1, - consumers = Consumers})}; - {undelivered, ActiveConsumersChanged, Consumers} -> - {undelivered, maybe_notify_decorators( - ActiveConsumersChanged, - State#q{consumers = Consumers})} - end. - -maybe_deliver_or_enqueue(Delivery = #delivery{message = Message}, - Delivered, - State = #q{overflow = Overflow, - backing_queue = BQ, - backing_queue_state = BQS, - dlx = DLX, - dlx_routing_key = RK}) -> - send_mandatory(Delivery), %% must do this before confirms - case {will_overflow(Delivery, State), Overflow} of - {true, 'reject-publish'} -> - %% Drop publish and nack to publisher - send_reject_publish(Delivery, Delivered, State); - {true, 'reject-publish-dlx'} -> - %% Publish to DLX - with_dlx( - DLX, - fun (X) -> - QName = qname(State), - rabbit_dead_letter:publish(Message, maxlen, X, RK, QName) - end, - fun () -> ok end), - %% Drop publish and nack to publisher - send_reject_publish(Delivery, Delivered, State); - _ -> - {IsDuplicate, BQS1} = BQ:is_duplicate(Message, BQS), - State1 = State#q{backing_queue_state = BQS1}, - case IsDuplicate of - true -> State1; - {true, drop} -> State1; - %% Drop publish and nack to publisher - {true, reject} -> - send_reject_publish(Delivery, Delivered, State1); - %% Enqueue and maybe drop head later - false -> - deliver_or_enqueue(Delivery, Delivered, State1) - end - end. - -deliver_or_enqueue(Delivery = #delivery{message = Message, - sender = SenderPid, - flow = Flow}, - Delivered, - State = #q{q = Q, backing_queue = BQ}) -> - {Confirm, State1} = send_or_record_confirm(Delivery, State), - Props = message_properties(Message, Confirm, State1), - case attempt_delivery(Delivery, Props, Delivered, State1) of - {delivered, State2} -> - State2; - %% The next one is an optimisation - {undelivered, State2 = #q{ttl = 0, dlx = undefined, - backing_queue_state = BQS, - msg_id_to_channel = MTC}} -> - {BQS1, MTC1} = discard(Delivery, BQ, BQS, MTC, amqqueue:get_name(Q)), - State2#q{backing_queue_state = BQS1, msg_id_to_channel = MTC1}; - {undelivered, State2 = #q{backing_queue_state = BQS}} -> - - BQS1 = BQ:publish(Message, Props, Delivered, SenderPid, Flow, BQS), - {Dropped, State3 = #q{backing_queue_state = BQS2}} = - maybe_drop_head(State2#q{backing_queue_state = BQS1}), - QLen = BQ:len(BQS2), - %% optimisation: it would be perfectly safe to always - %% invoke drop_expired_msgs here, but that is expensive so - %% we only do that if a new message that might have an - %% expiry ends up at the head of the queue. If the head - %% remains unchanged, or if the newly published message - %% has no expiry and becomes the head of the queue then - %% the call is unnecessary. - case {Dropped, QLen =:= 1, Props#message_properties.expiry} of - {false, false, _} -> State3; - {true, true, undefined} -> State3; - {_, _, _} -> drop_expired_msgs(State3) - end - end. - -maybe_drop_head(State = #q{max_length = undefined, - max_bytes = undefined}) -> - {false, State}; -maybe_drop_head(State = #q{overflow = 'reject-publish'}) -> - {false, State}; -maybe_drop_head(State = #q{overflow = 'reject-publish-dlx'}) -> - {false, State}; -maybe_drop_head(State = #q{overflow = 'drop-head'}) -> - maybe_drop_head(false, State). - -maybe_drop_head(AlreadyDropped, State = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - case over_max_length(State) of - true -> - maybe_drop_head(true, - with_dlx( - State#q.dlx, - fun (X) -> dead_letter_maxlen_msg(X, State) end, - fun () -> - {_, BQS1} = BQ:drop(false, BQS), - State#q{backing_queue_state = BQS1} - end)); - false -> - {AlreadyDropped, State} - end. - -send_reject_publish(#delivery{confirm = true, - sender = SenderPid, - flow = Flow, - msg_seq_no = MsgSeqNo, - message = #basic_message{id = MsgId}}, - _Delivered, - State = #q{ q = Q, - backing_queue = BQ, - backing_queue_state = BQS, - msg_id_to_channel = MTC}) -> - ok = rabbit_classic_queue:send_rejection(SenderPid, - amqqueue:get_name(Q), MsgSeqNo), - - MTC1 = maps:remove(MsgId, MTC), - BQS1 = BQ:discard(MsgId, SenderPid, Flow, BQS), - State#q{ backing_queue_state = BQS1, msg_id_to_channel = MTC1 }; -send_reject_publish(#delivery{confirm = false}, - _Delivered, State) -> - State. - -will_overflow(_, #q{max_length = undefined, - max_bytes = undefined}) -> false; -will_overflow(#delivery{message = Message}, - #q{max_length = MaxLen, - max_bytes = MaxBytes, - backing_queue = BQ, - backing_queue_state = BQS}) -> - ExpectedQueueLength = BQ:len(BQS) + 1, - - #basic_message{content = #content{payload_fragments_rev = PFR}} = Message, - MessageSize = iolist_size(PFR), - ExpectedQueueSizeBytes = BQ:info(message_bytes_ready, BQS) + MessageSize, - - ExpectedQueueLength > MaxLen orelse ExpectedQueueSizeBytes > MaxBytes. - -over_max_length(#q{max_length = MaxLen, - max_bytes = MaxBytes, - backing_queue = BQ, - backing_queue_state = BQS}) -> - BQ:len(BQS) > MaxLen orelse BQ:info(message_bytes_ready, BQS) > MaxBytes. - -requeue_and_run(AckTags, State = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - WasEmpty = BQ:is_empty(BQS), - {_MsgIds, BQS1} = BQ:requeue(AckTags, BQS), - {_Dropped, State1} = maybe_drop_head(State#q{backing_queue_state = BQS1}), - run_message_queue(maybe_send_drained(WasEmpty, drop_expired_msgs(State1))). - -fetch(AckRequired, State = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - {Result, BQS1} = BQ:fetch(AckRequired, BQS), - State1 = drop_expired_msgs(State#q{backing_queue_state = BQS1}), - {Result, maybe_send_drained(Result =:= empty, State1)}. - -ack(AckTags, ChPid, State) -> - subtract_acks(ChPid, AckTags, State, - fun (State1 = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - {_Guids, BQS1} = BQ:ack(AckTags, BQS), - State1#q{backing_queue_state = BQS1} - end). - -requeue(AckTags, ChPid, State) -> - subtract_acks(ChPid, AckTags, State, - fun (State1) -> requeue_and_run(AckTags, State1) end). - -possibly_unblock(Update, ChPid, State = #q{consumers = Consumers}) -> - case rabbit_queue_consumers:possibly_unblock(Update, ChPid, Consumers) of - unchanged -> State; - {unblocked, Consumers1} -> State1 = State#q{consumers = Consumers1}, - run_message_queue(true, State1) - end. - -should_auto_delete(#q{q = Q}) - when not ?amqqueue_is_auto_delete(Q) -> false; -should_auto_delete(#q{has_had_consumers = false}) -> false; -should_auto_delete(State) -> is_unused(State). - -handle_ch_down(DownPid, State = #q{consumers = Consumers, - active_consumer = Holder, - single_active_consumer_on = SingleActiveConsumerOn, - senders = Senders}) -> - State1 = State#q{senders = case pmon:is_monitored(DownPid, Senders) of - false -> - Senders; - true -> - %% A rabbit_channel process died. Here credit_flow will take care - %% of cleaning up the rabbit_amqqueue_process process dictionary - %% with regards to the credit we were tracking for the channel - %% process. See handle_cast({deliver, Deliver}, State) in this - %% module. In that cast function we process deliveries from the - %% channel, which means we credit_flow:ack/1 said - %% messages. credit_flow:ack'ing messages means we are increasing - %% a counter to know when we need to send MoreCreditAfter. Since - %% the process died, the credit_flow flow module will clean up - %% that for us. - credit_flow:peer_down(DownPid), - pmon:demonitor(DownPid, Senders) - end}, - case rabbit_queue_consumers:erase_ch(DownPid, Consumers) of - not_found -> - {ok, State1}; - {ChAckTags, ChCTags, Consumers1} -> - QName = qname(State1), - [emit_consumer_deleted(DownPid, CTag, QName, ?INTERNAL_USER) || CTag <- ChCTags], - Holder1 = new_single_active_consumer_after_channel_down(DownPid, Holder, SingleActiveConsumerOn, Consumers1), - State2 = State1#q{consumers = Consumers1, - active_consumer = Holder1}, - maybe_notify_consumer_updated(State2, Holder, Holder1), - notify_decorators(State2), - case should_auto_delete(State2) of - true -> - log_auto_delete( - io_lib:format( - "because all of its consumers (~p) were on a channel that was closed", - [length(ChCTags)]), - State), - {stop, State2}; - false -> {ok, requeue_and_run(ChAckTags, - ensure_expiry_timer(State2))} - end - end. - -new_single_active_consumer_after_channel_down(DownChPid, CurrentSingleActiveConsumer, _SingleActiveConsumerIsOn = true, Consumers) -> - case CurrentSingleActiveConsumer of - {DownChPid, _} -> - % the single active consumer is on the down channel, we have to replace it - case rabbit_queue_consumers:get_consumer(Consumers) of - undefined -> none; - Consumer -> Consumer - end; - _ -> - CurrentSingleActiveConsumer - end; -new_single_active_consumer_after_channel_down(DownChPid, CurrentSingleActiveConsumer, _SingleActiveConsumerIsOn = false, _Consumers) -> - case CurrentSingleActiveConsumer of - {DownChPid, _} -> none; - Other -> Other - end. - -check_exclusive_access({_ChPid, _ConsumerTag}, _ExclusiveConsume, _State) -> - in_use; -check_exclusive_access(none, false, _State) -> - ok; -check_exclusive_access(none, true, State) -> - case is_unused(State) of - true -> ok; - false -> in_use - end. - -is_unused(_State) -> rabbit_queue_consumers:count() == 0. - -maybe_send_reply(_ChPid, undefined) -> ok; -maybe_send_reply(ChPid, Msg) -> ok = rabbit_channel:send_command(ChPid, Msg). - -qname(#q{q = Q}) -> amqqueue:get_name(Q). - -backing_queue_timeout(State = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - State#q{backing_queue_state = BQ:timeout(BQS)}. - -subtract_acks(ChPid, AckTags, State = #q{consumers = Consumers}, Fun) -> - case rabbit_queue_consumers:subtract_acks(ChPid, AckTags, Consumers) of - not_found -> State; - unchanged -> Fun(State); - {unblocked, Consumers1} -> State1 = State#q{consumers = Consumers1}, - run_message_queue(true, Fun(State1)) - end. - -message_properties(Message = #basic_message{content = Content}, - Confirm, #q{ttl = TTL}) -> - #content{payload_fragments_rev = PFR} = Content, - #message_properties{expiry = calculate_msg_expiry(Message, TTL), - needs_confirming = Confirm == eventually, - size = iolist_size(PFR)}. - -calculate_msg_expiry(#basic_message{content = Content}, TTL) -> - #content{properties = Props} = - rabbit_binary_parser:ensure_content_decoded(Content), - %% We assert that the expiration must be valid - we check in the channel. - {ok, MsgTTL} = rabbit_basic:parse_expiration(Props), - case lists:min([TTL, MsgTTL]) of - undefined -> undefined; - T -> os:system_time(micro_seconds) + T * 1000 - end. - -%% Logically this function should invoke maybe_send_drained/2. -%% However, that is expensive. Since some frequent callers of -%% drop_expired_msgs/1, in particular deliver_or_enqueue/3, cannot -%% possibly cause the queue to become empty, we push the -%% responsibility to the callers. So be cautious when adding new ones. -drop_expired_msgs(State) -> - case is_empty(State) of - true -> State; - false -> drop_expired_msgs(os:system_time(micro_seconds), - State) - end. - -drop_expired_msgs(Now, State = #q{backing_queue_state = BQS, - backing_queue = BQ }) -> - ExpirePred = fun (#message_properties{expiry = Exp}) -> Now >= Exp end, - {Props, State1} = - with_dlx( - State#q.dlx, - fun (X) -> dead_letter_expired_msgs(ExpirePred, X, State) end, - fun () -> {Next, BQS1} = BQ:dropwhile(ExpirePred, BQS), - {Next, State#q{backing_queue_state = BQS1}} end), - ensure_ttl_timer(case Props of - undefined -> undefined; - #message_properties{expiry = Exp} -> Exp - end, State1). - -with_dlx(undefined, _With, Without) -> Without(); -with_dlx(DLX, With, Without) -> case rabbit_exchange:lookup(DLX) of - {ok, X} -> With(X); - {error, not_found} -> Without() - end. - -dead_letter_expired_msgs(ExpirePred, X, State = #q{backing_queue = BQ}) -> - dead_letter_msgs(fun (DLFun, Acc, BQS1) -> - BQ:fetchwhile(ExpirePred, DLFun, Acc, BQS1) - end, expired, X, State). - -dead_letter_rejected_msgs(AckTags, X, State = #q{backing_queue = BQ}) -> - {ok, State1} = - dead_letter_msgs( - fun (DLFun, Acc, BQS) -> - {Acc1, BQS1} = BQ:ackfold(DLFun, Acc, BQS, AckTags), - {ok, Acc1, BQS1} - end, rejected, X, State), - State1. - -dead_letter_maxlen_msg(X, State = #q{backing_queue = BQ}) -> - {ok, State1} = - dead_letter_msgs( - fun (DLFun, Acc, BQS) -> - {{Msg, _, AckTag}, BQS1} = BQ:fetch(true, BQS), - {ok, DLFun(Msg, AckTag, Acc), BQS1} - end, maxlen, X, State), - State1. - -dead_letter_msgs(Fun, Reason, X, State = #q{dlx_routing_key = RK, - backing_queue_state = BQS, - backing_queue = BQ}) -> - QName = qname(State), - {Res, Acks1, BQS1} = - Fun(fun (Msg, AckTag, Acks) -> - rabbit_dead_letter:publish(Msg, Reason, X, RK, QName), - [AckTag | Acks] - end, [], BQS), - {_Guids, BQS2} = BQ:ack(Acks1, BQS1), - {Res, State#q{backing_queue_state = BQS2}}. - -stop(State) -> stop(noreply, State). - -stop(noreply, State) -> {stop, normal, State}; -stop(Reply, State) -> {stop, normal, Reply, State}. - -infos(Items, #q{q = Q} = State) -> - lists:foldr(fun(totals, Acc) -> - [{messages_ready, i(messages_ready, State)}, - {messages, i(messages, State)}, - {messages_unacknowledged, i(messages_unacknowledged, State)}] ++ Acc; - (type_specific, Acc) -> - format(Q) ++ Acc; - (Item, Acc) -> - [{Item, i(Item, State)} | Acc] - end, [], Items). - -i(name, #q{q = Q}) -> amqqueue:get_name(Q); -i(durable, #q{q = Q}) -> amqqueue:is_durable(Q); -i(auto_delete, #q{q = Q}) -> amqqueue:is_auto_delete(Q); -i(arguments, #q{q = Q}) -> amqqueue:get_arguments(Q); -i(pid, _) -> - self(); -i(owner_pid, #q{q = Q}) when ?amqqueue_exclusive_owner_is(Q, none) -> - ''; -i(owner_pid, #q{q = Q}) -> - amqqueue:get_exclusive_owner(Q); -i(exclusive, #q{q = Q}) -> - ExclusiveOwner = amqqueue:get_exclusive_owner(Q), - is_pid(ExclusiveOwner); -i(policy, #q{q = Q}) -> - case rabbit_policy:name(Q) of - none -> ''; - Policy -> Policy - end; -i(operator_policy, #q{q = Q}) -> - case rabbit_policy:name_op(Q) of - none -> ''; - Policy -> Policy - end; -i(effective_policy_definition, #q{q = Q}) -> - case rabbit_policy:effective_definition(Q) of - undefined -> []; - Def -> Def - end; -i(exclusive_consumer_pid, #q{active_consumer = {ChPid, _ConsumerTag}, single_active_consumer_on = false}) -> - ChPid; -i(exclusive_consumer_pid, _) -> - ''; -i(exclusive_consumer_tag, #q{active_consumer = {_ChPid, ConsumerTag}, single_active_consumer_on = false}) -> - ConsumerTag; -i(exclusive_consumer_tag, _) -> - ''; -i(single_active_consumer_pid, #q{active_consumer = {ChPid, _Consumer}, single_active_consumer_on = true}) -> - ChPid; -i(single_active_consumer_pid, _) -> - ''; -i(single_active_consumer_tag, #q{active_consumer = {_ChPid, Consumer}, single_active_consumer_on = true}) -> - rabbit_queue_consumers:consumer_tag(Consumer); -i(single_active_consumer_tag, _) -> - ''; -i(messages_ready, #q{backing_queue_state = BQS, backing_queue = BQ}) -> - BQ:len(BQS); -i(messages_unacknowledged, _) -> - rabbit_queue_consumers:unacknowledged_message_count(); -i(messages, State) -> - lists:sum([i(Item, State) || Item <- [messages_ready, - messages_unacknowledged]]); -i(consumers, _) -> - rabbit_queue_consumers:count(); -i(consumer_utilisation, #q{consumers = Consumers}) -> - case rabbit_queue_consumers:count() of - 0 -> ''; - _ -> rabbit_queue_consumers:utilisation(Consumers) - end; -i(memory, _) -> - {memory, M} = process_info(self(), memory), - M; -i(slave_pids, #q{q = Q0}) -> - Name = amqqueue:get_name(Q0), - {ok, Q} = rabbit_amqqueue:lookup(Name), - case rabbit_mirror_queue_misc:is_mirrored(Q) of - false -> ''; - true -> amqqueue:get_slave_pids(Q) - end; -i(synchronised_slave_pids, #q{q = Q0}) -> - Name = amqqueue:get_name(Q0), - {ok, Q} = rabbit_amqqueue:lookup(Name), - case rabbit_mirror_queue_misc:is_mirrored(Q) of - false -> ''; - true -> amqqueue:get_sync_slave_pids(Q) - end; -i(recoverable_slaves, #q{q = Q0}) -> - Name = amqqueue:get_name(Q0), - Durable = amqqueue:is_durable(Q0), - {ok, Q} = rabbit_amqqueue:lookup(Name), - case Durable andalso rabbit_mirror_queue_misc:is_mirrored(Q) of - false -> ''; - true -> amqqueue:get_recoverable_slaves(Q) - end; -i(state, #q{status = running}) -> credit_flow:state(); -i(state, #q{status = State}) -> State; -i(garbage_collection, _State) -> - rabbit_misc:get_gc_info(self()); -i(reductions, _State) -> - {reductions, Reductions} = erlang:process_info(self(), reductions), - Reductions; -i(user_who_performed_action, #q{q = Q}) -> - Opts = amqqueue:get_options(Q), - maps:get(user, Opts, ?UNKNOWN_USER); -i(type, _) -> classic; -i(Item, #q{backing_queue_state = BQS, backing_queue = BQ}) -> - BQ:info(Item, BQS). - -emit_stats(State) -> - emit_stats(State, []). - -emit_stats(State, Extra) -> - ExtraKs = [K || {K, _} <- Extra], - [{messages_ready, MR}, {messages_unacknowledged, MU}, {messages, M}, - {reductions, R}, {name, Name} | Infos] = All - = [{K, V} || {K, V} <- infos(statistics_keys(), State), - not lists:member(K, ExtraKs)], - rabbit_core_metrics:queue_stats(Name, Extra ++ Infos), - rabbit_core_metrics:queue_stats(Name, MR, MU, M, R), - rabbit_event:notify(queue_stats, Extra ++ All). - -emit_consumer_created(ChPid, CTag, Exclusive, AckRequired, QName, - PrefetchCount, Args, Ref, ActingUser) -> - rabbit_event:notify(consumer_created, - [{consumer_tag, CTag}, - {exclusive, Exclusive}, - {ack_required, AckRequired}, - {channel, ChPid}, - {queue, QName}, - {prefetch_count, PrefetchCount}, - {arguments, Args}, - {user_who_performed_action, ActingUser}], - Ref). - -emit_consumer_deleted(ChPid, ConsumerTag, QName, ActingUser) -> - rabbit_core_metrics:consumer_deleted(ChPid, ConsumerTag, QName), - rabbit_event:notify(consumer_deleted, - [{consumer_tag, ConsumerTag}, - {channel, ChPid}, - {queue, QName}, - {user_who_performed_action, ActingUser}]). - -%%---------------------------------------------------------------------------- - -prioritise_call(Msg, _From, _Len, State) -> - case Msg of - info -> 9; - {info, _Items} -> 9; - consumers -> 9; - stat -> 7; - {basic_consume, _, _, _, _, _, _, _, _, _} -> consumer_bias(State, 0, 2); - {basic_cancel, _, _, _} -> consumer_bias(State, 0, 2); - _ -> 0 - end. - -prioritise_cast(Msg, _Len, State) -> - case Msg of - delete_immediately -> 8; - {delete_exclusive, _Pid} -> 8; - {set_ram_duration_target, _Duration} -> 8; - {set_maximum_since_use, _Age} -> 8; - {run_backing_queue, _Mod, _Fun} -> 6; - {ack, _AckTags, _ChPid} -> 4; %% [1] - {resume, _ChPid} -> 3; - {notify_sent, _ChPid, _Credit} -> consumer_bias(State, 0, 2); - _ -> 0 - end. - -%% [1] It should be safe to always prioritise ack / resume since they -%% will be rate limited by how fast consumers receive messages - -%% i.e. by notify_sent. We prioritise ack and resume to discourage -%% starvation caused by prioritising notify_sent. We don't vary their -%% priority since acks should stay in order (some parts of the queue -%% stack are optimised for that) and to make things easier to reason -%% about. Finally, we prioritise ack over resume since it should -%% always reduce memory use. -%% bump_reduce_memory_use is prioritised over publishes, because sending -%% credit to self is hard to reason about. Consumers can continue while -%% reduce_memory_use is in progress. - -consumer_bias(#q{backing_queue = BQ, backing_queue_state = BQS}, Low, High) -> - case BQ:msg_rates(BQS) of - {0.0, _} -> Low; - {Ingress, Egress} when Egress / Ingress < ?CONSUMER_BIAS_RATIO -> High; - {_, _} -> Low - end. - -prioritise_info(Msg, _Len, #q{q = Q}) -> - DownPid = amqqueue:get_exclusive_owner(Q), - case Msg of - {'DOWN', _, process, DownPid, _} -> 8; - update_ram_duration -> 8; - {maybe_expire, _Version} -> 8; - {drop_expired, _Version} -> 8; - emit_stats -> 7; - sync_timeout -> 6; - bump_reduce_memory_use -> 1; - _ -> 0 - end. - -handle_call({init, Recover}, From, State) -> - try - init_it(Recover, From, State) - catch - {coordinator_not_started, Reason} -> - %% The GM can shutdown before the coordinator has started up - %% (lost membership or missing group), thus the start_link of - %% the coordinator returns {error, shutdown} as rabbit_amqqueue_process - %% is trapping exists. The master captures this return value and - %% throws the current exception. - {stop, Reason, State} - end; - -handle_call(info, _From, State) -> - reply({ok, infos(info_keys(), State)}, State); - -handle_call({info, Items}, _From, State) -> - try - reply({ok, infos(Items, State)}, State) - catch Error -> reply({error, Error}, State) - end; - -handle_call(consumers, _From, State = #q{consumers = Consumers, single_active_consumer_on = false}) -> - reply(rabbit_queue_consumers:all(Consumers), State); -handle_call(consumers, _From, State = #q{consumers = Consumers, active_consumer = ActiveConsumer}) -> - reply(rabbit_queue_consumers:all(Consumers, ActiveConsumer, true), State); - -handle_call({notify_down, ChPid}, _From, State) -> - %% we want to do this synchronously, so that auto_deleted queues - %% are no longer visible by the time we send a response to the - %% client. The queue is ultimately deleted in terminate/2; if we - %% return stop with a reply, terminate/2 will be called by - %% gen_server2 *before* the reply is sent. - case handle_ch_down(ChPid, State) of - {ok, State1} -> reply(ok, State1); - {stop, State1} -> stop(ok, State1#q{status = {terminated_by, auto_delete}}) - end; - -handle_call({basic_get, ChPid, NoAck, LimiterPid}, _From, - State = #q{q = Q}) -> - QName = amqqueue:get_name(Q), - AckRequired = not NoAck, - State1 = ensure_expiry_timer(State), - case fetch(AckRequired, State1) of - {empty, State2} -> - reply(empty, State2); - {{Message, IsDelivered, AckTag}, - #q{backing_queue = BQ, backing_queue_state = BQS} = State2} -> - case AckRequired of - true -> ok = rabbit_queue_consumers:record_ack( - ChPid, LimiterPid, AckTag); - false -> ok - end, - Msg = {QName, self(), AckTag, IsDelivered, Message}, - reply({ok, BQ:len(BQS), Msg}, State2) - end; - -handle_call({basic_consume, NoAck, ChPid, LimiterPid, LimiterActive, - PrefetchCount, ConsumerTag, ExclusiveConsume, Args, OkMsg, ActingUser}, - _From, State = #q{consumers = Consumers, - active_consumer = Holder, - single_active_consumer_on = SingleActiveConsumerOn}) -> - ConsumerRegistration = case SingleActiveConsumerOn of - true -> - case ExclusiveConsume of - true -> - {error, reply({error, exclusive_consume_unavailable}, State)}; - false -> - Consumers1 = rabbit_queue_consumers:add( - ChPid, ConsumerTag, NoAck, - LimiterPid, LimiterActive, - PrefetchCount, Args, is_empty(State), - ActingUser, Consumers), - - case Holder of - none -> - NewConsumer = rabbit_queue_consumers:get(ChPid, ConsumerTag, Consumers1), - {state, State#q{consumers = Consumers1, - has_had_consumers = true, - active_consumer = NewConsumer}}; - _ -> - {state, State#q{consumers = Consumers1, - has_had_consumers = true}} - end - end; - false -> - case check_exclusive_access(Holder, ExclusiveConsume, State) of - in_use -> {error, reply({error, exclusive_consume_unavailable}, State)}; - ok -> - Consumers1 = rabbit_queue_consumers:add( - ChPid, ConsumerTag, NoAck, - LimiterPid, LimiterActive, - PrefetchCount, Args, is_empty(State), - ActingUser, Consumers), - ExclusiveConsumer = - if ExclusiveConsume -> {ChPid, ConsumerTag}; - true -> Holder - end, - {state, State#q{consumers = Consumers1, - has_had_consumers = true, - active_consumer = ExclusiveConsumer}} - end - end, - case ConsumerRegistration of - {error, Reply} -> - Reply; - {state, State1} -> - ok = maybe_send_reply(ChPid, OkMsg), - QName = qname(State1), - AckRequired = not NoAck, - TheConsumer = rabbit_queue_consumers:get(ChPid, ConsumerTag, State1#q.consumers), - {ConsumerIsActive, ActivityStatus} = - case {SingleActiveConsumerOn, State1#q.active_consumer} of - {true, TheConsumer} -> - {true, single_active}; - {true, _} -> - {false, waiting}; - {false, _} -> - {true, up} - end, - rabbit_core_metrics:consumer_created( - ChPid, ConsumerTag, ExclusiveConsume, AckRequired, QName, - PrefetchCount, ConsumerIsActive, ActivityStatus, Args), - emit_consumer_created(ChPid, ConsumerTag, ExclusiveConsume, - AckRequired, QName, PrefetchCount, - Args, none, ActingUser), - notify_decorators(State1), - reply(ok, run_message_queue(State1)) - end; - -handle_call({basic_cancel, ChPid, ConsumerTag, OkMsg, ActingUser}, _From, - State = #q{consumers = Consumers, - active_consumer = Holder, - single_active_consumer_on = SingleActiveConsumerOn }) -> - ok = maybe_send_reply(ChPid, OkMsg), - case rabbit_queue_consumers:remove(ChPid, ConsumerTag, Consumers) of - not_found -> - reply(ok, State); - Consumers1 -> - Holder1 = new_single_active_consumer_after_basic_cancel(ChPid, ConsumerTag, - Holder, SingleActiveConsumerOn, Consumers1 - ), - State1 = State#q{consumers = Consumers1, - active_consumer = Holder1}, - maybe_notify_consumer_updated(State1, Holder, Holder1), - emit_consumer_deleted(ChPid, ConsumerTag, qname(State1), ActingUser), - notify_decorators(State1), - case should_auto_delete(State1) of - false -> reply(ok, ensure_expiry_timer(State1)); - true -> - log_auto_delete( - io_lib:format( - "because its last consumer with tag '~s' was cancelled", - [ConsumerTag]), - State), - stop(ok, State1) - end - end; - -handle_call(stat, _From, State) -> - State1 = #q{backing_queue = BQ, backing_queue_state = BQS} = - ensure_expiry_timer(State), - reply({ok, BQ:len(BQS), rabbit_queue_consumers:count()}, State1); - -handle_call({delete, IfUnused, IfEmpty, ActingUser}, _From, - State = #q{backing_queue_state = BQS, backing_queue = BQ}) -> - IsEmpty = BQ:is_empty(BQS), - IsUnused = is_unused(State), - if - IfEmpty and not(IsEmpty) -> reply({error, not_empty}, State); - IfUnused and not(IsUnused) -> reply({error, in_use}, State); - true -> stop({ok, BQ:len(BQS)}, - State#q{status = {terminated_by, ActingUser}}) - end; - -handle_call(purge, _From, State = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - {Count, BQS1} = BQ:purge(BQS), - State1 = State#q{backing_queue_state = BQS1}, - reply({ok, Count}, maybe_send_drained(Count =:= 0, State1)); - -handle_call({requeue, AckTags, ChPid}, From, State) -> - gen_server2:reply(From, ok), - noreply(requeue(AckTags, ChPid, State)); - -handle_call(sync_mirrors, _From, - State = #q{backing_queue = rabbit_mirror_queue_master, - backing_queue_state = BQS}) -> - S = fun(BQSN) -> State#q{backing_queue_state = BQSN} end, - HandleInfo = fun (Status) -> - receive {'$gen_call', From, {info, Items}} -> - Infos = infos(Items, State#q{status = Status}), - gen_server2:reply(From, {ok, Infos}) - after 0 -> - ok - end - end, - EmitStats = fun (Status) -> - rabbit_event:if_enabled( - State, #q.stats_timer, - fun() -> emit_stats(State#q{status = Status}) end) - end, - case rabbit_mirror_queue_master:sync_mirrors(HandleInfo, EmitStats, BQS) of - {ok, BQS1} -> reply(ok, S(BQS1)); - {stop, Reason, BQS1} -> {stop, Reason, S(BQS1)} - end; - -handle_call(sync_mirrors, _From, State) -> - reply({error, not_mirrored}, State); - -%% By definition if we get this message here we do not have to do anything. -handle_call(cancel_sync_mirrors, _From, State) -> - reply({ok, not_syncing}, State). - -new_single_active_consumer_after_basic_cancel(ChPid, ConsumerTag, CurrentSingleActiveConsumer, - _SingleActiveConsumerIsOn = true, Consumers) -> - case rabbit_queue_consumers:is_same(ChPid, ConsumerTag, CurrentSingleActiveConsumer) of - true -> - case rabbit_queue_consumers:get_consumer(Consumers) of - undefined -> none; - Consumer -> Consumer - end; - false -> - CurrentSingleActiveConsumer - end; -new_single_active_consumer_after_basic_cancel(ChPid, ConsumerTag, CurrentSingleActiveConsumer, - _SingleActiveConsumerIsOn = false, _Consumers) -> - case CurrentSingleActiveConsumer of - {ChPid, ConsumerTag} -> none; - _ -> CurrentSingleActiveConsumer - end. - -maybe_notify_consumer_updated(#q{single_active_consumer_on = false}, _, _) -> - ok; -maybe_notify_consumer_updated(#q{single_active_consumer_on = true}, SingleActiveConsumer, SingleActiveConsumer) -> - % the single active consumer didn't change, nothing to do - ok; -maybe_notify_consumer_updated(#q{single_active_consumer_on = true} = State, _PreviousConsumer, NewConsumer) -> - case NewConsumer of - {ChPid, Consumer} -> - {Tag, Ack, Prefetch, Args} = rabbit_queue_consumers:get_infos(Consumer), - rabbit_core_metrics:consumer_updated( - ChPid, Tag, false, Ack, qname(State), - Prefetch, true, single_active, Args - ), - ok; - _ -> - ok - end. - -handle_cast(init, State) -> - try - init_it({no_barrier, non_clean_shutdown}, none, State) - catch - {coordinator_not_started, Reason} -> - %% The GM can shutdown before the coordinator has started up - %% (lost membership or missing group), thus the start_link of - %% the coordinator returns {error, shutdown} as rabbit_amqqueue_process - %% is trapping exists. The master captures this return value and - %% throws the current exception. - {stop, Reason, State} - end; - -handle_cast({run_backing_queue, Mod, Fun}, - State = #q{backing_queue = BQ, backing_queue_state = BQS}) -> - noreply(State#q{backing_queue_state = BQ:invoke(Mod, Fun, BQS)}); - -handle_cast({deliver, - Delivery = #delivery{sender = Sender, - flow = Flow}, - SlaveWhenPublished}, - State = #q{senders = Senders}) -> - Senders1 = case Flow of - %% In both credit_flow:ack/1 we are acking messages to the channel - %% process that sent us the message delivery. See handle_ch_down - %% for more info. - flow -> credit_flow:ack(Sender), - case SlaveWhenPublished of - true -> credit_flow:ack(Sender); %% [0] - false -> ok - end, - pmon:monitor(Sender, Senders); - noflow -> Senders - end, - State1 = State#q{senders = Senders1}, - noreply(maybe_deliver_or_enqueue(Delivery, SlaveWhenPublished, State1)); -%% [0] The second ack is since the channel thought we were a mirror at -%% the time it published this message, so it used two credits (see -%% rabbit_queue_type:deliver/2). - -handle_cast({ack, AckTags, ChPid}, State) -> - noreply(ack(AckTags, ChPid, State)); - -handle_cast({reject, true, AckTags, ChPid}, State) -> - noreply(requeue(AckTags, ChPid, State)); - -handle_cast({reject, false, AckTags, ChPid}, State) -> - noreply(with_dlx( - State#q.dlx, - fun (X) -> subtract_acks(ChPid, AckTags, State, - fun (State1) -> - dead_letter_rejected_msgs( - AckTags, X, State1) - end) end, - fun () -> ack(AckTags, ChPid, State) end)); - -handle_cast({delete_exclusive, ConnPid}, State) -> - log_delete_exclusive(ConnPid, State), - stop(State); - -handle_cast(delete_immediately, State) -> - stop(State); - -handle_cast({resume, ChPid}, State) -> - noreply(possibly_unblock(rabbit_queue_consumers:resume_fun(), - ChPid, State)); - -handle_cast({notify_sent, ChPid, Credit}, State) -> - noreply(possibly_unblock(rabbit_queue_consumers:notify_sent_fun(Credit), - ChPid, State)); - -handle_cast({activate_limit, ChPid}, State) -> - noreply(possibly_unblock(rabbit_queue_consumers:activate_limit_fun(), - ChPid, State)); - -handle_cast({set_ram_duration_target, Duration}, - State = #q{backing_queue = BQ, backing_queue_state = BQS}) -> - BQS1 = BQ:set_ram_duration_target(Duration, BQS), - noreply(State#q{backing_queue_state = BQS1}); - -handle_cast({set_maximum_since_use, Age}, State) -> - ok = file_handle_cache:set_maximum_since_use(Age), - noreply(State); - -handle_cast(update_mirroring, State = #q{q = Q, - mirroring_policy_version = Version}) -> - case needs_update_mirroring(Q, Version) of - false -> - noreply(State); - {Policy, NewVersion} -> - State1 = State#q{mirroring_policy_version = NewVersion}, - noreply(update_mirroring(Policy, State1)) - end; - -handle_cast({credit, ChPid, CTag, Credit, Drain}, - State = #q{consumers = Consumers, - backing_queue = BQ, - backing_queue_state = BQS, - q = Q}) -> - Len = BQ:len(BQS), - rabbit_classic_queue:send_queue_event(ChPid, amqqueue:get_name(Q), {send_credit_reply, Len}), - noreply( - case rabbit_queue_consumers:credit(Len == 0, Credit, Drain, ChPid, CTag, - Consumers) of - unchanged -> State; - {unblocked, Consumers1} -> State1 = State#q{consumers = Consumers1}, - run_message_queue(true, State1) - end); - -% Note: https://www.pivotaltracker.com/story/show/166962656 -% This event is necessary for the stats timer to be initialized with -% the correct values once the management agent has started -handle_cast({force_event_refresh, Ref}, - State = #q{consumers = Consumers, - active_consumer = Holder}) -> - rabbit_event:notify(queue_created, infos(?CREATION_EVENT_KEYS, State), Ref), - QName = qname(State), - AllConsumers = rabbit_queue_consumers:all(Consumers), - case Holder of - none -> - [emit_consumer_created( - Ch, CTag, false, AckRequired, QName, Prefetch, - Args, Ref, ActingUser) || - {Ch, CTag, AckRequired, Prefetch, _, _, Args, ActingUser} - <- AllConsumers]; - {Ch, CTag} -> - [{Ch, CTag, AckRequired, Prefetch, _, _, Args, ActingUser}] = AllConsumers, - emit_consumer_created( - Ch, CTag, true, AckRequired, QName, Prefetch, Args, Ref, ActingUser) - end, - noreply(rabbit_event:init_stats_timer(State, #q.stats_timer)); - -handle_cast(notify_decorators, State) -> - notify_decorators(State), - noreply(State); - -handle_cast(policy_changed, State = #q{q = Q0}) -> - Name = amqqueue:get_name(Q0), - %% We depend on the #q.q field being up to date at least WRT - %% policy (but not mirror pids) in various places, so when it - %% changes we go and read it from Mnesia again. - %% - %% This also has the side effect of waking us up so we emit a - %% stats event - so event consumers see the changed policy. - {ok, Q} = rabbit_amqqueue:lookup(Name), - noreply(process_args_policy(State#q{q = Q})); - -handle_cast({sync_start, _, _}, State = #q{q = Q}) -> - Name = amqqueue:get_name(Q), - %% Only a mirror should receive this, it means we are a duplicated master - rabbit_mirror_queue_misc:log_warning( - Name, "Stopping after receiving sync_start from another master", []), - stop(State). - -handle_info({maybe_expire, Vsn}, State = #q{args_policy_version = Vsn}) -> - case is_unused(State) of - true -> stop(State); - false -> noreply(State#q{expiry_timer_ref = undefined}) - end; - -handle_info({maybe_expire, _Vsn}, State) -> - noreply(State); - -handle_info({drop_expired, Vsn}, State = #q{args_policy_version = Vsn}) -> - WasEmpty = is_empty(State), - State1 = drop_expired_msgs(State#q{ttl_timer_ref = undefined}), - noreply(maybe_send_drained(WasEmpty, State1)); - -handle_info({drop_expired, _Vsn}, State) -> - noreply(State); - -handle_info(emit_stats, State) -> - emit_stats(State), - %% Don't call noreply/1, we don't want to set timers - {State1, Timeout} = next_state(rabbit_event:reset_stats_timer( - State, #q.stats_timer)), - {noreply, State1, Timeout}; - -handle_info({'DOWN', _MonitorRef, process, DownPid, _Reason}, - State = #q{q = Q}) when ?amqqueue_exclusive_owner_is(Q, DownPid) -> - %% Exclusively owned queues must disappear with their owner. In - %% the case of clean shutdown we delete the queue synchronously in - %% the reader - although not required by the spec this seems to - %% match what people expect (see bug 21824). However we need this - %% monitor-and-async- delete in case the connection goes away - %% unexpectedly. - log_delete_exclusive(DownPid, State), - stop(State); - -handle_info({'DOWN', _MonitorRef, process, DownPid, _Reason}, State) -> - case handle_ch_down(DownPid, State) of - {ok, State1} -> noreply(State1); - {stop, State1} -> stop(State1) - end; - -handle_info(update_ram_duration, State = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - {RamDuration, BQS1} = BQ:ram_duration(BQS), - DesiredDuration = - rabbit_memory_monitor:report_ram_duration(self(), RamDuration), - BQS2 = BQ:set_ram_duration_target(DesiredDuration, BQS1), - %% Don't call noreply/1, we don't want to set timers - {State1, Timeout} = next_state(State#q{rate_timer_ref = undefined, - backing_queue_state = BQS2}), - {noreply, State1, Timeout}; - -handle_info(sync_timeout, State) -> - noreply(backing_queue_timeout(State#q{sync_timer_ref = undefined})); - -handle_info(timeout, State) -> - noreply(backing_queue_timeout(State)); - -handle_info({'EXIT', _Pid, Reason}, State) -> - {stop, Reason, State}; - -handle_info({bump_credit, Msg}, State = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - %% The message_store is granting us more credit. This means the - %% backing queue (for the rabbit_variable_queue case) might - %% continue paging messages to disk if it still needs to. We - %% consume credits from the message_store whenever we need to - %% persist a message to disk. See: - %% rabbit_variable_queue:msg_store_write/4. - credit_flow:handle_bump_msg(Msg), - noreply(State#q{backing_queue_state = BQ:resume(BQS)}); -handle_info(bump_reduce_memory_use, State = #q{backing_queue = BQ, - backing_queue_state = BQS0}) -> - BQS1 = BQ:handle_info(bump_reduce_memory_use, BQS0), - noreply(State#q{backing_queue_state = BQ:resume(BQS1)}); - -handle_info(Info, State) -> - {stop, {unhandled_info, Info}, State}. - -handle_pre_hibernate(State = #q{backing_queue_state = undefined}) -> - {hibernate, State}; -handle_pre_hibernate(State = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - {RamDuration, BQS1} = BQ:ram_duration(BQS), - DesiredDuration = - rabbit_memory_monitor:report_ram_duration(self(), RamDuration), - BQS2 = BQ:set_ram_duration_target(DesiredDuration, BQS1), - BQS3 = BQ:handle_pre_hibernate(BQS2), - rabbit_event:if_enabled( - State, #q.stats_timer, - fun () -> emit_stats(State, - [{idle_since, - os:system_time(milli_seconds)}, - {consumer_utilisation, ''}]) - end), - State1 = rabbit_event:stop_stats_timer(State#q{backing_queue_state = BQS3}, - #q.stats_timer), - {hibernate, stop_rate_timer(State1)}. - -format_message_queue(Opt, MQ) -> rabbit_misc:format_message_queue(Opt, MQ). - -format(Q) when ?is_amqqueue(Q) -> - case rabbit_mirror_queue_misc:is_mirrored(Q) of - false -> - [{node, node(amqqueue:get_pid(Q))}]; - true -> - Slaves = amqqueue:get_slave_pids(Q), - SSlaves = amqqueue:get_sync_slave_pids(Q), - [{slave_nodes, [node(S) || S <- Slaves]}, - {synchronised_slave_nodes, [node(S) || S <- SSlaves]}, - {node, node(amqqueue:get_pid(Q))}] - end. - --spec is_policy_applicable(amqqueue:amqqueue(), any()) -> boolean(). -is_policy_applicable(_Q, _Policy) -> - true. - -log_delete_exclusive({ConPid, _ConRef}, State) -> - log_delete_exclusive(ConPid, State); -log_delete_exclusive(ConPid, #q{ q = Q }) -> - Resource = amqqueue:get_name(Q), - #resource{ name = QName, virtual_host = VHost } = Resource, - rabbit_log_queue:debug("Deleting exclusive queue '~s' in vhost '~s' " ++ - "because its declaring connection ~p was closed", - [QName, VHost, ConPid]). - -log_auto_delete(Reason, #q{ q = Q }) -> - Resource = amqqueue:get_name(Q), - #resource{ name = QName, virtual_host = VHost } = Resource, - rabbit_log_queue:debug("Deleting auto-delete queue '~s' in vhost '~s' " ++ - Reason, - [QName, VHost]). - -needs_update_mirroring(Q, Version) -> - {ok, UpQ} = rabbit_amqqueue:lookup(amqqueue:get_name(Q)), - DBVersion = amqqueue:get_policy_version(UpQ), - case DBVersion > Version of - true -> {rabbit_policy:get(<<"ha-mode">>, UpQ), DBVersion}; - false -> false - end. - - -update_mirroring(Policy, State = #q{backing_queue = BQ}) -> - case update_to(Policy, BQ) of - start_mirroring -> - start_mirroring(State); - stop_mirroring -> - stop_mirroring(State); - ignore -> - State; - update_ha_mode -> - update_ha_mode(State) - end. - -update_to(undefined, rabbit_mirror_queue_master) -> - stop_mirroring; -update_to(_, rabbit_mirror_queue_master) -> - update_ha_mode; -update_to(undefined, BQ) when BQ =/= rabbit_mirror_queue_master -> - ignore; -update_to(_, BQ) when BQ =/= rabbit_mirror_queue_master -> - start_mirroring. - -start_mirroring(State = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - %% lookup again to get policy for init_with_existing_bq - {ok, Q} = rabbit_amqqueue:lookup(qname(State)), - true = BQ =/= rabbit_mirror_queue_master, %% assertion - BQ1 = rabbit_mirror_queue_master, - BQS1 = BQ1:init_with_existing_bq(Q, BQ, BQS), - State#q{backing_queue = BQ1, - backing_queue_state = BQS1}. - -stop_mirroring(State = #q{backing_queue = BQ, - backing_queue_state = BQS}) -> - BQ = rabbit_mirror_queue_master, %% assertion - {BQ1, BQS1} = BQ:stop_mirroring(BQS), - State#q{backing_queue = BQ1, - backing_queue_state = BQS1}. - -update_ha_mode(State) -> - {ok, Q} = rabbit_amqqueue:lookup(qname(State)), - ok = rabbit_mirror_queue_misc:update_mirrors(Q), - State. - -confirm_to_sender(Pid, QName, MsgSeqNos) -> - rabbit_classic_queue:confirm_to_sender(Pid, QName, MsgSeqNos). - - diff --git a/src/rabbit_amqqueue_sup.erl b/src/rabbit_amqqueue_sup.erl deleted file mode 100644 index a9eaf4087f..0000000000 --- a/src/rabbit_amqqueue_sup.erl +++ /dev/null @@ -1,35 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_amqqueue_sup). - --behaviour(supervisor2). - --export([start_link/2]). - --export([init/1]). - --include("rabbit.hrl"). - -%%---------------------------------------------------------------------------- - --spec start_link(amqqueue:amqqueue(), rabbit_prequeue:start_mode()) -> - {'ok', pid(), pid()}. - -start_link(Q, StartMode) -> - Marker = spawn_link(fun() -> receive stop -> ok end end), - ChildSpec = {rabbit_amqqueue, - {rabbit_prequeue, start_link, [Q, StartMode, Marker]}, - intrinsic, ?WORKER_WAIT, worker, [rabbit_amqqueue_process, - rabbit_mirror_queue_slave]}, - {ok, SupPid} = supervisor2:start_link(?MODULE, []), - {ok, QPid} = supervisor2:start_child(SupPid, ChildSpec), - unlink(Marker), - Marker ! stop, - {ok, SupPid, QPid}. - -init([]) -> {ok, {{one_for_one, 5, 10}, []}}. diff --git a/src/rabbit_amqqueue_sup_sup.erl b/src/rabbit_amqqueue_sup_sup.erl deleted file mode 100644 index 732816b79f..0000000000 --- a/src/rabbit_amqqueue_sup_sup.erl +++ /dev/null @@ -1,84 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_amqqueue_sup_sup). - --behaviour(supervisor2). - --export([start_link/0, start_queue_process/3]). --export([start_for_vhost/1, stop_for_vhost/1, - find_for_vhost/2, find_for_vhost/1]). - --export([init/1]). - --include("rabbit.hrl"). - --define(SERVER, ?MODULE). - -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - supervisor2:start_link(?MODULE, []). - --spec start_queue_process - (node(), amqqueue:amqqueue(), 'declare' | 'recovery' | 'slave') -> - pid(). - -start_queue_process(Node, Q, StartMode) -> - #resource{virtual_host = VHost} = amqqueue:get_name(Q), - {ok, Sup} = find_for_vhost(VHost, Node), - {ok, _SupPid, QPid} = supervisor2:start_child(Sup, [Q, StartMode]), - QPid. - -init([]) -> - {ok, {{simple_one_for_one, 10, 10}, - [{rabbit_amqqueue_sup, {rabbit_amqqueue_sup, start_link, []}, - temporary, ?SUPERVISOR_WAIT, supervisor, [rabbit_amqqueue_sup]}]}}. - --spec find_for_vhost(rabbit_types:vhost()) -> {ok, pid()} | {error, term()}. -find_for_vhost(VHost) -> - find_for_vhost(VHost, node()). - --spec find_for_vhost(rabbit_types:vhost(), atom()) -> {ok, pid()} | {error, term()}. -find_for_vhost(VHost, Node) -> - {ok, VHostSup} = rabbit_vhost_sup_sup:get_vhost_sup(VHost, Node), - case supervisor2:find_child(VHostSup, rabbit_amqqueue_sup_sup) of - [QSup] -> {ok, QSup}; - Result -> {error, {queue_supervisor_not_found, Result}} - end. - --spec start_for_vhost(rabbit_types:vhost()) -> {ok, pid()} | {error, term()}. -start_for_vhost(VHost) -> - case rabbit_vhost_sup_sup:get_vhost_sup(VHost) of - {ok, VHostSup} -> - supervisor2:start_child( - VHostSup, - {rabbit_amqqueue_sup_sup, - {rabbit_amqqueue_sup_sup, start_link, []}, - transient, infinity, supervisor, [rabbit_amqqueue_sup_sup]}); - %% we can get here if a vhost is added and removed concurrently - %% e.g. some integration tests do it - {error, {no_such_vhost, VHost}} -> - rabbit_log:error("Failed to start a queue process supervisor for vhost ~s: vhost no longer exists!", - [VHost]), - {error, {no_such_vhost, VHost}} - end. - --spec stop_for_vhost(rabbit_types:vhost()) -> ok. -stop_for_vhost(VHost) -> - case rabbit_vhost_sup_sup:get_vhost_sup(VHost) of - {ok, VHostSup} -> - ok = supervisor2:terminate_child(VHostSup, rabbit_amqqueue_sup_sup), - ok = supervisor2:delete_child(VHostSup, rabbit_amqqueue_sup_sup); - %% see start/1 - {error, {no_such_vhost, VHost}} -> - rabbit_log:error("Failed to stop a queue process supervisor for vhost ~s: vhost no longer exists!", - [VHost]), - ok - end. diff --git a/src/rabbit_auth_backend_internal.erl b/src/rabbit_auth_backend_internal.erl deleted file mode 100644 index cb930a1630..0000000000 --- a/src/rabbit_auth_backend_internal.erl +++ /dev/null @@ -1,1076 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_auth_backend_internal). --include("rabbit.hrl"). - --behaviour(rabbit_authn_backend). --behaviour(rabbit_authz_backend). - --export([user_login_authentication/2, user_login_authorization/2, - check_vhost_access/3, check_resource_access/4, check_topic_access/4]). - --export([add_user/3, delete_user/2, lookup_user/1, exists/1, - change_password/3, clear_password/2, - hash_password/2, change_password_hash/2, change_password_hash/3, - set_tags/3, set_permissions/6, clear_permissions/3, - set_topic_permissions/6, clear_topic_permissions/3, clear_topic_permissions/4, - add_user_sans_validation/3, put_user/2, put_user/3]). - --export([set_user_limits/3, clear_user_limits/3, is_over_connection_limit/1, - is_over_channel_limit/1, get_user_limits/0, get_user_limits/1]). - --export([user_info_keys/0, perms_info_keys/0, - user_perms_info_keys/0, vhost_perms_info_keys/0, - user_vhost_perms_info_keys/0, all_users/0, - list_users/0, list_users/2, list_permissions/0, - list_user_permissions/1, list_user_permissions/3, - list_topic_permissions/0, - list_vhost_permissions/1, list_vhost_permissions/3, - list_user_vhost_permissions/2, - list_user_topic_permissions/1, list_vhost_topic_permissions/1, list_user_vhost_topic_permissions/2]). - --export([state_can_expire/0]). - -%% for testing --export([hashing_module_for_user/1, expand_topic_permission/2]). - -%%---------------------------------------------------------------------------- - --type regexp() :: binary(). - -%%---------------------------------------------------------------------------- -%% Implementation of rabbit_auth_backend - -%% Returns a password hashing module for the user record provided. If -%% there is no information in the record, we consider it to be legacy -%% (inserted by a version older than 3.6.0) and fall back to MD5, the -%% now obsolete hashing function. -hashing_module_for_user(User) -> - ModOrUndefined = internal_user:get_hashing_algorithm(User), - rabbit_password:hashing_mod(ModOrUndefined). - --define(BLANK_PASSWORD_REJECTION_MESSAGE, - "user '~s' attempted to log in with a blank password, which is prohibited by the internal authN backend. " - "To use TLS/x509 certificate-based authentication, see the rabbitmq_auth_mechanism_ssl plugin and configure the client to use the EXTERNAL authentication mechanism. " - "Alternatively change the password for the user to be non-blank."). - -%% For cases when we do not have a set of credentials, -%% namely when x509 (TLS) certificates are used. This should only be -%% possible when the EXTERNAL authentication mechanism is used, see -%% rabbit_auth_mechanism_plain:handle_response/2 and rabbit_reader:auth_phase/2. -user_login_authentication(Username, []) -> - internal_check_user_login(Username, fun(_) -> true end); -%% For cases when we do have a set of credentials. rabbit_auth_mechanism_plain:handle_response/2 -%% performs initial validation. -user_login_authentication(Username, AuthProps) -> - case lists:keyfind(password, 1, AuthProps) of - {password, <<"">>} -> - {refused, ?BLANK_PASSWORD_REJECTION_MESSAGE, - [Username]}; - {password, ""} -> - {refused, ?BLANK_PASSWORD_REJECTION_MESSAGE, - [Username]}; - {password, Cleartext} -> - internal_check_user_login( - Username, - fun(User) -> - case internal_user:get_password_hash(User) of - <<Salt:4/binary, Hash/binary>> -> - Hash =:= rabbit_password:salted_hash( - hashing_module_for_user(User), Salt, Cleartext); - _ -> - false - end - end); - false -> exit({unknown_auth_props, Username, AuthProps}) - end. - -state_can_expire() -> false. - -user_login_authorization(Username, _AuthProps) -> - case user_login_authentication(Username, []) of - {ok, #auth_user{impl = Impl, tags = Tags}} -> {ok, Impl, Tags}; - Else -> Else - end. - -internal_check_user_login(Username, Fun) -> - Refused = {refused, "user '~s' - invalid credentials", [Username]}, - case lookup_user(Username) of - {ok, User} -> - Tags = internal_user:get_tags(User), - case Fun(User) of - true -> {ok, #auth_user{username = Username, - tags = Tags, - impl = none}}; - _ -> Refused - end; - {error, not_found} -> - Refused - end. - -check_vhost_access(#auth_user{username = Username}, VHostPath, _AuthzData) -> - case mnesia:dirty_read({rabbit_user_permission, - #user_vhost{username = Username, - virtual_host = VHostPath}}) of - [] -> false; - [_R] -> true - end. - -check_resource_access(#auth_user{username = Username}, - #resource{virtual_host = VHostPath, name = Name}, - Permission, - _AuthContext) -> - case mnesia:dirty_read({rabbit_user_permission, - #user_vhost{username = Username, - virtual_host = VHostPath}}) of - [] -> - false; - [#user_permission{permission = P}] -> - PermRegexp = case element(permission_index(Permission), P) of - %% <<"^$">> breaks Emacs' erlang mode - <<"">> -> <<$^, $$>>; - RE -> RE - end, - case re:run(Name, PermRegexp, [{capture, none}]) of - match -> true; - nomatch -> false - end - end. - -check_topic_access(#auth_user{username = Username}, - #resource{virtual_host = VHostPath, name = Name, kind = topic}, - Permission, - Context) -> - case mnesia:dirty_read({rabbit_topic_permission, - #topic_permission_key{user_vhost = #user_vhost{username = Username, - virtual_host = VHostPath}, - exchange = Name - }}) of - [] -> - true; - [#topic_permission{permission = P}] -> - PermRegexp = case element(permission_index(Permission), P) of - %% <<"^$">> breaks Emacs' erlang mode - <<"">> -> <<$^, $$>>; - RE -> RE - end, - PermRegexpExpanded = expand_topic_permission( - PermRegexp, - maps:get(variable_map, Context, undefined) - ), - case re:run(maps:get(routing_key, Context), PermRegexpExpanded, [{capture, none}]) of - match -> true; - nomatch -> false - end - end. - -expand_topic_permission(Permission, ToExpand) when is_map(ToExpand) -> - Opening = <<"{">>, - Closing = <<"}">>, - ReplaceFun = fun(K, V, Acc) -> - Placeholder = <<Opening/binary, K/binary, Closing/binary>>, - binary:replace(Acc, Placeholder, V, [global]) - end, - maps:fold(ReplaceFun, Permission, ToExpand); -expand_topic_permission(Permission, _ToExpand) -> - Permission. - -permission_index(configure) -> #permission.configure; -permission_index(write) -> #permission.write; -permission_index(read) -> #permission.read. - -%%---------------------------------------------------------------------------- -%% Manipulation of the user database - -validate_credentials(Username, Password) -> - rabbit_credential_validation:validate(Username, Password). - -validate_and_alternate_credentials(Username, Password, ActingUser, Fun) -> - case validate_credentials(Username, Password) of - ok -> - Fun(Username, Password, ActingUser); - {error, Err} -> - rabbit_log:error("Credential validation for '~s' failed!~n", [Username]), - {error, Err} - end. - --spec add_user(rabbit_types:username(), rabbit_types:password(), - rabbit_types:username()) -> 'ok' | {'error', string()}. - -add_user(Username, Password, ActingUser) -> - validate_and_alternate_credentials(Username, Password, ActingUser, - fun add_user_sans_validation/3). - -add_user_sans_validation(Username, Password, ActingUser) -> - rabbit_log:debug("Asked to create a new user '~s', password length in bytes: ~p", [Username, bit_size(Password)]), - %% hash_password will pick the hashing function configured for us - %% but we also need to store a hint as part of the record, so we - %% retrieve it here one more time - HashingMod = rabbit_password:hashing_mod(), - PasswordHash = hash_password(HashingMod, Password), - User = internal_user:create_user(Username, PasswordHash, HashingMod), - try - R = rabbit_misc:execute_mnesia_transaction( - fun () -> - case mnesia:wread({rabbit_user, Username}) of - [] -> - ok = mnesia:write(rabbit_user, User, write); - _ -> - mnesia:abort({user_already_exists, Username}) - end - end), - rabbit_log:info("Created user '~s'", [Username]), - rabbit_event:notify(user_created, [{name, Username}, - {user_who_performed_action, ActingUser}]), - R - catch - throw:{error, {user_already_exists, _}} = Error -> - rabbit_log:warning("Failed to add user '~s': the user already exists", [Username]), - throw(Error); - throw:Error -> - rabbit_log:warning("Failed to add user '~s': ~p", [Username, Error]), - throw(Error); - exit:Error -> - rabbit_log:warning("Failed to add user '~s': ~p", [Username, Error]), - exit(Error) - end . - --spec delete_user(rabbit_types:username(), rabbit_types:username()) -> 'ok'. - -delete_user(Username, ActingUser) -> - rabbit_log:debug("Asked to delete user '~s'", [Username]), - try - R = rabbit_misc:execute_mnesia_transaction( - rabbit_misc:with_user( - Username, - fun () -> - ok = mnesia:delete({rabbit_user, Username}), - [ok = mnesia:delete_object( - rabbit_user_permission, R, write) || - R <- mnesia:match_object( - rabbit_user_permission, - #user_permission{user_vhost = #user_vhost{ - username = Username, - virtual_host = '_'}, - permission = '_'}, - write)], - UserTopicPermissionsQuery = match_user_vhost_topic_permission(Username, '_'), - UserTopicPermissions = UserTopicPermissionsQuery(), - [ok = mnesia:delete_object(rabbit_topic_permission, R, write) || R <- UserTopicPermissions], - ok - end)), - rabbit_log:info("Deleted user '~s'", [Username]), - rabbit_event:notify(user_deleted, - [{name, Username}, - {user_who_performed_action, ActingUser}]), - R - catch - throw:{error, {no_such_user, _}} = Error -> - rabbit_log:warning("Failed to delete user '~s': the user does not exist", [Username]), - throw(Error); - throw:Error -> - rabbit_log:warning("Failed to delete user '~s': ~p", [Username, Error]), - throw(Error); - exit:Error -> - rabbit_log:warning("Failed to delete user '~s': ~p", [Username, Error]), - exit(Error) - end . - --spec lookup_user - (rabbit_types:username()) -> - rabbit_types:ok(internal_user:internal_user()) | - rabbit_types:error('not_found'). - -lookup_user(Username) -> - rabbit_misc:dirty_read({rabbit_user, Username}). - --spec exists(rabbit_types:username()) -> boolean(). - -exists(Username) -> - case lookup_user(Username) of - {error, not_found} -> false; - _ -> true - end. - --spec change_password - (rabbit_types:username(), rabbit_types:password(), rabbit_types:username()) -> 'ok'. - -change_password(Username, Password, ActingUser) -> - validate_and_alternate_credentials(Username, Password, ActingUser, - fun change_password_sans_validation/3). - -change_password_sans_validation(Username, Password, ActingUser) -> - try - rabbit_log:debug("Asked to change password of user '~s', new password length in bytes: ~p", [Username, bit_size(Password)]), - HashingAlgorithm = rabbit_password:hashing_mod(), - R = change_password_hash(Username, - hash_password(rabbit_password:hashing_mod(), - Password), - HashingAlgorithm), - rabbit_log:info("Successfully changed password for user '~s'", [Username]), - rabbit_event:notify(user_password_changed, - [{name, Username}, - {user_who_performed_action, ActingUser}]), - R - catch - throw:{error, {no_such_user, _}} = Error -> - rabbit_log:warning("Failed to change password for user '~s': the user does not exist", [Username]), - throw(Error); - throw:Error -> - rabbit_log:warning("Failed to change password for user '~s': ~p", [Username, Error]), - throw(Error); - exit:Error -> - rabbit_log:warning("Failed to change password for user '~s': ~p", [Username, Error]), - exit(Error) - end. - --spec clear_password(rabbit_types:username(), rabbit_types:username()) -> 'ok'. - -clear_password(Username, ActingUser) -> - rabbit_log:info("Clearing password for '~s'~n", [Username]), - R = change_password_hash(Username, <<"">>), - rabbit_event:notify(user_password_cleared, - [{name, Username}, - {user_who_performed_action, ActingUser}]), - R. - --spec hash_password - (module(), rabbit_types:password()) -> rabbit_types:password_hash(). - -hash_password(HashingMod, Cleartext) -> - rabbit_password:hash(HashingMod, Cleartext). - --spec change_password_hash - (rabbit_types:username(), rabbit_types:password_hash()) -> 'ok'. - -change_password_hash(Username, PasswordHash) -> - change_password_hash(Username, PasswordHash, rabbit_password:hashing_mod()). - - -change_password_hash(Username, PasswordHash, HashingAlgorithm) -> - update_user(Username, fun(User) -> - internal_user:set_password_hash(User, - PasswordHash, HashingAlgorithm) - end). - --spec set_tags(rabbit_types:username(), [atom()], rabbit_types:username()) -> 'ok'. - -set_tags(Username, Tags, ActingUser) -> - ConvertedTags = [rabbit_data_coercion:to_atom(I) || I <- Tags], - rabbit_log:debug("Asked to set user tags for user '~s' to ~p", [Username, ConvertedTags]), - try - R = update_user(Username, fun(User) -> - internal_user:set_tags(User, ConvertedTags) - end), - rabbit_log:info("Successfully set user tags for user '~s' to ~p", [Username, ConvertedTags]), - rabbit_event:notify(user_tags_set, [{name, Username}, {tags, ConvertedTags}, - {user_who_performed_action, ActingUser}]), - R - catch - throw:{error, {no_such_user, _}} = Error -> - rabbit_log:warning("Failed to set tags for user '~s': the user does not exist", [Username]), - throw(Error); - throw:Error -> - rabbit_log:warning("Failed to set tags for user '~s': ~p", [Username, Error]), - throw(Error); - exit:Error -> - rabbit_log:warning("Failed to set tags for user '~s': ~p", [Username, Error]), - exit(Error) - end . - --spec set_permissions - (rabbit_types:username(), rabbit_types:vhost(), regexp(), regexp(), - regexp(), rabbit_types:username()) -> - 'ok'. - -set_permissions(Username, VirtualHost, ConfigurePerm, WritePerm, ReadPerm, ActingUser) -> - rabbit_log:debug("Asked to set permissions for " - "'~s' in virtual host '~s' to '~s', '~s', '~s'", - [Username, VirtualHost, ConfigurePerm, WritePerm, ReadPerm]), - lists:map( - fun (RegexpBin) -> - Regexp = binary_to_list(RegexpBin), - case re:compile(Regexp) of - {ok, _} -> ok; - {error, Reason} -> - rabbit_log:warning("Failed to set permissions for '~s' in virtual host '~s': " - "regular expression '~s' is invalid", - [Username, VirtualHost, RegexpBin]), - throw({error, {invalid_regexp, Regexp, Reason}}) - end - end, [ConfigurePerm, WritePerm, ReadPerm]), - try - R = rabbit_misc:execute_mnesia_transaction( - rabbit_vhost:with_user_and_vhost( - Username, VirtualHost, - fun () -> ok = mnesia:write( - rabbit_user_permission, - #user_permission{user_vhost = #user_vhost{ - username = Username, - virtual_host = VirtualHost}, - permission = #permission{ - configure = ConfigurePerm, - write = WritePerm, - read = ReadPerm}}, - write) - end)), - rabbit_log:info("Successfully set permissions for " - "'~s' in virtual host '~s' to '~s', '~s', '~s'", - [Username, VirtualHost, ConfigurePerm, WritePerm, ReadPerm]), - rabbit_event:notify(permission_created, [{user, Username}, - {vhost, VirtualHost}, - {configure, ConfigurePerm}, - {write, WritePerm}, - {read, ReadPerm}, - {user_who_performed_action, ActingUser}]), - R - catch - throw:{error, {no_such_vhost, _}} = Error -> - rabbit_log:warning("Failed to set permissions for '~s': virtual host '~s' does not exist", - [Username, VirtualHost]), - throw(Error); - throw:{error, {no_such_user, _}} = Error -> - rabbit_log:warning("Failed to set permissions for '~s': the user does not exist", - [Username]), - throw(Error); - throw:Error -> - rabbit_log:warning("Failed to set permissions for '~s' in virtual host '~s': ~p", - [Username, VirtualHost, Error]), - throw(Error); - exit:Error -> - rabbit_log:warning("Failed to set permissions for '~s' in virtual host '~s': ~p", - [Username, VirtualHost, Error]), - exit(Error) - end. - --spec clear_permissions - (rabbit_types:username(), rabbit_types:vhost(), rabbit_types:username()) -> 'ok'. - -clear_permissions(Username, VirtualHost, ActingUser) -> - rabbit_log:debug("Asked to clear permissions for '~s' in virtual host '~s'", - [Username, VirtualHost]), - try - R = rabbit_misc:execute_mnesia_transaction( - rabbit_vhost:with_user_and_vhost( - Username, VirtualHost, - fun () -> - ok = mnesia:delete({rabbit_user_permission, - #user_vhost{username = Username, - virtual_host = VirtualHost}}) - end)), - rabbit_log:info("Successfully cleared permissions for '~s' in virtual host '~s'", - [Username, VirtualHost]), - rabbit_event:notify(permission_deleted, [{user, Username}, - {vhost, VirtualHost}, - {user_who_performed_action, ActingUser}]), - R - catch - throw:{error, {no_such_vhost, _}} = Error -> - rabbit_log:warning("Failed to clear permissions for '~s': virtual host '~s' does not exist", - [Username, VirtualHost]), - throw(Error); - throw:{error, {no_such_user, _}} = Error -> - rabbit_log:warning("Failed to clear permissions for '~s': the user does not exist", - [Username]), - throw(Error); - throw:Error -> - rabbit_log:warning("Failed to clear permissions for '~s' in virtual host '~s': ~p", - [Username, VirtualHost, Error]), - throw(Error); - exit:Error -> - rabbit_log:warning("Failed to clear permissions for '~s' in virtual host '~s': ~p", - [Username, VirtualHost, Error]), - exit(Error) - end. - - -update_user(Username, Fun) -> - rabbit_misc:execute_mnesia_transaction( - rabbit_misc:with_user( - Username, - fun () -> - {ok, User} = lookup_user(Username), - ok = mnesia:write(rabbit_user, Fun(User), write) - end)). - -set_topic_permissions(Username, VirtualHost, Exchange, WritePerm, ReadPerm, ActingUser) -> - rabbit_log:debug("Asked to set topic permissions on exchange '~s' for " - "user '~s' in virtual host '~s' to '~s', '~s'", - [Exchange, Username, VirtualHost, WritePerm, ReadPerm]), - WritePermRegex = rabbit_data_coercion:to_binary(WritePerm), - ReadPermRegex = rabbit_data_coercion:to_binary(ReadPerm), - lists:map( - fun (RegexpBin) -> - case re:compile(RegexpBin) of - {ok, _} -> ok; - {error, Reason} -> - rabbit_log:warning("Failed to set topic permissions on exchange '~s' for " - "'~s' in virtual host '~s': regular expression '~s' is invalid", - [Exchange, Username, VirtualHost, RegexpBin]), - throw({error, {invalid_regexp, RegexpBin, Reason}}) - end - end, [WritePerm, ReadPerm]), - try - R = rabbit_misc:execute_mnesia_transaction( - rabbit_vhost:with_user_and_vhost( - Username, VirtualHost, - fun () -> ok = mnesia:write( - rabbit_topic_permission, - #topic_permission{ - topic_permission_key = #topic_permission_key{ - user_vhost = #user_vhost{ - username = Username, - virtual_host = VirtualHost}, - exchange = Exchange - }, - permission = #permission{ - write = WritePermRegex, - read = ReadPermRegex - } - }, - write) - end)), - rabbit_log:info("Successfully set topic permissions on exchange '~s' for " - "'~s' in virtual host '~s' to '~s', '~s'", - [Exchange, Username, VirtualHost, WritePerm, ReadPerm]), - rabbit_event:notify(topic_permission_created, [ - {user, Username}, - {vhost, VirtualHost}, - {exchange, Exchange}, - {write, WritePermRegex}, - {read, ReadPermRegex}, - {user_who_performed_action, ActingUser}]), - R - catch - throw:{error, {no_such_vhost, _}} = Error -> - rabbit_log:warning("Failed to set topic permissions on exchange '~s' for '~s': virtual host '~s' does not exist.", - [Exchange, Username, VirtualHost]), - throw(Error); - throw:{error, {no_such_user, _}} = Error -> - rabbit_log:warning("Failed to set topic permissions on exchange '~s' for '~s': the user does not exist.", - [Exchange, Username]), - throw(Error); - throw:Error -> - rabbit_log:warning("Failed to set topic permissions on exchange '~s' for '~s' in virtual host '~s': ~p.", - [Exchange, Username, VirtualHost, Error]), - throw(Error); - exit:Error -> - rabbit_log:warning("Failed to set topic permissions on exchange '~s' for '~s' in virtual host '~s': ~p.", - [Exchange, Username, VirtualHost, Error]), - exit(Error) - end . - -clear_topic_permissions(Username, VirtualHost, ActingUser) -> - rabbit_log:debug("Asked to clear topic permissions for '~s' in virtual host '~s'", - [Username, VirtualHost]), - try - R = rabbit_misc:execute_mnesia_transaction( - rabbit_vhost:with_user_and_vhost( - Username, VirtualHost, - fun () -> - ListFunction = match_user_vhost_topic_permission(Username, VirtualHost), - List = ListFunction(), - lists:foreach(fun(X) -> - ok = mnesia:delete_object(rabbit_topic_permission, X, write) - end, List) - end)), - rabbit_log:info("Successfully cleared topic permissions for '~s' in virtual host '~s'", - [Username, VirtualHost]), - rabbit_event:notify(topic_permission_deleted, [{user, Username}, - {vhost, VirtualHost}, - {user_who_performed_action, ActingUser}]), - R - catch - throw:{error, {no_such_vhost, _}} = Error -> - rabbit_log:warning("Failed to clear topic permissions for '~s': virtual host '~s' does not exist", - [Username, VirtualHost]), - throw(Error); - throw:{error, {no_such_user, _}} = Error -> - rabbit_log:warning("Failed to clear topic permissions for '~s': the user does not exist", - [Username]), - throw(Error); - throw:Error -> - rabbit_log:warning("Failed to clear topic permissions for '~s' in virtual host '~s': ~p", - [Username, VirtualHost, Error]), - throw(Error); - exit:Error -> - rabbit_log:warning("Failed to clear topic permissions for '~s' in virtual host '~s': ~p", - [Username, VirtualHost, Error]), - exit(Error) - end. - -clear_topic_permissions(Username, VirtualHost, Exchange, ActingUser) -> - rabbit_log:debug("Asked to clear topic permissions on exchange '~s' for '~s' in virtual host '~s'", - [Exchange, Username, VirtualHost]), - try - R = rabbit_misc:execute_mnesia_transaction( - rabbit_vhost:with_user_and_vhost( - Username, VirtualHost, - fun () -> - ok = mnesia:delete(rabbit_topic_permission, - #topic_permission_key{ - user_vhost = #user_vhost{ - username = Username, - virtual_host = VirtualHost}, - exchange = Exchange - }, write) - end)), - rabbit_log:info("Successfully cleared topic permissions on exchange '~s' for '~s' in virtual host '~s'", - [Exchange, Username, VirtualHost]), - rabbit_event:notify(permission_deleted, [{user, Username}, - {vhost, VirtualHost}, - {user_who_performed_action, ActingUser}]), - R - catch - throw:{error, {no_such_vhost, _}} = Error -> - rabbit_log:warning("Failed to clear topic permissions on exchange '~s' for '~s': virtual host '~s' does not exist", - [Exchange, Username, VirtualHost]), - throw(Error); - throw:{error, {no_such_user, _}} = Error -> - rabbit_log:warning("Failed to clear topic permissions on exchange '~s' for '~s': the user does not exist", - [Exchange, Username]), - throw(Error); - throw:Error -> - rabbit_log:warning("Failed to clear topic permissions on exchange '~s' for '~s' in virtual host '~s': ~p", - [Exchange, Username, VirtualHost, Error]), - throw(Error); - exit:Error -> - rabbit_log:warning("Failed to clear topic permissions on exchange '~s' for '~s' in virtual host '~s': ~p", - [Exchange, Username, VirtualHost, Error]), - exit(Error) - end. - -put_user(User, ActingUser) -> put_user(User, undefined, ActingUser). - -put_user(User, Version, ActingUser) -> - Username = maps:get(name, User), - HasPassword = maps:is_key(password, User), - HasPasswordHash = maps:is_key(password_hash, User), - Password = maps:get(password, User, undefined), - PasswordHash = maps:get(password_hash, User, undefined), - - Tags = case {maps:get(tags, User, undefined), maps:get(administrator, User, undefined)} of - {undefined, undefined} -> - throw({error, tags_not_present}); - {undefined, AdminS} -> - case rabbit_misc:parse_bool(AdminS) of - true -> [administrator]; - false -> [] - end; - {TagsS, _} -> - [list_to_atom(string:strip(T)) || - T <- string:tokens(binary_to_list(TagsS), ",")] - end, - - %% pre-configured, only applies to newly created users - Permissions = maps:get(permissions, User, undefined), - - PassedCredentialValidation = - case {HasPassword, HasPasswordHash} of - {true, false} -> - rabbit_credential_validation:validate(Username, Password) =:= ok; - {false, true} -> true; - _ -> - rabbit_credential_validation:validate(Username, Password) =:= ok - end, - - case exists(Username) of - true -> - case {HasPassword, HasPasswordHash} of - {true, false} -> - update_user_password(PassedCredentialValidation, Username, Password, Tags, ActingUser); - {false, true} -> - update_user_password_hash(Username, PasswordHash, Tags, User, Version, ActingUser); - {true, true} -> - throw({error, both_password_and_password_hash_are_provided}); - %% clear password, update tags if needed - _ -> - rabbit_auth_backend_internal:set_tags(Username, Tags, ActingUser), - rabbit_auth_backend_internal:clear_password(Username, ActingUser) - end; - false -> - case {HasPassword, HasPasswordHash} of - {true, false} -> - create_user_with_password(PassedCredentialValidation, Username, Password, Tags, Permissions, ActingUser); - {false, true} -> - create_user_with_password_hash(Username, PasswordHash, Tags, User, Version, Permissions, ActingUser); - {true, true} -> - throw({error, both_password_and_password_hash_are_provided}); - {false, false} -> - %% this user won't be able to sign in using - %% a username/password pair but can be used for x509 certificate authentication, - %% with authn backends such as HTTP or LDAP and so on. - create_user_with_password(PassedCredentialValidation, Username, <<"">>, Tags, Permissions, ActingUser) - end - end. - -update_user_password(_PassedCredentialValidation = true, Username, Password, Tags, ActingUser) -> - rabbit_auth_backend_internal:change_password(Username, Password, ActingUser), - rabbit_auth_backend_internal:set_tags(Username, Tags, ActingUser); -update_user_password(_PassedCredentialValidation = false, _Username, _Password, _Tags, _ActingUser) -> - %% we don't log here because - %% rabbit_auth_backend_internal will do it - throw({error, credential_validation_failed}). - -update_user_password_hash(Username, PasswordHash, Tags, User, Version, ActingUser) -> - %% when a hash this provided, credential validation - %% is not applied - HashingAlgorithm = hashing_algorithm(User, Version), - - Hash = rabbit_misc:b64decode_or_throw(PasswordHash), - rabbit_auth_backend_internal:change_password_hash( - Username, Hash, HashingAlgorithm), - rabbit_auth_backend_internal:set_tags(Username, Tags, ActingUser). - -create_user_with_password(_PassedCredentialValidation = true, Username, Password, Tags, undefined, ActingUser) -> - rabbit_auth_backend_internal:add_user(Username, Password, ActingUser), - rabbit_auth_backend_internal:set_tags(Username, Tags, ActingUser); -create_user_with_password(_PassedCredentialValidation = true, Username, Password, Tags, PreconfiguredPermissions, ActingUser) -> - rabbit_auth_backend_internal:add_user(Username, Password, ActingUser), - rabbit_auth_backend_internal:set_tags(Username, Tags, ActingUser), - preconfigure_permissions(Username, PreconfiguredPermissions, ActingUser); -create_user_with_password(_PassedCredentialValidation = false, _Username, _Password, _Tags, _, _) -> - %% we don't log here because - %% rabbit_auth_backend_internal will do it - throw({error, credential_validation_failed}). - -create_user_with_password_hash(Username, PasswordHash, Tags, User, Version, PreconfiguredPermissions, ActingUser) -> - %% when a hash this provided, credential validation - %% is not applied - HashingAlgorithm = hashing_algorithm(User, Version), - Hash = rabbit_misc:b64decode_or_throw(PasswordHash), - - %% first we create a user with dummy credentials and no - %% validation applied, then we update password hash - TmpPassword = rabbit_guid:binary(rabbit_guid:gen_secure(), "tmp"), - rabbit_auth_backend_internal:add_user_sans_validation(Username, TmpPassword, ActingUser), - - rabbit_auth_backend_internal:change_password_hash( - Username, Hash, HashingAlgorithm), - rabbit_auth_backend_internal:set_tags(Username, Tags, ActingUser), - preconfigure_permissions(Username, PreconfiguredPermissions, ActingUser). - -preconfigure_permissions(_Username, undefined, _ActingUser) -> - ok; -preconfigure_permissions(Username, Map, ActingUser) when is_map(Map) -> - maps:map(fun(VHost, M) -> - rabbit_auth_backend_internal:set_permissions(Username, VHost, - maps:get(<<"configure">>, M), - maps:get(<<"write">>, M), - maps:get(<<"read">>, M), - ActingUser) - end, - Map), - ok. - -set_user_limits(Username, Definition, ActingUser) when is_list(Definition); is_binary(Definition) -> - case rabbit_feature_flags:is_enabled(user_limits) of - true -> - case rabbit_json:try_decode(rabbit_data_coercion:to_binary(Definition)) of - {ok, Term} -> - validate_parameters_and_update_limit(Username, Term, ActingUser); - {error, Reason} -> - {error_string, rabbit_misc:format( - "JSON decoding error. Reason: ~ts", [Reason])} - end; - false -> {error_string, "cannot set any user limits: the user_limits feature flag is not enabled"} - end; -set_user_limits(Username, Definition, ActingUser) when is_map(Definition) -> - case rabbit_feature_flags:is_enabled(user_limits) of - true -> validate_parameters_and_update_limit(Username, Definition, ActingUser); - false -> {error_string, "cannot set any user limits: the user_limits feature flag is not enabled"} - end. - -validate_parameters_and_update_limit(Username, Term, ActingUser) -> - case flatten_errors(rabbit_parameter_validation:proplist( - <<"user-limits">>, user_limit_validation(), Term)) of - ok -> - update_user(Username, fun(User) -> - internal_user:update_limits(add, User, Term) - end), - notify_limit_set(Username, ActingUser, Term); - {errors, [{Reason, Arguments}]} -> - {error_string, rabbit_misc:format(Reason, Arguments)} - end. - -user_limit_validation() -> - [{<<"max-connections">>, fun rabbit_parameter_validation:integer/2, optional}, - {<<"max-channels">>, fun rabbit_parameter_validation:integer/2, optional}]. - -clear_user_limits(Username, <<"all">>, ActingUser) -> - update_user(Username, fun(User) -> - internal_user:clear_limits(User) - end), - notify_limit_clear(Username, ActingUser); -clear_user_limits(Username, LimitType, ActingUser) -> - update_user(Username, fun(User) -> - internal_user:update_limits(remove, User, LimitType) - end), - notify_limit_clear(Username, ActingUser). - -flatten_errors(L) -> - case [{F, A} || I <- lists:flatten([L]), {error, F, A} <- [I]] of - [] -> ok; - E -> {errors, E} - end. - -%%---------------------------------------------------------------------------- -%% Listing - --define(PERMS_INFO_KEYS, [configure, write, read]). --define(USER_INFO_KEYS, [user, tags]). - --spec user_info_keys() -> rabbit_types:info_keys(). - -user_info_keys() -> ?USER_INFO_KEYS. - --spec perms_info_keys() -> rabbit_types:info_keys(). - -perms_info_keys() -> [user, vhost | ?PERMS_INFO_KEYS]. - --spec vhost_perms_info_keys() -> rabbit_types:info_keys(). - -vhost_perms_info_keys() -> [user | ?PERMS_INFO_KEYS]. - --spec user_perms_info_keys() -> rabbit_types:info_keys(). - -user_perms_info_keys() -> [vhost | ?PERMS_INFO_KEYS]. - --spec user_vhost_perms_info_keys() -> rabbit_types:info_keys(). - -user_vhost_perms_info_keys() -> ?PERMS_INFO_KEYS. - -topic_perms_info_keys() -> [user, vhost, exchange, write, read]. -user_topic_perms_info_keys() -> [vhost, exchange, write, read]. -vhost_topic_perms_info_keys() -> [user, exchange, write, read]. -user_vhost_topic_perms_info_keys() -> [exchange, write, read]. - -all_users() -> mnesia:dirty_match_object(rabbit_user, internal_user:pattern_match_all()). - --spec list_users() -> [rabbit_types:infos()]. - -list_users() -> - [extract_internal_user_params(U) || - U <- all_users()]. - --spec list_users(reference(), pid()) -> 'ok'. - -list_users(Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map( - AggregatorPid, Ref, - fun(U) -> extract_internal_user_params(U) end, - all_users()). - --spec list_permissions() -> [rabbit_types:infos()]. - -list_permissions() -> - list_permissions(perms_info_keys(), match_user_vhost('_', '_')). - -list_permissions(Keys, QueryThunk) -> - [extract_user_permission_params(Keys, U) || - U <- rabbit_misc:execute_mnesia_transaction(QueryThunk)]. - -list_permissions(Keys, QueryThunk, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map( - AggregatorPid, Ref, fun(U) -> extract_user_permission_params(Keys, U) end, - rabbit_misc:execute_mnesia_transaction(QueryThunk)). - -filter_props(Keys, Props) -> [T || T = {K, _} <- Props, lists:member(K, Keys)]. - --spec list_user_permissions - (rabbit_types:username()) -> [rabbit_types:infos()]. - -list_user_permissions(Username) -> - list_permissions( - user_perms_info_keys(), - rabbit_misc:with_user(Username, match_user_vhost(Username, '_'))). - --spec list_user_permissions - (rabbit_types:username(), reference(), pid()) -> 'ok'. - -list_user_permissions(Username, Ref, AggregatorPid) -> - list_permissions( - user_perms_info_keys(), - rabbit_misc:with_user(Username, match_user_vhost(Username, '_')), - Ref, AggregatorPid). - --spec list_vhost_permissions - (rabbit_types:vhost()) -> [rabbit_types:infos()]. - -list_vhost_permissions(VHostPath) -> - list_permissions( - vhost_perms_info_keys(), - rabbit_vhost:with(VHostPath, match_user_vhost('_', VHostPath))). - --spec list_vhost_permissions - (rabbit_types:vhost(), reference(), pid()) -> 'ok'. - -list_vhost_permissions(VHostPath, Ref, AggregatorPid) -> - list_permissions( - vhost_perms_info_keys(), - rabbit_vhost:with(VHostPath, match_user_vhost('_', VHostPath)), - Ref, AggregatorPid). - --spec list_user_vhost_permissions - (rabbit_types:username(), rabbit_types:vhost()) -> [rabbit_types:infos()]. - -list_user_vhost_permissions(Username, VHostPath) -> - list_permissions( - user_vhost_perms_info_keys(), - rabbit_vhost:with_user_and_vhost( - Username, VHostPath, match_user_vhost(Username, VHostPath))). - -extract_user_permission_params(Keys, #user_permission{ - user_vhost = - #user_vhost{username = Username, - virtual_host = VHostPath}, - permission = #permission{ - configure = ConfigurePerm, - write = WritePerm, - read = ReadPerm}}) -> - filter_props(Keys, [{user, Username}, - {vhost, VHostPath}, - {configure, ConfigurePerm}, - {write, WritePerm}, - {read, ReadPerm}]). - -extract_internal_user_params(User) -> - [{user, internal_user:get_username(User)}, - {tags, internal_user:get_tags(User)}]. - -match_user_vhost(Username, VHostPath) -> - fun () -> mnesia:match_object( - rabbit_user_permission, - #user_permission{user_vhost = #user_vhost{ - username = Username, - virtual_host = VHostPath}, - permission = '_'}, - read) - end. - -list_topic_permissions() -> - list_topic_permissions(topic_perms_info_keys(), match_user_vhost_topic_permission('_', '_')). - -list_user_topic_permissions(Username) -> - list_topic_permissions(user_topic_perms_info_keys(), - rabbit_misc:with_user(Username, match_user_vhost_topic_permission(Username, '_'))). - -list_vhost_topic_permissions(VHost) -> - list_topic_permissions(vhost_topic_perms_info_keys(), - rabbit_vhost:with(VHost, match_user_vhost_topic_permission('_', VHost))). - -list_user_vhost_topic_permissions(Username, VHost) -> - list_topic_permissions(user_vhost_topic_perms_info_keys(), - rabbit_vhost:with_user_and_vhost(Username, VHost, match_user_vhost_topic_permission(Username, VHost))). - -list_topic_permissions(Keys, QueryThunk) -> - [extract_topic_permission_params(Keys, U) || - U <- rabbit_misc:execute_mnesia_transaction(QueryThunk)]. - -match_user_vhost_topic_permission(Username, VHostPath) -> - match_user_vhost_topic_permission(Username, VHostPath, '_'). - -match_user_vhost_topic_permission(Username, VHostPath, Exchange) -> - fun () -> mnesia:match_object( - rabbit_topic_permission, - #topic_permission{topic_permission_key = #topic_permission_key{ - user_vhost = #user_vhost{ - username = Username, - virtual_host = VHostPath}, - exchange = Exchange}, - permission = '_'}, - read) - end. - -extract_topic_permission_params(Keys, #topic_permission{ - topic_permission_key = #topic_permission_key{ - user_vhost = #user_vhost{username = Username, - virtual_host = VHostPath}, - exchange = Exchange}, - permission = #permission{ - write = WritePerm, - read = ReadPerm}}) -> - filter_props(Keys, [{user, Username}, - {vhost, VHostPath}, - {exchange, Exchange}, - {write, WritePerm}, - {read, ReadPerm}]). - -hashing_algorithm(User, Version) -> - case maps:get(hashing_algorithm, User, undefined) of - undefined -> - case Version of - %% 3.6.1 and later versions are supposed to have - %% the algorithm exported and thus not need a default - <<"3.6.0">> -> rabbit_password_hashing_sha256; - <<"3.5.", _/binary>> -> rabbit_password_hashing_md5; - <<"3.4.", _/binary>> -> rabbit_password_hashing_md5; - <<"3.3.", _/binary>> -> rabbit_password_hashing_md5; - <<"3.2.", _/binary>> -> rabbit_password_hashing_md5; - <<"3.1.", _/binary>> -> rabbit_password_hashing_md5; - <<"3.0.", _/binary>> -> rabbit_password_hashing_md5; - _ -> rabbit_password:hashing_mod() - end; - Alg -> rabbit_data_coercion:to_atom(Alg, utf8) - end. - -is_over_connection_limit(Username) -> - Fun = fun() -> - rabbit_connection_tracking:count_tracked_items_in({user, Username}) - end, - is_over_limit(Username, <<"max-connections">>, Fun). - -is_over_channel_limit(Username) -> - Fun = fun() -> - rabbit_channel_tracking:count_tracked_items_in({user, Username}) - end, - is_over_limit(Username, <<"max-channels">>, Fun). - -is_over_limit(Username, LimitType, Fun) -> - case get_user_limit(Username, LimitType) of - undefined -> false; - {ok, 0} -> {true, 0}; - {ok, Limit} -> - case Fun() >= Limit of - false -> false; - true -> {true, Limit} - end - end. - -get_user_limit(Username, LimitType) -> - case lookup_user(Username) of - {ok, User} -> - case rabbit_misc:pget(LimitType, internal_user:get_limits(User)) of - undefined -> undefined; - N when N < 0 -> undefined; - N when N >= 0 -> {ok, N} - end; - _ -> - undefined - end. - -get_user_limits() -> - [{internal_user:get_username(U), internal_user:get_limits(U)} || - U <- all_users(), - internal_user:get_limits(U) =/= #{}]. - -get_user_limits(Username) -> - case lookup_user(Username) of - {ok, User} -> internal_user:get_limits(User); - _ -> undefined - end. - -notify_limit_set(Username, ActingUser, Term) -> - rabbit_event:notify(user_limits_set, - [{name, <<"limits">>}, {user_who_performed_action, ActingUser}, - {username, Username} | maps:to_list(Term)]). - -notify_limit_clear(Username, ActingUser) -> - rabbit_event:notify(user_limits_cleared, - [{name, <<"limits">>}, {user_who_performed_action, ActingUser}, - {username, Username}]). diff --git a/src/rabbit_auth_mechanism_amqplain.erl b/src/rabbit_auth_mechanism_amqplain.erl deleted file mode 100644 index c81a337153..0000000000 --- a/src/rabbit_auth_mechanism_amqplain.erl +++ /dev/null @@ -1,54 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_auth_mechanism_amqplain). --include("rabbit.hrl"). - --behaviour(rabbit_auth_mechanism). - --export([description/0, should_offer/1, init/1, handle_response/2]). - --rabbit_boot_step({?MODULE, - [{description, "auth mechanism amqplain"}, - {mfa, {rabbit_registry, register, - [auth_mechanism, <<"AMQPLAIN">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -%% AMQPLAIN, as used by Qpid Python test suite. The 0-8 spec actually -%% defines this as PLAIN, but in 0-9 that definition is gone, instead -%% referring generically to "SASL security mechanism", i.e. the above. - -description() -> - [{description, <<"QPid AMQPLAIN mechanism">>}]. - -should_offer(_Sock) -> - true. - -init(_Sock) -> - []. - --define(IS_STRING_TYPE(Type), Type =:= longstr orelse Type =:= shortstr). - -handle_response(Response, _State) -> - LoginTable = rabbit_binary_parser:parse_table(Response), - case {lists:keysearch(<<"LOGIN">>, 1, LoginTable), - lists:keysearch(<<"PASSWORD">>, 1, LoginTable)} of - {{value, {_, UserType, User}}, - {value, {_, PassType, Pass}}} when ?IS_STRING_TYPE(UserType); - ?IS_STRING_TYPE(PassType) -> - rabbit_access_control:check_user_pass_login(User, Pass); - {{value, {_, _UserType, _User}}, - {value, {_, _PassType, _Pass}}} -> - {protocol_error, - "AMQPLAIN auth info ~w uses unsupported type for LOGIN or PASSWORD field", - [LoginTable]}; - _ -> - {protocol_error, - "AMQPLAIN auth info ~w is missing LOGIN or PASSWORD field", - [LoginTable]} - end. diff --git a/src/rabbit_auth_mechanism_cr_demo.erl b/src/rabbit_auth_mechanism_cr_demo.erl deleted file mode 100644 index 15439c461f..0000000000 --- a/src/rabbit_auth_mechanism_cr_demo.erl +++ /dev/null @@ -1,48 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_auth_mechanism_cr_demo). --include("rabbit.hrl"). - --behaviour(rabbit_auth_mechanism). - --export([description/0, should_offer/1, init/1, handle_response/2]). - --rabbit_boot_step({?MODULE, - [{description, "auth mechanism cr-demo"}, - {mfa, {rabbit_registry, register, - [auth_mechanism, <<"RABBIT-CR-DEMO">>, - ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - --record(state, {username = undefined}). - -%% Provides equivalent security to PLAIN but demos use of Connection.Secure(Ok) -%% START-OK: Username -%% SECURE: "Please tell me your password" -%% SECURE-OK: "My password is ~s", [Password] - -description() -> - [{description, <<"RabbitMQ Demo challenge-response authentication " - "mechanism">>}]. - -should_offer(_Sock) -> - true. - -init(_Sock) -> - #state{}. - -handle_response(Response, State = #state{username = undefined}) -> - {challenge, <<"Please tell me your password">>, - State#state{username = Response}}; - -handle_response(<<"My password is ", Password/binary>>, - #state{username = Username}) -> - rabbit_access_control:check_user_pass_login(Username, Password); -handle_response(Response, _State) -> - {protocol_error, "Invalid response '~s'", [Response]}. diff --git a/src/rabbit_auth_mechanism_plain.erl b/src/rabbit_auth_mechanism_plain.erl deleted file mode 100644 index d704c72400..0000000000 --- a/src/rabbit_auth_mechanism_plain.erl +++ /dev/null @@ -1,60 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_auth_mechanism_plain). --include("rabbit.hrl"). - --behaviour(rabbit_auth_mechanism). - --export([description/0, should_offer/1, init/1, handle_response/2]). - --rabbit_boot_step({?MODULE, - [{description, "auth mechanism plain"}, - {mfa, {rabbit_registry, register, - [auth_mechanism, <<"PLAIN">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -%% SASL PLAIN, as used by the Qpid Java client and our clients. Also, -%% apparently, by OpenAMQ. - -description() -> - [{description, <<"SASL PLAIN authentication mechanism">>}]. - -should_offer(_Sock) -> - true. - -init(_Sock) -> - []. - -handle_response(Response, _State) -> - case extract_user_pass(Response) of - {ok, User, Pass} -> - rabbit_access_control:check_user_pass_login(User, Pass); - error -> - {protocol_error, "response ~p invalid", [Response]} - end. - -extract_user_pass(Response) -> - case extract_elem(Response) of - {ok, User, Response1} -> case extract_elem(Response1) of - {ok, Pass, <<>>} -> {ok, User, Pass}; - _ -> error - end; - error -> error - end. - -extract_elem(<<0:8, Rest/binary>>) -> - Count = next_null_pos(Rest, 0), - <<Elem:Count/binary, Rest1/binary>> = Rest, - {ok, Elem, Rest1}; -extract_elem(_) -> - error. - -next_null_pos(<<>>, Count) -> Count; -next_null_pos(<<0:8, _Rest/binary>>, Count) -> Count; -next_null_pos(<<_:8, Rest/binary>>, Count) -> next_null_pos(Rest, Count + 1). diff --git a/src/rabbit_autoheal.erl b/src/rabbit_autoheal.erl deleted file mode 100644 index 6380d71895..0000000000 --- a/src/rabbit_autoheal.erl +++ /dev/null @@ -1,456 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_autoheal). - --export([init/0, enabled/0, maybe_start/1, rabbit_down/2, node_down/2, - handle_msg/3, process_down/2]). - -%% The named process we are running in. --define(SERVER, rabbit_node_monitor). - --define(MNESIA_STOPPED_PING_INTERNAL, 200). - --define(AUTOHEAL_STATE_AFTER_RESTART, rabbit_autoheal_state_after_restart). - -%%---------------------------------------------------------------------------- - -%% In order to autoheal we want to: -%% -%% * Find the winning partition -%% * Stop all nodes in other partitions -%% * Wait for them all to be stopped -%% * Start them again -%% -%% To keep things simple, we assume all nodes are up. We don't start -%% unless all nodes are up, and if a node goes down we abandon the -%% whole process. To further keep things simple we also defer the -%% decision as to the winning node to the "leader" - arbitrarily -%% selected as the first node in the cluster. -%% -%% To coordinate the restarting nodes we pick a special node from the -%% winning partition - the "winner". Restarting nodes then stop, and -%% wait for it to tell them it is safe to start again. The winner -%% determines that a node has stopped just by seeing if its rabbit app -%% stops - if a node stops for any other reason it just gets a message -%% it will ignore, and otherwise we carry on. -%% -%% Meanwhile, the leader may continue to receive new autoheal requests: -%% all of them are ignored. The winner notifies the leader when the -%% current autoheal process is finished (ie. when all losers stopped and -%% were asked to start again) or was aborted. When the leader receives -%% the notification or if it looses contact with the winner, it can -%% accept new autoheal requests. -%% -%% The winner and the leader are not necessarily the same node. -%% -%% The leader can be a loser and will restart in this case. It remembers -%% there is an autoheal in progress by temporarily saving the autoheal -%% state to the application environment. -%% -%% == Possible states == -%% -%% not_healing -%% - the default -%% -%% {winner_waiting, OutstandingStops, Notify} -%% - we are the winner and are waiting for all losing nodes to stop -%% before telling them they can restart -%% -%% {leader_waiting, Winner, Notify} -%% - we are the leader, and have already assigned the winner and losers. -%% We are waiting for a confirmation from the winner that the autoheal -%% process has ended. Meanwhile we can ignore autoheal requests. -%% Because we may be a loser too, this state is saved to the application -%% environment and restored on startup. -%% -%% restarting -%% - we are restarting. Of course the node monitor immediately dies -%% then so this state does not last long. We therefore send the -%% autoheal_safe_to_start message to the rabbit_outside_app_process -%% instead. -%% -%% == Message flow == -%% -%% 1. Any node (leader included) >> {request_start, node()} >> Leader -%% When Mnesia detects it is running partitioned or -%% when a remote node starts, rabbit_node_monitor calls -%% rabbit_autoheal:maybe_start/1. The message above is sent to the -%% leader so the leader can take a decision. -%% -%% 2. Leader >> {become_winner, Losers} >> Winner -%% The leader notifies the winner so the latter can proceed with -%% the autoheal. -%% -%% 3. Winner >> {winner_is, Winner} >> All losers -%% The winner notifies losers they must stop. -%% -%% 4. Winner >> autoheal_safe_to_start >> All losers -%% When either all losers stopped or the autoheal process was -%% aborted, the winner notifies losers they can start again. -%% -%% 5. Leader >> report_autoheal_status >> Winner -%% The leader asks the autoheal status to the winner. This only -%% happens when the leader is a loser too. If this is not the case, -%% this message is never sent. -%% -%% 6. Winner >> {autoheal_finished, Winner} >> Leader -%% The winner notifies the leader that the autoheal process was -%% either finished or aborted (ie. autoheal_safe_to_start was sent -%% to losers). - -%%---------------------------------------------------------------------------- - -init() -> - %% We check the application environment for a saved autoheal state - %% saved during a restart. If this node is a leader, it is used - %% to determine if it needs to ask the winner to report about the - %% autoheal progress. - State = case application:get_env(rabbit, ?AUTOHEAL_STATE_AFTER_RESTART) of - {ok, S} -> S; - undefined -> not_healing - end, - ok = application:unset_env(rabbit, ?AUTOHEAL_STATE_AFTER_RESTART), - case State of - {leader_waiting, Winner, _} -> - rabbit_log:info( - "Autoheal: in progress, requesting report from ~p~n", [Winner]), - send(Winner, report_autoheal_status); - _ -> - ok - end, - State. - -maybe_start(not_healing) -> - case enabled() of - true -> Leader = leader(), - send(Leader, {request_start, node()}), - rabbit_log:info("Autoheal request sent to ~p~n", [Leader]), - not_healing; - false -> not_healing - end; -maybe_start(State) -> - State. - -enabled() -> - case application:get_env(rabbit, cluster_partition_handling) of - {ok, autoheal} -> true; - {ok, {pause_if_all_down, _, autoheal}} -> true; - _ -> false - end. - -leader() -> - [Leader | _] = lists:usort(rabbit_mnesia:cluster_nodes(all)), - Leader. - -%% This is the winner receiving its last notification that a node has -%% stopped - all nodes can now start again -rabbit_down(Node, {winner_waiting, [Node], Notify}) -> - rabbit_log:info("Autoheal: final node has stopped, starting...~n",[]), - winner_finish(Notify); - -rabbit_down(Node, {winner_waiting, WaitFor, Notify}) -> - {winner_waiting, WaitFor -- [Node], Notify}; - -rabbit_down(Winner, {leader_waiting, Winner, Losers}) -> - abort([Winner], Losers); - -rabbit_down(_Node, State) -> - %% Ignore. Either: - %% o we already cancelled the autoheal process; - %% o we are still waiting the winner's report. - State. - -node_down(_Node, not_healing) -> - not_healing; - -node_down(Node, {winner_waiting, _, Notify}) -> - abort([Node], Notify); - -node_down(Node, {leader_waiting, Node, _Notify}) -> - %% The winner went down, we don't know what to do so we simply abort. - rabbit_log:info("Autoheal: aborting - winner ~p went down~n", [Node]), - not_healing; - -node_down(Node, {leader_waiting, _, _} = St) -> - %% If it is a partial partition, the winner might continue with the - %% healing process. If it is a full partition, the winner will also - %% see it and abort. Let's wait for it. - rabbit_log:info("Autoheal: ~p went down, waiting for winner decision ~n", [Node]), - St; - -node_down(Node, _State) -> - rabbit_log:info("Autoheal: aborting - ~p went down~n", [Node]), - not_healing. - -%% If the process that has to restart the node crashes for an unexpected reason, -%% we go back to a not healing state so the node is able to recover. -process_down({'EXIT', Pid, Reason}, {restarting, Pid}) when Reason =/= normal -> - rabbit_log:info("Autoheal: aborting - the process responsible for restarting the " - "node terminated with reason: ~p~n", [Reason]), - not_healing; - -process_down(_, State) -> - State. - -%% By receiving this message we become the leader -%% TODO should we try to debounce this? -handle_msg({request_start, Node}, - not_healing, Partitions) -> - rabbit_log:info("Autoheal request received from ~p~n", [Node]), - case check_other_nodes(Partitions) of - {error, E} -> - rabbit_log:info("Autoheal request denied: ~s~n", [fmt_error(E)]), - not_healing; - {ok, AllPartitions} -> - {Winner, Losers} = make_decision(AllPartitions), - rabbit_log:info("Autoheal decision~n" - " * Partitions: ~p~n" - " * Winner: ~p~n" - " * Losers: ~p~n", - [AllPartitions, Winner, Losers]), - case node() =:= Winner of - true -> handle_msg({become_winner, Losers}, - not_healing, Partitions); - false -> send(Winner, {become_winner, Losers}), - {leader_waiting, Winner, Losers} - end - end; - -handle_msg({request_start, Node}, - State, _Partitions) -> - rabbit_log:info("Autoheal request received from ~p when healing; " - "ignoring~n", [Node]), - State; - -handle_msg({become_winner, Losers}, - not_healing, _Partitions) -> - rabbit_log:info("Autoheal: I am the winner, waiting for ~p to stop~n", - [Losers]), - stop_partition(Losers); - -handle_msg({become_winner, Losers}, - {winner_waiting, _, Losers}, _Partitions) -> - %% The leader has aborted the healing, might have seen us down but - %% we didn't see the same. Let's try again as it is the same partition. - rabbit_log:info("Autoheal: I am the winner and received a duplicated " - "request, waiting again for ~p to stop~n", [Losers]), - stop_partition(Losers); - -handle_msg({become_winner, _}, - {winner_waiting, _, Losers}, _Partitions) -> - %% Something has happened to the leader, it might have seen us down but we - %% are still alive. Partitions have changed, cannot continue. - rabbit_log:info("Autoheal: I am the winner and received another healing " - "request, partitions have changed to ~p. Aborting ~n", [Losers]), - winner_finish(Losers), - not_healing; - -handle_msg({winner_is, Winner}, State = not_healing, - _Partitions) -> - %% This node is a loser, nothing else. - Pid = restart_loser(State, Winner), - {restarting, Pid}; -handle_msg({winner_is, Winner}, State = {leader_waiting, Winner, _}, - _Partitions) -> - %% This node is the leader and a loser at the same time. - Pid = restart_loser(State, Winner), - {restarting, Pid}; - -handle_msg(Request, {restarting, Pid} = St, _Partitions) -> - %% ignore, we can contribute no further - rabbit_log:info("Autoheal: Received the request ~p while waiting for ~p " - "to restart the node. Ignoring it ~n", [Request, Pid]), - St; - -handle_msg(report_autoheal_status, not_healing, _Partitions) -> - %% The leader is asking about the autoheal status to us (the - %% winner). This happens when the leader is a loser and it just - %% restarted. We are in the "not_healing" state, so the previous - %% autoheal process ended: let's tell this to the leader. - send(leader(), {autoheal_finished, node()}), - not_healing; - -handle_msg(report_autoheal_status, State, _Partitions) -> - %% Like above, the leader is asking about the autoheal status. We - %% are not finished with it. There is no need to send anything yet - %% to the leader: we will send the notification when it is over. - State; - -handle_msg({autoheal_finished, Winner}, - {leader_waiting, Winner, _}, _Partitions) -> - %% The winner is finished with the autoheal process and notified us - %% (the leader). We can transition to the "not_healing" state and - %% accept new requests. - rabbit_log:info("Autoheal finished according to winner ~p~n", [Winner]), - not_healing; - -handle_msg({autoheal_finished, Winner}, not_healing, _Partitions) - when Winner =:= node() -> - %% We are the leader and the winner. The state already transitioned - %% to "not_healing" at the end of the autoheal process. - rabbit_log:info("Autoheal finished according to winner ~p~n", [node()]), - not_healing; - -handle_msg({autoheal_finished, Winner}, not_healing, _Partitions) -> - %% We might have seen the winner down during a partial partition and - %% transitioned to not_healing. However, the winner was still able - %% to finish. Let it pass. - rabbit_log:info("Autoheal finished according to winner ~p." - " Unexpected, I might have previously seen the winner down~n", [Winner]), - not_healing. - -%%---------------------------------------------------------------------------- - -send(Node, Msg) -> {?SERVER, Node} ! {autoheal_msg, Msg}. - -abort(Down, Notify) -> - rabbit_log:info("Autoheal: aborting - ~p down~n", [Down]), - %% Make sure any nodes waiting for us start - it won't necessarily - %% heal the partition but at least they won't get stuck. - %% If we are executing this, we are not stopping. Thus, don't wait - %% for ourselves! - winner_finish(Notify -- [node()]). - -winner_finish(Notify) -> - %% There is a race in Mnesia causing a starting loser to hang - %% forever if another loser stops at the same time: the starting - %% node connects to the other node, negotiates the protocol and - %% attempts to acquire a write lock on the schema on the other node. - %% If the other node stops between the protocol negotiation and lock - %% request, the starting node never gets an answer to its lock - %% request. - %% - %% To work around the problem, we make sure Mnesia is stopped on all - %% losing nodes before sending the "autoheal_safe_to_start" signal. - wait_for_mnesia_shutdown(Notify), - [{rabbit_outside_app_process, N} ! autoheal_safe_to_start || N <- Notify], - send(leader(), {autoheal_finished, node()}), - not_healing. - -%% This improves the previous implementation, but could still potentially enter an infinity -%% loop. If it also possible that for when it finishes some of the nodes have been -%% manually restarted, but we can't do much more (apart from stop them again). So let it -%% continue and notify all the losers to restart. -wait_for_mnesia_shutdown(AllNodes) -> - Monitors = lists:foldl(fun(Node, Monitors0) -> - pmon:monitor({mnesia_sup, Node}, Monitors0) - end, pmon:new(), AllNodes), - wait_for_supervisors(Monitors). - -wait_for_supervisors(Monitors) -> - case pmon:is_empty(Monitors) of - true -> - ok; - false -> - receive - {'DOWN', _MRef, process, {mnesia_sup, _} = I, _Reason} -> - wait_for_supervisors(pmon:erase(I, Monitors)) - after - 60000 -> - AliveLosers = [Node || {_, Node} <- pmon:monitored(Monitors)], - rabbit_log:info("Autoheal: mnesia in nodes ~p is still up, sending " - "winner notification again to these ~n", [AliveLosers]), - [send(L, {winner_is, node()}) || L <- AliveLosers], - wait_for_mnesia_shutdown(AliveLosers) - end - end. - -restart_loser(State, Winner) -> - rabbit_log:warning( - "Autoheal: we were selected to restart; winner is ~p~n", [Winner]), - NextStateTimeout = application:get_env(rabbit, autoheal_state_transition_timeout, 60000), - rabbit_node_monitor:run_outside_applications( - fun () -> - MRef = erlang:monitor(process, {?SERVER, Winner}), - rabbit:stop(), - NextState = receive - {'DOWN', MRef, process, {?SERVER, Winner}, _Reason} -> - not_healing; - autoheal_safe_to_start -> - State - after NextStateTimeout -> - rabbit_log:warning( - "Autoheal: timed out waiting for a safe-to-start message from the winner (~p); will retry", - [Winner]), - not_healing - end, - erlang:demonitor(MRef, [flush]), - %% During the restart, the autoheal state is lost so we - %% store it in the application environment temporarily so - %% init/0 can pick it up. - %% - %% This is useful to the leader which is a loser at the - %% same time: because the leader is restarting, there - %% is a great chance it misses the "autoheal finished!" - %% notification from the winner. Thanks to the saved - %% state, it knows it needs to ask the winner if the - %% autoheal process is finished or not. - application:set_env(rabbit, - ?AUTOHEAL_STATE_AFTER_RESTART, NextState), - rabbit:start() - end, true). - -make_decision(AllPartitions) -> - Sorted = lists:sort([{partition_value(P), P} || P <- AllPartitions]), - [[Winner | _] | Rest] = lists:reverse([P || {_, P} <- Sorted]), - {Winner, lists:append(Rest)}. - -partition_value(Partition) -> - Connections = [Res || Node <- Partition, - Res <- [rpc:call(Node, rabbit_networking, - connections_local, [])], - is_list(Res)], - {length(lists:append(Connections)), length(Partition)}. - -%% We have our local understanding of what partitions exist; but we -%% only know which nodes we have been partitioned from, not which -%% nodes are partitioned from each other. -check_other_nodes(LocalPartitions) -> - Nodes = rabbit_mnesia:cluster_nodes(all), - {Results, Bad} = rabbit_node_monitor:status(Nodes -- [node()]), - RemotePartitions = [{Node, proplists:get_value(partitions, Res)} - || {Node, Res} <- Results], - RemoteDown = [{Node, Down} - || {Node, Res} <- Results, - Down <- [Nodes -- proplists:get_value(nodes, Res)], - Down =/= []], - case {Bad, RemoteDown} of - {[], []} -> Partitions = [{node(), LocalPartitions} | RemotePartitions], - {ok, all_partitions(Partitions, [Nodes])}; - {[], _} -> {error, {remote_down, RemoteDown}}; - {_, _} -> {error, {nodes_down, Bad}} - end. - -all_partitions([], Partitions) -> - Partitions; -all_partitions([{Node, CantSee} | Rest], Partitions) -> - {[Containing], Others} = - lists:partition(fun (Part) -> lists:member(Node, Part) end, Partitions), - A = Containing -- CantSee, - B = Containing -- A, - Partitions1 = case {A, B} of - {[], _} -> Partitions; - {_, []} -> Partitions; - _ -> [A, B | Others] - end, - all_partitions(Rest, Partitions1). - -fmt_error({remote_down, RemoteDown}) -> - rabbit_misc:format("Remote nodes disconnected:~n ~p", [RemoteDown]); -fmt_error({nodes_down, NodesDown}) -> - rabbit_misc:format("Local nodes down: ~p", [NodesDown]). - -stop_partition(Losers) -> - %% The leader said everything was ready - do we agree? If not then - %% give up. - Down = Losers -- rabbit_node_monitor:alive_rabbit_nodes(Losers), - case Down of - [] -> [send(L, {winner_is, node()}) || L <- Losers], - {winner_waiting, Losers, Losers}; - _ -> abort(Down, Losers) - end. diff --git a/src/rabbit_backing_queue.erl b/src/rabbit_backing_queue.erl deleted file mode 100644 index 4d709e14d0..0000000000 --- a/src/rabbit_backing_queue.erl +++ /dev/null @@ -1,264 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_backing_queue). - --export([info_keys/0]). - --define(INFO_KEYS, [messages_ram, messages_ready_ram, - messages_unacknowledged_ram, messages_persistent, - message_bytes, message_bytes_ready, - message_bytes_unacknowledged, message_bytes_ram, - message_bytes_persistent, head_message_timestamp, - disk_reads, disk_writes, backing_queue_status, - messages_paged_out, message_bytes_paged_out]). - -%% We can't specify a per-queue ack/state with callback signatures --type ack() :: any(). --type state() :: any(). - --type flow() :: 'flow' | 'noflow'. --type msg_ids() :: [rabbit_types:msg_id()]. --type publish() :: {rabbit_types:basic_message(), - rabbit_types:message_properties(), boolean()}. --type delivered_publish() :: {rabbit_types:basic_message(), - rabbit_types:message_properties()}. --type fetch_result(Ack) :: - ('empty' | {rabbit_types:basic_message(), boolean(), Ack}). --type drop_result(Ack) :: - ('empty' | {rabbit_types:msg_id(), Ack}). --type recovery_terms() :: [term()] | 'non_clean_shutdown'. --type recovery_info() :: 'new' | recovery_terms(). --type purged_msg_count() :: non_neg_integer(). --type async_callback() :: - fun ((atom(), fun ((atom(), state()) -> state())) -> 'ok'). --type duration() :: ('undefined' | 'infinity' | number()). - --type msg_fun(A) :: fun ((rabbit_types:basic_message(), ack(), A) -> A). --type msg_pred() :: fun ((rabbit_types:message_properties()) -> boolean()). - --type queue_mode() :: atom(). - -%% Called on startup with a vhost and a list of durable queue names on this vhost. -%% The queues aren't being started at this point, but this call allows the -%% backing queue to perform any checking necessary for the consistency -%% of those queues, or initialise any other shared resources. -%% -%% The list of queue recovery terms returned as {ok, Terms} must be given -%% in the same order as the list of queue names supplied. --callback start(rabbit_types:vhost(), [rabbit_amqqueue:name()]) -> rabbit_types:ok(recovery_terms()). - -%% Called to tear down any state/resources for vhost. NB: Implementations should -%% not depend on this function being called on shutdown and instead -%% should hook into the rabbit supervision hierarchy. --callback stop(rabbit_types:vhost()) -> 'ok'. - -%% Initialise the backing queue and its state. -%% -%% Takes -%% 1. the amqqueue record -%% 2. a term indicating whether the queue is an existing queue that -%% should be recovered or not. When 'new' is given, no recovery is -%% taking place, otherwise a list of recovery terms is given, or -%% the atom 'non_clean_shutdown' if no recovery terms are available. -%% 3. an asynchronous callback which accepts a function of type -%% backing-queue-state to backing-queue-state. This callback -%% function can be safely invoked from any process, which makes it -%% useful for passing messages back into the backing queue, -%% especially as the backing queue does not have control of its own -%% mailbox. --callback init(amqqueue:amqqueue(), recovery_info(), - async_callback()) -> state(). - -%% Called on queue shutdown when queue isn't being deleted. --callback terminate(any(), state()) -> state(). - -%% Called when the queue is terminating and needs to delete all its -%% content. --callback delete_and_terminate(any(), state()) -> state(). - -%% Called to clean up after a crashed queue. In this case we don't -%% have a process and thus a state(), we are just removing on-disk data. --callback delete_crashed(amqqueue:amqqueue()) -> 'ok'. - -%% Remove all 'fetchable' messages from the queue, i.e. all messages -%% except those that have been fetched already and are pending acks. --callback purge(state()) -> {purged_msg_count(), state()}. - -%% Remove all messages in the queue which have been fetched and are -%% pending acks. --callback purge_acks(state()) -> state(). - -%% Publish a message. --callback publish(rabbit_types:basic_message(), - rabbit_types:message_properties(), boolean(), pid(), flow(), - state()) -> state(). - -%% Like publish/6 but for batches of publishes. --callback batch_publish([publish()], pid(), flow(), state()) -> state(). - -%% Called for messages which have already been passed straight -%% out to a client. The queue will be empty for these calls -%% (i.e. saves the round trip through the backing queue). --callback publish_delivered(rabbit_types:basic_message(), - rabbit_types:message_properties(), pid(), flow(), - state()) - -> {ack(), state()}. - -%% Like publish_delivered/5 but for batches of publishes. --callback batch_publish_delivered([delivered_publish()], pid(), flow(), - state()) - -> {[ack()], state()}. - -%% Called to inform the BQ about messages which have reached the -%% queue, but are not going to be further passed to BQ. --callback discard(rabbit_types:msg_id(), pid(), flow(), state()) -> state(). - -%% Return ids of messages which have been confirmed since the last -%% invocation of this function (or initialisation). -%% -%% Message ids should only appear in the result of drain_confirmed -%% under the following circumstances: -%% -%% 1. The message appears in a call to publish_delivered/4 and the -%% first argument (ack_required) is false; or -%% 2. The message is fetched from the queue with fetch/2 and the first -%% argument (ack_required) is false; or -%% 3. The message is acked (ack/2 is called for the message); or -%% 4. The message is fully fsync'd to disk in such a way that the -%% recovery of the message is guaranteed in the event of a crash of -%% this rabbit node (excluding hardware failure). -%% -%% In addition to the above conditions, a message id may only appear -%% in the result of drain_confirmed if -%% #message_properties.needs_confirming = true when the msg was -%% published (through whichever means) to the backing queue. -%% -%% It is legal for the same message id to appear in the results of -%% multiple calls to drain_confirmed, which means that the backing -%% queue is not required to keep track of which messages it has -%% already confirmed. The confirm will be issued to the publisher the -%% first time the message id appears in the result of -%% drain_confirmed. All subsequent appearances of that message id will -%% be ignored. --callback drain_confirmed(state()) -> {msg_ids(), state()}. - -%% Drop messages from the head of the queue while the supplied -%% predicate on message properties returns true. Returns the first -%% message properties for which the predicate returned false, or -%% 'undefined' if the whole backing queue was traversed w/o the -%% predicate ever returning false. --callback dropwhile(msg_pred(), state()) - -> {rabbit_types:message_properties() | undefined, state()}. - -%% Like dropwhile, except messages are fetched in "require -%% acknowledgement" mode and are passed, together with their ack tag, -%% to the supplied function. The function is also fed an -%% accumulator. The result of fetchwhile is as for dropwhile plus the -%% accumulator. --callback fetchwhile(msg_pred(), msg_fun(A), A, state()) - -> {rabbit_types:message_properties() | undefined, - A, state()}. - -%% Produce the next message. --callback fetch(true, state()) -> {fetch_result(ack()), state()}; - (false, state()) -> {fetch_result(undefined), state()}. - -%% Remove the next message. --callback drop(true, state()) -> {drop_result(ack()), state()}; - (false, state()) -> {drop_result(undefined), state()}. - -%% Acktags supplied are for messages which can now be forgotten -%% about. Must return 1 msg_id per Ack, in the same order as Acks. --callback ack([ack()], state()) -> {msg_ids(), state()}. - -%% Reinsert messages into the queue which have already been delivered -%% and were pending acknowledgement. --callback requeue([ack()], state()) -> {msg_ids(), state()}. - -%% Fold over messages by ack tag. The supplied function is called with -%% each message, its ack tag, and an accumulator. --callback ackfold(msg_fun(A), A, state(), [ack()]) -> {A, state()}. - -%% Fold over all the messages in a queue and return the accumulated -%% results, leaving the queue undisturbed. --callback fold(fun((rabbit_types:basic_message(), - rabbit_types:message_properties(), - boolean(), A) -> {('stop' | 'cont'), A}), - A, state()) -> {A, state()}. - -%% How long is my queue? --callback len(state()) -> non_neg_integer(). - -%% Is my queue empty? --callback is_empty(state()) -> boolean(). - -%% What's the queue depth, where depth = length + number of pending acks --callback depth(state()) -> non_neg_integer(). - -%% For the next three functions, the assumption is that you're -%% monitoring something like the ingress and egress rates of the -%% queue. The RAM duration is thus the length of time represented by -%% the messages held in RAM given the current rates. If you want to -%% ignore all of this stuff, then do so, and return 0 in -%% ram_duration/1. - -%% The target is to have no more messages in RAM than indicated by the -%% duration and the current queue rates. --callback set_ram_duration_target(duration(), state()) -> state(). - -%% Optionally recalculate the duration internally (likely to be just -%% update your internal rates), and report how many seconds the -%% messages in RAM represent given the current rates of the queue. --callback ram_duration(state()) -> {duration(), state()}. - -%% Should 'timeout' be called as soon as the queue process can manage -%% (either on an empty mailbox, or when a timer fires)? --callback needs_timeout(state()) -> 'false' | 'timed' | 'idle'. - -%% Called (eventually) after needs_timeout returns 'idle' or 'timed'. -%% Note this may be called more than once for each 'idle' or 'timed' -%% returned from needs_timeout --callback timeout(state()) -> state(). - -%% Called immediately before the queue hibernates. --callback handle_pre_hibernate(state()) -> state(). - -%% Called when more credit has become available for credit_flow. --callback resume(state()) -> state(). - -%% Used to help prioritisation in rabbit_amqqueue_process. The rate of -%% inbound messages and outbound messages at the moment. --callback msg_rates(state()) -> {float(), float()}. - --callback info(atom(), state()) -> any(). - -%% Passed a function to be invoked with the relevant backing queue's -%% state. Useful for when the backing queue or other components need -%% to pass functions into the backing queue. --callback invoke(atom(), fun ((atom(), A) -> A), state()) -> state(). - -%% Called prior to a publish or publish_delivered call. Allows the BQ -%% to signal that it's already seen this message, (e.g. it was published -%% or discarded previously) specifying whether to drop the message or reject it. --callback is_duplicate(rabbit_types:basic_message(), state()) - -> {{true, drop} | {true, reject} | boolean(), state()}. - --callback set_queue_mode(queue_mode(), state()) -> state(). - --callback zip_msgs_and_acks([delivered_publish()], - [ack()], Acc, state()) - -> Acc. - -%% Called when rabbit_amqqueue_process receives a message via -%% handle_info and it should be processed by the backing -%% queue --callback handle_info(term(), state()) -> state(). - --spec info_keys() -> rabbit_types:info_keys(). - -info_keys() -> ?INFO_KEYS. diff --git a/src/rabbit_basic.erl b/src/rabbit_basic.erl deleted file mode 100644 index cdc9e082e4..0000000000 --- a/src/rabbit_basic.erl +++ /dev/null @@ -1,354 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_basic). --include("rabbit.hrl"). --include("rabbit_framing.hrl"). - --export([publish/4, publish/5, publish/1, - message/3, message/4, properties/1, prepend_table_header/3, - extract_headers/1, extract_timestamp/1, map_headers/2, delivery/4, - header_routes/1, parse_expiration/1, header/2, header/3]). --export([build_content/2, from_content/1, msg_size/1, - maybe_gc_large_msg/1, maybe_gc_large_msg/2]). --export([add_header/4, - peek_fmt_message/1]). - -%%---------------------------------------------------------------------------- - --type properties_input() :: - rabbit_framing:amqp_property_record() | [{atom(), any()}]. --type publish_result() :: - ok | rabbit_types:error('not_found'). --type header() :: any(). --type headers() :: rabbit_framing:amqp_table() | 'undefined'. - --type exchange_input() :: rabbit_types:exchange() | rabbit_exchange:name(). --type body_input() :: binary() | [binary()]. - -%%---------------------------------------------------------------------------- - -%% Convenience function, for avoiding round-trips in calls across the -%% erlang distributed network. - --spec publish - (exchange_input(), rabbit_router:routing_key(), properties_input(), - body_input()) -> - publish_result(). - -publish(Exchange, RoutingKeyBin, Properties, Body) -> - publish(Exchange, RoutingKeyBin, false, Properties, Body). - -%% Convenience function, for avoiding round-trips in calls across the -%% erlang distributed network. - --spec publish - (exchange_input(), rabbit_router:routing_key(), boolean(), - properties_input(), body_input()) -> - publish_result(). - -publish(X = #exchange{name = XName}, RKey, Mandatory, Props, Body) -> - Message = message(XName, RKey, properties(Props), Body), - publish(X, delivery(Mandatory, false, Message, undefined)); -publish(XName, RKey, Mandatory, Props, Body) -> - Message = message(XName, RKey, properties(Props), Body), - publish(delivery(Mandatory, false, Message, undefined)). - --spec publish(rabbit_types:delivery()) -> publish_result(). - -publish(Delivery = #delivery{ - message = #basic_message{exchange_name = XName}}) -> - case rabbit_exchange:lookup(XName) of - {ok, X} -> publish(X, Delivery); - Err -> Err - end. - -publish(X, Delivery) -> - Qs = rabbit_amqqueue:lookup(rabbit_exchange:route(X, Delivery)), - _ = rabbit_queue_type:deliver(Qs, Delivery, stateless), - ok. - --spec delivery - (boolean(), boolean(), rabbit_types:message(), undefined | integer()) -> - rabbit_types:delivery(). - -delivery(Mandatory, Confirm, Message, MsgSeqNo) -> - #delivery{mandatory = Mandatory, confirm = Confirm, sender = self(), - message = Message, msg_seq_no = MsgSeqNo, flow = noflow}. - --spec build_content - (rabbit_framing:amqp_property_record(), binary() | [binary()]) -> - rabbit_types:content(). - -build_content(Properties, BodyBin) when is_binary(BodyBin) -> - build_content(Properties, [BodyBin]); - -build_content(Properties, PFR) -> - %% basic.publish hasn't changed so we can just hard-code amqp_0_9_1 - {ClassId, _MethodId} = - rabbit_framing_amqp_0_9_1:method_id('basic.publish'), - #content{class_id = ClassId, - properties = Properties, - properties_bin = none, - protocol = none, - payload_fragments_rev = PFR}. - --spec from_content - (rabbit_types:content()) -> - {rabbit_framing:amqp_property_record(), binary()}. - -from_content(Content) -> - #content{class_id = ClassId, - properties = Props, - payload_fragments_rev = FragmentsRev} = - rabbit_binary_parser:ensure_content_decoded(Content), - %% basic.publish hasn't changed so we can just hard-code amqp_0_9_1 - {ClassId, _MethodId} = - rabbit_framing_amqp_0_9_1:method_id('basic.publish'), - {Props, list_to_binary(lists:reverse(FragmentsRev))}. - -%% This breaks the spec rule forbidding message modification -strip_header(#content{properties = #'P_basic'{headers = undefined}} - = DecodedContent, _Key) -> - DecodedContent; -strip_header(#content{properties = Props = #'P_basic'{headers = Headers}} - = DecodedContent, Key) -> - case lists:keysearch(Key, 1, Headers) of - false -> DecodedContent; - {value, Found} -> Headers0 = lists:delete(Found, Headers), - rabbit_binary_generator:clear_encoded_content( - DecodedContent#content{ - properties = Props#'P_basic'{ - headers = Headers0}}) - end. - --spec message - (rabbit_exchange:name(), rabbit_router:routing_key(), - rabbit_types:decoded_content()) -> - rabbit_types:ok_or_error2(rabbit_types:message(), any()). - -message(XName, RoutingKey, #content{properties = Props} = DecodedContent) -> - try - {ok, #basic_message{ - exchange_name = XName, - content = strip_header(DecodedContent, ?DELETED_HEADER), - id = rabbit_guid:gen(), - is_persistent = is_message_persistent(DecodedContent), - routing_keys = [RoutingKey | - header_routes(Props#'P_basic'.headers)]}} - catch - {error, _Reason} = Error -> Error - end. - --spec message - (rabbit_exchange:name(), rabbit_router:routing_key(), properties_input(), - binary()) -> - rabbit_types:message(). - -message(XName, RoutingKey, RawProperties, Body) -> - Properties = properties(RawProperties), - Content = build_content(Properties, Body), - {ok, Msg} = message(XName, RoutingKey, Content), - Msg. - --spec properties - (properties_input()) -> rabbit_framing:amqp_property_record(). - -properties(P = #'P_basic'{}) -> - P; -properties(P) when is_list(P) -> - %% Yes, this is O(length(P) * record_info(size, 'P_basic') / 2), - %% i.e. slow. Use the definition of 'P_basic' directly if - %% possible! - lists:foldl(fun ({Key, Value}, Acc) -> - case indexof(record_info(fields, 'P_basic'), Key) of - 0 -> throw({unknown_basic_property, Key}); - N -> setelement(N + 1, Acc, Value) - end - end, #'P_basic'{}, P). - --spec prepend_table_header - (binary(), rabbit_framing:amqp_table(), headers()) -> headers(). - -prepend_table_header(Name, Info, undefined) -> - prepend_table_header(Name, Info, []); -prepend_table_header(Name, Info, Headers) -> - case rabbit_misc:table_lookup(Headers, Name) of - {array, Existing} -> - prepend_table(Name, Info, Existing, Headers); - undefined -> - prepend_table(Name, Info, [], Headers); - Other -> - Headers2 = prepend_table(Name, Info, [], Headers), - set_invalid_header(Name, Other, Headers2) - end. - -prepend_table(Name, Info, Prior, Headers) -> - rabbit_misc:set_table_value(Headers, Name, array, [{table, Info} | Prior]). - -set_invalid_header(Name, {_, _}=Value, Headers) when is_list(Headers) -> - case rabbit_misc:table_lookup(Headers, ?INVALID_HEADERS_KEY) of - undefined -> - set_invalid([{Name, array, [Value]}], Headers); - {table, ExistingHdr} -> - update_invalid(Name, Value, ExistingHdr, Headers); - Other -> - %% somehow the x-invalid-headers header is corrupt - Invalid = [{?INVALID_HEADERS_KEY, array, [Other]}], - set_invalid_header(Name, Value, set_invalid(Invalid, Headers)) - end. - -set_invalid(NewHdr, Headers) -> - rabbit_misc:set_table_value(Headers, ?INVALID_HEADERS_KEY, table, NewHdr). - -update_invalid(Name, Value, ExistingHdr, Header) -> - Values = case rabbit_misc:table_lookup(ExistingHdr, Name) of - undefined -> [Value]; - {array, Prior} -> [Value | Prior] - end, - NewHdr = rabbit_misc:set_table_value(ExistingHdr, Name, array, Values), - set_invalid(NewHdr, Header). - --spec header(header(), headers()) -> 'undefined' | any(). - -header(_Header, undefined) -> - undefined; -header(_Header, []) -> - undefined; -header(Header, Headers) -> - header(Header, Headers, undefined). - --spec header(header(), headers(), any()) -> 'undefined' | any(). - -header(Header, Headers, Default) -> - case lists:keysearch(Header, 1, Headers) of - false -> Default; - {value, Val} -> Val - end. - --spec extract_headers(rabbit_types:content()) -> headers(). - -extract_headers(Content) -> - #content{properties = #'P_basic'{headers = Headers}} = - rabbit_binary_parser:ensure_content_decoded(Content), - Headers. - -extract_timestamp(Content) -> - #content{properties = #'P_basic'{timestamp = Timestamp}} = - rabbit_binary_parser:ensure_content_decoded(Content), - Timestamp. - --spec map_headers - (fun((headers()) -> headers()), rabbit_types:content()) -> - rabbit_types:content(). - -map_headers(F, Content) -> - Content1 = rabbit_binary_parser:ensure_content_decoded(Content), - #content{properties = #'P_basic'{headers = Headers} = Props} = Content1, - Headers1 = F(Headers), - rabbit_binary_generator:clear_encoded_content( - Content1#content{properties = Props#'P_basic'{headers = Headers1}}). - -indexof(L, Element) -> indexof(L, Element, 1). - -indexof([], _Element, _N) -> 0; -indexof([Element | _Rest], Element, N) -> N; -indexof([_ | Rest], Element, N) -> indexof(Rest, Element, N + 1). - -is_message_persistent(#content{properties = #'P_basic'{ - delivery_mode = Mode}}) -> - case Mode of - 1 -> false; - 2 -> true; - undefined -> false; - Other -> throw({error, {delivery_mode_unknown, Other}}) - end. - -%% Extract CC routes from headers - --spec header_routes(undefined | rabbit_framing:amqp_table()) -> [string()]. - -header_routes(undefined) -> - []; -header_routes(HeadersTable) -> - lists:append( - [case rabbit_misc:table_lookup(HeadersTable, HeaderKey) of - {array, Routes} -> [Route || {longstr, Route} <- Routes]; - undefined -> []; - {Type, _Val} -> throw({error, {unacceptable_type_in_header, - binary_to_list(HeaderKey), Type}}) - end || HeaderKey <- ?ROUTING_HEADERS]). - --spec parse_expiration - (rabbit_framing:amqp_property_record()) -> - rabbit_types:ok_or_error2('undefined' | non_neg_integer(), any()). - -parse_expiration(#'P_basic'{expiration = undefined}) -> - {ok, undefined}; -parse_expiration(#'P_basic'{expiration = Expiration}) -> - case string:to_integer(binary_to_list(Expiration)) of - {error, no_integer} = E -> - E; - {N, ""} -> - case rabbit_misc:check_expiry(N) of - ok -> {ok, N}; - E = {error, _} -> E - end; - {_, S} -> - {error, {leftover_string, S}} - end. - -maybe_gc_large_msg(Content) -> - rabbit_writer:maybe_gc_large_msg(Content). - -maybe_gc_large_msg(Content, undefined) -> - rabbit_writer:msg_size(Content); -maybe_gc_large_msg(Content, GCThreshold) -> - rabbit_writer:maybe_gc_large_msg(Content, GCThreshold). - -msg_size(Content) -> - rabbit_writer:msg_size(Content). - -add_header(Name, Type, Value, #basic_message{content = Content0} = Msg) -> - Content = rabbit_basic:map_headers( - fun(undefined) -> - rabbit_misc:set_table_value([], Name, Type, Value); - (Headers) -> - rabbit_misc:set_table_value(Headers, Name, Type, Value) - end, Content0), - Msg#basic_message{content = Content}. - -peek_fmt_message(#basic_message{exchange_name = Ex, - routing_keys = RKeys, - content = - #content{payload_fragments_rev = Payl0, - properties = Props}}) -> - Fields = [atom_to_binary(F, utf8) || F <- record_info(fields, 'P_basic')], - T = lists:zip(Fields, tl(tuple_to_list(Props))), - lists:foldl( - fun ({<<"headers">>, Hdrs}, Acc) -> - case Hdrs of - [] -> - Acc; - _ -> - Acc ++ [{header_key(H), V} || {H, _T, V} <- Hdrs] - end; - ({_, undefined}, Acc) -> - Acc; - (KV, Acc) -> - [KV | Acc] - end, [], [{<<"payload (max 64 bytes)">>, - %% restric payload to 64 bytes - binary_prefix_64(iolist_to_binary(lists:reverse(Payl0)), 64)}, - {<<"exchange">>, Ex#resource.name}, - {<<"routing_keys">>, RKeys} | T]). - -header_key(A) -> - <<"header.", A/binary>>. - -binary_prefix_64(Bin, Len) -> - binary:part(Bin, 0, min(byte_size(Bin), Len)). diff --git a/src/rabbit_binding.erl b/src/rabbit_binding.erl deleted file mode 100644 index 6ef25c4e60..0000000000 --- a/src/rabbit_binding.erl +++ /dev/null @@ -1,691 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_binding). --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --export([recover/0, recover/2, exists/1, add/2, add/3, remove/1, remove/2, remove/3, remove/4]). --export([list/1, list_for_source/1, list_for_destination/1, - list_for_source_and_destination/2, list_explicit/0]). --export([new_deletions/0, combine_deletions/2, add_deletion/3, - process_deletions/2, binding_action/3]). --export([info_keys/0, info/1, info/2, info_all/1, info_all/2, info_all/4]). -%% these must all be run inside a mnesia tx --export([has_for_source/1, remove_for_source/1, - remove_for_destination/2, remove_transient_for_destination/1, - remove_default_exchange_binding_rows_of/1]). - --export([implicit_for_destination/1, reverse_binding/1]). --export([new/4]). - --define(DEFAULT_EXCHANGE(VHostPath), #resource{virtual_host = VHostPath, - kind = exchange, - name = <<>>}). - -%%---------------------------------------------------------------------------- - --export_type([key/0, deletions/0]). - --type key() :: binary(). - --type bind_errors() :: rabbit_types:error( - {'resources_missing', - [{'not_found', (rabbit_types:binding_source() | - rabbit_types:binding_destination())} | - {'absent', amqqueue:amqqueue()}]}). - --type bind_ok_or_error() :: 'ok' | bind_errors() | - rabbit_types:error( - {'binding_invalid', string(), [any()]}). --type bind_res() :: bind_ok_or_error() | rabbit_misc:thunk(bind_ok_or_error()). --type inner_fun() :: - fun((rabbit_types:exchange(), - rabbit_types:exchange() | amqqueue:amqqueue()) -> - rabbit_types:ok_or_error(rabbit_types:amqp_error())). --type bindings() :: [rabbit_types:binding()]. - -%% TODO this should really be opaque but that seems to confuse 17.1's -%% dialyzer into objecting to everything that uses it. --type deletions() :: dict:dict(). - -%%---------------------------------------------------------------------------- - --spec new(rabbit_types:exchange(), - key(), - rabbit_types:exchange() | amqqueue:amqqueue(), - rabbit_framing:amqp_table()) -> - rabbit_types:binding(). - -new(Src, RoutingKey, Dst, #{}) -> - new(Src, RoutingKey, Dst, []); -new(Src, RoutingKey, Dst, Arguments) when is_map(Arguments) -> - new(Src, RoutingKey, Dst, maps:to_list(Arguments)); -new(Src, RoutingKey, Dst, Arguments) -> - #binding{source = Src, key = RoutingKey, destination = Dst, args = Arguments}. - - --define(INFO_KEYS, [source_name, source_kind, - destination_name, destination_kind, - routing_key, arguments, - vhost]). - -%% Global table recovery - --spec recover([rabbit_exchange:name()], [rabbit_amqqueue:name()]) -> - 'ok'. - -recover() -> - rabbit_misc:table_filter( - fun (Route) -> - mnesia:read({rabbit_semi_durable_route, Route}) =:= [] - end, - fun (Route, true) -> - ok = mnesia:write(rabbit_semi_durable_route, Route, write); - (_Route, false) -> - ok - end, rabbit_durable_route). - -%% Virtual host-specific recovery -recover(XNames, QNames) -> - XNameSet = sets:from_list(XNames), - QNameSet = sets:from_list(QNames), - SelectSet = fun (#resource{kind = exchange}) -> XNameSet; - (#resource{kind = queue}) -> QNameSet - end, - {ok, Gatherer} = gatherer:start_link(), - [recover_semi_durable_route(Gatherer, R, SelectSet(Dst)) || - R = #route{binding = #binding{destination = Dst}} <- - rabbit_misc:dirty_read_all(rabbit_semi_durable_route)], - empty = gatherer:out(Gatherer), - ok = gatherer:stop(Gatherer), - ok. - -recover_semi_durable_route(Gatherer, R = #route{binding = B}, ToRecover) -> - #binding{source = Src, destination = Dst} = B, - case sets:is_element(Dst, ToRecover) of - true -> {ok, X} = rabbit_exchange:lookup(Src), - ok = gatherer:fork(Gatherer), - ok = worker_pool:submit_async( - fun () -> - recover_semi_durable_route_txn(R, X), - gatherer:finish(Gatherer) - end); - false -> ok - end. - -recover_semi_durable_route_txn(R = #route{binding = B}, X) -> - rabbit_misc:execute_mnesia_transaction( - fun () -> - case mnesia:read(rabbit_semi_durable_route, B, read) of - [] -> no_recover; - _ -> ok = sync_transient_route(R, fun mnesia:write/3), - rabbit_exchange:serial(X) - end - end, - fun (no_recover, _) -> ok; - (_Serial, true) -> x_callback(transaction, X, add_binding, B); - (Serial, false) -> x_callback(Serial, X, add_binding, B) - end). - --spec exists(rabbit_types:binding()) -> boolean() | bind_errors(). - -exists(#binding{source = ?DEFAULT_EXCHANGE(_), - destination = #resource{kind = queue, name = QName} = Queue, - key = QName, - args = []}) -> - case rabbit_amqqueue:lookup(Queue) of - {ok, _} -> true; - {error, not_found} -> false - end; -exists(Binding) -> - binding_action( - Binding, fun (_Src, _Dst, B) -> - rabbit_misc:const(mnesia:read({rabbit_route, B}) /= []) - end, fun not_found_or_absent_errs/1). - --spec add(rabbit_types:binding(), rabbit_types:username()) -> bind_res(). - -add(Binding, ActingUser) -> add(Binding, fun (_Src, _Dst) -> ok end, ActingUser). - --spec add(rabbit_types:binding(), inner_fun(), rabbit_types:username()) -> bind_res(). - -add(Binding, InnerFun, ActingUser) -> - binding_action( - Binding, - fun (Src, Dst, B) -> - case rabbit_exchange:validate_binding(Src, B) of - ok -> - lock_resource(Src, read), - lock_resource(Dst, read), - %% this argument is used to check queue exclusivity; - %% in general, we want to fail on that in preference to - %% anything else - case InnerFun(Src, Dst) of - ok -> - case mnesia:read({rabbit_route, B}) of - [] -> add(Src, Dst, B, ActingUser); - [_] -> fun () -> ok end - end; - {error, _} = Err -> - rabbit_misc:const(Err) - end; - {error, _} = Err -> - rabbit_misc:const(Err) - end - end, fun not_found_or_absent_errs/1). - -add(Src, Dst, B, ActingUser) -> - [SrcDurable, DstDurable] = [durable(E) || E <- [Src, Dst]], - ok = sync_route(#route{binding = B}, SrcDurable, DstDurable, - fun mnesia:write/3), - x_callback(transaction, Src, add_binding, B), - Serial = rabbit_exchange:serial(Src), - fun () -> - x_callback(Serial, Src, add_binding, B), - ok = rabbit_event:notify( - binding_created, - info(B) ++ [{user_who_performed_action, ActingUser}]) - end. - --spec remove(rabbit_types:binding()) -> bind_res(). -remove(Binding) -> remove(Binding, fun (_Src, _Dst) -> ok end, ?INTERNAL_USER). - --spec remove(rabbit_types:binding(), rabbit_types:username()) -> bind_res(). -remove(Binding, ActingUser) -> remove(Binding, fun (_Src, _Dst) -> ok end, ActingUser). - - --spec remove(rabbit_types:binding(), inner_fun(), rabbit_types:username()) -> bind_res(). -remove(Binding, InnerFun, ActingUser) -> - binding_action( - Binding, - fun (Src, Dst, B) -> - lock_resource(Src, read), - lock_resource(Dst, read), - case mnesia:read(rabbit_route, B, write) of - [] -> case mnesia:read(rabbit_durable_route, B, write) of - [] -> rabbit_misc:const(ok); - %% We still delete the binding and run - %% all post-delete functions if there is only - %% a durable route in the database - _ -> remove(Src, Dst, B, ActingUser) - end; - _ -> case InnerFun(Src, Dst) of - ok -> remove(Src, Dst, B, ActingUser); - {error, _} = Err -> rabbit_misc:const(Err) - end - end - end, fun absent_errs_only/1). - -remove(Src, Dst, B, ActingUser) -> - ok = sync_route(#route{binding = B}, durable(Src), durable(Dst), - fun delete/3), - Deletions = maybe_auto_delete( - B#binding.source, [B], new_deletions(), false), - process_deletions(Deletions, ActingUser). - -%% Implicit bindings are implicit as of rabbitmq/rabbitmq-server#1721. -remove_default_exchange_binding_rows_of(Dst = #resource{}) -> - case implicit_for_destination(Dst) of - [Binding] -> - mnesia:dirty_delete(rabbit_durable_route, Binding), - mnesia:dirty_delete(rabbit_semi_durable_route, Binding), - mnesia:dirty_delete(rabbit_reverse_route, - reverse_binding(Binding)), - mnesia:dirty_delete(rabbit_route, Binding); - _ -> - %% no binding to remove or - %% a competing tx has beaten us to it? - ok - end, - ok. - --spec list_explicit() -> bindings(). - -list_explicit() -> - mnesia:async_dirty( - fun () -> - AllRoutes = mnesia:dirty_match_object(rabbit_route, #route{_ = '_'}), - %% if there are any default exchange bindings left after an upgrade - %% of a pre-3.8 database, filter them out - AllBindings = [B || #route{binding = B} <- AllRoutes], - lists:filter(fun(#binding{source = S}) -> - not (S#resource.kind =:= exchange andalso S#resource.name =:= <<>>) - end, AllBindings) - end). - --spec list(rabbit_types:vhost()) -> bindings(). - -list(VHostPath) -> - VHostResource = rabbit_misc:r(VHostPath, '_'), - Route = #route{binding = #binding{source = VHostResource, - destination = VHostResource, - _ = '_'}, - _ = '_'}, - %% if there are any default exchange bindings left after an upgrade - %% of a pre-3.8 database, filter them out - AllBindings = [B || #route{binding = B} <- mnesia:dirty_match_object(rabbit_route, - Route)], - Filtered = lists:filter(fun(#binding{source = S}) -> - S =/= ?DEFAULT_EXCHANGE(VHostPath) - end, AllBindings), - implicit_bindings(VHostPath) ++ Filtered. - --spec list_for_source - (rabbit_types:binding_source()) -> bindings(). - -list_for_source(?DEFAULT_EXCHANGE(VHostPath)) -> - implicit_bindings(VHostPath); -list_for_source(SrcName) -> - mnesia:async_dirty( - fun() -> - Route = #route{binding = #binding{source = SrcName, _ = '_'}}, - [B || #route{binding = B} - <- mnesia:match_object(rabbit_route, Route, read)] - end). - --spec list_for_destination - (rabbit_types:binding_destination()) -> bindings(). - -list_for_destination(DstName = #resource{virtual_host = VHostPath}) -> - AllBindings = mnesia:async_dirty( - fun() -> - Route = #route{binding = #binding{destination = DstName, - _ = '_'}}, - [reverse_binding(B) || - #reverse_route{reverse_binding = B} <- - mnesia:match_object(rabbit_reverse_route, - reverse_route(Route), read)] - end), - Filtered = lists:filter(fun(#binding{source = S}) -> - S =/= ?DEFAULT_EXCHANGE(VHostPath) - end, AllBindings), - implicit_for_destination(DstName) ++ Filtered. - -implicit_bindings(VHostPath) -> - DstQueues = rabbit_amqqueue:list_names(VHostPath), - [ #binding{source = ?DEFAULT_EXCHANGE(VHostPath), - destination = DstQueue, - key = QName, - args = []} - || DstQueue = #resource{name = QName} <- DstQueues ]. - -implicit_for_destination(DstQueue = #resource{kind = queue, - virtual_host = VHostPath, - name = QName}) -> - [#binding{source = ?DEFAULT_EXCHANGE(VHostPath), - destination = DstQueue, - key = QName, - args = []}]; -implicit_for_destination(_) -> - []. - --spec list_for_source_and_destination - (rabbit_types:binding_source(), rabbit_types:binding_destination()) -> - bindings(). - -list_for_source_and_destination(?DEFAULT_EXCHANGE(VHostPath), - #resource{kind = queue, - virtual_host = VHostPath, - name = QName} = DstQueue) -> - [#binding{source = ?DEFAULT_EXCHANGE(VHostPath), - destination = DstQueue, - key = QName, - args = []}]; -list_for_source_and_destination(SrcName, DstName) -> - mnesia:async_dirty( - fun() -> - Route = #route{binding = #binding{source = SrcName, - destination = DstName, - _ = '_'}}, - [B || #route{binding = B} <- mnesia:match_object(rabbit_route, - Route, read)] - end). - --spec info_keys() -> rabbit_types:info_keys(). - -info_keys() -> ?INFO_KEYS. - -map(VHostPath, F) -> - %% TODO: there is scope for optimisation here, e.g. using a - %% cursor, parallelising the function invocation - lists:map(F, list(VHostPath)). - -infos(Items, B) -> [{Item, i(Item, B)} || Item <- Items]. - -i(source_name, #binding{source = SrcName}) -> SrcName#resource.name; -i(source_kind, #binding{source = SrcName}) -> SrcName#resource.kind; -i(vhost, #binding{source = SrcName}) -> SrcName#resource.virtual_host; -i(destination_name, #binding{destination = DstName}) -> DstName#resource.name; -i(destination_kind, #binding{destination = DstName}) -> DstName#resource.kind; -i(routing_key, #binding{key = RoutingKey}) -> RoutingKey; -i(arguments, #binding{args = Arguments}) -> Arguments; -i(Item, _) -> throw({bad_argument, Item}). - --spec info(rabbit_types:binding()) -> rabbit_types:infos(). - -info(B = #binding{}) -> infos(?INFO_KEYS, B). - --spec info(rabbit_types:binding(), rabbit_types:info_keys()) -> - rabbit_types:infos(). - -info(B = #binding{}, Items) -> infos(Items, B). - --spec info_all(rabbit_types:vhost()) -> [rabbit_types:infos()]. - -info_all(VHostPath) -> map(VHostPath, fun (B) -> info(B) end). - --spec info_all(rabbit_types:vhost(), rabbit_types:info_keys()) -> - [rabbit_types:infos()]. - -info_all(VHostPath, Items) -> map(VHostPath, fun (B) -> info(B, Items) end). - --spec info_all(rabbit_types:vhost(), rabbit_types:info_keys(), - reference(), pid()) -> 'ok'. - -info_all(VHostPath, Items, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map( - AggregatorPid, Ref, fun(B) -> info(B, Items) end, list(VHostPath)). - --spec has_for_source(rabbit_types:binding_source()) -> boolean(). - -has_for_source(SrcName) -> - Match = #route{binding = #binding{source = SrcName, _ = '_'}}, - %% we need to check for semi-durable routes (which subsumes - %% durable routes) here too in case a bunch of routes to durable - %% queues have been removed temporarily as a result of a node - %% failure - contains(rabbit_route, Match) orelse - contains(rabbit_semi_durable_route, Match). - --spec remove_for_source(rabbit_types:binding_source()) -> bindings(). - -remove_for_source(SrcName) -> - lock_resource(SrcName), - Match = #route{binding = #binding{source = SrcName, _ = '_'}}, - remove_routes( - lists:usort( - mnesia:dirty_match_object(rabbit_route, Match) ++ - mnesia:dirty_match_object(rabbit_semi_durable_route, Match))). - --spec remove_for_destination - (rabbit_types:binding_destination(), boolean()) -> deletions(). - -remove_for_destination(DstName, OnlyDurable) -> - remove_for_destination(DstName, OnlyDurable, fun remove_routes/1). - --spec remove_transient_for_destination - (rabbit_types:binding_destination()) -> deletions(). - -remove_transient_for_destination(DstName) -> - remove_for_destination(DstName, false, fun remove_transient_routes/1). - -%%---------------------------------------------------------------------------- - -durable(#exchange{durable = D}) -> D; -durable(Q) when ?is_amqqueue(Q) -> - amqqueue:is_durable(Q). - -binding_action(Binding = #binding{source = SrcName, - destination = DstName, - args = Arguments}, Fun, ErrFun) -> - call_with_source_and_destination( - SrcName, DstName, - fun (Src, Dst) -> - SortedArgs = rabbit_misc:sort_field_table(Arguments), - Fun(Src, Dst, Binding#binding{args = SortedArgs}) - end, ErrFun). - -sync_route(Route, true, true, Fun) -> - ok = Fun(rabbit_durable_route, Route, write), - sync_route(Route, false, true, Fun); - -sync_route(Route, false, true, Fun) -> - ok = Fun(rabbit_semi_durable_route, Route, write), - sync_route(Route, false, false, Fun); - -sync_route(Route, _SrcDurable, false, Fun) -> - sync_transient_route(Route, Fun). - -sync_transient_route(Route, Fun) -> - ok = Fun(rabbit_route, Route, write), - ok = Fun(rabbit_reverse_route, reverse_route(Route), write). - -call_with_source_and_destination(SrcName, DstName, Fun, ErrFun) -> - SrcTable = table_for_resource(SrcName), - DstTable = table_for_resource(DstName), - rabbit_misc:execute_mnesia_tx_with_tail( - fun () -> - case {mnesia:read({SrcTable, SrcName}), - mnesia:read({DstTable, DstName})} of - {[Src], [Dst]} -> Fun(Src, Dst); - {[], [_] } -> ErrFun([SrcName]); - {[_], [] } -> ErrFun([DstName]); - {[], [] } -> ErrFun([SrcName, DstName]) - end - end). - -not_found_or_absent_errs(Names) -> - Errs = [not_found_or_absent(Name) || Name <- Names], - rabbit_misc:const({error, {resources_missing, Errs}}). - -absent_errs_only(Names) -> - Errs = [E || Name <- Names, - {absent, _Q, _Reason} = E <- [not_found_or_absent(Name)]], - rabbit_misc:const(case Errs of - [] -> ok; - _ -> {error, {resources_missing, Errs}} - end). - -table_for_resource(#resource{kind = exchange}) -> rabbit_exchange; -table_for_resource(#resource{kind = queue}) -> rabbit_queue. - -not_found_or_absent(#resource{kind = exchange} = Name) -> - {not_found, Name}; -not_found_or_absent(#resource{kind = queue} = Name) -> - case rabbit_amqqueue:not_found_or_absent(Name) of - not_found -> {not_found, Name}; - {absent, _Q, _Reason} = R -> R - end. - -contains(Table, MatchHead) -> - continue(mnesia:select(Table, [{MatchHead, [], ['$_']}], 1, read)). - -continue('$end_of_table') -> false; -continue({[_|_], _}) -> true; -continue({[], Continuation}) -> continue(mnesia:select(Continuation)). - -remove_routes(Routes) -> - %% This partitioning allows us to suppress unnecessary delete - %% operations on disk tables, which require an fsync. - {RamRoutes, DiskRoutes} = - lists:partition(fun (R) -> mnesia:read( - rabbit_durable_route, R#route.binding, read) == [] end, - Routes), - {RamOnlyRoutes, SemiDurableRoutes} = - lists:partition(fun (R) -> mnesia:read( - rabbit_semi_durable_route, R#route.binding, read) == [] end, - RamRoutes), - %% Of course the destination might not really be durable but it's - %% just as easy to try to delete it from the semi-durable table - %% than check first - [ok = sync_route(R, true, true, fun delete/3) || - R <- DiskRoutes], - [ok = sync_route(R, false, true, fun delete/3) || - R <- SemiDurableRoutes], - [ok = sync_route(R, false, false, fun delete/3) || - R <- RamOnlyRoutes], - [R#route.binding || R <- Routes]. - - -delete(Tab, #route{binding = B}, LockKind) -> - mnesia:delete(Tab, B, LockKind); -delete(Tab, #reverse_route{reverse_binding = B}, LockKind) -> - mnesia:delete(Tab, B, LockKind). - -remove_transient_routes(Routes) -> - [begin - ok = sync_transient_route(R, fun delete/3), - R#route.binding - end || R <- Routes]. - -remove_for_destination(DstName, OnlyDurable, Fun) -> - lock_resource(DstName), - MatchFwd = #route{binding = #binding{destination = DstName, _ = '_'}}, - MatchRev = reverse_route(MatchFwd), - Routes = case OnlyDurable of - false -> - [reverse_route(R) || - R <- mnesia:dirty_match_object( - rabbit_reverse_route, MatchRev)]; - true -> lists:usort( - mnesia:dirty_match_object( - rabbit_durable_route, MatchFwd) ++ - mnesia:dirty_match_object( - rabbit_semi_durable_route, MatchFwd)) - end, - Bindings = Fun(Routes), - group_bindings_fold(fun maybe_auto_delete/4, new_deletions(), - lists:keysort(#binding.source, Bindings), OnlyDurable). - -%% Instead of locking entire table on remove operations we can lock the -%% affected resource only. -lock_resource(Name) -> lock_resource(Name, write). - -lock_resource(Name, LockKind) -> - mnesia:lock({global, Name, mnesia:table_info(rabbit_route, where_to_write)}, - LockKind). - -%% Requires that its input binding list is sorted in exchange-name -%% order, so that the grouping of bindings (for passing to -%% group_bindings_and_auto_delete1) works properly. -group_bindings_fold(_Fun, Acc, [], _OnlyDurable) -> - Acc; -group_bindings_fold(Fun, Acc, [B = #binding{source = SrcName} | Bs], - OnlyDurable) -> - group_bindings_fold(Fun, SrcName, Acc, Bs, [B], OnlyDurable). - -group_bindings_fold( - Fun, SrcName, Acc, [B = #binding{source = SrcName} | Bs], Bindings, - OnlyDurable) -> - group_bindings_fold(Fun, SrcName, Acc, Bs, [B | Bindings], OnlyDurable); -group_bindings_fold(Fun, SrcName, Acc, Removed, Bindings, OnlyDurable) -> - %% Either Removed is [], or its head has a non-matching SrcName. - group_bindings_fold(Fun, Fun(SrcName, Bindings, Acc, OnlyDurable), Removed, - OnlyDurable). - -maybe_auto_delete(XName, Bindings, Deletions, OnlyDurable) -> - {Entry, Deletions1} = - case mnesia:read({case OnlyDurable of - true -> rabbit_durable_exchange; - false -> rabbit_exchange - end, XName}) of - [] -> {{undefined, not_deleted, Bindings}, Deletions}; - [X] -> case rabbit_exchange:maybe_auto_delete(X, OnlyDurable) of - not_deleted -> - {{X, not_deleted, Bindings}, Deletions}; - {deleted, Deletions2} -> - {{X, deleted, Bindings}, - combine_deletions(Deletions, Deletions2)} - end - end, - add_deletion(XName, Entry, Deletions1). - -reverse_route(#route{binding = Binding}) -> - #reverse_route{reverse_binding = reverse_binding(Binding)}; - -reverse_route(#reverse_route{reverse_binding = Binding}) -> - #route{binding = reverse_binding(Binding)}. - -reverse_binding(#reverse_binding{source = SrcName, - destination = DstName, - key = Key, - args = Args}) -> - #binding{source = SrcName, - destination = DstName, - key = Key, - args = Args}; - -reverse_binding(#binding{source = SrcName, - destination = DstName, - key = Key, - args = Args}) -> - #reverse_binding{source = SrcName, - destination = DstName, - key = Key, - args = Args}. - -%% ---------------------------------------------------------------------------- -%% Binding / exchange deletion abstraction API -%% ---------------------------------------------------------------------------- - -anything_but( NotThis, NotThis, NotThis) -> NotThis; -anything_but( NotThis, NotThis, This) -> This; -anything_but( NotThis, This, NotThis) -> This; -anything_but(_NotThis, This, This) -> This. - --spec new_deletions() -> deletions(). - -new_deletions() -> dict:new(). - --spec add_deletion - (rabbit_exchange:name(), - {'undefined' | rabbit_types:exchange(), - 'deleted' | 'not_deleted', - bindings()}, - deletions()) -> - deletions(). - -add_deletion(XName, Entry, Deletions) -> - dict:update(XName, fun (Entry1) -> merge_entry(Entry1, Entry) end, - Entry, Deletions). - --spec combine_deletions(deletions(), deletions()) -> deletions(). - -combine_deletions(Deletions1, Deletions2) -> - dict:merge(fun (_XName, Entry1, Entry2) -> merge_entry(Entry1, Entry2) end, - Deletions1, Deletions2). - -merge_entry({X1, Deleted1, Bindings1}, {X2, Deleted2, Bindings2}) -> - {anything_but(undefined, X1, X2), - anything_but(not_deleted, Deleted1, Deleted2), - [Bindings1 | Bindings2]}. - --spec process_deletions(deletions(), rabbit_types:username()) -> rabbit_misc:thunk('ok'). - -process_deletions(Deletions, ActingUser) -> - AugmentedDeletions = - dict:map(fun (_XName, {X, deleted, Bindings}) -> - Bs = lists:flatten(Bindings), - x_callback(transaction, X, delete, Bs), - {X, deleted, Bs, none}; - (_XName, {X, not_deleted, Bindings}) -> - Bs = lists:flatten(Bindings), - x_callback(transaction, X, remove_bindings, Bs), - {X, not_deleted, Bs, rabbit_exchange:serial(X)} - end, Deletions), - fun() -> - dict:fold(fun (XName, {X, deleted, Bs, Serial}, ok) -> - ok = rabbit_event:notify( - exchange_deleted, - [{name, XName}, - {user_who_performed_action, ActingUser}]), - del_notify(Bs, ActingUser), - x_callback(Serial, X, delete, Bs); - (_XName, {X, not_deleted, Bs, Serial}, ok) -> - del_notify(Bs, ActingUser), - x_callback(Serial, X, remove_bindings, Bs) - end, ok, AugmentedDeletions) - end. - -del_notify(Bs, ActingUser) -> [rabbit_event:notify( - binding_deleted, - info(B) ++ [{user_who_performed_action, ActingUser}]) - || B <- Bs]. - -x_callback(Serial, X, F, Bs) -> - ok = rabbit_exchange:callback(X, F, Serial, [X, Bs]). diff --git a/src/rabbit_boot_steps.erl b/src/rabbit_boot_steps.erl deleted file mode 100644 index f87448edb7..0000000000 --- a/src/rabbit_boot_steps.erl +++ /dev/null @@ -1,91 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_boot_steps). - --export([run_boot_steps/0, run_boot_steps/1, run_cleanup_steps/1]). --export([find_steps/0, find_steps/1]). - -run_boot_steps() -> - run_boot_steps(loaded_applications()). - -run_boot_steps(Apps) -> - [begin - rabbit_log:info("Running boot step ~s defined by app ~s", [Step, App]), - ok = run_step(Attrs, mfa) - end || {App, Step, Attrs} <- find_steps(Apps)], - ok. - -run_cleanup_steps(Apps) -> - [run_step(Attrs, cleanup) || {_, _, Attrs} <- find_steps(Apps)], - ok. - -loaded_applications() -> - [App || {App, _, _} <- application:loaded_applications()]. - -find_steps() -> - find_steps(loaded_applications()). - -find_steps(Apps) -> - All = sort_boot_steps(rabbit_misc:all_module_attributes(rabbit_boot_step)), - [Step || {App, _, _} = Step <- All, lists:member(App, Apps)]. - -run_step(Attributes, AttributeName) -> - [begin - rabbit_log:debug("Applying MFA: M = ~s, F = ~s, A = ~p", - [M, F, A]), - case apply(M,F,A) of - ok -> ok; - {error, Reason} -> exit({error, Reason}) - end - end - || {Key, {M,F,A}} <- Attributes, - Key =:= AttributeName], - ok. - -vertices({AppName, _Module, Steps}) -> - [{StepName, {AppName, StepName, Atts}} || {StepName, Atts} <- Steps]. - -edges({_AppName, _Module, Steps}) -> - EnsureList = fun (L) when is_list(L) -> L; - (T) -> [T] - end, - [case Key of - requires -> {StepName, OtherStep}; - enables -> {OtherStep, StepName} - end || {StepName, Atts} <- Steps, - {Key, OtherStepOrSteps} <- Atts, - OtherStep <- EnsureList(OtherStepOrSteps), - Key =:= requires orelse Key =:= enables]. - -sort_boot_steps(UnsortedSteps) -> - case rabbit_misc:build_acyclic_graph(fun vertices/1, fun edges/1, - UnsortedSteps) of - {ok, G} -> - %% Use topological sort to find a consistent ordering (if - %% there is one, otherwise fail). - SortedSteps = lists:reverse( - [begin - {StepName, Step} = digraph:vertex(G, - StepName), - Step - end || StepName <- digraph_utils:topsort(G)]), - digraph:delete(G), - %% Check that all mentioned {M,F,A} triples are exported. - case [{StepName, {M,F,A}} || - {_App, StepName, Attributes} <- SortedSteps, - {mfa, {M,F,A}} <- Attributes, - code:ensure_loaded(M) =/= {module, M} orelse - not erlang:function_exported(M, F, length(A))] of - [] -> SortedSteps; - MissingFns -> exit({boot_functions_not_exported, MissingFns}) - end; - {error, {vertex, duplicate, StepName}} -> - exit({duplicate_boot_step, StepName}); - {error, {edge, Reason, From, To}} -> - exit({invalid_boot_step_dependency, From, To, Reason}) - end. diff --git a/src/rabbit_channel.erl b/src/rabbit_channel.erl deleted file mode 100644 index 8e7828a7c0..0000000000 --- a/src/rabbit_channel.erl +++ /dev/null @@ -1,2797 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_channel). - -%% Transitional step until we can require Erlang/OTP 21 and -%% use the now recommended try/catch syntax for obtaining the stack trace. --compile(nowarn_deprecated_function). - -%% rabbit_channel processes represent an AMQP 0-9-1 channels. -%% -%% Connections parse protocol frames coming from clients and -%% dispatch them to channel processes. -%% Channels are responsible for implementing the logic behind -%% the various protocol methods, involving other processes as -%% needed: -%% -%% * Routing messages (using functions in various exchange type -%% modules) to queue processes. -%% * Managing queues, exchanges, and bindings. -%% * Keeping track of consumers -%% * Keeping track of unacknowledged deliveries to consumers -%% * Keeping track of publisher confirms -%% * Transaction management -%% * Authorisation (enforcing permissions) -%% * Publishing trace events if tracing is enabled -%% -%% Every channel has a number of dependent processes: -%% -%% * A writer which is responsible for sending frames to clients. -%% * A limiter which controls how many messages can be delivered -%% to consumers according to active QoS prefetch and internal -%% flow control logic. -%% -%% Channels are also aware of their connection's queue collector. -%% When a queue is declared as exclusive on a channel, the channel -%% will notify queue collector of that queue. - --include_lib("rabbit_common/include/rabbit_framing.hrl"). --include_lib("rabbit_common/include/rabbit.hrl"). --include_lib("rabbit_common/include/rabbit_misc.hrl"). - --include("amqqueue.hrl"). - --behaviour(gen_server2). - --export([start_link/11, start_link/12, do/2, do/3, do_flow/3, flush/1, shutdown/1]). --export([send_command/2, deliver/4, deliver_reply/2, - send_credit_reply/2, send_drained/2]). --export([list/0, info_keys/0, info/1, info/2, info_all/0, info_all/1, - emit_info_all/4, info_local/1]). --export([refresh_config_local/0, ready_for_close/1]). --export([refresh_interceptors/0]). --export([force_event_refresh/1]). --export([update_user_state/2]). - --export([init/1, terminate/2, code_change/3, handle_call/3, handle_cast/2, - handle_info/2, handle_pre_hibernate/1, handle_post_hibernate/1, - prioritise_call/4, prioritise_cast/3, prioritise_info/3, - format_message_queue/2]). - -%% Internal --export([list_local/0, emit_info_local/3, deliver_reply_local/3]). --export([get_vhost/1, get_user/1]). -%% For testing --export([build_topic_variable_map/3]). --export([list_queue_states/1, get_max_message_size/0]). - -%% Mgmt HTTP API refactor --export([handle_method/6]). - --record(conf, { - %% starting | running | flow | closing - state, - %% same as reader's protocol. Used when instantiating - %% (protocol) exceptions. - protocol, - %% channel number - channel, - %% reader process - reader_pid, - %% writer process - writer_pid, - %% - conn_pid, - %% same as reader's name, see #v1.name - %% in rabbit_reader - conn_name, - %% channel's originating source e.g. rabbit_reader | rabbit_direct | undefined - %% or any other channel creating/spawning entity - source, - %% same as #v1.user in the reader, used in - %% authorisation checks - user, - %% same as #v1.user in the reader - virtual_host, - %% when queue.bind's queue field is empty, - %% this name will be used instead - most_recently_declared_queue, - %% when a queue is declared as exclusive, queue - %% collector must be notified. - %% see rabbit_queue_collector for more info. - queue_collector_pid, - - %% same as capabilities in the reader - capabilities, - %% tracing exchange resource if tracing is enabled, - %% 'none' otherwise - trace_state, - consumer_prefetch, - %% Message content size limit - max_message_size, - consumer_timeout, - authz_context, - %% defines how ofter gc will be executed - writer_gc_threshold - }). - --record(pending_ack, {delivery_tag, - tag, - delivered_at, - queue, %% queue name - msg_id}). - --record(ch, {cfg :: #conf{}, - %% limiter state, see rabbit_limiter - limiter, - %% none | {Msgs, Acks} | committing | failed | - tx, - %% (consumer) delivery tag sequence - next_tag, - %% messages pending consumer acknowledgement - unacked_message_q, - %% queue processes are monitored to update - %% queue names - queue_monitors, - %% a map of consumer tags to - %% consumer details: #amqqueue record, acknowledgement mode, - %% consumer exclusivity, etc - consumer_mapping, - %% a map of queue names to consumer tag lists - queue_consumers, - %% timer used to emit statistics - stats_timer, - %% are publisher confirms enabled for this channel? - confirm_enabled, - %% publisher confirm delivery tag sequence - publish_seqno, - %% an unconfirmed_messages data structure used to track unconfirmed - %% (to publishers) messages - unconfirmed, - %% a list of tags for published messages that were - %% delivered but are yet to be confirmed to the client - confirmed, - %% a list of tags for published messages that were - %% rejected but are yet to be sent to the client - rejected, - %% used by "one shot RPC" (amq. - reply_consumer, - %% flow | noflow, see rabbitmq-server#114 - delivery_flow, - interceptor_state, - queue_states, - tick_timer - }). - --define(QUEUE, lqueue). - --define(MAX_PERMISSION_CACHE_SIZE, 12). - --define(REFRESH_TIMEOUT, 15000). - --define(STATISTICS_KEYS, - [reductions, - pid, - transactional, - confirm, - consumer_count, - messages_unacknowledged, - messages_unconfirmed, - messages_uncommitted, - acks_uncommitted, - pending_raft_commands, - prefetch_count, - global_prefetch_count, - state, - garbage_collection]). - - --define(CREATION_EVENT_KEYS, - [pid, - name, - connection, - number, - user, - vhost, - user_who_performed_action]). - --define(INFO_KEYS, ?CREATION_EVENT_KEYS ++ ?STATISTICS_KEYS -- [pid]). - --define(INCR_STATS(Type, Key, Inc, Measure, State), - case rabbit_event:stats_level(State, #ch.stats_timer) of - fine -> - rabbit_core_metrics:channel_stats(Type, Measure, {self(), Key}, Inc), - %% Keys in the process dictionary are used to clean up the core metrics - put({Type, Key}, none); - _ -> - ok - end). - --define(INCR_STATS(Type, Key, Inc, Measure), - begin - rabbit_core_metrics:channel_stats(Type, Measure, {self(), Key}, Inc), - %% Keys in the process dictionary are used to clean up the core metrics - put({Type, Key}, none) - end). - -%%---------------------------------------------------------------------------- - --export_type([channel_number/0]). - --type channel_number() :: non_neg_integer(). - --export_type([channel/0]). - --type channel() :: #ch{}. - -%%---------------------------------------------------------------------------- - --spec start_link - (channel_number(), pid(), pid(), pid(), string(), rabbit_types:protocol(), - rabbit_types:user(), rabbit_types:vhost(), rabbit_framing:amqp_table(), - pid(), pid()) -> - rabbit_types:ok_pid_or_error(). - -start_link(Channel, ReaderPid, WriterPid, ConnPid, ConnName, Protocol, User, - VHost, Capabilities, CollectorPid, Limiter) -> - start_link(Channel, ReaderPid, WriterPid, ConnPid, ConnName, Protocol, User, - VHost, Capabilities, CollectorPid, Limiter, undefined). - --spec start_link - (channel_number(), pid(), pid(), pid(), string(), rabbit_types:protocol(), - rabbit_types:user(), rabbit_types:vhost(), rabbit_framing:amqp_table(), - pid(), pid(), any()) -> - rabbit_types:ok_pid_or_error(). - -start_link(Channel, ReaderPid, WriterPid, ConnPid, ConnName, Protocol, User, - VHost, Capabilities, CollectorPid, Limiter, AmqpParams) -> - gen_server2:start_link( - ?MODULE, [Channel, ReaderPid, WriterPid, ConnPid, ConnName, Protocol, - User, VHost, Capabilities, CollectorPid, Limiter, AmqpParams], []). - --spec do(pid(), rabbit_framing:amqp_method_record()) -> 'ok'. - -do(Pid, Method) -> - rabbit_channel_common:do(Pid, Method). - --spec do - (pid(), rabbit_framing:amqp_method_record(), - rabbit_types:maybe(rabbit_types:content())) -> - 'ok'. - -do(Pid, Method, Content) -> - rabbit_channel_common:do(Pid, Method, Content). - --spec do_flow - (pid(), rabbit_framing:amqp_method_record(), - rabbit_types:maybe(rabbit_types:content())) -> - 'ok'. - -do_flow(Pid, Method, Content) -> - rabbit_channel_common:do_flow(Pid, Method, Content). - --spec flush(pid()) -> 'ok'. - -flush(Pid) -> - gen_server2:call(Pid, flush, infinity). - --spec shutdown(pid()) -> 'ok'. - -shutdown(Pid) -> - gen_server2:cast(Pid, terminate). - --spec send_command(pid(), rabbit_framing:amqp_method_record()) -> 'ok'. - -send_command(Pid, Msg) -> - gen_server2:cast(Pid, {command, Msg}). - --spec deliver - (pid(), rabbit_types:ctag(), boolean(), rabbit_amqqueue:qmsg()) -> 'ok'. - -deliver(Pid, ConsumerTag, AckRequired, Msg) -> - gen_server2:cast(Pid, {deliver, ConsumerTag, AckRequired, Msg}). - --spec deliver_reply(binary(), rabbit_types:delivery()) -> 'ok'. - -deliver_reply(<<"amq.rabbitmq.reply-to.", Rest/binary>>, Delivery) -> - case decode_fast_reply_to(Rest) of - {ok, Pid, Key} -> - delegate:invoke_no_result( - Pid, {?MODULE, deliver_reply_local, [Key, Delivery]}); - error -> - ok - end. - -%% We want to ensure people can't use this mechanism to send a message -%% to an arbitrary process and kill it! - --spec deliver_reply_local(pid(), binary(), rabbit_types:delivery()) -> 'ok'. - -deliver_reply_local(Pid, Key, Delivery) -> - case pg_local:in_group(rabbit_channels, Pid) of - true -> gen_server2:cast(Pid, {deliver_reply, Key, Delivery}); - false -> ok - end. - -declare_fast_reply_to(<<"amq.rabbitmq.reply-to">>) -> - exists; -declare_fast_reply_to(<<"amq.rabbitmq.reply-to.", Rest/binary>>) -> - case decode_fast_reply_to(Rest) of - {ok, Pid, Key} -> - Msg = {declare_fast_reply_to, Key}, - rabbit_misc:with_exit_handler( - rabbit_misc:const(not_found), - fun() -> gen_server2:call(Pid, Msg, infinity) end); - error -> - not_found - end; -declare_fast_reply_to(_) -> - not_found. - -decode_fast_reply_to(Rest) -> - case string:tokens(binary_to_list(Rest), ".") of - [PidEnc, Key] -> Pid = binary_to_term(base64:decode(PidEnc)), - {ok, Pid, Key}; - _ -> error - end. - --spec send_credit_reply(pid(), non_neg_integer()) -> 'ok'. - -send_credit_reply(Pid, Len) -> - gen_server2:cast(Pid, {send_credit_reply, Len}). - --spec send_drained(pid(), [{rabbit_types:ctag(), non_neg_integer()}]) -> 'ok'. - -send_drained(Pid, CTagCredit) -> - gen_server2:cast(Pid, {send_drained, CTagCredit}). - --spec list() -> [pid()]. - -list() -> - Nodes = rabbit_nodes:all_running(), - rabbit_misc:append_rpc_all_nodes(Nodes, rabbit_channel, list_local, [], ?RPC_TIMEOUT). - --spec list_local() -> [pid()]. - -list_local() -> - pg_local:get_members(rabbit_channels). - --spec info_keys() -> rabbit_types:info_keys(). - -info_keys() -> ?INFO_KEYS. - --spec info(pid()) -> rabbit_types:infos(). - -info(Pid) -> - {Timeout, Deadline} = get_operation_timeout_and_deadline(), - try - case gen_server2:call(Pid, {info, Deadline}, Timeout) of - {ok, Res} -> Res; - {error, Error} -> throw(Error) - end - catch - exit:{timeout, _} -> - rabbit_log:error("Timed out getting channel ~p info", [Pid]), - throw(timeout) - end. - --spec info(pid(), rabbit_types:info_keys()) -> rabbit_types:infos(). - -info(Pid, Items) -> - {Timeout, Deadline} = get_operation_timeout_and_deadline(), - try - case gen_server2:call(Pid, {{info, Items}, Deadline}, Timeout) of - {ok, Res} -> Res; - {error, Error} -> throw(Error) - end - catch - exit:{timeout, _} -> - rabbit_log:error("Timed out getting channel ~p info", [Pid]), - throw(timeout) - end. - --spec info_all() -> [rabbit_types:infos()]. - -info_all() -> - rabbit_misc:filter_exit_map(fun (C) -> info(C) end, list()). - --spec info_all(rabbit_types:info_keys()) -> [rabbit_types:infos()]. - -info_all(Items) -> - rabbit_misc:filter_exit_map(fun (C) -> info(C, Items) end, list()). - -info_local(Items) -> - rabbit_misc:filter_exit_map(fun (C) -> info(C, Items) end, list_local()). - -emit_info_all(Nodes, Items, Ref, AggregatorPid) -> - Pids = [ spawn_link(Node, rabbit_channel, emit_info_local, [Items, Ref, AggregatorPid]) || Node <- Nodes ], - rabbit_control_misc:await_emitters_termination(Pids). - -emit_info_local(Items, Ref, AggregatorPid) -> - emit_info(list_local(), Items, Ref, AggregatorPid). - -emit_info(PidList, InfoItems, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map_with_exit_handler( - AggregatorPid, Ref, fun(C) -> info(C, InfoItems) end, PidList). - --spec refresh_config_local() -> 'ok'. - -refresh_config_local() -> - rabbit_misc:upmap( - fun (C) -> - try - gen_server2:call(C, refresh_config, infinity) - catch _:Reason -> - rabbit_log:error("Failed to refresh channel config " - "for channel ~p. Reason ~p", - [C, Reason]) - end - end, - list_local()), - ok. - -refresh_interceptors() -> - rabbit_misc:upmap( - fun (C) -> - try - gen_server2:call(C, refresh_interceptors, ?REFRESH_TIMEOUT) - catch _:Reason -> - rabbit_log:error("Failed to refresh channel interceptors " - "for channel ~p. Reason ~p", - [C, Reason]) - end - end, - list_local()), - ok. - --spec ready_for_close(pid()) -> 'ok'. - -ready_for_close(Pid) -> - rabbit_channel_common:ready_for_close(Pid). - --spec force_event_refresh(reference()) -> 'ok'. - -% Note: https://www.pivotaltracker.com/story/show/166962656 -% This event is necessary for the stats timer to be initialized with -% the correct values once the management agent has started -force_event_refresh(Ref) -> - [gen_server2:cast(C, {force_event_refresh, Ref}) || C <- list()], - ok. - -list_queue_states(Pid) -> - gen_server2:call(Pid, list_queue_states). - --spec update_user_state(pid(), rabbit_types:auth_user()) -> 'ok' | {error, channel_terminated}. - -update_user_state(Pid, UserState) when is_pid(Pid) -> - case erlang:is_process_alive(Pid) of - true -> Pid ! {update_user_state, UserState}, - ok; - false -> {error, channel_terminated} - end. - -%%--------------------------------------------------------------------------- - -init([Channel, ReaderPid, WriterPid, ConnPid, ConnName, Protocol, User, VHost, - Capabilities, CollectorPid, LimiterPid, AmqpParams]) -> - process_flag(trap_exit, true), - ?LG_PROCESS_TYPE(channel), - ?store_proc_name({ConnName, Channel}), - ok = pg_local:join(rabbit_channels, self()), - Flow = case rabbit_misc:get_env(rabbit, mirroring_flow_control, true) of - true -> flow; - false -> noflow - end, - {ok, {Global, Prefetch}} = application:get_env(rabbit, default_consumer_prefetch), - Limiter0 = rabbit_limiter:new(LimiterPid), - Limiter = case {Global, Prefetch} of - {true, 0} -> - rabbit_limiter:unlimit_prefetch(Limiter0); - {true, _} -> - rabbit_limiter:limit_prefetch(Limiter0, Prefetch, 0); - _ -> - Limiter0 - end, - %% Process dictionary is used here because permission cache already uses it. MK. - put(permission_cache_can_expire, rabbit_access_control:permission_cache_can_expire(User)), - MaxMessageSize = get_max_message_size(), - ConsumerTimeout = get_consumer_timeout(), - OptionalVariables = extract_variable_map_from_amqp_params(AmqpParams), - {ok, GCThreshold} = application:get_env(rabbit, writer_gc_threshold), - State = #ch{cfg = #conf{state = starting, - protocol = Protocol, - channel = Channel, - reader_pid = ReaderPid, - writer_pid = WriterPid, - conn_pid = ConnPid, - conn_name = ConnName, - user = User, - virtual_host = VHost, - most_recently_declared_queue = <<>>, - queue_collector_pid = CollectorPid, - capabilities = Capabilities, - trace_state = rabbit_trace:init(VHost), - consumer_prefetch = Prefetch, - max_message_size = MaxMessageSize, - consumer_timeout = ConsumerTimeout, - authz_context = OptionalVariables, - writer_gc_threshold = GCThreshold - }, - limiter = Limiter, - tx = none, - next_tag = 1, - unacked_message_q = ?QUEUE:new(), - queue_monitors = pmon:new(), - consumer_mapping = #{}, - queue_consumers = #{}, - confirm_enabled = false, - publish_seqno = 1, - unconfirmed = rabbit_confirms:init(), - rejected = [], - confirmed = [], - reply_consumer = none, - delivery_flow = Flow, - interceptor_state = undefined, - queue_states = rabbit_queue_type:init() - }, - State1 = State#ch{ - interceptor_state = rabbit_channel_interceptor:init(State)}, - State2 = rabbit_event:init_stats_timer(State1, #ch.stats_timer), - Infos = infos(?CREATION_EVENT_KEYS, State2), - rabbit_core_metrics:channel_created(self(), Infos), - rabbit_event:notify(channel_created, Infos), - rabbit_event:if_enabled(State2, #ch.stats_timer, - fun() -> emit_stats(State2) end), - put_operation_timeout(), - State3 = init_tick_timer(State2), - {ok, State3, hibernate, - {backoff, ?HIBERNATE_AFTER_MIN, ?HIBERNATE_AFTER_MIN, ?DESIRED_HIBERNATE}}. - -prioritise_call(Msg, _From, _Len, _State) -> - case Msg of - info -> 9; - {info, _Items} -> 9; - _ -> 0 - end. - -prioritise_cast(Msg, _Len, _State) -> - case Msg of - {confirm, _MsgSeqNos, _QPid} -> 5; - {reject_publish, _MsgSeqNos, _QPid} -> 5; - {queue_event, _, {confirm, _MsgSeqNos, _QPid}} -> 5; - {queue_event, _, {reject_publish, _MsgSeqNos, _QPid}} -> 5; - _ -> 0 - end. - -prioritise_info(Msg, _Len, _State) -> - case Msg of - emit_stats -> 7; - _ -> 0 - end. - -handle_call(flush, _From, State) -> - reply(ok, State); - -handle_call({info, Deadline}, _From, State) -> - try - reply({ok, infos(?INFO_KEYS, Deadline, State)}, State) - catch - Error -> - reply({error, Error}, State) - end; - -handle_call({{info, Items}, Deadline}, _From, State) -> - try - reply({ok, infos(Items, Deadline, State)}, State) - catch - Error -> - reply({error, Error}, State) - end; - -handle_call(refresh_config, _From, - State = #ch{cfg = #conf{virtual_host = VHost} = Cfg}) -> - reply(ok, State#ch{cfg = Cfg#conf{trace_state = rabbit_trace:init(VHost)}}); - -handle_call(refresh_interceptors, _From, State) -> - IState = rabbit_channel_interceptor:init(State), - reply(ok, State#ch{interceptor_state = IState}); - -handle_call({declare_fast_reply_to, Key}, _From, - State = #ch{reply_consumer = Consumer}) -> - reply(case Consumer of - {_, _, Key} -> exists; - _ -> not_found - end, State); - -handle_call(list_queue_states, _From, State = #ch{queue_states = QueueStates}) -> - %% For testing of cleanup only - %% HACK - {reply, maps:keys(element(2, QueueStates)), State}; - -handle_call(_Request, _From, State) -> - noreply(State). - -handle_cast({method, Method, Content, Flow}, - State = #ch{cfg = #conf{reader_pid = Reader}, - interceptor_state = IState}) -> - case Flow of - %% We are going to process a message from the rabbit_reader - %% process, so here we ack it. In this case we are accessing - %% the rabbit_channel process dictionary. - flow -> credit_flow:ack(Reader); - noflow -> ok - end, - try handle_method(rabbit_channel_interceptor:intercept_in( - expand_shortcuts(Method, State), Content, IState), - State) of - {reply, Reply, NewState} -> - ok = send(Reply, NewState), - noreply(NewState); - {noreply, NewState} -> - noreply(NewState); - stop -> - {stop, normal, State} - catch - exit:Reason = #amqp_error{} -> - MethodName = rabbit_misc:method_record_type(Method), - handle_exception(Reason#amqp_error{method = MethodName}, State); - _:Reason:Stacktrace -> - {stop, {Reason, Stacktrace}, State} - end; - -handle_cast(ready_for_close, - State = #ch{cfg = #conf{state = closing, - writer_pid = WriterPid}}) -> - ok = rabbit_writer:send_command_sync(WriterPid, #'channel.close_ok'{}), - {stop, normal, State}; - -handle_cast(terminate, State = #ch{cfg = #conf{writer_pid = WriterPid}}) -> - ok = rabbit_writer:flush(WriterPid), - {stop, normal, State}; - -handle_cast({command, #'basic.consume_ok'{consumer_tag = CTag} = Msg}, State) -> - ok = send(Msg, State), - noreply(consumer_monitor(CTag, State)); - -handle_cast({command, Msg}, State) -> - ok = send(Msg, State), - noreply(State); - -handle_cast({deliver, _CTag, _AckReq, _Msg}, - State = #ch{cfg = #conf{state = closing}}) -> - noreply(State); -handle_cast({deliver, ConsumerTag, AckRequired, Msg}, State) -> - % TODO: handle as action - noreply(handle_deliver(ConsumerTag, AckRequired, Msg, State)); - -handle_cast({deliver_reply, _K, _Del}, - State = #ch{cfg = #conf{state = closing}}) -> - noreply(State); -handle_cast({deliver_reply, _K, _Del}, State = #ch{reply_consumer = none}) -> - noreply(State); -handle_cast({deliver_reply, Key, #delivery{message = - #basic_message{exchange_name = ExchangeName, - routing_keys = [RoutingKey | _CcRoutes], - content = Content}}}, - State = #ch{cfg = #conf{writer_pid = WriterPid}, - next_tag = DeliveryTag, - reply_consumer = {ConsumerTag, _Suffix, Key}}) -> - ok = rabbit_writer:send_command( - WriterPid, - #'basic.deliver'{consumer_tag = ConsumerTag, - delivery_tag = DeliveryTag, - redelivered = false, - exchange = ExchangeName#resource.name, - routing_key = RoutingKey}, - Content), - noreply(State); -handle_cast({deliver_reply, _K1, _}, State=#ch{reply_consumer = {_, _, _K2}}) -> - noreply(State); - -handle_cast({send_credit_reply, Len}, - State = #ch{cfg = #conf{writer_pid = WriterPid}}) -> - ok = rabbit_writer:send_command( - WriterPid, #'basic.credit_ok'{available = Len}), - noreply(State); - -handle_cast({send_drained, CTagCredit}, - State = #ch{cfg = #conf{writer_pid = WriterPid}}) -> - [ok = rabbit_writer:send_command( - WriterPid, #'basic.credit_drained'{consumer_tag = ConsumerTag, - credit_drained = CreditDrained}) - || {ConsumerTag, CreditDrained} <- CTagCredit], - noreply(State); - -% Note: https://www.pivotaltracker.com/story/show/166962656 -% This event is necessary for the stats timer to be initialized with -% the correct values once the management agent has started -handle_cast({force_event_refresh, Ref}, State) -> - rabbit_event:notify(channel_created, infos(?CREATION_EVENT_KEYS, State), - Ref), - noreply(rabbit_event:init_stats_timer(State, #ch.stats_timer)); - -handle_cast({mandatory_received, _MsgSeqNo}, State) -> - %% This feature was used by `rabbit_amqqueue_process` and - %% `rabbit_mirror_queue_slave` up-to and including RabbitMQ 3.7.x. - %% It is unused in 3.8.x and thus deprecated. We keep it to support - %% in-place upgrades to 3.8.x (i.e. mixed-version clusters), but it - %% is a no-op starting with that version. - %% - %% NB: don't call noreply/1 since we don't want to send confirms. - noreply_coalesce(State); - -handle_cast({reject_publish, _MsgSeqNo, QPid} = Evt, State) -> - %% For backwards compatibility - QRef = find_queue_name_from_pid(QPid, State#ch.queue_states), - case QRef of - undefined -> - %% ignore if no queue could be found for the given pid - noreply(State); - _ -> - handle_cast({queue_event, QRef, Evt}, State) - end; - -handle_cast({confirm, _MsgSeqNo, QPid} = Evt, State) -> - %% For backwards compatibility - QRef = find_queue_name_from_pid(QPid, State#ch.queue_states), - case QRef of - undefined -> - %% ignore if no queue could be found for the given pid - noreply(State); - _ -> - handle_cast({queue_event, QRef, Evt}, State) - end; -handle_cast({queue_event, QRef, Evt}, - #ch{queue_states = QueueStates0} = State0) -> - case rabbit_queue_type:handle_event(QRef, Evt, QueueStates0) of - {ok, QState1, Actions} -> - State1 = State0#ch{queue_states = QState1}, - State = handle_queue_actions(Actions, State1), - noreply_coalesce(State); - eol -> - State1 = handle_consuming_queue_down_or_eol(QRef, State0), - {ConfirmMXs, UC1} = - rabbit_confirms:remove_queue(QRef, State1#ch.unconfirmed), - %% Deleted queue is a special case. - %% Do not nack the "rejected" messages. - State2 = record_confirms(ConfirmMXs, - State1#ch{unconfirmed = UC1}), - erase_queue_stats(QRef), - noreply_coalesce( - State2#ch{queue_states = rabbit_queue_type:remove(QRef, QueueStates0)}); - {protocol_error, Type, Reason, ReasonArgs} -> - rabbit_misc:protocol_error(Type, Reason, ReasonArgs) - end. - -handle_info({ra_event, {Name, _} = From, Evt}, State) -> - %% For backwards compatibility - QRef = find_queue_name_from_quorum_name(Name, State#ch.queue_states), - handle_cast({queue_event, QRef, {From, Evt}}, State); - -handle_info({bump_credit, Msg}, State) -> - %% A rabbit_amqqueue_process is granting credit to our channel. If - %% our channel was being blocked by this process, and no other - %% process is blocking our channel, then this channel will be - %% unblocked. This means that any credit that was deferred will be - %% sent to rabbit_reader processs that might be blocked by this - %% particular channel. - credit_flow:handle_bump_msg(Msg), - noreply(State); - -handle_info(timeout, State) -> - noreply(State); - -handle_info(emit_stats, State) -> - emit_stats(State), - State1 = rabbit_event:reset_stats_timer(State, #ch.stats_timer), - %% NB: don't call noreply/1 since we don't want to kick off the - %% stats timer. - {noreply, send_confirms_and_nacks(State1), hibernate}; - -handle_info({'DOWN', _MRef, process, QPid, Reason}, - #ch{queue_states = QStates0, - queue_monitors = _QMons} = State0) -> - credit_flow:peer_down(QPid), - case rabbit_queue_type:handle_down(QPid, Reason, QStates0) of - {ok, QState1, Actions} -> - State1 = State0#ch{queue_states = QState1}, - State = handle_queue_actions(Actions, State1), - noreply_coalesce(State); - {eol, QRef} -> - State1 = handle_consuming_queue_down_or_eol(QRef, State0), - {ConfirmMXs, UC1} = - rabbit_confirms:remove_queue(QRef, State1#ch.unconfirmed), - %% Deleted queue is a special case. - %% Do not nack the "rejected" messages. - State2 = record_confirms(ConfirmMXs, - State1#ch{unconfirmed = UC1}), - erase_queue_stats(QRef), - noreply_coalesce( - State2#ch{queue_states = rabbit_queue_type:remove(QRef, QStates0)}) - end; - -handle_info({'EXIT', _Pid, Reason}, State) -> - {stop, Reason, State}; - -handle_info({{Ref, Node}, LateAnswer}, - State = #ch{cfg = #conf{channel = Channel}}) - when is_reference(Ref) -> - rabbit_log_channel:warning("Channel ~p ignoring late answer ~p from ~p", - [Channel, LateAnswer, Node]), - noreply(State); - -handle_info(tick, State0 = #ch{queue_states = QueueStates0}) -> - case get(permission_cache_can_expire) of - true -> ok = clear_permission_cache(); - _ -> ok - end, - case evaluate_consumer_timeout(State0#ch{queue_states = QueueStates0}) of - {noreply, State} -> - noreply(init_tick_timer(reset_tick_timer(State))); - Return -> - Return - end; -handle_info({update_user_state, User}, State = #ch{cfg = Cfg}) -> - noreply(State#ch{cfg = Cfg#conf{user = User}}). - - -handle_pre_hibernate(State0) -> - ok = clear_permission_cache(), - State = maybe_cancel_tick_timer(State0), - rabbit_event:if_enabled( - State, #ch.stats_timer, - fun () -> emit_stats(State, - [{idle_since, - os:system_time(milli_seconds)}]) - end), - {hibernate, rabbit_event:stop_stats_timer(State, #ch.stats_timer)}. - -handle_post_hibernate(State0) -> - State = init_tick_timer(State0), - {noreply, State}. - -terminate(_Reason, - State = #ch{cfg = #conf{user = #user{username = Username}}, - queue_states = QueueCtxs}) -> - _ = rabbit_queue_type:close(QueueCtxs), - {_Res, _State1} = notify_queues(State), - pg_local:leave(rabbit_channels, self()), - rabbit_event:if_enabled(State, #ch.stats_timer, - fun() -> emit_stats(State) end), - [delete_stats(Tag) || {Tag, _} <- get()], - rabbit_core_metrics:channel_closed(self()), - rabbit_event:notify(channel_closed, [{pid, self()}, - {user_who_performed_action, Username}]). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -format_message_queue(Opt, MQ) -> rabbit_misc:format_message_queue(Opt, MQ). - --spec get_max_message_size() -> non_neg_integer(). - -get_max_message_size() -> - case application:get_env(rabbit, max_message_size) of - {ok, MS} when is_integer(MS) -> - erlang:min(MS, ?MAX_MSG_SIZE); - _ -> - ?MAX_MSG_SIZE - end. - -get_consumer_timeout() -> - case application:get_env(rabbit, consumer_timeout) of - {ok, MS} when is_integer(MS) -> - MS; - _ -> - undefined - end. -%%--------------------------------------------------------------------------- - -reply(Reply, NewState) -> {reply, Reply, next_state(NewState), hibernate}. - -noreply(NewState) -> {noreply, next_state(NewState), hibernate}. - -next_state(State) -> ensure_stats_timer(send_confirms_and_nacks(State)). - -noreply_coalesce(State = #ch{confirmed = C, rejected = R}) -> - Timeout = case {C, R} of {[], []} -> hibernate; _ -> 0 end, - {noreply, ensure_stats_timer(State), Timeout}. - -ensure_stats_timer(State) -> - rabbit_event:ensure_stats_timer(State, #ch.stats_timer, emit_stats). - -return_ok(State, true, _Msg) -> {noreply, State}; -return_ok(State, false, Msg) -> {reply, Msg, State}. - -ok_msg(true, _Msg) -> undefined; -ok_msg(false, Msg) -> Msg. - -send(_Command, #ch{cfg = #conf{state = closing}}) -> - ok; -send(Command, #ch{cfg = #conf{writer_pid = WriterPid}}) -> - ok = rabbit_writer:send_command(WriterPid, Command). - -format_soft_error(#amqp_error{name = N, explanation = E, method = M}) -> - io_lib:format("operation ~s caused a channel exception ~s: ~ts", [M, N, E]). - -handle_exception(Reason, State = #ch{cfg = #conf{protocol = Protocol, - channel = Channel, - writer_pid = WriterPid, - reader_pid = ReaderPid, - conn_pid = ConnPid, - conn_name = ConnName, - virtual_host = VHost, - user = User - }}) -> - %% something bad's happened: notify_queues may not be 'ok' - {_Result, State1} = notify_queues(State), - case rabbit_binary_generator:map_exception(Channel, Reason, Protocol) of - {Channel, CloseMethod} -> - rabbit_log_channel:error( - "Channel error on connection ~p (~s, vhost: '~s'," - " user: '~s'), channel ~p:~n~s~n", - [ConnPid, ConnName, VHost, User#user.username, - Channel, format_soft_error(Reason)]), - ok = rabbit_writer:send_command(WriterPid, CloseMethod), - {noreply, State1}; - {0, _} -> - ReaderPid ! {channel_exit, Channel, Reason}, - {stop, normal, State1} - end. - --spec precondition_failed(string()) -> no_return(). - -precondition_failed(Format) -> precondition_failed(Format, []). - --spec precondition_failed(string(), [any()]) -> no_return(). - -precondition_failed(Format, Params) -> - rabbit_misc:protocol_error(precondition_failed, Format, Params). - -return_queue_declare_ok(#resource{name = ActualName}, - NoWait, MessageCount, ConsumerCount, - #ch{cfg = Cfg} = State) -> - return_ok(State#ch{cfg = Cfg#conf{most_recently_declared_queue = ActualName}}, - NoWait, #'queue.declare_ok'{queue = ActualName, - message_count = MessageCount, - consumer_count = ConsumerCount}). - -check_resource_access(User, Resource, Perm, Context) -> - V = {Resource, Context, Perm}, - - Cache = case get(permission_cache) of - undefined -> []; - Other -> Other - end, - case lists:member(V, Cache) of - true -> ok; - false -> ok = rabbit_access_control:check_resource_access( - User, Resource, Perm, Context), - CacheTail = lists:sublist(Cache, ?MAX_PERMISSION_CACHE_SIZE-1), - put(permission_cache, [V | CacheTail]) - end. - -clear_permission_cache() -> erase(permission_cache), - erase(topic_permission_cache), - ok. - -check_configure_permitted(Resource, User, Context) -> - check_resource_access(User, Resource, configure, Context). - -check_write_permitted(Resource, User, Context) -> - check_resource_access(User, Resource, write, Context). - -check_read_permitted(Resource, User, Context) -> - check_resource_access(User, Resource, read, Context). - -check_write_permitted_on_topic(Resource, User, RoutingKey, AuthzContext) -> - check_topic_authorisation(Resource, User, RoutingKey, AuthzContext, write). - -check_read_permitted_on_topic(Resource, User, RoutingKey, AuthzContext) -> - check_topic_authorisation(Resource, User, RoutingKey, AuthzContext, read). - -check_user_id_header(#'P_basic'{user_id = undefined}, _) -> - ok; -check_user_id_header(#'P_basic'{user_id = Username}, - #ch{cfg = #conf{user = #user{username = Username}}}) -> - ok; -check_user_id_header( - #'P_basic'{}, #ch{cfg = #conf{user = #user{authz_backends = - [{rabbit_auth_backend_dummy, _}]}}}) -> - ok; -check_user_id_header(#'P_basic'{user_id = Claimed}, - #ch{cfg = #conf{user = #user{username = Actual, - tags = Tags}}}) -> - case lists:member(impersonator, Tags) of - true -> ok; - false -> precondition_failed( - "user_id property set to '~s' but authenticated user was " - "'~s'", [Claimed, Actual]) - end. - -check_expiration_header(Props) -> - case rabbit_basic:parse_expiration(Props) of - {ok, _} -> ok; - {error, E} -> precondition_failed("invalid expiration '~s': ~p", - [Props#'P_basic'.expiration, E]) - end. - -check_internal_exchange(#exchange{name = Name, internal = true}) -> - rabbit_misc:protocol_error(access_refused, - "cannot publish to internal ~s", - [rabbit_misc:rs(Name)]); -check_internal_exchange(_) -> - ok. - -check_topic_authorisation(#exchange{name = Name = #resource{virtual_host = VHost}, type = topic}, - User = #user{username = Username}, - RoutingKey, AuthzContext, Permission) -> - Resource = Name#resource{kind = topic}, - VariableMap = build_topic_variable_map(AuthzContext, VHost, Username), - Context = #{routing_key => RoutingKey, - variable_map => VariableMap}, - Cache = case get(topic_permission_cache) of - undefined -> []; - Other -> Other - end, - case lists:member({Resource, Context, Permission}, Cache) of - true -> ok; - false -> ok = rabbit_access_control:check_topic_access( - User, Resource, Permission, Context), - CacheTail = lists:sublist(Cache, ?MAX_PERMISSION_CACHE_SIZE-1), - put(topic_permission_cache, [{Resource, Context, Permission} | CacheTail]) - end; -check_topic_authorisation(_, _, _, _, _) -> - ok. - - -build_topic_variable_map(AuthzContext, VHost, Username) when is_map(AuthzContext) -> - maps:merge(AuthzContext, #{<<"vhost">> => VHost, <<"username">> => Username}); -build_topic_variable_map(AuthzContext, VHost, Username) -> - maps:merge(extract_variable_map_from_amqp_params(AuthzContext), #{<<"vhost">> => VHost, <<"username">> => Username}). - -%% Use tuple representation of amqp_params to avoid a dependency on amqp_client. -%% Extracts variable map only from amqp_params_direct, not amqp_params_network. -%% amqp_params_direct records are usually used by plugins (e.g. MQTT, STOMP) -extract_variable_map_from_amqp_params({amqp_params, {amqp_params_direct, _, _, _, _, - {amqp_adapter_info, _,_,_,_,_,_,AdditionalInfo}, _}}) -> - proplists:get_value(variable_map, AdditionalInfo, #{}); -extract_variable_map_from_amqp_params({amqp_params_direct, _, _, _, _, - {amqp_adapter_info, _,_,_,_,_,_,AdditionalInfo}, _}) -> - proplists:get_value(variable_map, AdditionalInfo, #{}); -extract_variable_map_from_amqp_params([Value]) -> - extract_variable_map_from_amqp_params(Value); -extract_variable_map_from_amqp_params(_) -> - #{}. - -check_msg_size(Content, MaxMessageSize, GCThreshold) -> - Size = rabbit_basic:maybe_gc_large_msg(Content, GCThreshold), - case Size of - S when S > MaxMessageSize -> - ErrorMessage = case MaxMessageSize of - ?MAX_MSG_SIZE -> - "message size ~B is larger than max size ~B"; - _ -> - "message size ~B is larger than configured max size ~B" - end, - precondition_failed(ErrorMessage, - [Size, MaxMessageSize]); - _ -> ok - end. - -check_vhost_queue_limit(#resource{name = QueueName}, VHost) -> - case rabbit_vhost_limit:is_over_queue_limit(VHost) of - false -> ok; - {true, Limit} -> precondition_failed("cannot declare queue '~s': " - "queue limit in vhost '~s' (~p) is reached", - [QueueName, VHost, Limit]) - - end. - -qbin_to_resource(QueueNameBin, VHostPath) -> - name_to_resource(queue, QueueNameBin, VHostPath). - -name_to_resource(Type, NameBin, VHostPath) -> - rabbit_misc:r(VHostPath, Type, NameBin). - -expand_queue_name_shortcut(<<>>, #ch{cfg = #conf{most_recently_declared_queue = <<>>}}) -> - rabbit_misc:protocol_error(not_found, "no previously declared queue", []); -expand_queue_name_shortcut(<<>>, #ch{cfg = #conf{most_recently_declared_queue = MRDQ}}) -> - MRDQ; -expand_queue_name_shortcut(QueueNameBin, _) -> - QueueNameBin. - -expand_routing_key_shortcut(<<>>, <<>>, - #ch{cfg = #conf{most_recently_declared_queue = <<>>}}) -> - rabbit_misc:protocol_error(not_found, "no previously declared queue", []); -expand_routing_key_shortcut(<<>>, <<>>, - #ch{cfg = #conf{most_recently_declared_queue = MRDQ}}) -> - MRDQ; -expand_routing_key_shortcut(_QueueNameBin, RoutingKey, _State) -> - RoutingKey. - -expand_shortcuts(#'basic.get' {queue = Q} = M, State) -> - M#'basic.get' {queue = expand_queue_name_shortcut(Q, State)}; -expand_shortcuts(#'basic.consume'{queue = Q} = M, State) -> - M#'basic.consume'{queue = expand_queue_name_shortcut(Q, State)}; -expand_shortcuts(#'queue.delete' {queue = Q} = M, State) -> - M#'queue.delete' {queue = expand_queue_name_shortcut(Q, State)}; -expand_shortcuts(#'queue.purge' {queue = Q} = M, State) -> - M#'queue.purge' {queue = expand_queue_name_shortcut(Q, State)}; -expand_shortcuts(#'queue.bind' {queue = Q, routing_key = K} = M, State) -> - M#'queue.bind' {queue = expand_queue_name_shortcut(Q, State), - routing_key = expand_routing_key_shortcut(Q, K, State)}; -expand_shortcuts(#'queue.unbind' {queue = Q, routing_key = K} = M, State) -> - M#'queue.unbind' {queue = expand_queue_name_shortcut(Q, State), - routing_key = expand_routing_key_shortcut(Q, K, State)}; -expand_shortcuts(M, _State) -> - M. - -check_not_default_exchange(#resource{kind = exchange, name = <<"">>}) -> - rabbit_misc:protocol_error( - access_refused, "operation not permitted on the default exchange", []); -check_not_default_exchange(_) -> - ok. - -check_exchange_deletion(XName = #resource{name = <<"amq.", _/binary>>, - kind = exchange}) -> - rabbit_misc:protocol_error( - access_refused, "deletion of system ~s not allowed", - [rabbit_misc:rs(XName)]); -check_exchange_deletion(_) -> - ok. - -%% check that an exchange/queue name does not contain the reserved -%% "amq." prefix. -%% -%% As per the AMQP 0-9-1 spec, the exclusion of "amq." prefixed names -%% only applies on actual creation, and not in the cases where the -%% entity already exists or passive=true. -%% -%% NB: We deliberately do not enforce the other constraints on names -%% required by the spec. -check_name(Kind, NameBin = <<"amq.", _/binary>>) -> - rabbit_misc:protocol_error( - access_refused, - "~s name '~s' contains reserved prefix 'amq.*'",[Kind, NameBin]); -check_name(_Kind, NameBin) -> - NameBin. - -strip_cr_lf(NameBin) -> - binary:replace(NameBin, [<<"\n">>, <<"\r">>], <<"">>, [global]). - - -maybe_set_fast_reply_to( - C = #content{properties = P = #'P_basic'{reply_to = - <<"amq.rabbitmq.reply-to">>}}, - #ch{reply_consumer = ReplyConsumer}) -> - case ReplyConsumer of - none -> rabbit_misc:protocol_error( - precondition_failed, - "fast reply consumer does not exist", []); - {_, Suf, _K} -> Rep = <<"amq.rabbitmq.reply-to.", Suf/binary>>, - rabbit_binary_generator:clear_encoded_content( - C#content{properties = P#'P_basic'{reply_to = Rep}}) - end; -maybe_set_fast_reply_to(C, _State) -> - C. - -record_rejects([], State) -> - State; -record_rejects(MXs, State = #ch{rejected = R, tx = Tx}) -> - Tx1 = case Tx of - none -> none; - _ -> failed - end, - State#ch{rejected = [MXs | R], tx = Tx1}. - -record_confirms([], State) -> - State; -record_confirms(MXs, State = #ch{confirmed = C}) -> - State#ch{confirmed = [MXs | C]}. - -handle_method({Method, Content}, State) -> - handle_method(Method, Content, State). - -handle_method(#'channel.open'{}, _, - State = #ch{cfg = #conf{state = starting} = Cfg}) -> - %% Don't leave "starting" as the state for 5s. TODO is this TRTTD? - State1 = State#ch{cfg = Cfg#conf{state = running}}, - rabbit_event:if_enabled(State1, #ch.stats_timer, - fun() -> emit_stats(State1) end), - {reply, #'channel.open_ok'{}, State1}; - -handle_method(#'channel.open'{}, _, _State) -> - rabbit_misc:protocol_error( - channel_error, "second 'channel.open' seen", []); - -handle_method(_Method, _, #ch{cfg = #conf{state = starting}}) -> - rabbit_misc:protocol_error(channel_error, "expected 'channel.open'", []); - -handle_method(#'channel.close_ok'{}, _, #ch{cfg = #conf{state = closing}}) -> - stop; - -handle_method(#'channel.close'{}, _, - State = #ch{cfg = #conf{state = closing, - writer_pid = WriterPid}}) -> - ok = rabbit_writer:send_command(WriterPid, #'channel.close_ok'{}), - {noreply, State}; - -handle_method(_Method, _, State = #ch{cfg = #conf{state = closing}}) -> - {noreply, State}; - -handle_method(#'channel.close'{}, _, - State = #ch{cfg = #conf{reader_pid = ReaderPid}}) -> - {_Result, State1} = notify_queues(State), - %% We issue the channel.close_ok response after a handshake with - %% the reader, the other half of which is ready_for_close. That - %% way the reader forgets about the channel before we send the - %% response (and this channel process terminates). If we didn't do - %% that, a channel.open for the same channel number, which a - %% client is entitled to send as soon as it has received the - %% close_ok, might be received by the reader before it has seen - %% the termination and hence be sent to the old, now dead/dying - %% channel process, instead of a new process, and thus lost. - ReaderPid ! {channel_closing, self()}, - {noreply, State1}; - -%% Even though the spec prohibits the client from sending commands -%% while waiting for the reply to a synchronous command, we generally -%% do allow this...except in the case of a pending tx.commit, where -%% it could wreak havoc. -handle_method(_Method, _, #ch{tx = Tx}) - when Tx =:= committing orelse Tx =:= failed -> - rabbit_misc:protocol_error( - channel_error, "unexpected command while processing 'tx.commit'", []); - -handle_method(#'access.request'{},_, State) -> - {reply, #'access.request_ok'{ticket = 1}, State}; - -handle_method(#'basic.publish'{immediate = true}, _Content, _State) -> - rabbit_misc:protocol_error(not_implemented, "immediate=true", []); - -handle_method(#'basic.publish'{exchange = ExchangeNameBin, - routing_key = RoutingKey, - mandatory = Mandatory}, - Content, State = #ch{cfg = #conf{channel = ChannelNum, - conn_name = ConnName, - virtual_host = VHostPath, - user = #user{username = Username} = User, - trace_state = TraceState, - max_message_size = MaxMessageSize, - authz_context = AuthzContext, - writer_gc_threshold = GCThreshold - }, - tx = Tx, - confirm_enabled = ConfirmEnabled, - delivery_flow = Flow - }) -> - check_msg_size(Content, MaxMessageSize, GCThreshold), - ExchangeName = rabbit_misc:r(VHostPath, exchange, ExchangeNameBin), - check_write_permitted(ExchangeName, User, AuthzContext), - Exchange = rabbit_exchange:lookup_or_die(ExchangeName), - check_internal_exchange(Exchange), - check_write_permitted_on_topic(Exchange, User, RoutingKey, AuthzContext), - %% We decode the content's properties here because we're almost - %% certain to want to look at delivery-mode and priority. - DecodedContent = #content {properties = Props} = - maybe_set_fast_reply_to( - rabbit_binary_parser:ensure_content_decoded(Content), State), - check_user_id_header(Props, State), - check_expiration_header(Props), - DoConfirm = Tx =/= none orelse ConfirmEnabled, - {MsgSeqNo, State1} = - case DoConfirm orelse Mandatory of - false -> {undefined, State}; - true -> SeqNo = State#ch.publish_seqno, - {SeqNo, State#ch{publish_seqno = SeqNo + 1}} - end, - case rabbit_basic:message(ExchangeName, RoutingKey, DecodedContent) of - {ok, Message} -> - Delivery = rabbit_basic:delivery( - Mandatory, DoConfirm, Message, MsgSeqNo), - QNames = rabbit_exchange:route(Exchange, Delivery), - rabbit_trace:tap_in(Message, QNames, ConnName, ChannelNum, - Username, TraceState), - DQ = {Delivery#delivery{flow = Flow}, QNames}, - {noreply, case Tx of - none -> deliver_to_queues(DQ, State1); - {Msgs, Acks} -> Msgs1 = ?QUEUE:in(DQ, Msgs), - State1#ch{tx = {Msgs1, Acks}} - end}; - {error, Reason} -> - precondition_failed("invalid message: ~p", [Reason]) - end; - -handle_method(#'basic.nack'{delivery_tag = DeliveryTag, - multiple = Multiple, - requeue = Requeue}, _, State) -> - reject(DeliveryTag, Requeue, Multiple, State); - -handle_method(#'basic.ack'{delivery_tag = DeliveryTag, - multiple = Multiple}, - _, State = #ch{unacked_message_q = UAMQ, tx = Tx}) -> - {Acked, Remaining} = collect_acks(UAMQ, DeliveryTag, Multiple), - State1 = State#ch{unacked_message_q = Remaining}, - {noreply, case Tx of - none -> {State2, Actions} = ack(Acked, State1), - handle_queue_actions(Actions, State2); - {Msgs, Acks} -> Acks1 = ack_cons(ack, Acked, Acks), - State1#ch{tx = {Msgs, Acks1}} - end}; - -handle_method(#'basic.get'{queue = QueueNameBin, no_ack = NoAck}, - _, State = #ch{cfg = #conf{writer_pid = WriterPid, - conn_pid = ConnPid, - user = User, - virtual_host = VHostPath, - authz_context = AuthzContext - }, - limiter = Limiter, - next_tag = DeliveryTag, - queue_states = QueueStates0}) -> - QueueName = qbin_to_resource(QueueNameBin, VHostPath), - check_read_permitted(QueueName, User, AuthzContext), - case rabbit_amqqueue:with_exclusive_access_or_die( - QueueName, ConnPid, - %% Use the delivery tag as consumer tag for quorum queues - fun (Q) -> - rabbit_queue_type:dequeue( - Q, NoAck, rabbit_limiter:pid(Limiter), - DeliveryTag, QueueStates0) - end) of - {ok, MessageCount, Msg, QueueStates} -> - handle_basic_get(WriterPid, DeliveryTag, NoAck, MessageCount, Msg, - State#ch{queue_states = QueueStates}); - {empty, QueueStates} -> - ?INCR_STATS(queue_stats, QueueName, 1, get_empty, State), - {reply, #'basic.get_empty'{}, State#ch{queue_states = QueueStates}}; - empty -> - ?INCR_STATS(queue_stats, QueueName, 1, get_empty, State), - {reply, #'basic.get_empty'{}, State}; - {error, {unsupported, single_active_consumer}} -> - rabbit_misc:protocol_error( - resource_locked, - "cannot obtain access to locked ~s. basic.get operations " - "are not supported by quorum queues with single active consumer", - [rabbit_misc:rs(QueueName)]); - {error, Reason} -> - %% TODO add queue type to error message - rabbit_misc:protocol_error(internal_error, - "Cannot get a message from queue '~s': ~p", - [rabbit_misc:rs(QueueName), Reason]); - {protocol_error, Type, Reason, ReasonArgs} -> - rabbit_misc:protocol_error(Type, Reason, ReasonArgs) - end; - -handle_method(#'basic.consume'{queue = <<"amq.rabbitmq.reply-to">>, - consumer_tag = CTag0, - no_ack = NoAck, - nowait = NoWait}, - _, State = #ch{reply_consumer = ReplyConsumer, - consumer_mapping = ConsumerMapping}) -> - case maps:find(CTag0, ConsumerMapping) of - error -> - case {ReplyConsumer, NoAck} of - {none, true} -> - CTag = case CTag0 of - <<>> -> rabbit_guid:binary( - rabbit_guid:gen_secure(), "amq.ctag"); - Other -> Other - end, - %% Precalculate both suffix and key; base64 encoding is - %% expensive - Key = base64:encode(rabbit_guid:gen_secure()), - PidEnc = base64:encode(term_to_binary(self())), - Suffix = <<PidEnc/binary, ".", Key/binary>>, - Consumer = {CTag, Suffix, binary_to_list(Key)}, - State1 = State#ch{reply_consumer = Consumer}, - case NoWait of - true -> {noreply, State1}; - false -> Rep = #'basic.consume_ok'{consumer_tag = CTag}, - {reply, Rep, State1} - end; - {_, false} -> - rabbit_misc:protocol_error( - precondition_failed, - "reply consumer cannot acknowledge", []); - _ -> - rabbit_misc:protocol_error( - precondition_failed, "reply consumer already set", []) - end; - {ok, _} -> - %% Attempted reuse of consumer tag. - rabbit_misc:protocol_error( - not_allowed, "attempt to reuse consumer tag '~s'", [CTag0]) - end; - -handle_method(#'basic.cancel'{consumer_tag = ConsumerTag, nowait = NoWait}, - _, State = #ch{reply_consumer = {ConsumerTag, _, _}}) -> - State1 = State#ch{reply_consumer = none}, - case NoWait of - true -> {noreply, State1}; - false -> Rep = #'basic.cancel_ok'{consumer_tag = ConsumerTag}, - {reply, Rep, State1} - end; - -handle_method(#'basic.consume'{queue = QueueNameBin, - consumer_tag = ConsumerTag, - no_local = _, % FIXME: implement - no_ack = NoAck, - exclusive = ExclusiveConsume, - nowait = NoWait, - arguments = Args}, - _, State = #ch{cfg = #conf{consumer_prefetch = ConsumerPrefetch, - user = User, - virtual_host = VHostPath, - authz_context = AuthzContext}, - consumer_mapping = ConsumerMapping - }) -> - case maps:find(ConsumerTag, ConsumerMapping) of - error -> - QueueName = qbin_to_resource(QueueNameBin, VHostPath), - check_read_permitted(QueueName, User, AuthzContext), - ActualConsumerTag = - case ConsumerTag of - <<>> -> rabbit_guid:binary(rabbit_guid:gen_secure(), - "amq.ctag"); - Other -> Other - end, - case basic_consume( - QueueName, NoAck, ConsumerPrefetch, ActualConsumerTag, - ExclusiveConsume, Args, NoWait, State) of - {ok, State1} -> - {noreply, State1}; - {error, exclusive_consume_unavailable} -> - rabbit_misc:protocol_error( - access_refused, "~s in exclusive use", - [rabbit_misc:rs(QueueName)]); - {error, global_qos_not_supported_for_queue_type} -> - rabbit_misc:protocol_error( - not_implemented, "~s does not support global qos", - [rabbit_misc:rs(QueueName)]) - end; - {ok, _} -> - %% Attempted reuse of consumer tag. - rabbit_misc:protocol_error( - not_allowed, "attempt to reuse consumer tag '~s'", [ConsumerTag]) - end; - -handle_method(#'basic.cancel'{consumer_tag = ConsumerTag, nowait = NoWait}, - _, State = #ch{cfg = #conf{user = #user{username = Username}}, - consumer_mapping = ConsumerMapping, - queue_consumers = QCons, - queue_states = QueueStates0}) -> - OkMsg = #'basic.cancel_ok'{consumer_tag = ConsumerTag}, - case maps:find(ConsumerTag, ConsumerMapping) of - error -> - %% Spec requires we ignore this situation. - return_ok(State, NoWait, OkMsg); - {ok, {Q, _CParams}} when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - - ConsumerMapping1 = maps:remove(ConsumerTag, ConsumerMapping), - QCons1 = - case maps:find(QName, QCons) of - error -> QCons; - {ok, CTags} -> CTags1 = gb_sets:delete(ConsumerTag, CTags), - case gb_sets:is_empty(CTags1) of - true -> maps:remove(QName, QCons); - false -> maps:put(QName, CTags1, QCons) - end - end, - NewState = State#ch{consumer_mapping = ConsumerMapping1, - queue_consumers = QCons1}, - %% In order to ensure that no more messages are sent to - %% the consumer after the cancel_ok has been sent, we get - %% the queue process to send the cancel_ok on our - %% behalf. If we were sending the cancel_ok ourselves it - %% might overtake a message sent previously by the queue. - case rabbit_misc:with_exit_handler( - fun () -> {error, not_found} end, - fun () -> - rabbit_queue_type:cancel( - Q, ConsumerTag, ok_msg(NoWait, OkMsg), - Username, QueueStates0) - end) of - {ok, QueueStates} -> - {noreply, NewState#ch{queue_states = QueueStates}}; - {error, not_found} -> - %% Spec requires we ignore this situation. - return_ok(NewState, NoWait, OkMsg) - end - end; - -handle_method(#'basic.qos'{prefetch_size = Size}, _, _State) when Size /= 0 -> - rabbit_misc:protocol_error(not_implemented, - "prefetch_size!=0 (~w)", [Size]); - -handle_method(#'basic.qos'{global = false, - prefetch_count = PrefetchCount}, - _, State = #ch{cfg = Cfg, - limiter = Limiter}) -> - %% Ensures that if default was set, it's overridden - Limiter1 = rabbit_limiter:unlimit_prefetch(Limiter), - {reply, #'basic.qos_ok'{}, State#ch{cfg = Cfg#conf{consumer_prefetch = PrefetchCount}, - limiter = Limiter1}}; - -handle_method(#'basic.qos'{global = true, - prefetch_count = 0}, - _, State = #ch{limiter = Limiter}) -> - Limiter1 = rabbit_limiter:unlimit_prefetch(Limiter), - {reply, #'basic.qos_ok'{}, State#ch{limiter = Limiter1}}; - -handle_method(#'basic.qos'{global = true, - prefetch_count = PrefetchCount}, - _, State = #ch{limiter = Limiter, unacked_message_q = UAMQ}) -> - %% TODO ?QUEUE:len(UAMQ) is not strictly right since that counts - %% unacked messages from basic.get too. Pretty obscure though. - Limiter1 = rabbit_limiter:limit_prefetch(Limiter, - PrefetchCount, ?QUEUE:len(UAMQ)), - case ((not rabbit_limiter:is_active(Limiter)) andalso - rabbit_limiter:is_active(Limiter1)) of - true -> rabbit_amqqueue:activate_limit_all( - classic_consumer_queue_pids(State#ch.consumer_mapping), self()); - false -> ok - end, - {reply, #'basic.qos_ok'{}, State#ch{limiter = Limiter1}}; - -handle_method(#'basic.recover_async'{requeue = true}, - _, State = #ch{unacked_message_q = UAMQ, - limiter = Limiter, - queue_states = QueueStates0}) -> - OkFun = fun () -> ok end, - UAMQL = ?QUEUE:to_list(UAMQ), - {QueueStates, Actions} = - foreach_per_queue( - fun ({QPid, CTag}, MsgIds, {Acc0, Actions0}) -> - rabbit_misc:with_exit_handler( - OkFun, - fun () -> - {ok, Acc, Act} = rabbit_amqqueue:requeue(QPid, {CTag, MsgIds}, Acc0), - {Acc, Act ++ Actions0} - end) - end, lists:reverse(UAMQL), {QueueStates0, []}), - ok = notify_limiter(Limiter, UAMQL), - State1 = handle_queue_actions(Actions, State#ch{unacked_message_q = ?QUEUE:new(), - queue_states = QueueStates}), - %% No answer required - basic.recover is the newer, synchronous - %% variant of this method - {noreply, State1}; - -handle_method(#'basic.recover_async'{requeue = false}, _, _State) -> - rabbit_misc:protocol_error(not_implemented, "requeue=false", []); - -handle_method(#'basic.recover'{requeue = Requeue}, Content, State) -> - {noreply, State1} = handle_method(#'basic.recover_async'{requeue = Requeue}, - Content, State), - {reply, #'basic.recover_ok'{}, State1}; - -handle_method(#'basic.reject'{delivery_tag = DeliveryTag, requeue = Requeue}, - _, State) -> - reject(DeliveryTag, Requeue, false, State); - -handle_method(#'exchange.declare'{nowait = NoWait} = Method, - _, State = #ch{cfg = #conf{virtual_host = VHostPath, - user = User, - queue_collector_pid = CollectorPid, - conn_pid = ConnPid, - authz_context = AuthzContext}}) -> - handle_method(Method, ConnPid, AuthzContext, CollectorPid, VHostPath, User), - return_ok(State, NoWait, #'exchange.declare_ok'{}); - -handle_method(#'exchange.delete'{nowait = NoWait} = Method, - _, State = #ch{cfg = #conf{conn_pid = ConnPid, - authz_context = AuthzContext, - virtual_host = VHostPath, - queue_collector_pid = CollectorPid, - user = User}}) -> - handle_method(Method, ConnPid, AuthzContext, CollectorPid, VHostPath, User), - return_ok(State, NoWait, #'exchange.delete_ok'{}); - -handle_method(#'exchange.bind'{nowait = NoWait} = Method, - _, State = #ch{cfg = #conf{virtual_host = VHostPath, - conn_pid = ConnPid, - authz_context = AuthzContext, - queue_collector_pid = CollectorPid, - user = User}}) -> - handle_method(Method, ConnPid, AuthzContext, CollectorPid, VHostPath, User), - return_ok(State, NoWait, #'exchange.bind_ok'{}); - -handle_method(#'exchange.unbind'{nowait = NoWait} = Method, - _, State = #ch{cfg = #conf{virtual_host = VHostPath, - conn_pid = ConnPid, - authz_context = AuthzContext, - queue_collector_pid = CollectorPid, - user = User}}) -> - handle_method(Method, ConnPid, AuthzContext, CollectorPid, VHostPath, User), - return_ok(State, NoWait, #'exchange.unbind_ok'{}); - -handle_method(#'queue.declare'{nowait = NoWait} = Method, - _, State = #ch{cfg = #conf{virtual_host = VHostPath, - conn_pid = ConnPid, - authz_context = AuthzContext, - queue_collector_pid = CollectorPid, - user = User}}) -> - {ok, QueueName, MessageCount, ConsumerCount} = - handle_method(Method, ConnPid, AuthzContext, CollectorPid, VHostPath, User), - return_queue_declare_ok(QueueName, NoWait, MessageCount, - ConsumerCount, State); - -handle_method(#'queue.delete'{nowait = NoWait} = Method, _, - State = #ch{cfg = #conf{conn_pid = ConnPid, - authz_context = AuthzContext, - virtual_host = VHostPath, - queue_collector_pid = CollectorPid, - user = User}}) -> - {ok, PurgedMessageCount} = - handle_method(Method, ConnPid, AuthzContext, CollectorPid, VHostPath, User), - return_ok(State, NoWait, - #'queue.delete_ok'{message_count = PurgedMessageCount}); - -handle_method(#'queue.bind'{nowait = NoWait} = Method, _, - State = #ch{cfg = #conf{conn_pid = ConnPid, - authz_context = AuthzContext, - user = User, - queue_collector_pid = CollectorPid, - virtual_host = VHostPath}}) -> - handle_method(Method, ConnPid, AuthzContext, CollectorPid, VHostPath, User), - return_ok(State, NoWait, #'queue.bind_ok'{}); - -handle_method(#'queue.unbind'{} = Method, _, - State = #ch{cfg = #conf{conn_pid = ConnPid, - authz_context = AuthzContext, - user = User, - queue_collector_pid = CollectorPid, - virtual_host = VHostPath}}) -> - handle_method(Method, ConnPid, AuthzContext, CollectorPid, VHostPath, User), - return_ok(State, false, #'queue.unbind_ok'{}); - -handle_method(#'queue.purge'{nowait = NoWait} = Method, - _, State = #ch{cfg = #conf{conn_pid = ConnPid, - authz_context = AuthzContext, - user = User, - queue_collector_pid = CollectorPid, - virtual_host = VHostPath}}) -> - case handle_method(Method, ConnPid, AuthzContext, CollectorPid, - VHostPath, User) of - {ok, PurgedMessageCount} -> - return_ok(State, NoWait, - #'queue.purge_ok'{message_count = PurgedMessageCount}) - end; - -handle_method(#'tx.select'{}, _, #ch{confirm_enabled = true}) -> - precondition_failed("cannot switch from confirm to tx mode"); - -handle_method(#'tx.select'{}, _, State = #ch{tx = none}) -> - {reply, #'tx.select_ok'{}, State#ch{tx = new_tx()}}; - -handle_method(#'tx.select'{}, _, State) -> - {reply, #'tx.select_ok'{}, State}; - -handle_method(#'tx.commit'{}, _, #ch{tx = none}) -> - precondition_failed("channel is not transactional"); - -handle_method(#'tx.commit'{}, _, State = #ch{tx = {Msgs, Acks}, - limiter = Limiter}) -> - State1 = queue_fold(fun deliver_to_queues/2, State, Msgs), - Rev = fun (X) -> lists:reverse(lists:sort(X)) end, - {State2, Actions2} = - lists:foldl(fun ({ack, A}, {Acc, Actions}) -> - {Acc0, Actions0} = ack(Rev(A), Acc), - {Acc0, Actions ++ Actions0}; - ({Requeue, A}, {Acc, Actions}) -> - {Acc0, Actions0} = internal_reject(Requeue, Rev(A), Limiter, Acc), - {Acc0, Actions ++ Actions0} - end, {State1, []}, lists:reverse(Acks)), - State3 = handle_queue_actions(Actions2, State2), - {noreply, maybe_complete_tx(State3#ch{tx = committing})}; - -handle_method(#'tx.rollback'{}, _, #ch{tx = none}) -> - precondition_failed("channel is not transactional"); - -handle_method(#'tx.rollback'{}, _, State = #ch{unacked_message_q = UAMQ, - tx = {_Msgs, Acks}}) -> - AcksL = lists:append(lists:reverse([lists:reverse(L) || {_, L} <- Acks])), - UAMQ1 = ?QUEUE:from_list(lists:usort(AcksL ++ ?QUEUE:to_list(UAMQ))), - {reply, #'tx.rollback_ok'{}, State#ch{unacked_message_q = UAMQ1, - tx = new_tx()}}; - -handle_method(#'confirm.select'{}, _, #ch{tx = {_, _}}) -> - precondition_failed("cannot switch from tx to confirm mode"); - -handle_method(#'confirm.select'{nowait = NoWait}, _, State) -> - return_ok(State#ch{confirm_enabled = true}, - NoWait, #'confirm.select_ok'{}); - -handle_method(#'channel.flow'{active = true}, _, State) -> - {reply, #'channel.flow_ok'{active = true}, State}; - -handle_method(#'channel.flow'{active = false}, _, _State) -> - rabbit_misc:protocol_error(not_implemented, "active=false", []); - -handle_method(#'basic.credit'{consumer_tag = CTag, - credit = Credit, - drain = Drain}, - _, State = #ch{consumer_mapping = Consumers, - queue_states = QStates0}) -> - case maps:find(CTag, Consumers) of - {ok, {Q, _CParams}} -> - {ok, QStates, Actions} = rabbit_queue_type:credit(Q, CTag, Credit, Drain, QStates0), - {noreply, handle_queue_actions(Actions, State#ch{queue_states = QStates})}; - error -> precondition_failed( - "unknown consumer tag '~s'", [CTag]) - end; - -handle_method(_MethodRecord, _Content, _State) -> - rabbit_misc:protocol_error( - command_invalid, "unimplemented method", []). - -%%---------------------------------------------------------------------------- - -%% We get the queue process to send the consume_ok on our behalf. This -%% is for symmetry with basic.cancel - see the comment in that method -%% for why. -basic_consume(QueueName, NoAck, ConsumerPrefetch, ActualConsumerTag, - ExclusiveConsume, Args, NoWait, - State = #ch{cfg = #conf{conn_pid = ConnPid, - user = #user{username = Username}}, - limiter = Limiter, - consumer_mapping = ConsumerMapping, - queue_states = QueueStates0}) -> - case rabbit_amqqueue:with_exclusive_access_or_die( - QueueName, ConnPid, - fun (Q) -> - {rabbit_amqqueue:basic_consume( - Q, NoAck, self(), - rabbit_limiter:pid(Limiter), - rabbit_limiter:is_active(Limiter), - ConsumerPrefetch, ActualConsumerTag, - ExclusiveConsume, Args, - ok_msg(NoWait, #'basic.consume_ok'{ - consumer_tag = ActualConsumerTag}), - Username, QueueStates0), - Q} - end) of - {{ok, QueueStates, Actions}, Q} when ?is_amqqueue(Q) -> - CM1 = maps:put( - ActualConsumerTag, - {Q, {NoAck, ConsumerPrefetch, ExclusiveConsume, Args}}, - ConsumerMapping), - - State1 = State#ch{consumer_mapping = CM1, - queue_states = QueueStates}, - State2 = handle_queue_actions(Actions, State1), - {ok, case NoWait of - true -> consumer_monitor(ActualConsumerTag, State2); - false -> State2 - end}; - {{error, exclusive_consume_unavailable} = E, _Q} -> - E; - {{error, global_qos_not_supported_for_queue_type} = E, _Q} -> - E; - {{protocol_error, Type, Reason, ReasonArgs}, _Q} -> - rabbit_misc:protocol_error(Type, Reason, ReasonArgs) - end. - -maybe_stat(false, Q) -> rabbit_amqqueue:stat(Q); -maybe_stat(true, _Q) -> {ok, 0, 0}. - -consumer_monitor(ConsumerTag, - State = #ch{consumer_mapping = ConsumerMapping, - queue_consumers = QCons}) -> - {Q, _} = maps:get(ConsumerTag, ConsumerMapping), - QRef = amqqueue:get_name(Q), - CTags1 = case maps:find(QRef, QCons) of - {ok, CTags} -> gb_sets:insert(ConsumerTag, CTags); - error -> gb_sets:singleton(ConsumerTag) - end, - QCons1 = maps:put(QRef, CTags1, QCons), - State#ch{queue_consumers = QCons1}. - -handle_consuming_queue_down_or_eol(QName, - State = #ch{queue_consumers = QCons}) -> - ConsumerTags = case maps:find(QName, QCons) of - error -> gb_sets:new(); - {ok, CTags} -> CTags - end, - gb_sets:fold( - fun (CTag, StateN = #ch{consumer_mapping = CMap}) -> - case queue_down_consumer_action(CTag, CMap) of - remove -> - cancel_consumer(CTag, QName, StateN); - {recover, {NoAck, ConsumerPrefetch, Exclusive, Args}} -> - case catch basic_consume( - QName, NoAck, ConsumerPrefetch, CTag, - Exclusive, Args, true, StateN) of - {ok, StateN1} -> - StateN1; - _Err -> - cancel_consumer(CTag, QName, StateN) - end - end - end, State#ch{queue_consumers = maps:remove(QName, QCons)}, ConsumerTags). - -%% [0] There is a slight danger here that if a queue is deleted and -%% then recreated again the reconsume will succeed even though it was -%% not an HA failover. But the likelihood is not great and most users -%% are unlikely to care. - -cancel_consumer(CTag, QName, - State = #ch{cfg = #conf{capabilities = Capabilities}, - consumer_mapping = CMap}) -> - case rabbit_misc:table_lookup( - Capabilities, <<"consumer_cancel_notify">>) of - {bool, true} -> ok = send(#'basic.cancel'{consumer_tag = CTag, - nowait = true}, State); - _ -> ok - end, - rabbit_event:notify(consumer_deleted, [{consumer_tag, CTag}, - {channel, self()}, - {queue, QName}]), - State#ch{consumer_mapping = maps:remove(CTag, CMap)}. - -queue_down_consumer_action(CTag, CMap) -> - {_, {_, _, _, Args} = ConsumeSpec} = maps:get(CTag, CMap), - case rabbit_misc:table_lookup(Args, <<"x-cancel-on-ha-failover">>) of - {bool, true} -> remove; - _ -> {recover, ConsumeSpec} - end. - -binding_action(Fun, SourceNameBin0, DestinationType, DestinationNameBin0, - RoutingKey, Arguments, VHostPath, ConnPid, AuthzContext, - #user{username = Username} = User) -> - ExchangeNameBin = strip_cr_lf(SourceNameBin0), - DestinationNameBin = strip_cr_lf(DestinationNameBin0), - DestinationName = name_to_resource(DestinationType, DestinationNameBin, VHostPath), - check_write_permitted(DestinationName, User, AuthzContext), - ExchangeName = rabbit_misc:r(VHostPath, exchange, ExchangeNameBin), - [check_not_default_exchange(N) || N <- [DestinationName, ExchangeName]], - check_read_permitted(ExchangeName, User, AuthzContext), - case rabbit_exchange:lookup(ExchangeName) of - {error, not_found} -> - ok; - {ok, Exchange} -> - check_read_permitted_on_topic(Exchange, User, RoutingKey, AuthzContext) - end, - case Fun(#binding{source = ExchangeName, - destination = DestinationName, - key = RoutingKey, - args = Arguments}, - fun (_X, Q) when ?is_amqqueue(Q) -> - try rabbit_amqqueue:check_exclusive_access(Q, ConnPid) - catch exit:Reason -> {error, Reason} - end; - (_X, #exchange{}) -> - ok - end, - Username) of - {error, {resources_missing, [{not_found, Name} | _]}} -> - rabbit_amqqueue:not_found(Name); - {error, {resources_missing, [{absent, Q, Reason} | _]}} -> - rabbit_amqqueue:absent(Q, Reason); - {error, binding_not_found} -> - rabbit_misc:protocol_error( - not_found, "no binding ~s between ~s and ~s", - [RoutingKey, rabbit_misc:rs(ExchangeName), - rabbit_misc:rs(DestinationName)]); - {error, {binding_invalid, Fmt, Args}} -> - rabbit_misc:protocol_error(precondition_failed, Fmt, Args); - {error, #amqp_error{} = Error} -> - rabbit_misc:protocol_error(Error); - ok -> - ok - end. - -basic_return(#basic_message{exchange_name = ExchangeName, - routing_keys = [RoutingKey | _CcRoutes], - content = Content}, - State = #ch{cfg = #conf{protocol = Protocol, - writer_pid = WriterPid}}, - Reason) -> - ?INCR_STATS(exchange_stats, ExchangeName, 1, return_unroutable, State), - {_Close, ReplyCode, ReplyText} = Protocol:lookup_amqp_exception(Reason), - ok = rabbit_writer:send_command( - WriterPid, - #'basic.return'{reply_code = ReplyCode, - reply_text = ReplyText, - exchange = ExchangeName#resource.name, - routing_key = RoutingKey}, - Content). - -reject(DeliveryTag, Requeue, Multiple, - State = #ch{unacked_message_q = UAMQ, tx = Tx}) -> - {Acked, Remaining} = collect_acks(UAMQ, DeliveryTag, Multiple), - State1 = State#ch{unacked_message_q = Remaining}, - {noreply, case Tx of - none -> - {State2, Actions} = internal_reject(Requeue, Acked, State1#ch.limiter, State1), - handle_queue_actions(Actions, State2); - {Msgs, Acks} -> - Acks1 = ack_cons(Requeue, Acked, Acks), - State1#ch{tx = {Msgs, Acks1}} - end}. - -%% NB: Acked is in youngest-first order -internal_reject(Requeue, Acked, Limiter, - State = #ch{queue_states = QueueStates0}) -> - {QueueStates, Actions} = - foreach_per_queue( - fun({QRef, CTag}, MsgIds, {Acc0, Actions0}) -> - Op = case Requeue of - false -> discard; - true -> requeue - end, - case rabbit_queue_type:settle(QRef, Op, CTag, MsgIds, Acc0) of - {ok, Acc, Actions} -> - {Acc, Actions0 ++ Actions}; - {protocol_error, ErrorType, Reason, ReasonArgs} -> - rabbit_misc:protocol_error(ErrorType, Reason, ReasonArgs) - end - end, Acked, {QueueStates0, []}), - ok = notify_limiter(Limiter, Acked), - {State#ch{queue_states = QueueStates}, Actions}. - -record_sent(Type, Tag, AckRequired, - Msg = {QName, _QPid, MsgId, Redelivered, _Message}, - State = #ch{cfg = #conf{channel = ChannelNum, - trace_state = TraceState, - user = #user{username = Username}, - conn_name = ConnName - }, - unacked_message_q = UAMQ, - next_tag = DeliveryTag - }) -> - ?INCR_STATS(queue_stats, QName, 1, case {Type, AckRequired} of - {get, true} -> get; - {get, false} -> get_no_ack; - {deliver, true} -> deliver; - {deliver, false} -> deliver_no_ack - end, State), - case Redelivered of - true -> ?INCR_STATS(queue_stats, QName, 1, redeliver, State); - false -> ok - end, - DeliveredAt = os:system_time(millisecond), - rabbit_trace:tap_out(Msg, ConnName, ChannelNum, Username, TraceState), - UAMQ1 = case AckRequired of - true -> - ?QUEUE:in(#pending_ack{delivery_tag = DeliveryTag, - tag = Tag, - delivered_at = DeliveredAt, - queue = QName, - msg_id = MsgId}, UAMQ); - false -> - UAMQ - end, - State#ch{unacked_message_q = UAMQ1, next_tag = DeliveryTag + 1}. - -%% NB: returns acks in youngest-first order -collect_acks(Q, 0, true) -> - {lists:reverse(?QUEUE:to_list(Q)), ?QUEUE:new()}; -collect_acks(Q, DeliveryTag, Multiple) -> - collect_acks([], [], Q, DeliveryTag, Multiple). - -collect_acks(ToAcc, PrefixAcc, Q, DeliveryTag, Multiple) -> - case ?QUEUE:out(Q) of - {{value, UnackedMsg = #pending_ack{delivery_tag = CurrentDeliveryTag}}, - QTail} -> - if CurrentDeliveryTag == DeliveryTag -> - {[UnackedMsg | ToAcc], - case PrefixAcc of - [] -> QTail; - _ -> ?QUEUE:join( - ?QUEUE:from_list(lists:reverse(PrefixAcc)), - QTail) - end}; - Multiple -> - collect_acks([UnackedMsg | ToAcc], PrefixAcc, - QTail, DeliveryTag, Multiple); - true -> - collect_acks(ToAcc, [UnackedMsg | PrefixAcc], - QTail, DeliveryTag, Multiple) - end; - {empty, _} -> - precondition_failed("unknown delivery tag ~w", [DeliveryTag]) - end. - -%% NB: Acked is in youngest-first order -ack(Acked, State = #ch{queue_states = QueueStates0}) -> - {QueueStates, Actions} = - foreach_per_queue( - fun ({QRef, CTag}, MsgIds, {Acc0, ActionsAcc0}) -> - case rabbit_queue_type:settle(QRef, complete, CTag, - MsgIds, Acc0) of - {ok, Acc, ActionsAcc} -> - incr_queue_stats(QRef, MsgIds, State), - {Acc, ActionsAcc0 ++ ActionsAcc}; - {protocol_error, ErrorType, Reason, ReasonArgs} -> - rabbit_misc:protocol_error(ErrorType, Reason, ReasonArgs) - end - end, Acked, {QueueStates0, []}), - ok = notify_limiter(State#ch.limiter, Acked), - {State#ch{queue_states = QueueStates}, Actions}. - -incr_queue_stats(QName, MsgIds, State) -> - Count = length(MsgIds), - ?INCR_STATS(queue_stats, QName, Count, ack, State). - -%% {Msgs, Acks} -%% -%% Msgs is a queue. -%% -%% Acks looks s.t. like this: -%% [{false,[5,4]},{true,[3]},{ack,[2,1]}, ...] -%% -%% Each element is a pair consisting of a tag and a list of -%% ack'ed/reject'ed msg ids. The tag is one of 'ack' (to ack), 'true' -%% (reject w requeue), 'false' (reject w/o requeue). The msg ids, as -%% well as the list overall, are in "most-recent (generally youngest) -%% ack first" order. -new_tx() -> {?QUEUE:new(), []}. - -notify_queues(State = #ch{cfg = #conf{state = closing}}) -> - {ok, State}; -notify_queues(State = #ch{consumer_mapping = Consumers, - cfg = Cfg}) -> - QPids = classic_consumer_queue_pids(Consumers), - Timeout = get_operation_timeout(), - {rabbit_amqqueue:notify_down_all(QPids, self(), Timeout), - State#ch{cfg = Cfg#conf{state = closing}}}. - -foreach_per_queue(_F, [], Acc) -> - Acc; -foreach_per_queue(F, [#pending_ack{tag = CTag, - queue = QName, - msg_id = MsgId}], Acc) -> - %% quorum queue, needs the consumer tag - F({QName, CTag}, [MsgId], Acc); -foreach_per_queue(F, UAL, Acc) -> - T = lists:foldl(fun (#pending_ack{tag = CTag, - queue = QName, - msg_id = MsgId}, T) -> - rabbit_misc:gb_trees_cons({QName, CTag}, MsgId, T) - end, gb_trees:empty(), UAL), - rabbit_misc:gb_trees_fold(fun (Key, Val, Acc0) -> F(Key, Val, Acc0) end, Acc, T). - -%% hack to patch up missing queue type behaviour for classic queue -classic_consumer_queue_pids(Consumers) -> - lists:usort([amqqueue:get_pid(Q) - || {Q, _CParams} <- maps:values(Consumers), - amqqueue:get_type(Q) == rabbit_classic_queue]). - -%% tell the limiter about the number of acks that have been received -%% for messages delivered to subscribed consumers, but not acks for -%% messages sent in a response to a basic.get (identified by their -%% consumer tag as an integer (the same as the delivery tag, required -%% quorum queues)) -notify_limiter(Limiter, Acked) -> - %% optimisation: avoid the potentially expensive 'foldl' in the - %% common case. - case rabbit_limiter:is_active(Limiter) of - false -> ok; - true -> case lists:foldl(fun ({_, CTag, _, _}, Acc) when is_integer(CTag) -> - %% Quorum queues use integer CTags - %% classic queues use binaries - %% Quorum queues do not interact - %% with limiters - Acc; - ({_, _, _, _}, Acc) -> Acc + 1 - end, 0, Acked) of - 0 -> ok; - Count -> rabbit_limiter:ack(Limiter, Count) - end - end. - -deliver_to_queues({#delivery{message = #basic_message{exchange_name = XName}, - confirm = false, - mandatory = false}, - _RoutedToQs = []}, State) -> %% optimisation - ?INCR_STATS(exchange_stats, XName, 1, publish, State), - ?INCR_STATS(exchange_stats, XName, 1, drop_unroutable, State), - State; -deliver_to_queues({Delivery = #delivery{message = Message = #basic_message{ - exchange_name = XName}, - mandatory = Mandatory, - confirm = Confirm, - msg_seq_no = MsgSeqNo}, - DelQNames}, State0 = #ch{queue_states = QueueStates0}) -> - Qs = rabbit_amqqueue:lookup(DelQNames), - AllQueueNames = lists:foldl(fun (Q, Acc) -> - QRef = amqqueue:get_name(Q), - [QRef | Acc] - end, [], Qs), - {ok, QueueStates, Actions} = - rabbit_queue_type:deliver(Qs, Delivery, QueueStates0), - %% NB: the order here is important since basic.returns must be - %% sent before confirms. - ok = process_routing_mandatory(Mandatory, Qs, Message, State0), - State1 = process_routing_confirm(Confirm, AllQueueNames, - MsgSeqNo, XName, State0), - %% Actions must be processed after registering confirms as actions may - %% contain rejections of publishes - State = handle_queue_actions(Actions, - State1#ch{queue_states = QueueStates}), - case rabbit_event:stats_level(State, #ch.stats_timer) of - fine -> - ?INCR_STATS(exchange_stats, XName, 1, publish), - [?INCR_STATS(queue_exchange_stats, - {amqqueue:get_name(Q), XName}, 1, publish) - || Q <- Qs]; - _ -> - ok - end, - State. - -process_routing_mandatory(_Mandatory = true, - _RoutedToQs = [], - Msg, State) -> - ok = basic_return(Msg, State, no_route), - ok; -process_routing_mandatory(_Mandatory = false, - _RoutedToQs = [], - #basic_message{exchange_name = ExchangeName}, State) -> - ?INCR_STATS(exchange_stats, ExchangeName, 1, drop_unroutable, State), - ok; -process_routing_mandatory(_, _, _, _) -> - ok. - -process_routing_confirm(false, _, _, _, State) -> - State; -process_routing_confirm(true, [], MsgSeqNo, XName, State) -> - record_confirms([{MsgSeqNo, XName}], State); -process_routing_confirm(true, QRefs, MsgSeqNo, XName, State) -> - State#ch{unconfirmed = - rabbit_confirms:insert(MsgSeqNo, QRefs, XName, State#ch.unconfirmed)}. - -confirm(MsgSeqNos, QRef, State = #ch{unconfirmed = UC}) -> - %% NOTE: if queue name does not exist here it's likely that the ref also - %% does not exist in unconfirmed messages. - %% Neither does the 'ignore' atom, so it's a reasonable fallback. - {ConfirmMXs, UC1} = rabbit_confirms:confirm(MsgSeqNos, QRef, UC), - %% NB: don't call noreply/1 since we don't want to send confirms. - record_confirms(ConfirmMXs, State#ch{unconfirmed = UC1}). - -send_confirms_and_nacks(State = #ch{tx = none, confirmed = [], rejected = []}) -> - State; -send_confirms_and_nacks(State = #ch{tx = none, confirmed = C, rejected = R}) -> - case rabbit_node_monitor:pause_partition_guard() of - ok -> - Confirms = lists:append(C), - Rejects = lists:append(R), - ConfirmMsgSeqNos = - lists:foldl( - fun ({MsgSeqNo, XName}, MSNs) -> - ?INCR_STATS(exchange_stats, XName, 1, confirm, State), - [MsgSeqNo | MSNs] - end, [], Confirms), - RejectMsgSeqNos = [MsgSeqNo || {MsgSeqNo, _} <- Rejects], - - State1 = send_confirms(ConfirmMsgSeqNos, - RejectMsgSeqNos, - State#ch{confirmed = []}), - %% TODO: msg seq nos, same as for confirms. Need to implement - %% nack rates first. - send_nacks(RejectMsgSeqNos, - ConfirmMsgSeqNos, - State1#ch{rejected = []}); - pausing -> State - end; -send_confirms_and_nacks(State) -> - case rabbit_node_monitor:pause_partition_guard() of - ok -> maybe_complete_tx(State); - pausing -> State - end. - -send_nacks([], _, State) -> - State; -send_nacks(_Rs, _, State = #ch{cfg = #conf{state = closing}}) -> %% optimisation - State; -send_nacks(Rs, Cs, State) -> - coalesce_and_send(Rs, Cs, - fun(MsgSeqNo, Multiple) -> - #'basic.nack'{delivery_tag = MsgSeqNo, - multiple = Multiple} - end, State). - -send_confirms([], _, State) -> - State; -send_confirms(_Cs, _, State = #ch{cfg = #conf{state = closing}}) -> %% optimisation - State; -send_confirms([MsgSeqNo], _, State) -> - ok = send(#'basic.ack'{delivery_tag = MsgSeqNo}, State), - State; -send_confirms(Cs, Rs, State) -> - coalesce_and_send(Cs, Rs, - fun(MsgSeqNo, Multiple) -> - #'basic.ack'{delivery_tag = MsgSeqNo, - multiple = Multiple} - end, State). - -coalesce_and_send(MsgSeqNos, NegativeMsgSeqNos, MkMsgFun, State = #ch{unconfirmed = UC}) -> - SMsgSeqNos = lists:usort(MsgSeqNos), - UnconfirmedCutoff = case rabbit_confirms:is_empty(UC) of - true -> lists:last(SMsgSeqNos) + 1; - false -> rabbit_confirms:smallest(UC) - end, - Cutoff = lists:min([UnconfirmedCutoff | NegativeMsgSeqNos]), - {Ms, Ss} = lists:splitwith(fun(X) -> X < Cutoff end, SMsgSeqNos), - case Ms of - [] -> ok; - _ -> ok = send(MkMsgFun(lists:last(Ms), true), State) - end, - [ok = send(MkMsgFun(SeqNo, false), State) || SeqNo <- Ss], - State. - -ack_cons(Tag, Acked, [{Tag, Acks} | L]) -> [{Tag, Acked ++ Acks} | L]; -ack_cons(Tag, Acked, Acks) -> [{Tag, Acked} | Acks]. - -ack_len(Acks) -> lists:sum([length(L) || {ack, L} <- Acks]). - -maybe_complete_tx(State = #ch{tx = {_, _}}) -> - State; -maybe_complete_tx(State = #ch{unconfirmed = UC}) -> - case rabbit_confirms:is_empty(UC) of - false -> State; - true -> complete_tx(State#ch{confirmed = []}) - end. - -complete_tx(State = #ch{tx = committing}) -> - ok = send(#'tx.commit_ok'{}, State), - State#ch{tx = new_tx()}; -complete_tx(State = #ch{tx = failed}) -> - {noreply, State1} = handle_exception( - rabbit_misc:amqp_error( - precondition_failed, "partial tx completion", [], - 'tx.commit'), - State), - State1#ch{tx = new_tx()}. - -infos(Items, State) -> [{Item, i(Item, State)} || Item <- Items]. - -infos(Items, Deadline, State) -> - [begin - Now = now_millis(), - if - Now > Deadline -> - throw(timeout); - true -> - {Item, i(Item, State)} - end - end || Item <- Items]. - -i(pid, _) -> self(); -i(connection, #ch{cfg = #conf{conn_pid = ConnPid}}) -> ConnPid; -i(number, #ch{cfg = #conf{channel = Channel}}) -> Channel; -i(user, #ch{cfg = #conf{user = User}}) -> User#user.username; -i(user_who_performed_action, Ch) -> i(user, Ch); -i(vhost, #ch{cfg = #conf{virtual_host = VHost}}) -> VHost; -i(transactional, #ch{tx = Tx}) -> Tx =/= none; -i(confirm, #ch{confirm_enabled = CE}) -> CE; -i(name, State) -> name(State); -i(consumer_count, #ch{consumer_mapping = CM}) -> maps:size(CM); -i(messages_unconfirmed, #ch{unconfirmed = UC}) -> rabbit_confirms:size(UC); -i(messages_unacknowledged, #ch{unacked_message_q = UAMQ}) -> ?QUEUE:len(UAMQ); -i(messages_uncommitted, #ch{tx = {Msgs, _Acks}}) -> ?QUEUE:len(Msgs); -i(messages_uncommitted, #ch{}) -> 0; -i(acks_uncommitted, #ch{tx = {_Msgs, Acks}}) -> ack_len(Acks); -i(acks_uncommitted, #ch{}) -> 0; -i(pending_raft_commands, #ch{queue_states = QS}) -> - pending_raft_commands(QS); -i(state, #ch{cfg = #conf{state = running}}) -> credit_flow:state(); -i(state, #ch{cfg = #conf{state = State}}) -> State; -i(prefetch_count, #ch{cfg = #conf{consumer_prefetch = C}}) -> C; -i(global_prefetch_count, #ch{limiter = Limiter}) -> - rabbit_limiter:get_prefetch_limit(Limiter); -i(interceptors, #ch{interceptor_state = IState}) -> - IState; -i(garbage_collection, _State) -> - rabbit_misc:get_gc_info(self()); -i(reductions, _State) -> - {reductions, Reductions} = erlang:process_info(self(), reductions), - Reductions; -i(Item, _) -> - throw({bad_argument, Item}). - -pending_raft_commands(QStates) -> - Fun = fun(_, V, Acc) -> - case rabbit_queue_type:state_info(V) of - #{pending_raft_commands := P} -> - Acc + P; - _ -> - Acc - end - end, - rabbit_queue_type:fold_state(Fun, 0, QStates). - -name(#ch{cfg = #conf{conn_name = ConnName, channel = Channel}}) -> - list_to_binary(rabbit_misc:format("~s (~p)", [ConnName, Channel])). - -emit_stats(State) -> emit_stats(State, []). - -emit_stats(State, Extra) -> - [{reductions, Red} | Coarse0] = infos(?STATISTICS_KEYS, State), - %% First metric must be `idle_since` (if available), as expected by - %% `rabbit_mgmt_format:format_channel_stats`. This is a performance - %% optimisation that avoids traversing the whole list when only - %% one element has to be formatted. - rabbit_core_metrics:channel_stats(self(), Extra ++ Coarse0), - rabbit_core_metrics:channel_stats(reductions, self(), Red). - -erase_queue_stats(QName) -> - rabbit_core_metrics:channel_queue_down({self(), QName}), - erase({queue_stats, QName}), - [begin - rabbit_core_metrics:channel_queue_exchange_down({self(), QX}), - erase({queue_exchange_stats, QX}) - end || {{queue_exchange_stats, QX = {QName0, _}}, _} <- get(), - QName0 =:= QName]. - -get_vhost(#ch{cfg = #conf{virtual_host = VHost}}) -> VHost. - -get_user(#ch{cfg = #conf{user = User}}) -> User. - -delete_stats({queue_stats, QName}) -> - rabbit_core_metrics:channel_queue_down({self(), QName}); -delete_stats({exchange_stats, XName}) -> - rabbit_core_metrics:channel_exchange_down({self(), XName}); -delete_stats({queue_exchange_stats, QX}) -> - rabbit_core_metrics:channel_queue_exchange_down({self(), QX}); -delete_stats(_) -> - ok. - -put_operation_timeout() -> - put(channel_operation_timeout, ?CHANNEL_OPERATION_TIMEOUT). - -get_operation_timeout() -> - get(channel_operation_timeout). - -%% Refactored and exported to allow direct calls from the HTTP API, -%% avoiding the usage of AMQP 0-9-1 from the management. - -handle_method(#'exchange.bind'{destination = DestinationNameBin, - source = SourceNameBin, - routing_key = RoutingKey, - arguments = Arguments}, - ConnPid, AuthzContext, _CollectorId, VHostPath, User) -> - binding_action(fun rabbit_binding:add/3, - SourceNameBin, exchange, DestinationNameBin, - RoutingKey, Arguments, VHostPath, ConnPid, AuthzContext, User); -handle_method(#'exchange.unbind'{destination = DestinationNameBin, - source = SourceNameBin, - routing_key = RoutingKey, - arguments = Arguments}, - ConnPid, AuthzContext, _CollectorId, VHostPath, User) -> - binding_action(fun rabbit_binding:remove/3, - SourceNameBin, exchange, DestinationNameBin, - RoutingKey, Arguments, VHostPath, ConnPid, AuthzContext, User); -handle_method(#'queue.unbind'{queue = QueueNameBin, - exchange = ExchangeNameBin, - routing_key = RoutingKey, - arguments = Arguments}, - ConnPid, AuthzContext, _CollectorId, VHostPath, User) -> - binding_action(fun rabbit_binding:remove/3, - ExchangeNameBin, queue, QueueNameBin, - RoutingKey, Arguments, VHostPath, ConnPid, AuthzContext, User); -handle_method(#'queue.bind'{queue = QueueNameBin, - exchange = ExchangeNameBin, - routing_key = RoutingKey, - arguments = Arguments}, - ConnPid, AuthzContext, _CollectorId, VHostPath, User) -> - binding_action(fun rabbit_binding:add/3, - ExchangeNameBin, queue, QueueNameBin, - RoutingKey, Arguments, VHostPath, ConnPid, AuthzContext, User); -%% Note that all declares to these are effectively passive. If it -%% exists it by definition has one consumer. -handle_method(#'queue.declare'{queue = <<"amq.rabbitmq.reply-to", - _/binary>> = QueueNameBin}, - _ConnPid, _AuthzContext, _CollectorPid, VHost, _User) -> - StrippedQueueNameBin = strip_cr_lf(QueueNameBin), - QueueName = rabbit_misc:r(VHost, queue, StrippedQueueNameBin), - case declare_fast_reply_to(StrippedQueueNameBin) of - exists -> {ok, QueueName, 0, 1}; - not_found -> rabbit_amqqueue:not_found(QueueName) - end; -handle_method(#'queue.declare'{queue = QueueNameBin, - passive = false, - durable = DurableDeclare, - exclusive = ExclusiveDeclare, - auto_delete = AutoDelete, - nowait = NoWait, - arguments = Args} = Declare, - ConnPid, AuthzContext, CollectorPid, VHostPath, - #user{username = Username} = User) -> - Owner = case ExclusiveDeclare of - true -> ConnPid; - false -> none - end, - StrippedQueueNameBin = strip_cr_lf(QueueNameBin), - Durable = DurableDeclare andalso not ExclusiveDeclare, - ActualNameBin = case StrippedQueueNameBin of - <<>> -> - case rabbit_amqqueue:is_server_named_allowed(Args) of - true -> - rabbit_guid:binary(rabbit_guid:gen_secure(), "amq.gen"); - false -> - rabbit_misc:protocol_error( - precondition_failed, - "Cannot declare a server-named queue for type ~p", - [rabbit_amqqueue:get_queue_type(Args)]) - end; - Other -> check_name('queue', Other) - end, - QueueName = rabbit_misc:r(VHostPath, queue, ActualNameBin), - check_configure_permitted(QueueName, User, AuthzContext), - rabbit_core_metrics:queue_declared(QueueName), - case rabbit_amqqueue:with( - QueueName, - fun (Q) -> ok = rabbit_amqqueue:assert_equivalence( - Q, Durable, AutoDelete, Args, Owner), - maybe_stat(NoWait, Q) - end) of - {ok, MessageCount, ConsumerCount} -> - {ok, QueueName, MessageCount, ConsumerCount}; - {error, not_found} -> - %% enforce the limit for newly declared queues only - check_vhost_queue_limit(QueueName, VHostPath), - DlxKey = <<"x-dead-letter-exchange">>, - case rabbit_misc:r_arg(VHostPath, exchange, Args, DlxKey) of - undefined -> - ok; - {error, {invalid_type, Type}} -> - precondition_failed( - "invalid type '~s' for arg '~s' in ~s", - [Type, DlxKey, rabbit_misc:rs(QueueName)]); - DLX -> - check_read_permitted(QueueName, User, AuthzContext), - check_write_permitted(DLX, User, AuthzContext), - ok - end, - case rabbit_amqqueue:declare(QueueName, Durable, AutoDelete, - Args, Owner, Username) of - {new, Q} when ?is_amqqueue(Q) -> - %% We need to notify the reader within the channel - %% process so that we can be sure there are no - %% outstanding exclusive queues being declared as - %% the connection shuts down. - QPid = amqqueue:get_pid(Q), - ok = case {Owner, CollectorPid} of - {none, _} -> ok; - {_, none} -> ok; %% Supports call from mgmt API - _ -> rabbit_queue_collector:register( - CollectorPid, QPid) - end, - rabbit_core_metrics:queue_created(QueueName), - {ok, QueueName, 0, 0}; - {existing, _Q} -> - %% must have been created between the stat and the - %% declare. Loop around again. - handle_method(Declare, ConnPid, AuthzContext, CollectorPid, VHostPath, - User); - {absent, Q, Reason} -> - rabbit_amqqueue:absent(Q, Reason); - {owner_died, _Q} -> - %% Presumably our own days are numbered since the - %% connection has died. Pretend the queue exists though, - %% just so nothing fails. - {ok, QueueName, 0, 0}; - {protocol_error, ErrorType, Reason, ReasonArgs} -> - rabbit_misc:protocol_error(ErrorType, Reason, ReasonArgs) - end; - {error, {absent, Q, Reason}} -> - rabbit_amqqueue:absent(Q, Reason) - end; -handle_method(#'queue.declare'{queue = QueueNameBin, - nowait = NoWait, - passive = true}, - ConnPid, _AuthzContext, _CollectorPid, VHostPath, _User) -> - StrippedQueueNameBin = strip_cr_lf(QueueNameBin), - QueueName = rabbit_misc:r(VHostPath, queue, StrippedQueueNameBin), - Fun = fun (Q0) -> - QStat = maybe_stat(NoWait, Q0), - {QStat, Q0} - end, - %% Note: no need to check if Q is an #amqqueue, with_or_die does it - {{ok, MessageCount, ConsumerCount}, Q} = rabbit_amqqueue:with_or_die(QueueName, Fun), - ok = rabbit_amqqueue:check_exclusive_access(Q, ConnPid), - {ok, QueueName, MessageCount, ConsumerCount}; -handle_method(#'queue.delete'{queue = QueueNameBin, - if_unused = IfUnused, - if_empty = IfEmpty}, - ConnPid, AuthzContext, _CollectorPid, VHostPath, - User = #user{username = Username}) -> - StrippedQueueNameBin = strip_cr_lf(QueueNameBin), - QueueName = qbin_to_resource(StrippedQueueNameBin, VHostPath), - - check_configure_permitted(QueueName, User, AuthzContext), - case rabbit_amqqueue:with( - QueueName, - fun (Q) -> - rabbit_amqqueue:check_exclusive_access(Q, ConnPid), - rabbit_queue_type:delete(Q, IfUnused, IfEmpty, Username) - end, - fun (not_found) -> - {ok, 0}; - ({absent, Q, crashed}) -> - _ = rabbit_classic_queue:delete_crashed(Q, Username), - {ok, 0}; - ({absent, Q, stopped}) -> - _ = rabbit_classic_queue:delete_crashed(Q, Username), - {ok, 0}; - ({absent, Q, Reason}) -> - rabbit_amqqueue:absent(Q, Reason) - end) of - {error, in_use} -> - precondition_failed("~s in use", [rabbit_misc:rs(QueueName)]); - {error, not_empty} -> - precondition_failed("~s not empty", [rabbit_misc:rs(QueueName)]); - {ok, Count} -> - {ok, Count}; - {protocol_error, Type, Reason, ReasonArgs} -> - rabbit_misc:protocol_error(Type, Reason, ReasonArgs) - end; -handle_method(#'exchange.delete'{exchange = ExchangeNameBin, - if_unused = IfUnused}, - _ConnPid, AuthzContext, _CollectorPid, VHostPath, - User = #user{username = Username}) -> - StrippedExchangeNameBin = strip_cr_lf(ExchangeNameBin), - ExchangeName = rabbit_misc:r(VHostPath, exchange, StrippedExchangeNameBin), - check_not_default_exchange(ExchangeName), - check_exchange_deletion(ExchangeName), - check_configure_permitted(ExchangeName, User, AuthzContext), - case rabbit_exchange:delete(ExchangeName, IfUnused, Username) of - {error, not_found} -> - ok; - {error, in_use} -> - precondition_failed("~s in use", [rabbit_misc:rs(ExchangeName)]); - ok -> - ok - end; -handle_method(#'queue.purge'{queue = QueueNameBin}, - ConnPid, AuthzContext, _CollectorPid, VHostPath, User) -> - QueueName = qbin_to_resource(QueueNameBin, VHostPath), - check_read_permitted(QueueName, User, AuthzContext), - rabbit_amqqueue:with_exclusive_access_or_die( - QueueName, ConnPid, - fun (Q) -> - case rabbit_queue_type:purge(Q) of - {ok, _} = Res -> - Res; - {error, not_supported} -> - rabbit_misc:protocol_error( - not_implemented, - "queue.purge not supported by stream queues ~s", - [rabbit_misc:rs(amqqueue:get_name(Q))]) - end - end); -handle_method(#'exchange.declare'{exchange = ExchangeNameBin, - type = TypeNameBin, - passive = false, - durable = Durable, - auto_delete = AutoDelete, - internal = Internal, - arguments = Args}, - _ConnPid, AuthzContext, _CollectorPid, VHostPath, - #user{username = Username} = User) -> - CheckedType = rabbit_exchange:check_type(TypeNameBin), - ExchangeName = rabbit_misc:r(VHostPath, exchange, strip_cr_lf(ExchangeNameBin)), - check_not_default_exchange(ExchangeName), - check_configure_permitted(ExchangeName, User, AuthzContext), - X = case rabbit_exchange:lookup(ExchangeName) of - {ok, FoundX} -> FoundX; - {error, not_found} -> - check_name('exchange', strip_cr_lf(ExchangeNameBin)), - AeKey = <<"alternate-exchange">>, - case rabbit_misc:r_arg(VHostPath, exchange, Args, AeKey) of - undefined -> ok; - {error, {invalid_type, Type}} -> - precondition_failed( - "invalid type '~s' for arg '~s' in ~s", - [Type, AeKey, rabbit_misc:rs(ExchangeName)]); - AName -> check_read_permitted(ExchangeName, User, AuthzContext), - check_write_permitted(AName, User, AuthzContext), - ok - end, - rabbit_exchange:declare(ExchangeName, - CheckedType, - Durable, - AutoDelete, - Internal, - Args, - Username) - end, - ok = rabbit_exchange:assert_equivalence(X, CheckedType, Durable, - AutoDelete, Internal, Args); -handle_method(#'exchange.declare'{exchange = ExchangeNameBin, - passive = true}, - _ConnPid, _AuthzContext, _CollectorPid, VHostPath, _User) -> - ExchangeName = rabbit_misc:r(VHostPath, exchange, strip_cr_lf(ExchangeNameBin)), - check_not_default_exchange(ExchangeName), - _ = rabbit_exchange:lookup_or_die(ExchangeName). - -handle_deliver(CTag, Ack, Msgs, State) when is_list(Msgs) -> - lists:foldl(fun(Msg, S) -> - handle_deliver0(CTag, Ack, Msg, S) - end, State, Msgs); -handle_deliver(CTag, Ack, Msg, State) -> - %% backwards compatibility clause - handle_deliver0(CTag, Ack, Msg, State). - -handle_deliver0(ConsumerTag, AckRequired, - Msg = {QName, QPid, _MsgId, Redelivered, - #basic_message{exchange_name = ExchangeName, - routing_keys = [RoutingKey | _CcRoutes], - content = Content}}, - State = #ch{cfg = #conf{writer_pid = WriterPid, - writer_gc_threshold = GCThreshold}, - next_tag = DeliveryTag, - queue_states = Qs}) -> - Deliver = #'basic.deliver'{consumer_tag = ConsumerTag, - delivery_tag = DeliveryTag, - redelivered = Redelivered, - exchange = ExchangeName#resource.name, - routing_key = RoutingKey}, - case rabbit_queue_type:module(QName, Qs) of - {ok, rabbit_classic_queue} -> - ok = rabbit_writer:send_command_and_notify( - WriterPid, QPid, self(), Deliver, Content); - _ -> - ok = rabbit_writer:send_command(WriterPid, Deliver, Content) - end, - case GCThreshold of - undefined -> ok; - _ -> rabbit_basic:maybe_gc_large_msg(Content, GCThreshold) - end, - record_sent(deliver, ConsumerTag, AckRequired, Msg, State). - -handle_basic_get(WriterPid, DeliveryTag, NoAck, MessageCount, - Msg = {_QName, _QPid, _MsgId, Redelivered, - #basic_message{exchange_name = ExchangeName, - routing_keys = [RoutingKey | _CcRoutes], - content = Content}}, State) -> - ok = rabbit_writer:send_command( - WriterPid, - #'basic.get_ok'{delivery_tag = DeliveryTag, - redelivered = Redelivered, - exchange = ExchangeName#resource.name, - routing_key = RoutingKey, - message_count = MessageCount}, - Content), - {noreply, record_sent(get, DeliveryTag, not(NoAck), Msg, State)}. - -init_tick_timer(State = #ch{tick_timer = undefined}) -> - {ok, Interval} = application:get_env(rabbit, channel_tick_interval), - State#ch{tick_timer = erlang:send_after(Interval, self(), tick)}; -init_tick_timer(State) -> - State. - -reset_tick_timer(State) -> - State#ch{tick_timer = undefined}. - -maybe_cancel_tick_timer(#ch{tick_timer = undefined} = State) -> - State; -maybe_cancel_tick_timer(#ch{tick_timer = TRef, - unacked_message_q = UMQ} = State) -> - case ?QUEUE:len(UMQ) of - 0 -> - %% we can only cancel the tick timer if the unacked messages - %% queue is empty. - _ = erlang:cancel_timer(TRef), - State#ch{tick_timer = undefined}; - _ -> - %% let the timer continue - State - end. - -now_millis() -> - erlang:monotonic_time(millisecond). - -get_operation_timeout_and_deadline() -> - % NB: can't use get_operation_timeout because - % this code may not be running via the channel Pid - Timeout = ?CHANNEL_OPERATION_TIMEOUT, - Deadline = now_millis() + Timeout, - {Timeout, Deadline}. - -queue_fold(Fun, Init, Q) -> - case ?QUEUE:out(Q) of - {empty, _Q} -> Init; - {{value, V}, Q1} -> queue_fold(Fun, Fun(V, Init), Q1) - end. - -evaluate_consumer_timeout(State0 = #ch{cfg = #conf{channel = Channel, - consumer_timeout = Timeout}, - unacked_message_q = UAMQ}) -> - Now = os:system_time(millisecond), - case ?QUEUE:peek(UAMQ) of - {value, #pending_ack{delivery_tag = ConsumerTag, - delivered_at = Time}} - when is_integer(Timeout) - andalso Time < Now - Timeout -> - rabbit_log_channel:warning("Consumer ~s on channel ~w has timed out " - "waiting on consumer acknowledgement. Timeout used: ~p ms", - [rabbit_data_coercion:to_binary(ConsumerTag), - Channel, Timeout]), - Ex = rabbit_misc:amqp_error(precondition_failed, - "consumer ack timed out on channel ~w", - [Channel], none), - handle_exception(Ex, State0); - _ -> - {noreply, State0} - end. - -handle_queue_actions(Actions, #ch{} = State0) -> - WriterPid = State0#ch.cfg#conf.writer_pid, - lists:foldl( - fun ({send_credit_reply, Avail}, S0) -> - ok = rabbit_writer:send_command(WriterPid, - #'basic.credit_ok'{available = Avail}), - S0; - ({send_drained, {CTag, Credit}}, S0) -> - ok = rabbit_writer:send_command( - WriterPid, - #'basic.credit_drained'{consumer_tag = CTag, - credit_drained = Credit}), - S0; - ({settled, QRef, MsgSeqNos}, S0) -> - confirm(MsgSeqNos, QRef, S0); - ({rejected, _QRef, MsgSeqNos}, S0) -> - {U, Rej} = - lists:foldr( - fun(SeqNo, {U1, Acc}) -> - case rabbit_confirms:reject(SeqNo, U1) of - {ok, MX, U2} -> - {U2, [MX | Acc]}; - {error, not_found} -> - {U1, Acc} - end - end, {S0#ch.unconfirmed, []}, MsgSeqNos), - S = S0#ch{unconfirmed = U}, - record_rejects(Rej, S); - ({deliver, CTag, AckRequired, Msgs}, S0) -> - handle_deliver(CTag, AckRequired, Msgs, S0); - ({queue_down, QRef}, S0) -> - handle_consuming_queue_down_or_eol(QRef, S0) - - end, State0, Actions). - -find_queue_name_from_pid(Pid, QStates) when is_pid(Pid) -> - Fun = fun(K, _V, undefined) -> - case rabbit_amqqueue:lookup(K) of - {error, not_found} -> - undefined; - {ok, Q} -> - Pids = get_queue_pids(Q), - case lists:member(Pid, Pids) of - true -> - K; - false -> - undefined - end - end; - (_K, _V, Acc) -> - Acc - end, - rabbit_queue_type:fold_state(Fun, undefined, QStates). - -get_queue_pids(Q) when ?amqqueue_is_quorum(Q) -> - [amqqueue:get_leader(Q)]; -get_queue_pids(Q) -> - [amqqueue:get_pid(Q) | amqqueue:get_slave_pids(Q)]. - -find_queue_name_from_quorum_name(Name, QStates) -> - Fun = fun(K, _V, undefined) -> - {ok, Q} = rabbit_amqqueue:lookup(K), - case amqqueue:get_pid(Q) of - {Name, _} -> - amqqueue:get_name(Q); - _ -> - undefined - end - end, - rabbit_queue_type:fold_state(Fun, undefined, QStates). diff --git a/src/rabbit_channel_interceptor.erl b/src/rabbit_channel_interceptor.erl deleted file mode 100644 index c40b437f10..0000000000 --- a/src/rabbit_channel_interceptor.erl +++ /dev/null @@ -1,104 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_channel_interceptor). - --include("rabbit_framing.hrl"). --include("rabbit.hrl"). - --export([init/1, intercept_in/3]). - --behaviour(rabbit_registry_class). - --export([added_to_rabbit_registry/2, removed_from_rabbit_registry/1]). - --type(method_name() :: rabbit_framing:amqp_method_name()). --type(original_method() :: rabbit_framing:amqp_method_record()). --type(processed_method() :: rabbit_framing:amqp_method_record()). --type(original_content() :: rabbit_types:maybe(rabbit_types:content())). --type(processed_content() :: rabbit_types:maybe(rabbit_types:content())). --type(interceptor_state() :: term()). - --callback description() -> [proplists:property()]. -%% Derive some initial state from the channel. This will be passed back -%% as the third argument of intercept/3. --callback init(rabbit_channel:channel()) -> interceptor_state(). --callback intercept(original_method(), original_content(), - interceptor_state()) -> - {processed_method(), processed_content()} | - rabbit_misc:channel_or_connection_exit(). --callback applies_to() -> list(method_name()). - -added_to_rabbit_registry(_Type, _ModuleName) -> - rabbit_channel:refresh_interceptors(). -removed_from_rabbit_registry(_Type) -> - rabbit_channel:refresh_interceptors(). - -init(Ch) -> - Mods = [M || {_, M} <- rabbit_registry:lookup_all(channel_interceptor)], - check_no_overlap(Mods), - [{Mod, Mod:init(Ch)} || Mod <- Mods]. - -check_no_overlap(Mods) -> - check_no_overlap1([sets:from_list(Mod:applies_to()) || Mod <- Mods]). - -%% Check no non-empty pairwise intersection in a list of sets -check_no_overlap1(Sets) -> - lists:foldl(fun(Set, Union) -> - Is = sets:intersection(Set, Union), - case sets:size(Is) of - 0 -> ok; - _ -> - internal_error("Interceptor: more than one " - "module handles ~p~n", [Is]) - end, - sets:union(Set, Union) - end, - sets:new(), - Sets), - ok. - -intercept_in(M, C, Mods) -> - lists:foldl(fun({Mod, ModState}, {M1, C1}) -> - call_module(Mod, ModState, M1, C1) - end, - {M, C}, - Mods). - -call_module(Mod, St, M, C) -> - % this little dance is because Mod might be unloaded at any point - case (catch {ok, Mod:intercept(M, C, St)}) of - {ok, R} -> validate_response(Mod, M, C, R); - {'EXIT', {undef, [{Mod, intercept, _, _} | _]}} -> {M, C} - end. - -validate_response(Mod, M1, C1, R = {M2, C2}) -> - case {validate_method(M1, M2), validate_content(C1, C2)} of - {true, true} -> R; - {false, _} -> - internal_error("Interceptor: ~p expected to return " - "method: ~p but returned: ~p", - [Mod, rabbit_misc:method_record_type(M1), - rabbit_misc:method_record_type(M2)]); - {_, false} -> - internal_error("Interceptor: ~p expected to return " - "content iff content is provided but " - "content in = ~p; content out = ~p", - [Mod, C1, C2]) - end. - -validate_method(M, M2) -> - rabbit_misc:method_record_type(M) =:= rabbit_misc:method_record_type(M2). - -validate_content(none, none) -> true; -validate_content(#content{}, #content{}) -> true; -validate_content(_, _) -> false. - -%% keep dialyzer happy --spec internal_error(string(), [any()]) -> no_return(). -internal_error(Format, Args) -> - rabbit_misc:protocol_error(internal_error, Format, Args). diff --git a/src/rabbit_channel_sup.erl b/src/rabbit_channel_sup.erl deleted file mode 100644 index 0d405ad3a7..0000000000 --- a/src/rabbit_channel_sup.erl +++ /dev/null @@ -1,92 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_channel_sup). - -%% Supervises processes that implement AMQP 0-9-1 channels: -%% -%% * Channel process itself -%% * Network writer (for network connections) -%% * Limiter (handles channel QoS and flow control) -%% -%% Every rabbit_channel_sup is supervised by rabbit_channel_sup_sup. -%% -%% See also rabbit_channel, rabbit_writer, rabbit_limiter. - --behaviour(supervisor2). - --export([start_link/1]). - --export([init/1]). - --include("rabbit.hrl"). - -%%---------------------------------------------------------------------------- - --export_type([start_link_args/0]). - --type start_link_args() :: - {'tcp', rabbit_net:socket(), rabbit_channel:channel_number(), - non_neg_integer(), pid(), string(), rabbit_types:protocol(), - rabbit_types:user(), rabbit_types:vhost(), rabbit_framing:amqp_table(), - pid()} | - {'direct', rabbit_channel:channel_number(), pid(), string(), - rabbit_types:protocol(), rabbit_types:user(), rabbit_types:vhost(), - rabbit_framing:amqp_table(), pid()}. - --define(FAIR_WAIT, 70000). - -%%---------------------------------------------------------------------------- - --spec start_link(start_link_args()) -> {'ok', pid(), {pid(), any()}}. - -start_link({tcp, Sock, Channel, FrameMax, ReaderPid, ConnName, Protocol, User, - VHost, Capabilities, Collector}) -> - {ok, SupPid} = supervisor2:start_link( - ?MODULE, {tcp, Sock, Channel, FrameMax, - ReaderPid, Protocol, {ConnName, Channel}}), - [LimiterPid] = supervisor2:find_child(SupPid, limiter), - [WriterPid] = supervisor2:find_child(SupPid, writer), - {ok, ChannelPid} = - supervisor2:start_child( - SupPid, - {channel, {rabbit_channel, start_link, - [Channel, ReaderPid, WriterPid, ReaderPid, ConnName, - Protocol, User, VHost, Capabilities, Collector, - LimiterPid]}, - intrinsic, ?FAIR_WAIT, worker, [rabbit_channel]}), - {ok, AState} = rabbit_command_assembler:init(Protocol), - {ok, SupPid, {ChannelPid, AState}}; -start_link({direct, Channel, ClientChannelPid, ConnPid, ConnName, Protocol, - User, VHost, Capabilities, Collector, AmqpParams}) -> - {ok, SupPid} = supervisor2:start_link( - ?MODULE, {direct, {ConnName, Channel}}), - [LimiterPid] = supervisor2:find_child(SupPid, limiter), - {ok, ChannelPid} = - supervisor2:start_child( - SupPid, - {channel, {rabbit_channel, start_link, - [Channel, ClientChannelPid, ClientChannelPid, ConnPid, - ConnName, Protocol, User, VHost, Capabilities, Collector, - LimiterPid, AmqpParams]}, - intrinsic, ?FAIR_WAIT, worker, [rabbit_channel]}), - {ok, SupPid, {ChannelPid, none}}. - -%%---------------------------------------------------------------------------- - -init(Type) -> - ?LG_PROCESS_TYPE(channel_sup), - {ok, {{one_for_all, 0, 1}, child_specs(Type)}}. - -child_specs({tcp, Sock, Channel, FrameMax, ReaderPid, Protocol, Identity}) -> - [{writer, {rabbit_writer, start_link, - [Sock, Channel, FrameMax, Protocol, ReaderPid, Identity, true]}, - intrinsic, ?FAIR_WAIT, worker, [rabbit_writer]} - | child_specs({direct, Identity})]; -child_specs({direct, Identity}) -> - [{limiter, {rabbit_limiter, start_link, [Identity]}, - transient, ?FAIR_WAIT, worker, [rabbit_limiter]}]. diff --git a/src/rabbit_channel_sup_sup.erl b/src/rabbit_channel_sup_sup.erl deleted file mode 100644 index 72cf38d6c8..0000000000 --- a/src/rabbit_channel_sup_sup.erl +++ /dev/null @@ -1,42 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_channel_sup_sup). - -%% Supervisor for AMQP 0-9-1 channels. Every AMQP 0-9-1 connection has -%% one of these. -%% -%% See also rabbit_channel_sup, rabbit_connection_helper_sup, rabbit_reader. - --behaviour(supervisor2). - --export([start_link/0, start_channel/2]). - --export([init/1]). - --include("rabbit.hrl"). - -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - supervisor2:start_link(?MODULE, []). - --spec start_channel(pid(), rabbit_channel_sup:start_link_args()) -> - {'ok', pid(), {pid(), any()}}. - -start_channel(Pid, Args) -> - supervisor2:start_child(Pid, [Args]). - -%%---------------------------------------------------------------------------- - -init([]) -> - ?LG_PROCESS_TYPE(channel_sup_sup), - {ok, {{simple_one_for_one, 0, 1}, - [{channel_sup, {rabbit_channel_sup, start_link, []}, - temporary, infinity, supervisor, [rabbit_channel_sup]}]}}. diff --git a/src/rabbit_channel_tracking.erl b/src/rabbit_channel_tracking.erl deleted file mode 100644 index 42ab664a06..0000000000 --- a/src/rabbit_channel_tracking.erl +++ /dev/null @@ -1,291 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_channel_tracking). - -%% Abstracts away how tracked connection records are stored -%% and queried. -%% -%% See also: -%% -%% * rabbit_channel_tracking_handler -%% * rabbit_reader -%% * rabbit_event --behaviour(rabbit_tracking). - --export([boot/0, - update_tracked/1, - handle_cast/1, - register_tracked/1, - unregister_tracked/1, - count_tracked_items_in/1, - clear_tracking_tables/0, - shutdown_tracked_items/2]). - --export([list/0, list_of_user/1, list_on_node/1, - tracked_channel_table_name_for/1, - tracked_channel_per_user_table_name_for/1, - get_all_tracked_channel_table_names_for_node/1, - delete_tracked_channel_user_entry/1]). - --include_lib("rabbit.hrl"). - --import(rabbit_misc, [pget/2]). - -%% -%% API -%% - -%% Sets up and resets channel tracking tables for this node. --spec boot() -> ok. - -boot() -> - ensure_tracked_channels_table_for_this_node(), - rabbit_log:info("Setting up a table for channel tracking on this node: ~p", - [tracked_channel_table_name_for(node())]), - ensure_per_user_tracked_channels_table_for_node(), - rabbit_log:info("Setting up a table for channel tracking on this node: ~p", - [tracked_channel_per_user_table_name_for(node())]), - clear_tracking_tables(), - ok. - --spec update_tracked(term()) -> ok. - -update_tracked(Event) -> - spawn(?MODULE, handle_cast, [Event]), - ok. - -%% Asynchronously handle update events --spec handle_cast(term()) -> ok. - -handle_cast({channel_created, Details}) -> - ThisNode = node(), - case node(pget(pid, Details)) of - ThisNode -> - TrackedCh = #tracked_channel{id = TrackedChId} = - tracked_channel_from_channel_created_event(Details), - try - register_tracked(TrackedCh) - catch - error:{no_exists, _} -> - Msg = "Could not register channel ~p for tracking, " - "its table is not ready yet or the channel terminated prematurely", - rabbit_log_connection:warning(Msg, [TrackedChId]), - ok; - error:Err -> - Msg = "Could not register channel ~p for tracking: ~p", - rabbit_log_connection:warning(Msg, [TrackedChId, Err]), - ok - end; - _OtherNode -> - %% ignore - ok - end; -handle_cast({channel_closed, Details}) -> - %% channel has terminated, unregister iff local - case get_tracked_channel_by_pid(pget(pid, Details)) of - [#tracked_channel{name = Name}] -> - unregister_tracked(rabbit_tracking:id(node(), Name)); - _Other -> ok - end; -handle_cast({connection_closed, ConnDetails}) -> - ThisNode= node(), - ConnPid = pget(pid, ConnDetails), - - case pget(node, ConnDetails) of - ThisNode -> - TrackedChs = get_tracked_channels_by_connection_pid(ConnPid), - rabbit_log_connection:info( - "Closing all channels from connection '~p' " - "because it has been closed", [pget(name, ConnDetails)]), - %% Shutting down channels will take care of unregistering the - %% corresponding tracking. - shutdown_tracked_items(TrackedChs, undefined), - ok; - _DifferentNode -> - ok - end; -handle_cast({user_deleted, Details}) -> - Username = pget(name, Details), - %% Schedule user entry deletion, allowing time for connections to close - _ = timer:apply_after(?TRACKING_EXECUTION_TIMEOUT, ?MODULE, - delete_tracked_channel_user_entry, [Username]), - ok; -handle_cast({node_deleted, Details}) -> - Node = pget(node, Details), - rabbit_log_connection:info( - "Node '~s' was removed from the cluster, deleting" - " its channel tracking tables...", [Node]), - delete_tracked_channels_table_for_node(Node), - delete_per_user_tracked_channels_table_for_node(Node). - --spec register_tracked(rabbit_types:tracked_channel()) -> ok. --dialyzer([{nowarn_function, [register_tracked/1]}, race_conditions]). - -register_tracked(TrackedCh = - #tracked_channel{node = Node, name = Name, username = Username}) -> - ChId = rabbit_tracking:id(Node, Name), - TableName = tracked_channel_table_name_for(Node), - PerUserChTableName = tracked_channel_per_user_table_name_for(Node), - %% upsert - case mnesia:dirty_read(TableName, ChId) of - [] -> - mnesia:dirty_write(TableName, TrackedCh), - mnesia:dirty_update_counter(PerUserChTableName, Username, 1); - [#tracked_channel{}] -> - ok - end, - ok. - --spec unregister_tracked(rabbit_types:tracked_channel_id()) -> ok. - -unregister_tracked(ChId = {Node, _Name}) when Node =:= node() -> - TableName = tracked_channel_table_name_for(Node), - PerUserChannelTableName = tracked_channel_per_user_table_name_for(Node), - case mnesia:dirty_read(TableName, ChId) of - [] -> ok; - [#tracked_channel{username = Username}] -> - mnesia:dirty_update_counter(PerUserChannelTableName, Username, -1), - mnesia:dirty_delete(TableName, ChId) - end. - --spec count_tracked_items_in({atom(), rabbit_types:username()}) -> non_neg_integer(). - -count_tracked_items_in({user, Username}) -> - rabbit_tracking:count_tracked_items( - fun tracked_channel_per_user_table_name_for/1, - #tracked_channel_per_user.channel_count, Username, - "channels in vhost"). - --spec clear_tracking_tables() -> ok. - -clear_tracking_tables() -> - clear_tracked_channel_tables_for_this_node(), - ok. - --spec shutdown_tracked_items(list(), term()) -> ok. - -shutdown_tracked_items(TrackedItems, _Args) -> - close_channels(TrackedItems). - -%% helper functions --spec list() -> [rabbit_types:tracked_channel()]. - -list() -> - lists:foldl( - fun (Node, Acc) -> - Tab = tracked_channel_table_name_for(Node), - Acc ++ mnesia:dirty_match_object(Tab, #tracked_channel{_ = '_'}) - end, [], rabbit_nodes:all_running()). - --spec list_of_user(rabbit_types:username()) -> [rabbit_types:tracked_channel()]. - -list_of_user(Username) -> - rabbit_tracking:match_tracked_items( - fun tracked_channel_table_name_for/1, - #tracked_channel{username = Username, _ = '_'}). - --spec list_on_node(node()) -> [rabbit_types:tracked_channel()]. - -list_on_node(Node) -> - try mnesia:dirty_match_object( - tracked_channel_table_name_for(Node), - #tracked_channel{_ = '_'}) - catch exit:{aborted, {no_exists, _}} -> [] - end. - --spec tracked_channel_table_name_for(node()) -> atom(). - -tracked_channel_table_name_for(Node) -> - list_to_atom(rabbit_misc:format("tracked_channel_on_node_~s", [Node])). - --spec tracked_channel_per_user_table_name_for(node()) -> atom(). - -tracked_channel_per_user_table_name_for(Node) -> - list_to_atom(rabbit_misc:format( - "tracked_channel_table_per_user_on_node_~s", [Node])). - -%% internal -ensure_tracked_channels_table_for_this_node() -> - ensure_tracked_channels_table_for_node(node()). - -ensure_per_user_tracked_channels_table_for_node() -> - ensure_per_user_tracked_channels_table_for_node(node()). - -%% Create tables -ensure_tracked_channels_table_for_node(Node) -> - TableName = tracked_channel_table_name_for(Node), - case mnesia:create_table(TableName, [{record_name, tracked_channel}, - {attributes, record_info(fields, tracked_channel)}]) of - {atomic, ok} -> ok; - {aborted, {already_exists, _}} -> ok; - {aborted, Error} -> - rabbit_log:error("Failed to create a tracked channel table for node ~p: ~p", [Node, Error]), - ok - end. - -ensure_per_user_tracked_channels_table_for_node(Node) -> - TableName = tracked_channel_per_user_table_name_for(Node), - case mnesia:create_table(TableName, [{record_name, tracked_channel_per_user}, - {attributes, record_info(fields, tracked_channel_per_user)}]) of - {atomic, ok} -> ok; - {aborted, {already_exists, _}} -> ok; - {aborted, Error} -> - rabbit_log:error("Failed to create a per-user tracked channel table for node ~p: ~p", [Node, Error]), - ok - end. - -clear_tracked_channel_tables_for_this_node() -> - [rabbit_tracking:clear_tracking_table(T) - || T <- get_all_tracked_channel_table_names_for_node(node())]. - -delete_tracked_channels_table_for_node(Node) -> - TableName = tracked_channel_table_name_for(Node), - rabbit_tracking:delete_tracking_table(TableName, Node, "tracked channel"). - -delete_per_user_tracked_channels_table_for_node(Node) -> - TableName = tracked_channel_per_user_table_name_for(Node), - rabbit_tracking:delete_tracking_table(TableName, Node, - "per-user tracked channels"). - -get_all_tracked_channel_table_names_for_node(Node) -> - [tracked_channel_table_name_for(Node), - tracked_channel_per_user_table_name_for(Node)]. - -get_tracked_channels_by_connection_pid(ConnPid) -> - rabbit_tracking:match_tracked_items( - fun tracked_channel_table_name_for/1, - #tracked_channel{connection = ConnPid, _ = '_'}). - -get_tracked_channel_by_pid(ChPid) -> - rabbit_tracking:match_tracked_items( - fun tracked_channel_table_name_for/1, - #tracked_channel{pid = ChPid, _ = '_'}). - -delete_tracked_channel_user_entry(Username) -> - rabbit_tracking:delete_tracked_entry( - {rabbit_auth_backend_internal, exists, [Username]}, - fun tracked_channel_per_user_table_name_for/1, - Username). - -tracked_channel_from_channel_created_event(ChannelDetails) -> - Node = node(ChPid = pget(pid, ChannelDetails)), - Name = pget(name, ChannelDetails), - #tracked_channel{ - id = rabbit_tracking:id(Node, Name), - name = Name, - node = Node, - vhost = pget(vhost, ChannelDetails), - pid = ChPid, - connection = pget(connection, ChannelDetails), - username = pget(user, ChannelDetails)}. - -close_channels(TrackedChannels = [#tracked_channel{}|_]) -> - [rabbit_channel:shutdown(ChPid) - || #tracked_channel{pid = ChPid} <- TrackedChannels], - ok; -close_channels(_TrackedChannels = []) -> ok. diff --git a/src/rabbit_channel_tracking_handler.erl b/src/rabbit_channel_tracking_handler.erl deleted file mode 100644 index 0cbe02f39e..0000000000 --- a/src/rabbit_channel_tracking_handler.erl +++ /dev/null @@ -1,71 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_channel_tracking_handler). - -%% This module keeps track of channel creation and termination events -%% on its local node. Similar to the rabbit_connection_tracking_handler, -%% the primary goal here is to decouple channel tracking from rabbit_reader -%% and isolate channel tracking to its own process to avoid blocking connection -%% creation events. Additionaly, creation events are also non-blocking in that -%% they spawn a short-live process for updating the tracking tables in realtime. -%% -%% Events from other nodes are ignored. - --behaviour(gen_event). - --export([init/1, handle_call/2, handle_event/2, handle_info/2, - terminate/2, code_change/3]). - --include_lib("rabbit.hrl"). - --rabbit_boot_step({?MODULE, - [{description, "channel tracking event handler"}, - {mfa, {gen_event, add_handler, - [rabbit_event, ?MODULE, []]}}, - {cleanup, {gen_event, delete_handler, - [rabbit_event, ?MODULE, []]}}, - {requires, [channel_tracking]}, - {enables, recovery}]}). - -%% -%% API -%% - -init([]) -> - {ok, []}. - -handle_event(#event{type = channel_created, props = Details}, State) -> - ok = rabbit_channel_tracking:update_tracked({channel_created, Details}), - {ok, State}; -handle_event(#event{type = channel_closed, props = Details}, State) -> - ok = rabbit_channel_tracking:update_tracked({channel_closed, Details}), - {ok, State}; -handle_event(#event{type = connection_closed, props = Details}, State) -> - ok = rabbit_channel_tracking:update_tracked({connection_closed, Details}), - {ok, State}; -handle_event(#event{type = user_deleted, props = Details}, State) -> - ok = rabbit_channel_tracking:update_tracked({user_deleted, Details}), - {ok, State}; -%% A node had been deleted from the cluster. -handle_event(#event{type = node_deleted, props = Details}, State) -> - ok = rabbit_channel_tracking:update_tracked({node_deleted, Details}), - {ok, State}; -handle_event(_Event, State) -> - {ok, State}. - -handle_call(_Request, State) -> - {ok, not_understood, State}. - -handle_info(_Info, State) -> - {ok, State}. - -terminate(_Arg, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. diff --git a/src/rabbit_classic_queue.erl b/src/rabbit_classic_queue.erl deleted file mode 100644 index e53c0aecc2..0000000000 --- a/src/rabbit_classic_queue.erl +++ /dev/null @@ -1,527 +0,0 @@ --module(rabbit_classic_queue). --behaviour(rabbit_queue_type). - --include("amqqueue.hrl"). --include_lib("rabbit_common/include/rabbit.hrl"). - --record(msg_status, {pending :: [pid()], - confirmed = [] :: [pid()]}). - --record(?MODULE, {pid :: undefined | pid(), %% the current master pid - qref :: term(), %% TODO - unconfirmed = #{} :: - #{non_neg_integer() => #msg_status{}}}). --define(STATE, ?MODULE). - --opaque state() :: #?STATE{}. - --export_type([state/0]). - --export([ - is_enabled/0, - declare/2, - delete/4, - is_recoverable/1, - recover/2, - purge/1, - policy_changed/1, - stat/1, - init/1, - close/1, - update/2, - consume/3, - cancel/5, - handle_event/2, - deliver/2, - settle/4, - credit/4, - dequeue/4, - info/2, - state_info/1, - capabilities/0 - ]). - --export([delete_crashed/1, - delete_crashed/2, - delete_crashed_internal/2]). - --export([confirm_to_sender/3, - send_rejection/3, - send_queue_event/3]). - -is_enabled() -> true. - -declare(Q, Node) when ?amqqueue_is_classic(Q) -> - QName = amqqueue:get_name(Q), - VHost = amqqueue:get_vhost(Q), - Node1 = case Node of - {ignore_location, Node0} -> - Node0; - _ -> - case rabbit_queue_master_location_misc:get_location(Q) of - {ok, Node0} -> Node0; - _ -> Node - end - end, - Node1 = rabbit_mirror_queue_misc:initial_queue_node(Q, Node1), - case rabbit_vhost_sup_sup:get_vhost_sup(VHost, Node1) of - {ok, _} -> - gen_server2:call( - rabbit_amqqueue_sup_sup:start_queue_process(Node1, Q, declare), - {init, new}, infinity); - {error, Error} -> - {protocol_error, internal_error, "Cannot declare a queue '~s' on node '~s': ~255p", - [rabbit_misc:rs(QName), Node1, Error]} - end. - -delete(Q, IfUnused, IfEmpty, ActingUser) when ?amqqueue_is_classic(Q) -> - case wait_for_promoted_or_stopped(Q) of - {promoted, Q1} -> - QPid = amqqueue:get_pid(Q1), - delegate:invoke(QPid, {gen_server2, call, - [{delete, IfUnused, IfEmpty, ActingUser}, - infinity]}); - {stopped, Q1} -> - #resource{name = Name, virtual_host = Vhost} = amqqueue:get_name(Q1), - case IfEmpty of - true -> - rabbit_log:error("Queue ~s in vhost ~s has its master node down and " - "no mirrors available or eligible for promotion. " - "The queue may be non-empty. " - "Refusing to force-delete.", - [Name, Vhost]), - {error, not_empty}; - false -> - rabbit_log:warning("Queue ~s in vhost ~s has its master node is down and " - "no mirrors available or eligible for promotion. " - "Forcing queue deletion.", - [Name, Vhost]), - delete_crashed_internal(Q1, ActingUser), - {ok, 0} - end; - {error, not_found} -> - %% Assume the queue was deleted - {ok, 0} - end. - -is_recoverable(Q) when ?is_amqqueue(Q) -> - Node = node(), - Node =:= node(amqqueue:get_pid(Q)) andalso - %% Terminations on node down will not remove the rabbit_queue - %% record if it is a mirrored queue (such info is now obtained from - %% the policy). Thus, we must check if the local pid is alive - %% - if the record is present - in order to restart. - (mnesia:read(rabbit_queue, amqqueue:get_name(Q), read) =:= [] - orelse not rabbit_mnesia:is_process_alive(amqqueue:get_pid(Q))). - -recover(VHost, Queues) -> - {ok, BQ} = application:get_env(rabbit, backing_queue_module), - %% We rely on BQ:start/1 returning the recovery terms in the same - %% order as the supplied queue names, so that we can zip them together - %% for further processing in recover_durable_queues. - {ok, OrderedRecoveryTerms} = - BQ:start(VHost, [amqqueue:get_name(Q) || Q <- Queues]), - case rabbit_amqqueue_sup_sup:start_for_vhost(VHost) of - {ok, _} -> - RecoveredQs = recover_durable_queues(lists:zip(Queues, - OrderedRecoveryTerms)), - RecoveredNames = [amqqueue:get_name(Q) || Q <- RecoveredQs], - FailedQueues = [Q || Q <- Queues, - not lists:member(amqqueue:get_name(Q), RecoveredNames)], - {RecoveredQs, FailedQueues}; - {error, Reason} -> - rabbit_log:error("Failed to start queue supervisor for vhost '~s': ~s", [VHost, Reason]), - throw({error, Reason}) - end. - --spec policy_changed(amqqueue:amqqueue()) -> ok. -policy_changed(Q) -> - QPid = amqqueue:get_pid(Q), - gen_server2:cast(QPid, policy_changed). - -stat(Q) -> - delegate:invoke(amqqueue:get_pid(Q), - {gen_server2, call, [stat, infinity]}). - --spec init(amqqueue:amqqueue()) -> state(). -init(Q) when ?amqqueue_is_classic(Q) -> - QName = amqqueue:get_name(Q), - #?STATE{pid = amqqueue:get_pid(Q), - qref = QName}. - --spec close(state()) -> ok. -close(_State) -> - ok. - --spec update(amqqueue:amqqueue(), state()) -> state(). -update(Q, #?STATE{pid = Pid} = State) when ?amqqueue_is_classic(Q) -> - case amqqueue:get_pid(Q) of - Pid -> - State; - NewPid -> - %% master pid is different, update - State#?STATE{pid = NewPid} - end. - -consume(Q, Spec, State) when ?amqqueue_is_classic(Q) -> - QPid = amqqueue:get_pid(Q), - QRef = amqqueue:get_name(Q), - #{no_ack := NoAck, - channel_pid := ChPid, - limiter_pid := LimiterPid, - limiter_active := LimiterActive, - prefetch_count := ConsumerPrefetchCount, - consumer_tag := ConsumerTag, - exclusive_consume := ExclusiveConsume, - args := Args, - ok_msg := OkMsg, - acting_user := ActingUser} = Spec, - case delegate:invoke(QPid, - {gen_server2, call, - [{basic_consume, NoAck, ChPid, LimiterPid, - LimiterActive, ConsumerPrefetchCount, ConsumerTag, - ExclusiveConsume, Args, OkMsg, ActingUser}, - infinity]}) of - ok -> - %% ask the host process to monitor this pid - %% TODO: track pids as they change - {ok, State#?STATE{pid = QPid}, [{monitor, QPid, QRef}]}; - Err -> - Err - end. - -cancel(Q, ConsumerTag, OkMsg, ActingUser, State) -> - QPid = amqqueue:get_pid(Q), - case delegate:invoke(QPid, {gen_server2, call, - [{basic_cancel, self(), ConsumerTag, - OkMsg, ActingUser}, infinity]}) of - ok -> - {ok, State}; - Err -> Err - end. - --spec settle(rabbit_queue_type:settle_op(), rabbit_types:ctag(), - [non_neg_integer()], state()) -> - {state(), rabbit_queue_type:actions()}. -settle(complete, _CTag, MsgIds, State) -> - Pid = State#?STATE.pid, - delegate:invoke_no_result(Pid, - {gen_server2, cast, [{ack, MsgIds, self()}]}), - {State, []}; -settle(Op, _CTag, MsgIds, State) -> - ChPid = self(), - ok = delegate:invoke_no_result(State#?STATE.pid, - {gen_server2, cast, - [{reject, Op == requeue, MsgIds, ChPid}]}), - {State, []}. - -credit(CTag, Credit, Drain, State) -> - ChPid = self(), - delegate:invoke_no_result(State#?STATE.pid, - {gen_server2, cast, - [{credit, ChPid, CTag, Credit, Drain}]}), - {State, []}. - -handle_event({confirm, MsgSeqNos, Pid}, #?STATE{qref = QRef, - unconfirmed = U0} = State) -> - %% confirms should never result in rejections - {Unconfirmed, ConfirmedSeqNos, []} = - settle_seq_nos(MsgSeqNos, Pid, U0, confirm), - Actions = [{settled, QRef, ConfirmedSeqNos}], - %% handle confirm event from queues - %% in this case the classic queue should track each individual publish and - %% the processes involved and only emit a settle action once they have all - %% been received (or DOWN has been received). - %% Hence this part of the confirm logic is queue specific. - {ok, State#?STATE{unconfirmed = Unconfirmed}, Actions}; -handle_event({reject_publish, SeqNo, _QPid}, - #?STATE{qref = QRef, - unconfirmed = U0} = State) -> - %% It does not matter which queue rejected the message, - %% if any queue did, it should not be confirmed. - {U, Rejected} = reject_seq_no(SeqNo, U0), - Actions = [{rejected, QRef, Rejected}], - {ok, State#?STATE{unconfirmed = U}, Actions}; -handle_event({down, Pid, Info}, #?STATE{qref = QRef, - pid = MasterPid, - unconfirmed = U0} = State0) -> - Actions0 = case Pid =:= MasterPid of - true -> - [{queue_down, QRef}]; - false -> - [] - end, - case rabbit_misc:is_abnormal_exit(Info) of - false when Info =:= normal andalso Pid == MasterPid -> - %% queue was deleted and masterpid is down - eol; - false -> - %% this assumes the mirror isn't part of the active set - MsgSeqNos = maps:keys( - maps:filter(fun (_, #msg_status{pending = Pids}) -> - lists:member(Pid, Pids) - end, U0)), - {Unconfirmed, Settled, Rejected} = - settle_seq_nos(MsgSeqNos, Pid, U0, down), - Actions = settlement_action( - settled, QRef, Settled, - settlement_action(rejected, QRef, Rejected, Actions0)), - {ok, State0#?STATE{unconfirmed = Unconfirmed}, Actions}; - true -> - %% any abnormal exit should be considered a full reject of the - %% oustanding message ids - If the message didn't get to all - %% mirrors we have to assume it will never get there - MsgIds = maps:fold( - fun (SeqNo, Status, Acc) -> - case lists:member(Pid, Status#msg_status.pending) of - true -> - [SeqNo | Acc]; - false -> - Acc - end - end, [], U0), - U = maps:without(MsgIds, U0), - {ok, State0#?STATE{unconfirmed = U}, - [{rejected, QRef, MsgIds} | Actions0]} - end; -handle_event({send_credit_reply, _} = Action, State) -> - {ok, State, [Action]}. - -settlement_action(_Type, _QRef, [], Acc) -> - Acc; -settlement_action(Type, QRef, MsgSeqs, Acc) -> - [{Type, QRef, MsgSeqs} | Acc]. - --spec deliver([{amqqueue:amqqueue(), state()}], - Delivery :: term()) -> - {[{amqqueue:amqqueue(), state()}], rabbit_queue_type:actions()}. -deliver(Qs0, #delivery{flow = Flow, - msg_seq_no = MsgNo, - message = #basic_message{exchange_name = _Ex}, - confirm = _Confirm} = Delivery) -> - %% TODO: record master and slaves for confirm processing - {MPids, SPids, Qs, Actions} = qpids(Qs0, MsgNo), - QPids = MPids ++ SPids, - case Flow of - %% Here we are tracking messages sent by the rabbit_channel - %% process. We are accessing the rabbit_channel process - %% dictionary. - flow -> [credit_flow:send(QPid) || QPid <- QPids], - [credit_flow:send(QPid) || QPid <- SPids]; - noflow -> ok - end, - MMsg = {deliver, Delivery, false}, - SMsg = {deliver, Delivery, true}, - delegate:invoke_no_result(MPids, {gen_server2, cast, [MMsg]}), - delegate:invoke_no_result(SPids, {gen_server2, cast, [SMsg]}), - {Qs, Actions}. - - --spec dequeue(NoAck :: boolean(), LimiterPid :: pid(), - rabbit_types:ctag(), state()) -> - {ok, Count :: non_neg_integer(), rabbit_amqqueue:qmsg(), state()} | - {empty, state()}. -dequeue(NoAck, LimiterPid, _CTag, State) -> - QPid = State#?STATE.pid, - case delegate:invoke(QPid, {gen_server2, call, - [{basic_get, self(), NoAck, LimiterPid}, infinity]}) of - empty -> - {empty, State}; - {ok, Count, Msg} -> - {ok, Count, Msg, State} - end. - --spec state_info(state()) -> #{atom() := term()}. -state_info(_State) -> - #{}. - -%% general queue info --spec info(amqqueue:amqqueue(), all_keys | rabbit_types:info_keys()) -> - rabbit_types:infos(). -info(Q, Items) -> - QPid = amqqueue:get_pid(Q), - Req = case Items of - all_keys -> info; - _ -> {info, Items} - end, - case delegate:invoke(QPid, {gen_server2, call, [Req, infinity]}) of - {ok, Result} -> - Result; - {error, _Err} -> - []; - Result when is_list(Result) -> - %% this is a backwards compatibility clause - Result - end. - --spec purge(amqqueue:amqqueue()) -> - {ok, non_neg_integer()}. -purge(Q) when ?is_amqqueue(Q) -> - QPid = amqqueue:get_pid(Q), - delegate:invoke(QPid, {gen_server2, call, [purge, infinity]}). - -qpids(Qs, MsgNo) -> - lists:foldl( - fun ({Q, S0}, {MPidAcc, SPidAcc, Qs0, Actions0}) -> - QPid = amqqueue:get_pid(Q), - SPids = amqqueue:get_slave_pids(Q), - QRef = amqqueue:get_name(Q), - Actions = [{monitor, QPid, QRef} - | [{monitor, P, QRef} || P <- SPids]] ++ Actions0, - %% confirm record only if MsgNo isn't undefined - S = case S0 of - #?STATE{unconfirmed = U0} -> - Rec = [QPid | SPids], - U = case MsgNo of - undefined -> - U0; - _ -> - U0#{MsgNo => #msg_status{pending = Rec}} - end, - S0#?STATE{pid = QPid, - unconfirmed = U}; - stateless -> - S0 - end, - {[QPid | MPidAcc], SPidAcc ++ SPids, - [{Q, S} | Qs0], Actions} - end, {[], [], [], []}, Qs). - -%% internal-ish --spec wait_for_promoted_or_stopped(amqqueue:amqqueue()) -> - {promoted, amqqueue:amqqueue()} | - {stopped, amqqueue:amqqueue()} | - {error, not_found}. -wait_for_promoted_or_stopped(Q0) -> - QName = amqqueue:get_name(Q0), - case rabbit_amqqueue:lookup(QName) of - {ok, Q} -> - QPid = amqqueue:get_pid(Q), - SPids = amqqueue:get_slave_pids(Q), - case rabbit_mnesia:is_process_alive(QPid) of - true -> {promoted, Q}; - false -> - case lists:any(fun(Pid) -> - rabbit_mnesia:is_process_alive(Pid) - end, SPids) of - %% There is a live slave. May be promoted - true -> - timer:sleep(100), - wait_for_promoted_or_stopped(Q); - %% All slave pids are stopped. - %% No process left for the queue - false -> {stopped, Q} - end - end; - {error, not_found} -> - {error, not_found} - end. - --spec delete_crashed(amqqueue:amqqueue()) -> ok. -delete_crashed(Q) -> - delete_crashed(Q, ?INTERNAL_USER). - -delete_crashed(Q, ActingUser) -> - ok = rpc:call(amqqueue:qnode(Q), ?MODULE, delete_crashed_internal, - [Q, ActingUser]). - -delete_crashed_internal(Q, ActingUser) -> - QName = amqqueue:get_name(Q), - {ok, BQ} = application:get_env(rabbit, backing_queue_module), - BQ:delete_crashed(Q), - ok = rabbit_amqqueue:internal_delete(QName, ActingUser). - -recover_durable_queues(QueuesAndRecoveryTerms) -> - {Results, Failures} = - gen_server2:mcall( - [{rabbit_amqqueue_sup_sup:start_queue_process(node(), Q, recovery), - {init, {self(), Terms}}} || {Q, Terms} <- QueuesAndRecoveryTerms]), - [rabbit_log:error("Queue ~p failed to initialise: ~p~n", - [Pid, Error]) || {Pid, Error} <- Failures], - [Q || {_, {new, Q}} <- Results]. - -capabilities() -> - #{policies => [<<"expires">>, <<"message-ttl">>, <<"dead-letter-exchange">>, - <<"dead-letter-routing-key">>, <<"max-length">>, - <<"max-length-bytes">>, <<"max-in-memory-length">>, <<"max-in-memory-bytes">>, - <<"max-priority">>, <<"overflow">>, <<"queue-mode">>, - <<"single-active-consumer">>, <<"delivery-limit">>, - <<"ha-mode">>, <<"ha-params">>, <<"ha-sync-mode">>, - <<"ha-promote-on-shutdown">>, <<"ha-promote-on-failure">>, - <<"queue-master-locator">>], - queue_arguments => [<<"x-expires">>, <<"x-message-ttl">>, <<"x-dead-letter-exchange">>, - <<"x-dead-letter-routing-key">>, <<"x-max-length">>, - <<"x-max-length-bytes">>, <<"x-max-in-memory-length">>, - <<"x-max-in-memory-bytes">>, <<"x-max-priority">>, - <<"x-overflow">>, <<"x-queue-mode">>, <<"x-single-active-consumer">>, - <<"x-queue-type">>, <<"x-queue-master-locator">>], - consumer_arguments => [<<"x-cancel-on-ha-failover">>, - <<"x-priority">>, <<"x-credit">> - ], - server_named => true}. - -reject_seq_no(SeqNo, U0) -> - reject_seq_no(SeqNo, U0, []). - -reject_seq_no(SeqNo, U0, Acc) -> - case maps:take(SeqNo, U0) of - {_, U} -> - {U, [SeqNo | Acc]}; - error -> - {U0, Acc} - end. - -settle_seq_nos(MsgSeqNos, Pid, U0, Reason) -> - lists:foldl( - fun (SeqNo, {U, C0, R0}) -> - case U of - #{SeqNo := Status0} -> - case update_msg_status(Reason, Pid, Status0) of - #msg_status{pending = [], - confirmed = []} -> - %% no pending left and nothing confirmed - %% then we reject it - {maps:remove(SeqNo, U), C0, [SeqNo | R0]}; - #msg_status{pending = [], - confirmed = _} -> - %% this can be confirmed as there are no pending - %% and confirmed isn't empty - {maps:remove(SeqNo, U), [SeqNo | C0], R0}; - MsgStatus -> - {U#{SeqNo => MsgStatus}, C0, R0} - end; - _ -> - {U, C0, R0} - end - end, {U0, [], []}, MsgSeqNos). - -update_msg_status(confirm, Pid, #msg_status{pending = P, - confirmed = C} = S) -> - Rem = lists:delete(Pid, P), - S#msg_status{pending = Rem, confirmed = [Pid | C]}; -update_msg_status(down, Pid, #msg_status{pending = P} = S) -> - S#msg_status{pending = lists:delete(Pid, P)}. - -%% part of channel <-> queue api -confirm_to_sender(Pid, QName, MsgSeqNos) -> - %% the stream queue included the queue type refactoring and thus requires - %% a different message format - Evt = case rabbit_ff_registry:is_enabled(stream_queue) of - true -> - {queue_event, QName, {confirm, MsgSeqNos, self()}}; - false -> - {confirm, MsgSeqNos, self()} - end, - gen_server2:cast(Pid, Evt). - -send_rejection(Pid, QName, MsgSeqNo) -> - case rabbit_ff_registry:is_enabled(stream_queue) of - true -> - gen_server2:cast(Pid, {queue_event, QName, - {reject_publish, MsgSeqNo, self()}}); - false -> - gen_server2:cast(Pid, {reject_publish, MsgSeqNo, self()}) - end. - -send_queue_event(Pid, QName, Evt) -> - gen_server2:cast(Pid, {queue_event, QName, Evt}). diff --git a/src/rabbit_client_sup.erl b/src/rabbit_client_sup.erl deleted file mode 100644 index a28e4ce39c..0000000000 --- a/src/rabbit_client_sup.erl +++ /dev/null @@ -1,43 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_client_sup). - --behaviour(supervisor2). - --export([start_link/1, start_link/2, start_link_worker/2]). - --export([init/1]). - --include("rabbit.hrl"). - -%%---------------------------------------------------------------------------- - --spec start_link(rabbit_types:mfargs()) -> - rabbit_types:ok_pid_or_error(). - -start_link(Callback) -> - supervisor2:start_link(?MODULE, Callback). - --spec start_link({'local', atom()}, rabbit_types:mfargs()) -> - rabbit_types:ok_pid_or_error(). - -start_link(SupName, Callback) -> - supervisor2:start_link(SupName, ?MODULE, Callback). - --spec start_link_worker({'local', atom()}, rabbit_types:mfargs()) -> - rabbit_types:ok_pid_or_error(). - -start_link_worker(SupName, Callback) -> - supervisor2:start_link(SupName, ?MODULE, {Callback, worker}). - -init({M,F,A}) -> - {ok, {{simple_one_for_one, 0, 1}, - [{client, {M,F,A}, temporary, infinity, supervisor, [M]}]}}; -init({{M,F,A}, worker}) -> - {ok, {{simple_one_for_one, 0, 1}, - [{client, {M,F,A}, temporary, ?WORKER_WAIT, worker, [M]}]}}. diff --git a/src/rabbit_config.erl b/src/rabbit_config.erl deleted file mode 100644 index 1198035a7a..0000000000 --- a/src/rabbit_config.erl +++ /dev/null @@ -1,46 +0,0 @@ --module(rabbit_config). - --export([ - config_files/0, - get_advanced_config/0 - ]). - --export([schema_dir/0]). --deprecated([{schema_dir, 0, eventually}]). - --export_type([config_location/0]). - --type config_location() :: string(). - -get_confs() -> - case get_prelaunch_config_state() of - #{config_files := Confs} -> Confs; - _ -> [] - end. - -schema_dir() -> - undefined. - -get_advanced_config() -> - case get_prelaunch_config_state() of - %% There can be only one advanced.config - #{config_advanced_file := FileName} when FileName =/= undefined -> - case rabbit_file:is_file(FileName) of - true -> FileName; - false -> none - end; - _ -> none - end. - --spec config_files() -> [config_location()]. -config_files() -> - ConfFiles = [filename:absname(File) || File <- get_confs(), - filelib:is_regular(File)], - AdvancedFiles = case get_advanced_config() of - none -> []; - FileName -> [filename:absname(FileName)] - end, - AdvancedFiles ++ ConfFiles. - -get_prelaunch_config_state() -> - rabbit_prelaunch_conf:get_config_state(). diff --git a/src/rabbit_confirms.erl b/src/rabbit_confirms.erl deleted file mode 100644 index 2fe032d1f1..0000000000 --- a/src/rabbit_confirms.erl +++ /dev/null @@ -1,152 +0,0 @@ --module(rabbit_confirms). - --compile({no_auto_import, [size/1]}). - --include_lib("rabbit_common/include/rabbit.hrl"). - --export([init/0, - insert/4, - confirm/3, - reject/2, - - remove_queue/2, - - smallest/1, - size/1, - is_empty/1]). - --type seq_no() :: non_neg_integer(). --type queue_name() :: rabbit_amqqueue:name(). --type exchange_name() :: rabbit_exchange:name(). - --record(?MODULE, {smallest :: undefined | seq_no(), - unconfirmed = #{} :: #{seq_no() => - {exchange_name(), - #{queue_name() => ok}}} - }). - --type mx() :: {seq_no(), exchange_name()}. - --opaque state() :: #?MODULE{}. - --export_type([ - state/0 - ]). - --spec init() -> state(). -init() -> - #?MODULE{}. - --spec insert(seq_no(), [queue_name()], exchange_name(), state()) -> - state(). -insert(SeqNo, QNames, #resource{kind = exchange} = XName, - #?MODULE{smallest = S0, - unconfirmed = U0} = State) - when is_integer(SeqNo) - andalso is_list(QNames) - andalso is_map_key(SeqNo, U0) == false -> - U = U0#{SeqNo => {XName, maps:from_list([{Q, ok} || Q <- QNames])}}, - S = case S0 of - undefined -> SeqNo; - _ -> S0 - end, - State#?MODULE{smallest = S, - unconfirmed = U}. - --spec confirm([seq_no()], queue_name(), state()) -> - {[mx()], state()}. -confirm(SeqNos, QName, #?MODULE{smallest = Smallest0, - unconfirmed = U0} = State) - when is_list(SeqNos) -> - {Confirmed, U} = lists:foldr( - fun (SeqNo, Acc) -> - confirm_one(SeqNo, QName, Acc) - end, {[], U0}, SeqNos), - %% check if smallest is in Confirmed - %% TODO: this can be optimised by checking in the preceeding foldr - Smallest = - case lists:any(fun ({S, _}) -> S == Smallest0 end, Confirmed) of - true -> - %% work out new smallest - next_smallest(Smallest0, U); - false -> - Smallest0 - end, - {Confirmed, State#?MODULE{smallest = Smallest, - unconfirmed = U}}. - --spec reject(seq_no(), state()) -> - {ok, mx(), state()} | {error, not_found}. -reject(SeqNo, #?MODULE{smallest = Smallest0, - unconfirmed = U0} = State) - when is_integer(SeqNo) -> - case maps:take(SeqNo, U0) of - {{XName, _QS}, U} -> - Smallest = case SeqNo of - Smallest0 -> - %% need to scan as the smallest was removed - next_smallest(Smallest0, U); - _ -> - Smallest0 - end, - {ok, {SeqNo, XName}, State#?MODULE{unconfirmed = U, - smallest = Smallest}}; - error -> - {error, not_found} - end. - -%% idempotent --spec remove_queue(queue_name(), state()) -> - {[mx()], state()}. -remove_queue(QName, #?MODULE{unconfirmed = U} = State) -> - SeqNos = maps:fold( - fun (SeqNo, {_XName, QS0}, Acc) -> - case maps:is_key(QName, QS0) of - true -> - [SeqNo | Acc]; - false -> - Acc - end - end, [], U), - confirm(lists:sort(SeqNos), QName,State). - --spec smallest(state()) -> seq_no() | undefined. -smallest(#?MODULE{smallest = Smallest}) -> - Smallest. - --spec size(state()) -> non_neg_integer(). -size(#?MODULE{unconfirmed = U}) -> - maps:size(U). - --spec is_empty(state()) -> boolean(). -is_empty(State) -> - size(State) == 0. - -%% INTERNAL - -confirm_one(SeqNo, QName, {Acc, U0}) -> - case maps:take(SeqNo, U0) of - {{XName, QS}, U1} - when is_map_key(QName, QS) - andalso map_size(QS) == 1 -> - %% last queue confirm - {[{SeqNo, XName} | Acc], U1}; - {{XName, QS}, U1} -> - {Acc, U1#{SeqNo => {XName, maps:remove(QName, QS)}}}; - error -> - {Acc, U0} - end. - -next_smallest(_S, U) when map_size(U) == 0 -> - undefined; -next_smallest(S, U) when is_map_key(S, U) -> - S; -next_smallest(S, U) -> - %% TODO: this is potentially infinitely recursive if called incorrectly - next_smallest(S+1, U). - - - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --endif. diff --git a/src/rabbit_connection_helper_sup.erl b/src/rabbit_connection_helper_sup.erl deleted file mode 100644 index d0509029fd..0000000000 --- a/src/rabbit_connection_helper_sup.erl +++ /dev/null @@ -1,57 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_connection_helper_sup). - -%% Supervises auxiliary processes of AMQP 0-9-1 connections: -%% -%% * Channel supervisor -%% * Heartbeat receiver -%% * Heartbeat sender -%% * Exclusive queue collector -%% -%% See also rabbit_heartbeat, rabbit_channel_sup_sup, rabbit_queue_collector. - --behaviour(supervisor2). - --export([start_link/0]). --export([start_channel_sup_sup/1, - start_queue_collector/2]). - --export([init/1]). - --include("rabbit.hrl"). - -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - supervisor2:start_link(?MODULE, []). - --spec start_channel_sup_sup(pid()) -> rabbit_types:ok_pid_or_error(). - -start_channel_sup_sup(SupPid) -> - supervisor2:start_child( - SupPid, - {channel_sup_sup, {rabbit_channel_sup_sup, start_link, []}, - intrinsic, infinity, supervisor, [rabbit_channel_sup_sup]}). - --spec start_queue_collector(pid(), rabbit_types:proc_name()) -> - rabbit_types:ok_pid_or_error(). - -start_queue_collector(SupPid, Identity) -> - supervisor2:start_child( - SupPid, - {collector, {rabbit_queue_collector, start_link, [Identity]}, - intrinsic, ?WORKER_WAIT, worker, [rabbit_queue_collector]}). - -%%---------------------------------------------------------------------------- - -init([]) -> - ?LG_PROCESS_TYPE(connection_helper_sup), - {ok, {{one_for_one, 10, 10}, []}}. diff --git a/src/rabbit_connection_sup.erl b/src/rabbit_connection_sup.erl deleted file mode 100644 index c1d1bd0d77..0000000000 --- a/src/rabbit_connection_sup.erl +++ /dev/null @@ -1,66 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_connection_sup). - -%% Supervisor for a (network) AMQP 0-9-1 client connection. -%% -%% Supervises -%% -%% * rabbit_reader -%% * Auxiliary process supervisor -%% -%% See also rabbit_reader, rabbit_connection_helper_sup. - --behaviour(supervisor2). --behaviour(ranch_protocol). - --export([start_link/4, reader/1]). - --export([init/1]). - --include("rabbit.hrl"). - -%%---------------------------------------------------------------------------- - --spec start_link(any(), rabbit_net:socket(), module(), any()) -> - {'ok', pid(), pid()}. - -start_link(Ref, _Sock, _Transport, _Opts) -> - {ok, SupPid} = supervisor2:start_link(?MODULE, []), - %% We need to get channels in the hierarchy here so they get shut - %% down after the reader, so the reader gets a chance to terminate - %% them cleanly. But for 1.0 readers we can't start the real - %% ch_sup_sup (because we don't know if we will be 0-9-1 or 1.0) - - %% so we add another supervisor into the hierarchy. - %% - %% This supervisor also acts as an intermediary for heartbeaters and - %% the queue collector process, since these must not be siblings of the - %% reader due to the potential for deadlock if they are added/restarted - %% whilst the supervision tree is shutting down. - {ok, HelperSup} = - supervisor2:start_child( - SupPid, - {helper_sup, {rabbit_connection_helper_sup, start_link, []}, - intrinsic, infinity, supervisor, [rabbit_connection_helper_sup]}), - {ok, ReaderPid} = - supervisor2:start_child( - SupPid, - {reader, {rabbit_reader, start_link, [HelperSup, Ref]}, - intrinsic, ?WORKER_WAIT, worker, [rabbit_reader]}), - {ok, SupPid, ReaderPid}. - --spec reader(pid()) -> pid(). - -reader(Pid) -> - hd(supervisor2:find_child(Pid, reader)). - -%%-------------------------------------------------------------------------- - -init([]) -> - ?LG_PROCESS_TYPE(connection_sup), - {ok, {{one_for_all, 0, 1}, []}}. diff --git a/src/rabbit_connection_tracking.erl b/src/rabbit_connection_tracking.erl deleted file mode 100644 index c0704e6a7c..0000000000 --- a/src/rabbit_connection_tracking.erl +++ /dev/null @@ -1,515 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_connection_tracking). - -%% Abstracts away how tracked connection records are stored -%% and queried. -%% -%% See also: -%% -%% * rabbit_connection_tracking_handler -%% * rabbit_reader -%% * rabbit_event --behaviour(rabbit_tracking). - --export([boot/0, - update_tracked/1, - handle_cast/1, - register_tracked/1, - unregister_tracked/1, - count_tracked_items_in/1, - clear_tracking_tables/0, - shutdown_tracked_items/2]). - --export([ensure_tracked_connections_table_for_node/1, - ensure_per_vhost_tracked_connections_table_for_node/1, - ensure_per_user_tracked_connections_table_for_node/1, - - ensure_tracked_connections_table_for_this_node/0, - ensure_per_vhost_tracked_connections_table_for_this_node/0, - ensure_per_user_tracked_connections_table_for_this_node/0, - - tracked_connection_table_name_for/1, - tracked_connection_per_vhost_table_name_for/1, - tracked_connection_per_user_table_name_for/1, - get_all_tracked_connection_table_names_for_node/1, - - delete_tracked_connections_table_for_node/1, - delete_per_vhost_tracked_connections_table_for_node/1, - delete_per_user_tracked_connections_table_for_node/1, - delete_tracked_connection_user_entry/1, - delete_tracked_connection_vhost_entry/1, - - clear_tracked_connection_tables_for_this_node/0, - - list/0, list/1, list_on_node/1, list_on_node/2, list_of_user/1, - tracked_connection_from_connection_created/1, - tracked_connection_from_connection_state/1, - lookup/1, - count/0]). - --include_lib("rabbit.hrl"). - --import(rabbit_misc, [pget/2]). - --export([close_connections/3]). - -%% -%% API -%% - -%% Behaviour callbacks - --spec boot() -> ok. - -%% Sets up and resets connection tracking tables for this -%% node. -boot() -> - ensure_tracked_connections_table_for_this_node(), - rabbit_log:info("Setting up a table for connection tracking on this node: ~p", - [tracked_connection_table_name_for(node())]), - ensure_per_vhost_tracked_connections_table_for_this_node(), - rabbit_log:info("Setting up a table for per-vhost connection counting on this node: ~p", - [tracked_connection_per_vhost_table_name_for(node())]), - ensure_per_user_tracked_connections_table_for_this_node(), - rabbit_log:info("Setting up a table for per-user connection counting on this node: ~p", - [tracked_connection_per_user_table_name_for(node())]), - clear_tracking_tables(), - ok. - --spec update_tracked(term()) -> ok. - -update_tracked(Event) -> - spawn(?MODULE, handle_cast, [Event]), - ok. - -%% Asynchronously handle update events --spec handle_cast(term()) -> ok. - -handle_cast({connection_created, Details}) -> - ThisNode = node(), - case pget(node, Details) of - ThisNode -> - TConn = tracked_connection_from_connection_created(Details), - ConnId = TConn#tracked_connection.id, - try - register_tracked(TConn) - catch - error:{no_exists, _} -> - Msg = "Could not register connection ~p for tracking, " - "its table is not ready yet or the connection terminated prematurely", - rabbit_log_connection:warning(Msg, [ConnId]), - ok; - error:Err -> - Msg = "Could not register connection ~p for tracking: ~p", - rabbit_log_connection:warning(Msg, [ConnId, Err]), - ok - end; - _OtherNode -> - %% ignore - ok - end; -handle_cast({connection_closed, Details}) -> - ThisNode = node(), - case pget(node, Details) of - ThisNode -> - %% [{name,<<"127.0.0.1:64078 -> 127.0.0.1:5672">>}, - %% {pid,<0.1774.0>}, - %% {node, rabbit@hostname}] - unregister_tracked( - rabbit_tracking:id(ThisNode, pget(name, Details))); - _OtherNode -> - %% ignore - ok - end; -handle_cast({vhost_deleted, Details}) -> - VHost = pget(name, Details), - %% Schedule vhost entry deletion, allowing time for connections to close - _ = timer:apply_after(?TRACKING_EXECUTION_TIMEOUT, ?MODULE, - delete_tracked_connection_vhost_entry, [VHost]), - rabbit_log_connection:info("Closing all connections in vhost '~s' because it's being deleted", [VHost]), - shutdown_tracked_items( - rabbit_connection_tracking:list(VHost), - rabbit_misc:format("vhost '~s' is deleted", [VHost])); -%% Note: under normal circumstances this will be called immediately -%% after the vhost_deleted above. Therefore we should be careful about -%% what we log and be more defensive. -handle_cast({vhost_down, Details}) -> - VHost = pget(name, Details), - Node = pget(node, Details), - rabbit_log_connection:info("Closing all connections in vhost '~s' on node '~s'" - " because the vhost is stopping", - [VHost, Node]), - shutdown_tracked_items( - rabbit_connection_tracking:list_on_node(Node, VHost), - rabbit_misc:format("vhost '~s' is down", [VHost])); -handle_cast({user_deleted, Details}) -> - Username = pget(name, Details), - %% Schedule user entry deletion, allowing time for connections to close - _ = timer:apply_after(?TRACKING_EXECUTION_TIMEOUT, ?MODULE, - delete_tracked_connection_user_entry, [Username]), - rabbit_log_connection:info("Closing all connections from user '~s' because it's being deleted", [Username]), - shutdown_tracked_items( - rabbit_connection_tracking:list_of_user(Username), - rabbit_misc:format("user '~s' is deleted", [Username])); -%% A node had been deleted from the cluster. -handle_cast({node_deleted, Details}) -> - Node = pget(node, Details), - rabbit_log_connection:info("Node '~s' was removed from the cluster, deleting its connection tracking tables...", [Node]), - delete_tracked_connections_table_for_node(Node), - delete_per_vhost_tracked_connections_table_for_node(Node), - delete_per_user_tracked_connections_table_for_node(Node). - --spec register_tracked(rabbit_types:tracked_connection()) -> ok. --dialyzer([{nowarn_function, [register_tracked/1]}, race_conditions]). - -register_tracked(#tracked_connection{username = Username, vhost = VHost, id = ConnId, node = Node} = Conn) when Node =:= node() -> - TableName = tracked_connection_table_name_for(Node), - PerVhostTableName = tracked_connection_per_vhost_table_name_for(Node), - PerUserConnTableName = tracked_connection_per_user_table_name_for(Node), - %% upsert - case mnesia:dirty_read(TableName, ConnId) of - [] -> - mnesia:dirty_write(TableName, Conn), - mnesia:dirty_update_counter(PerVhostTableName, VHost, 1), - mnesia:dirty_update_counter(PerUserConnTableName, Username, 1); - [#tracked_connection{}] -> - ok - end, - ok. - --spec unregister_tracked(rabbit_types:tracked_connection_id()) -> ok. - -unregister_tracked(ConnId = {Node, _Name}) when Node =:= node() -> - TableName = tracked_connection_table_name_for(Node), - PerVhostTableName = tracked_connection_per_vhost_table_name_for(Node), - PerUserConnTableName = tracked_connection_per_user_table_name_for(Node), - case mnesia:dirty_read(TableName, ConnId) of - [] -> ok; - [#tracked_connection{vhost = VHost, username = Username}] -> - mnesia:dirty_update_counter(PerUserConnTableName, Username, -1), - mnesia:dirty_update_counter(PerVhostTableName, VHost, -1), - mnesia:dirty_delete(TableName, ConnId) - end. - --spec count_tracked_items_in({atom(), rabbit_types:vhost()}) -> non_neg_integer(). - -count_tracked_items_in({vhost, VirtualHost}) -> - rabbit_tracking:count_tracked_items( - fun tracked_connection_per_vhost_table_name_for/1, - #tracked_connection_per_vhost.connection_count, VirtualHost, - "connections in vhost"); -count_tracked_items_in({user, Username}) -> - rabbit_tracking:count_tracked_items( - fun tracked_connection_per_user_table_name_for/1, - #tracked_connection_per_user.connection_count, Username, - "connections for user"). - --spec clear_tracking_tables() -> ok. - -clear_tracking_tables() -> - clear_tracked_connection_tables_for_this_node(). - --spec shutdown_tracked_items(list(), term()) -> ok. - -shutdown_tracked_items(TrackedItems, Message) -> - close_connections(TrackedItems, Message). - -%% Extended API - --spec ensure_tracked_connections_table_for_this_node() -> ok. - -ensure_tracked_connections_table_for_this_node() -> - ensure_tracked_connections_table_for_node(node()). - - --spec ensure_per_vhost_tracked_connections_table_for_this_node() -> ok. - -ensure_per_vhost_tracked_connections_table_for_this_node() -> - ensure_per_vhost_tracked_connections_table_for_node(node()). - - --spec ensure_per_user_tracked_connections_table_for_this_node() -> ok. - -ensure_per_user_tracked_connections_table_for_this_node() -> - ensure_per_user_tracked_connections_table_for_node(node()). - - -%% Create tables --spec ensure_tracked_connections_table_for_node(node()) -> ok. - -ensure_tracked_connections_table_for_node(Node) -> - TableName = tracked_connection_table_name_for(Node), - case mnesia:create_table(TableName, [{record_name, tracked_connection}, - {attributes, record_info(fields, tracked_connection)}]) of - {atomic, ok} -> ok; - {aborted, {already_exists, _}} -> ok; - {aborted, Error} -> - rabbit_log:error("Failed to create a tracked connection table for node ~p: ~p", [Node, Error]), - ok - end. - --spec ensure_per_vhost_tracked_connections_table_for_node(node()) -> ok. - -ensure_per_vhost_tracked_connections_table_for_node(Node) -> - TableName = tracked_connection_per_vhost_table_name_for(Node), - case mnesia:create_table(TableName, [{record_name, tracked_connection_per_vhost}, - {attributes, record_info(fields, tracked_connection_per_vhost)}]) of - {atomic, ok} -> ok; - {aborted, {already_exists, _}} -> ok; - {aborted, Error} -> - rabbit_log:error("Failed to create a per-vhost tracked connection table for node ~p: ~p", [Node, Error]), - ok - end. - --spec ensure_per_user_tracked_connections_table_for_node(node()) -> ok. - -ensure_per_user_tracked_connections_table_for_node(Node) -> - TableName = tracked_connection_per_user_table_name_for(Node), - case mnesia:create_table(TableName, [{record_name, tracked_connection_per_user}, - {attributes, record_info(fields, tracked_connection_per_user)}]) of - {atomic, ok} -> ok; - {aborted, {already_exists, _}} -> ok; - {aborted, Error} -> - rabbit_log:error("Failed to create a per-user tracked connection table for node ~p: ~p", [Node, Error]), - ok - end. - --spec clear_tracked_connection_tables_for_this_node() -> ok. - -clear_tracked_connection_tables_for_this_node() -> - [rabbit_tracking:clear_tracking_table(T) - || T <- get_all_tracked_connection_table_names_for_node(node())], - ok. - --spec delete_tracked_connections_table_for_node(node()) -> ok. - -delete_tracked_connections_table_for_node(Node) -> - TableName = tracked_connection_table_name_for(Node), - rabbit_tracking:delete_tracking_table(TableName, Node, "tracked connection"). - --spec delete_per_vhost_tracked_connections_table_for_node(node()) -> ok. - -delete_per_vhost_tracked_connections_table_for_node(Node) -> - TableName = tracked_connection_per_vhost_table_name_for(Node), - rabbit_tracking:delete_tracking_table(TableName, Node, - "per-vhost tracked connection"). - --spec delete_per_user_tracked_connections_table_for_node(node()) -> ok. - -delete_per_user_tracked_connections_table_for_node(Node) -> - TableName = tracked_connection_per_user_table_name_for(Node), - rabbit_tracking:delete_tracking_table(TableName, Node, - "per-user tracked connection"). - --spec tracked_connection_table_name_for(node()) -> atom(). - -tracked_connection_table_name_for(Node) -> - list_to_atom(rabbit_misc:format("tracked_connection_on_node_~s", [Node])). - --spec tracked_connection_per_vhost_table_name_for(node()) -> atom(). - -tracked_connection_per_vhost_table_name_for(Node) -> - list_to_atom(rabbit_misc:format("tracked_connection_per_vhost_on_node_~s", [Node])). - --spec tracked_connection_per_user_table_name_for(node()) -> atom(). - -tracked_connection_per_user_table_name_for(Node) -> - list_to_atom(rabbit_misc:format( - "tracked_connection_table_per_user_on_node_~s", [Node])). - --spec get_all_tracked_connection_table_names_for_node(node()) -> [atom()]. - -get_all_tracked_connection_table_names_for_node(Node) -> - [tracked_connection_table_name_for(Node), - tracked_connection_per_vhost_table_name_for(Node), - tracked_connection_per_user_table_name_for(Node)]. - --spec lookup(rabbit_types:connection_name()) -> rabbit_types:tracked_connection() | 'not_found'. - -lookup(Name) -> - Nodes = rabbit_nodes:all_running(), - lookup(Name, Nodes). - -lookup(_, []) -> - not_found; -lookup(Name, [Node | Nodes]) -> - TableName = tracked_connection_table_name_for(Node), - case mnesia:dirty_read(TableName, {Node, Name}) of - [] -> lookup(Name, Nodes); - [Row] -> Row - end. - --spec list() -> [rabbit_types:tracked_connection()]. - -list() -> - lists:foldl( - fun (Node, Acc) -> - Tab = tracked_connection_table_name_for(Node), - Acc ++ mnesia:dirty_match_object(Tab, #tracked_connection{_ = '_'}) - end, [], rabbit_nodes:all_running()). - --spec count() -> non_neg_integer(). - -count() -> - lists:foldl( - fun (Node, Acc) -> - Tab = tracked_connection_table_name_for(Node), - Acc + mnesia:table_info(Tab, size) - end, 0, rabbit_nodes:all_running()). - --spec list(rabbit_types:vhost()) -> [rabbit_types:tracked_connection()]. - -list(VHost) -> - rabbit_tracking:match_tracked_items( - fun tracked_connection_table_name_for/1, - #tracked_connection{vhost = VHost, _ = '_'}). - --spec list_on_node(node()) -> [rabbit_types:tracked_connection()]. - -list_on_node(Node) -> - try mnesia:dirty_match_object( - tracked_connection_table_name_for(Node), - #tracked_connection{_ = '_'}) - catch exit:{aborted, {no_exists, _}} -> [] - end. - --spec list_on_node(node(), rabbit_types:vhost()) -> [rabbit_types:tracked_connection()]. - -list_on_node(Node, VHost) -> - try mnesia:dirty_match_object( - tracked_connection_table_name_for(Node), - #tracked_connection{vhost = VHost, _ = '_'}) - catch exit:{aborted, {no_exists, _}} -> [] - end. - - --spec list_of_user(rabbit_types:username()) -> [rabbit_types:tracked_connection()]. - -list_of_user(Username) -> - rabbit_tracking:match_tracked_items( - fun tracked_connection_table_name_for/1, - #tracked_connection{username = Username, _ = '_'}). - -%% Internal, delete tracked entries - -delete_tracked_connection_vhost_entry(Vhost) -> - rabbit_tracking:delete_tracked_entry( - {rabbit_vhost, exists, [Vhost]}, - fun tracked_connection_per_vhost_table_name_for/1, - Vhost). - -delete_tracked_connection_user_entry(Username) -> - rabbit_tracking:delete_tracked_entry( - {rabbit_auth_backend_internal, exists, [Username]}, - fun tracked_connection_per_user_table_name_for/1, - Username). - -%% Returns a #tracked_connection from connection_created -%% event details. -%% -%% @see rabbit_connection_tracking_handler. -tracked_connection_from_connection_created(EventDetails) -> - %% Example event: - %% - %% [{type,network}, - %% {pid,<0.329.0>}, - %% {name,<<"127.0.0.1:60998 -> 127.0.0.1:5672">>}, - %% {port,5672}, - %% {peer_port,60998}, - %% {host,{0,0,0,0,0,65535,32512,1}}, - %% {peer_host,{0,0,0,0,0,65535,32512,1}}, - %% {ssl,false}, - %% {peer_cert_subject,''}, - %% {peer_cert_issuer,''}, - %% {peer_cert_validity,''}, - %% {auth_mechanism,<<"PLAIN">>}, - %% {ssl_protocol,''}, - %% {ssl_key_exchange,''}, - %% {ssl_cipher,''}, - %% {ssl_hash,''}, - %% {protocol,{0,9,1}}, - %% {user,<<"guest">>}, - %% {vhost,<<"/">>}, - %% {timeout,14}, - %% {frame_max,131072}, - %% {channel_max,65535}, - %% {client_properties, - %% [{<<"capabilities">>,table, - %% [{<<"publisher_confirms">>,bool,true}, - %% {<<"consumer_cancel_notify">>,bool,true}, - %% {<<"exchange_exchange_bindings">>,bool,true}, - %% {<<"basic.nack">>,bool,true}, - %% {<<"connection.blocked">>,bool,true}, - %% {<<"authentication_failure_close">>,bool,true}]}, - %% {<<"product">>,longstr,<<"Bunny">>}, - %% {<<"platform">>,longstr, - %% <<"ruby 2.3.0p0 (2015-12-25 revision 53290) [x86_64-darwin15]">>}, - %% {<<"version">>,longstr,<<"2.3.0.pre">>}, - %% {<<"information">>,longstr, - %% <<"http://rubybunny.info">>}]}, - %% {connected_at,1453214290847}] - Name = pget(name, EventDetails), - Node = pget(node, EventDetails), - #tracked_connection{id = rabbit_tracking:id(Node, Name), - name = Name, - node = Node, - vhost = pget(vhost, EventDetails), - username = pget(user, EventDetails), - connected_at = pget(connected_at, EventDetails), - pid = pget(pid, EventDetails), - type = pget(type, EventDetails), - peer_host = pget(peer_host, EventDetails), - peer_port = pget(peer_port, EventDetails)}. - -tracked_connection_from_connection_state(#connection{ - vhost = VHost, - connected_at = Ts, - peer_host = PeerHost, - peer_port = PeerPort, - user = Username, - name = Name - }) -> - tracked_connection_from_connection_created( - [{name, Name}, - {node, node()}, - {vhost, VHost}, - {user, Username}, - {user_who_performed_action, Username}, - {connected_at, Ts}, - {pid, self()}, - {type, network}, - {peer_port, PeerPort}, - {peer_host, PeerHost}]). - -close_connections(Tracked, Message) -> - close_connections(Tracked, Message, 0). - -close_connections(Tracked, Message, Delay) -> - [begin - close_connection(Conn, Message), - timer:sleep(Delay) - end || Conn <- Tracked], - ok. - -close_connection(#tracked_connection{pid = Pid, type = network}, Message) -> - try - rabbit_networking:close_connection(Pid, Message) - catch error:{not_a_connection, _} -> - %% could has been closed concurrently, or the input - %% is bogus. In any case, we should not terminate - ok; - _:Err -> - %% ignore, don't terminate - rabbit_log:warning("Could not close connection ~p: ~p", [Pid, Err]), - ok - end; -close_connection(#tracked_connection{pid = Pid, type = direct}, Message) -> - %% Do an RPC call to the node running the direct client. - Node = node(Pid), - rpc:call(Node, amqp_direct_connection, server_close, [Pid, 320, Message]). diff --git a/src/rabbit_connection_tracking_handler.erl b/src/rabbit_connection_tracking_handler.erl deleted file mode 100644 index 17085d805a..0000000000 --- a/src/rabbit_connection_tracking_handler.erl +++ /dev/null @@ -1,80 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_connection_tracking_handler). - -%% This module keeps track of connection creation and termination events -%% on its local node. The primary goal here is to decouple connection -%% tracking from rabbit_reader in rabbit_common. -%% -%% Events from other nodes are ignored. - --behaviour(gen_event). - --export([init/1, handle_call/2, handle_event/2, handle_info/2, - terminate/2, code_change/3]). - -%% for compatibility with previous versions of CLI tools --export([close_connections/3]). - --include_lib("rabbit.hrl"). - --rabbit_boot_step({?MODULE, - [{description, "connection tracking event handler"}, - {mfa, {gen_event, add_handler, - [rabbit_event, ?MODULE, []]}}, - {cleanup, {gen_event, delete_handler, - [rabbit_event, ?MODULE, []]}}, - {requires, [connection_tracking]}, - {enables, recovery}]}). - -%% -%% API -%% - -init([]) -> - {ok, []}. - -handle_event(#event{type = connection_created, props = Details}, State) -> - ok = rabbit_connection_tracking:update_tracked({connection_created, Details}), - {ok, State}; -handle_event(#event{type = connection_closed, props = Details}, State) -> - ok = rabbit_connection_tracking:update_tracked({connection_closed, Details}), - {ok, State}; -handle_event(#event{type = vhost_deleted, props = Details}, State) -> - ok = rabbit_connection_tracking:update_tracked({vhost_deleted, Details}), - {ok, State}; -%% Note: under normal circumstances this will be called immediately -%% after the vhost_deleted above. Therefore we should be careful about -%% what we log and be more defensive. -handle_event(#event{type = vhost_down, props = Details}, State) -> - ok = rabbit_connection_tracking:update_tracked({vhost_down, Details}), - {ok, State}; -handle_event(#event{type = user_deleted, props = Details}, State) -> - ok = rabbit_connection_tracking:update_tracked({user_deleted, Details}), - {ok, State}; -%% A node had been deleted from the cluster. -handle_event(#event{type = node_deleted, props = Details}, State) -> - ok = rabbit_connection_tracking:update_tracked({node_deleted, Details}), - {ok, State}; -handle_event(_Event, State) -> - {ok, State}. - -handle_call(_Request, State) -> - {ok, not_understood, State}. - -handle_info(_Info, State) -> - {ok, State}. - -terminate(_Arg, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -close_connections(Tracked, Message, Delay) -> - rabbit_connection_tracking:close_connections(Tracked, Message, Delay). diff --git a/src/rabbit_control_pbe.erl b/src/rabbit_control_pbe.erl deleted file mode 100644 index 95c4fe41f1..0000000000 --- a/src/rabbit_control_pbe.erl +++ /dev/null @@ -1,82 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_control_pbe). - --export([decode/4, encode/4, list_ciphers/0, list_hashes/0]). - -% for testing purposes --export([evaluate_input_as_term/1]). - -list_ciphers() -> - {ok, io_lib:format("~p", [rabbit_pbe:supported_ciphers()])}. - -list_hashes() -> - {ok, io_lib:format("~p", [rabbit_pbe:supported_hashes()])}. - -validate(_Cipher, _Hash, Iterations, _Args) when Iterations =< 0 -> - {error, io_lib:format("The requested number of iterations is incorrect", [])}; -validate(_Cipher, _Hash, _Iterations, Args) when length(Args) < 2 -> - {error, io_lib:format("Please provide a value to encode/decode and a passphrase", [])}; -validate(_Cipher, _Hash, _Iterations, Args) when length(Args) > 2 -> - {error, io_lib:format("Too many arguments. Please provide a value to encode/decode and a passphrase", [])}; -validate(Cipher, Hash, _Iterations, _Args) -> - case lists:member(Cipher, rabbit_pbe:supported_ciphers()) of - false -> - {error, io_lib:format("The requested cipher is not supported", [])}; - true -> - case lists:member(Hash, rabbit_pbe:supported_hashes()) of - false -> - {error, io_lib:format("The requested hash is not supported", [])}; - true -> ok - end - end. - -encode(Cipher, Hash, Iterations, Args) -> - case validate(Cipher, Hash, Iterations, Args) of - {error, Err} -> {error, Err}; - ok -> - [Value, PassPhrase] = Args, - try begin - TermValue = evaluate_input_as_term(Value), - Result = {encrypted, _} = rabbit_pbe:encrypt_term(Cipher, Hash, Iterations, - list_to_binary(PassPhrase), TermValue), - {ok, io_lib:format("~p", [Result])} - end - catch - _:Msg -> {error, io_lib:format("Error during cipher operation: ~p", [Msg])} - end - end. - -decode(Cipher, Hash, Iterations, Args) -> - case validate(Cipher, Hash, Iterations, Args) of - {error, Err} -> {error, Err}; - ok -> - [Value, PassPhrase] = Args, - try begin - TermValue = evaluate_input_as_term(Value), - TermToDecrypt = case TermValue of - {encrypted, _}=EncryptedTerm -> - EncryptedTerm; - _ -> - {encrypted, TermValue} - end, - Result = rabbit_pbe:decrypt_term(Cipher, Hash, Iterations, - list_to_binary(PassPhrase), - TermToDecrypt), - {ok, io_lib:format("~p", [Result])} - end - catch - _:Msg -> {error, io_lib:format("Error during cipher operation: ~p", [Msg])} - end - end. - -evaluate_input_as_term(Input) -> - {ok,Tokens,_EndLine} = erl_scan:string(Input ++ "."), - {ok,AbsForm} = erl_parse:parse_exprs(Tokens), - {value,TermValue,_Bs} = erl_eval:exprs(AbsForm, erl_eval:new_bindings()), - TermValue. diff --git a/src/rabbit_core_ff.erl b/src/rabbit_core_ff.erl deleted file mode 100644 index 6d30846775..0000000000 --- a/src/rabbit_core_ff.erl +++ /dev/null @@ -1,179 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_core_ff). - --export([quorum_queue_migration/3, - stream_queue_migration/3, - implicit_default_bindings_migration/3, - virtual_host_metadata_migration/3, - maintenance_mode_status_migration/3, - user_limits_migration/3]). - --rabbit_feature_flag( - {quorum_queue, - #{desc => "Support queues of type `quorum`", - doc_url => "https://www.rabbitmq.com/quorum-queues.html", - stability => stable, - migration_fun => {?MODULE, quorum_queue_migration} - }}). - --rabbit_feature_flag( - {stream_queue, - #{desc => "Support queues of type `stream`", - doc_url => "https://www.rabbitmq.com/stream-queues.html", - stability => stable, - depends_on => [quorum_queue], - migration_fun => {?MODULE, stream_queue_migration} - }}). - --rabbit_feature_flag( - {implicit_default_bindings, - #{desc => "Default bindings are now implicit, instead of " - "being stored in the database", - stability => stable, - migration_fun => {?MODULE, implicit_default_bindings_migration} - }}). - --rabbit_feature_flag( - {virtual_host_metadata, - #{desc => "Virtual host metadata (description, tags, etc)", - stability => stable, - migration_fun => {?MODULE, virtual_host_metadata_migration} - }}). - --rabbit_feature_flag( - {maintenance_mode_status, - #{desc => "Maintenance mode status", - stability => stable, - migration_fun => {?MODULE, maintenance_mode_status_migration} - }}). - --rabbit_feature_flag( - {user_limits, - #{desc => "Configure connection and channel limits for a user", - stability => stable, - migration_fun => {?MODULE, user_limits_migration} - }}). - -%% ------------------------------------------------------------------- -%% Quorum queues. -%% ------------------------------------------------------------------- - --define(quorum_queue_tables, [rabbit_queue, - rabbit_durable_queue]). - -quorum_queue_migration(FeatureName, _FeatureProps, enable) -> - Tables = ?quorum_queue_tables, - rabbit_table:wait(Tables, _Retry = true), - Fields = amqqueue:fields(amqqueue_v2), - migrate_to_amqqueue_with_type(FeatureName, Tables, Fields); -quorum_queue_migration(_FeatureName, _FeatureProps, is_enabled) -> - Tables = ?quorum_queue_tables, - rabbit_table:wait(Tables, _Retry = true), - Fields = amqqueue:fields(amqqueue_v2), - mnesia:table_info(rabbit_queue, attributes) =:= Fields andalso - mnesia:table_info(rabbit_durable_queue, attributes) =:= Fields. - -stream_queue_migration(_FeatureName, _FeatureProps, _Enable) -> - ok. - -migrate_to_amqqueue_with_type(FeatureName, [Table | Rest], Fields) -> - rabbit_log_feature_flags:info( - "Feature flag `~s`: migrating Mnesia table ~s...", - [FeatureName, Table]), - Fun = fun(Queue) -> amqqueue:upgrade_to(amqqueue_v2, Queue) end, - case mnesia:transform_table(Table, Fun, Fields) of - {atomic, ok} -> migrate_to_amqqueue_with_type(FeatureName, - Rest, - Fields); - {aborted, Reason} -> {error, Reason} - end; -migrate_to_amqqueue_with_type(FeatureName, [], _) -> - rabbit_log_feature_flags:info( - "Feature flag `~s`: Mnesia tables migration done", - [FeatureName]), - ok. - -%% ------------------------------------------------------------------- -%% Default bindings. -%% ------------------------------------------------------------------- - -implicit_default_bindings_migration(FeatureName, _FeatureProps, - enable) -> - %% Default exchange bindings are now implicit (not stored in the - %% route tables). It should be safe to remove them outside of a - %% transaction. - rabbit_table:wait([rabbit_queue]), - Queues = mnesia:dirty_all_keys(rabbit_queue), - remove_explicit_default_bindings(FeatureName, Queues); -implicit_default_bindings_migration(_Feature_Name, _FeatureProps, - is_enabled) -> - undefined. - -remove_explicit_default_bindings(_FeatureName, []) -> - ok; -remove_explicit_default_bindings(FeatureName, Queues) -> - rabbit_log_feature_flags:info( - "Feature flag `~s`: deleting explicit default bindings " - "for ~b queues (it may take some time)...", - [FeatureName, length(Queues)]), - [rabbit_binding:remove_default_exchange_binding_rows_of(Q) - || Q <- Queues], - ok. - -%% ------------------------------------------------------------------- -%% Virtual host metadata. -%% ------------------------------------------------------------------- - -virtual_host_metadata_migration(_FeatureName, _FeatureProps, enable) -> - Tab = rabbit_vhost, - rabbit_table:wait([Tab], _Retry = true), - Fun = fun(Row) -> vhost:upgrade_to(vhost_v2, Row) end, - case mnesia:transform_table(Tab, Fun, vhost:fields(vhost_v2)) of - {atomic, ok} -> ok; - {aborted, Reason} -> {error, Reason} - end; -virtual_host_metadata_migration(_FeatureName, _FeatureProps, is_enabled) -> - mnesia:table_info(rabbit_vhost, attributes) =:= vhost:fields(vhost_v2). - -%% ------------------------------------------------------------------- -%% Maintenance mode. -%% ------------------------------------------------------------------- - -maintenance_mode_status_migration(FeatureName, _FeatureProps, enable) -> - TableName = rabbit_maintenance:status_table_name(), - rabbit_log:info( - "Creating table ~s for feature flag `~s`", - [TableName, FeatureName]), - try - _ = rabbit_table:create( - TableName, - rabbit_maintenance:status_table_definition()), - _ = rabbit_table:ensure_table_copy(TableName, node()) - catch throw:Reason -> - rabbit_log:error( - "Failed to create maintenance status table: ~p", - [Reason]) - end; -maintenance_mode_status_migration(_FeatureName, _FeatureProps, is_enabled) -> - rabbit_table:exists(rabbit_maintenance:status_table_name()). - -%% ------------------------------------------------------------------- -%% User limits. -%% ------------------------------------------------------------------- - -user_limits_migration(_FeatureName, _FeatureProps, enable) -> - Tab = rabbit_user, - rabbit_table:wait([Tab], _Retry = true), - Fun = fun(Row) -> internal_user:upgrade_to(internal_user_v2, Row) end, - case mnesia:transform_table(Tab, Fun, internal_user:fields(internal_user_v2)) of - {atomic, ok} -> ok; - {aborted, Reason} -> {error, Reason} - end; -user_limits_migration(_FeatureName, _FeatureProps, is_enabled) -> - mnesia:table_info(rabbit_user, attributes) =:= internal_user:fields(internal_user_v2). diff --git a/src/rabbit_core_metrics_gc.erl b/src/rabbit_core_metrics_gc.erl deleted file mode 100644 index 890c127586..0000000000 --- a/src/rabbit_core_metrics_gc.erl +++ /dev/null @@ -1,199 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% --module(rabbit_core_metrics_gc). - --record(state, {timer, - interval - }). - --export([start_link/0]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -init(_) -> - Interval = rabbit_misc:get_env(rabbit, core_metrics_gc_interval, 120000), - {ok, start_timer(#state{interval = Interval})}. - -handle_call(test, _From, State) -> - {reply, ok, State}. - -handle_cast(_Request, State) -> - {noreply, State}. - -handle_info(start_gc, State) -> - gc_connections(), - gc_channels(), - gc_queues(), - gc_exchanges(), - gc_nodes(), - gc_gen_server2(), - gc_auth_attempts(), - {noreply, start_timer(State)}. - -terminate(_Reason, #state{timer = TRef}) -> - erlang:cancel_timer(TRef), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -start_timer(#state{interval = Interval} = St) -> - TRef = erlang:send_after(Interval, self(), start_gc), - St#state{timer = TRef}. - -gc_connections() -> - gc_process(connection_created), - gc_process(connection_metrics), - gc_process(connection_coarse_metrics). - -gc_channels() -> - gc_process(channel_created), - gc_process(channel_metrics), - gc_process(channel_process_metrics), - ok. - -gc_queues() -> - gc_local_queues(), - gc_global_queues(). - -gc_local_queues() -> - Queues = rabbit_amqqueue:list_local_names(), - QueuesDown = rabbit_amqqueue:list_local_names_down(), - GbSet = gb_sets:from_list(Queues), - GbSetDown = gb_sets:from_list(QueuesDown), - gc_queue_metrics(GbSet, GbSetDown), - gc_entity(queue_coarse_metrics, GbSet), - Followers = gb_sets:from_list([amqqueue:get_name(Q) || Q <- rabbit_amqqueue:list_local_followers() ]), - gc_leader_data(Followers). - -gc_leader_data(Followers) -> - ets:foldl(fun({Id, _, _, _, _}, none) -> - gc_leader_data(Id, queue_coarse_metrics, Followers) - end, none, queue_coarse_metrics). - -gc_leader_data(Id, Table, GbSet) -> - case gb_sets:is_member(Id, GbSet) of - true -> - ets:delete(Table, Id), - none; - false -> - none - end. - -gc_global_queues() -> - GbSet = gb_sets:from_list(rabbit_amqqueue:list_names()), - gc_process_and_entity(channel_queue_metrics, GbSet), - gc_process_and_entity(consumer_created, GbSet), - ExchangeGbSet = gb_sets:from_list(rabbit_exchange:list_names()), - gc_process_and_entities(channel_queue_exchange_metrics, GbSet, ExchangeGbSet). - -gc_exchanges() -> - Exchanges = rabbit_exchange:list_names(), - GbSet = gb_sets:from_list(Exchanges), - gc_process_and_entity(channel_exchange_metrics, GbSet). - -gc_nodes() -> - Nodes = rabbit_mnesia:cluster_nodes(all), - GbSet = gb_sets:from_list(Nodes), - gc_entity(node_node_metrics, GbSet). - -gc_gen_server2() -> - gc_process(gen_server2_metrics). - -gc_process(Table) -> - ets:foldl(fun({Pid = Key, _}, none) -> - gc_process(Pid, Table, Key); - ({Pid = Key, _, _, _, _}, none) -> - gc_process(Pid, Table, Key); - ({Pid = Key, _, _, _}, none) -> - gc_process(Pid, Table, Key) - end, none, Table). - -gc_process(Pid, Table, Key) -> - case rabbit_misc:is_process_alive(Pid) of - true -> - none; - false -> - ets:delete(Table, Key), - none - end. - -gc_queue_metrics(GbSet, GbSetDown) -> - Table = queue_metrics, - ets:foldl(fun({Key, Props, Marker}, none) -> - case gb_sets:is_member(Key, GbSet) of - true -> - case gb_sets:is_member(Key, GbSetDown) of - true -> - ets:insert(Table, {Key, [{state, down} | lists:keydelete(state, 1, Props)], Marker}), - none; - false -> - none - end; - false -> - ets:delete(Table, Key), - none - end - end, none, Table). - -gc_entity(Table, GbSet) -> - ets:foldl(fun({{_, Id} = Key, _}, none) -> - gc_entity(Id, Table, Key, GbSet); - ({Id = Key, _}, none) -> - gc_entity(Id, Table, Key, GbSet); - ({Id = Key, _, _}, none) -> - gc_entity(Id, Table, Key, GbSet); - ({Id = Key, _, _, _, _}, none) -> - gc_entity(Id, Table, Key, GbSet) - end, none, Table). - -gc_entity(Id, Table, Key, GbSet) -> - case gb_sets:is_member(Id, GbSet) of - true -> - none; - false -> - ets:delete(Table, Key), - none - end. - -gc_process_and_entity(Table, GbSet) -> - ets:foldl(fun({{Pid, Id} = Key, _, _, _, _, _, _, _, _}, none) - when Table == channel_queue_metrics -> - gc_process_and_entity(Id, Pid, Table, Key, GbSet); - ({{Pid, Id} = Key, _, _, _, _, _}, none) - when Table == channel_exchange_metrics -> - gc_process_and_entity(Id, Pid, Table, Key, GbSet); - ({{Id, Pid, _} = Key, _, _, _, _, _, _}, none) - when Table == consumer_created -> - gc_process_and_entity(Id, Pid, Table, Key, GbSet); - ({{{Pid, Id}, _} = Key, _, _, _, _}, none) -> - gc_process_and_entity(Id, Pid, Table, Key, GbSet) - end, none, Table). - -gc_process_and_entity(Id, Pid, Table, Key, GbSet) -> - case rabbit_misc:is_process_alive(Pid) andalso gb_sets:is_member(Id, GbSet) of - true -> - none; - false -> - ets:delete(Table, Key), - none - end. - -gc_process_and_entities(Table, QueueGbSet, ExchangeGbSet) -> - ets:foldl(fun({{Pid, {Q, X}} = Key, _, _}, none) -> - gc_process(Pid, Table, Key), - gc_entity(Q, Table, Key, QueueGbSet), - gc_entity(X, Table, Key, ExchangeGbSet) - end, none, Table). - -gc_auth_attempts() -> - ets:delete_all_objects(auth_attempt_detailed_metrics). diff --git a/src/rabbit_credential_validation.erl b/src/rabbit_credential_validation.erl deleted file mode 100644 index 8712628ade..0000000000 --- a/src/rabbit_credential_validation.erl +++ /dev/null @@ -1,44 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_credential_validation). - --include("rabbit.hrl"). - -%% used for backwards compatibility --define(DEFAULT_BACKEND, rabbit_credential_validator_accept_everything). - -%% -%% API -%% - --export([validate/2, backend/0]). - -%% Validates a username/password pair by delegating to the effective -%% `rabbit_credential_validator`. Used by `rabbit_auth_backend_internal`. -%% Note that some validators may choose to only validate passwords. -%% -%% Possible return values: -%% -%% * ok: provided credentials passed validation. -%% * {error, Error, Args}: provided password password failed validation. - --spec validate(rabbit_types:username(), rabbit_types:password()) -> 'ok' | {'error', string()}. - -validate(Username, Password) -> - Backend = backend(), - Backend:validate(Username, Password). - --spec backend() -> atom(). - -backend() -> - case application:get_env(rabbit, credential_validator) of - undefined -> - ?DEFAULT_BACKEND; - {ok, Proplist} -> - proplists:get_value(validation_backend, Proplist, ?DEFAULT_BACKEND) - end. diff --git a/src/rabbit_credential_validator.erl b/src/rabbit_credential_validator.erl deleted file mode 100644 index 3b5d0752bf..0000000000 --- a/src/rabbit_credential_validator.erl +++ /dev/null @@ -1,19 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_credential_validator). - --include("rabbit.hrl"). - -%% Validates a password. Used by `rabbit_auth_backend_internal`. -%% -%% Possible return values: -%% -%% * ok: provided password passed validation. -%% * {error, Error, Args}: provided password password failed validation. - --callback validate(rabbit_types:username(), rabbit_types:password()) -> 'ok' | {'error', string()}. diff --git a/src/rabbit_credential_validator_accept_everything.erl b/src/rabbit_credential_validator_accept_everything.erl deleted file mode 100644 index fea10fd4b6..0000000000 --- a/src/rabbit_credential_validator_accept_everything.erl +++ /dev/null @@ -1,23 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_credential_validator_accept_everything). - --include("rabbit.hrl"). - --behaviour(rabbit_credential_validator). - -%% -%% API -%% - --export([validate/2]). - --spec validate(rabbit_types:username(), rabbit_types:password()) -> 'ok' | {'error', string()}. - -validate(_Username, _Password) -> - ok. diff --git a/src/rabbit_credential_validator_min_password_length.erl b/src/rabbit_credential_validator_min_password_length.erl deleted file mode 100644 index 463090127f..0000000000 --- a/src/rabbit_credential_validator_min_password_length.erl +++ /dev/null @@ -1,50 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_credential_validator_min_password_length). - --include("rabbit.hrl"). - --behaviour(rabbit_credential_validator). - -%% accommodates default (localhost-only) user credentials, -%% guest/guest --define(DEFAULT_MIN_LENGTH, 5). - -%% -%% API -%% - --export([validate/2]). -%% for tests --export([validate/3]). - --spec validate(rabbit_types:username(), rabbit_types:password()) -> 'ok' | {'error', string()}. - -validate(Username, Password) -> - MinLength = case application:get_env(rabbit, credential_validator) of - undefined -> - ?DEFAULT_MIN_LENGTH; - {ok, Proplist} -> - case proplists:get_value(min_length, Proplist) of - undefined -> ?DEFAULT_MIN_LENGTH; - Value -> rabbit_data_coercion:to_integer(Value) - end - end, - validate(Username, Password, MinLength). - - --spec validate(rabbit_types:username(), rabbit_types:password(), integer()) -> 'ok' | {'error', string(), [any()]}. - -%% passwordless users -validate(_Username, undefined, MinLength) -> - {error, rabbit_misc:format("minimum required password length is ~B", [MinLength])}; -validate(_Username, Password, MinLength) -> - case size(Password) >= MinLength of - true -> ok; - false -> {error, rabbit_misc:format("minimum required password length is ~B", [MinLength])} - end. diff --git a/src/rabbit_credential_validator_password_regexp.erl b/src/rabbit_credential_validator_password_regexp.erl deleted file mode 100644 index dc64cf1d31..0000000000 --- a/src/rabbit_credential_validator_password_regexp.erl +++ /dev/null @@ -1,42 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - - -%% A `rabbit_credential_validator` implementation that matches -%% password against a pre-configured regular expression. --module(rabbit_credential_validator_password_regexp). - --include("rabbit.hrl"). - --behaviour(rabbit_credential_validator). - -%% -%% API -%% - --export([validate/2]). -%% for tests --export([validate/3]). - --spec validate(rabbit_types:username(), rabbit_types:password()) -> 'ok' | {'error', string()}. - -validate(Username, Password) -> - {ok, Proplist} = application:get_env(rabbit, credential_validator), - Regexp = case proplists:get_value(regexp, Proplist) of - undefined -> {error, "rabbit.credential_validator.regexp config key is undefined"}; - Value -> rabbit_data_coercion:to_list(Value) - end, - validate(Username, Password, Regexp). - - --spec validate(rabbit_types:username(), rabbit_types:password(), string()) -> 'ok' | {'error', string(), [any()]}. - -validate(_Username, Password, Pattern) -> - case re:run(rabbit_data_coercion:to_list(Password), Pattern) of - {match, _} -> ok; - nomatch -> {error, "provided password does not match the validator regular expression"} - end. diff --git a/src/rabbit_dead_letter.erl b/src/rabbit_dead_letter.erl deleted file mode 100644 index 755de5cf53..0000000000 --- a/src/rabbit_dead_letter.erl +++ /dev/null @@ -1,253 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_dead_letter). - --export([publish/5]). - --include("rabbit.hrl"). --include("rabbit_framing.hrl"). - -%%---------------------------------------------------------------------------- - --type reason() :: 'expired' | 'rejected' | 'maxlen' | delivery_limit. - -%%---------------------------------------------------------------------------- - --spec publish(rabbit_types:message(), reason(), rabbit_types:exchange(), - 'undefined' | binary(), rabbit_amqqueue:name()) -> 'ok'. -publish(Msg, Reason, X, RK, QName) -> - DLMsg = make_msg(Msg, Reason, X#exchange.name, RK, QName), - Delivery = rabbit_basic:delivery(false, false, DLMsg, undefined), - {Queues, Cycles} = detect_cycles(Reason, DLMsg, - rabbit_exchange:route(X, Delivery)), - lists:foreach(fun log_cycle_once/1, Cycles), - _ = rabbit_queue_type:deliver(rabbit_amqqueue:lookup(Queues), - Delivery, stateless), - ok. - -make_msg(Msg = #basic_message{content = Content, - exchange_name = Exchange, - routing_keys = RoutingKeys}, - Reason, DLX, RK, #resource{name = QName}) -> - {DeathRoutingKeys, HeadersFun1} = - case RK of - undefined -> {RoutingKeys, fun (H) -> H end}; - _ -> {[RK], fun (H) -> lists:keydelete(<<"CC">>, 1, H) end} - end, - ReasonBin = list_to_binary(atom_to_list(Reason)), - TimeSec = os:system_time(seconds), - PerMsgTTL = per_msg_ttl_header(Content#content.properties), - HeadersFun2 = - fun (Headers) -> - %% The first routing key is the one specified in the - %% basic.publish; all others are CC or BCC keys. - RKs = [hd(RoutingKeys) | rabbit_basic:header_routes(Headers)], - RKs1 = [{longstr, Key} || Key <- RKs], - Info = [{<<"reason">>, longstr, ReasonBin}, - {<<"queue">>, longstr, QName}, - {<<"time">>, timestamp, TimeSec}, - {<<"exchange">>, longstr, Exchange#resource.name}, - {<<"routing-keys">>, array, RKs1}] ++ PerMsgTTL, - HeadersFun1(update_x_death_header(Info, Headers)) - end, - Content1 = #content{properties = Props} = - rabbit_basic:map_headers(HeadersFun2, Content), - Content2 = Content1#content{properties = - Props#'P_basic'{expiration = undefined}}, - Msg#basic_message{exchange_name = DLX, - id = rabbit_guid:gen(), - routing_keys = DeathRoutingKeys, - content = Content2}. - - -x_death_event_key(Info, Key) -> - case lists:keysearch(Key, 1, Info) of - false -> undefined; - {value, {Key, _KeyType, Val}} -> Val - end. - -maybe_append_to_event_group(Table, _Key, _SeenKeys, []) -> - [Table]; -maybe_append_to_event_group(Table, {_Queue, _Reason} = Key, SeenKeys, Acc) -> - case sets:is_element(Key, SeenKeys) of - true -> Acc; - false -> [Table | Acc] - end. - -group_by_queue_and_reason([]) -> - []; -group_by_queue_and_reason([Table]) -> - [Table]; -group_by_queue_and_reason(Tables) -> - {_, Grouped} = - lists:foldl( - fun ({table, Info}, {SeenKeys, Acc}) -> - Q = x_death_event_key(Info, <<"queue">>), - R = x_death_event_key(Info, <<"reason">>), - Matcher = queue_and_reason_matcher(Q, R), - {Matches, _} = lists:partition(Matcher, Tables), - {Augmented, N} = case Matches of - [X] -> {X, 1}; - [X|_] = Xs -> {X, length(Xs)} - end, - Key = {Q, R}, - Acc1 = maybe_append_to_event_group( - ensure_xdeath_event_count(Augmented, N), - Key, SeenKeys, Acc), - {sets:add_element(Key, SeenKeys), Acc1} - end, {sets:new(), []}, Tables), - Grouped. - -update_x_death_header(Info, undefined) -> - update_x_death_header(Info, []); -update_x_death_header(Info, Headers) -> - X = x_death_event_key(Info, <<"exchange">>), - Q = x_death_event_key(Info, <<"queue">>), - R = x_death_event_key(Info, <<"reason">>), - case rabbit_basic:header(<<"x-death">>, Headers) of - undefined -> - %% First x-death event gets its own top-level headers. - %% See rabbitmq/rabbitmq-server#1332. - Headers2 = rabbit_misc:set_table_value(Headers, <<"x-first-death-reason">>, - longstr, R), - Headers3 = rabbit_misc:set_table_value(Headers2, <<"x-first-death-queue">>, - longstr, Q), - Headers4 = rabbit_misc:set_table_value(Headers3, <<"x-first-death-exchange">>, - longstr, X), - rabbit_basic:prepend_table_header( - <<"x-death">>, - [{<<"count">>, long, 1} | Info], Headers4); - {<<"x-death">>, array, Tables} -> - %% group existing x-death headers in case we have some from - %% before rabbitmq-server#78 - GroupedTables = group_by_queue_and_reason(Tables), - {Matches, Others} = lists:partition( - queue_and_reason_matcher(Q, R), - GroupedTables), - Info1 = case Matches of - [] -> - [{<<"count">>, long, 1} | Info]; - [{table, M}] -> - increment_xdeath_event_count(M) - end, - rabbit_misc:set_table_value( - Headers, <<"x-death">>, array, - [{table, rabbit_misc:sort_field_table(Info1)} | Others]); - {<<"x-death">>, InvalidType, Header} -> - rabbit_log:warning("Message has invalid x-death header (type: ~p)." - " Resetting header ~p~n", - [InvalidType, Header]), - %% if x-death is something other than an array (list) - %% then we reset it: this happens when some clients consume - %% a message and re-publish is, converting header values - %% to strings, intentionally or not. - %% See rabbitmq/rabbitmq-server#767 for details. - rabbit_misc:set_table_value( - Headers, <<"x-death">>, array, - [{table, [{<<"count">>, long, 1} | Info]}]) - end. - -ensure_xdeath_event_count({table, Info}, InitialVal) when InitialVal >= 1 -> - {table, ensure_xdeath_event_count(Info, InitialVal)}; -ensure_xdeath_event_count(Info, InitialVal) when InitialVal >= 1 -> - case x_death_event_key(Info, <<"count">>) of - undefined -> - [{<<"count">>, long, InitialVal} | Info]; - _ -> - Info - end. - -increment_xdeath_event_count(Info) -> - case x_death_event_key(Info, <<"count">>) of - undefined -> - [{<<"count">>, long, 1} | Info]; - N -> - lists:keyreplace( - <<"count">>, 1, Info, - {<<"count">>, long, N + 1}) - end. - -queue_and_reason_matcher(Q, R) -> - F = fun(Info) -> - x_death_event_key(Info, <<"queue">>) =:= Q - andalso x_death_event_key(Info, <<"reason">>) =:= R - end, - fun({table, Info}) -> - F(Info); - (Info) when is_list(Info) -> - F(Info) - end. - -per_msg_ttl_header(#'P_basic'{expiration = undefined}) -> - []; -per_msg_ttl_header(#'P_basic'{expiration = Expiration}) -> - [{<<"original-expiration">>, longstr, Expiration}]; -per_msg_ttl_header(_) -> - []. - -detect_cycles(rejected, _Msg, Queues) -> - {Queues, []}; - -detect_cycles(_Reason, #basic_message{content = Content}, Queues) -> - #content{properties = #'P_basic'{headers = Headers}} = - rabbit_binary_parser:ensure_content_decoded(Content), - NoCycles = {Queues, []}, - case Headers of - undefined -> - NoCycles; - _ -> - case rabbit_misc:table_lookup(Headers, <<"x-death">>) of - {array, Deaths} -> - {Cycling, NotCycling} = - lists:partition(fun (#resource{name = Queue}) -> - is_cycle(Queue, Deaths) - end, Queues), - OldQueues = [rabbit_misc:table_lookup(D, <<"queue">>) || - {table, D} <- Deaths], - OldQueues1 = [QName || {longstr, QName} <- OldQueues], - {NotCycling, [[QName | OldQueues1] || - #resource{name = QName} <- Cycling]}; - _ -> - NoCycles - end - end. - -is_cycle(Queue, Deaths) -> - {Cycle, Rest} = - lists:splitwith( - fun ({table, D}) -> - {longstr, Queue} =/= rabbit_misc:table_lookup(D, <<"queue">>); - (_) -> - true - end, Deaths), - %% Is there a cycle, and if so, is it "fully automatic", i.e. with - %% no reject in it? - case Rest of - [] -> false; - [H|_] -> lists:all( - fun ({table, D}) -> - {longstr, <<"rejected">>} =/= - rabbit_misc:table_lookup(D, <<"reason">>); - (_) -> - %% There was something we didn't expect, therefore - %% a client must have put it there, therefore the - %% cycle was not "fully automatic". - false - end, Cycle ++ [H]) - end. - -log_cycle_once(Queues) -> - Key = {queue_cycle, Queues}, - case get(Key) of - true -> ok; - undefined -> rabbit_log:warning( - "Message dropped. Dead-letter queues cycle detected" ++ - ": ~p~nThis cycle will NOT be reported again.~n", - [Queues]), - put(Key, true) - end. diff --git a/src/rabbit_definitions.erl b/src/rabbit_definitions.erl deleted file mode 100644 index 0d0212dbae..0000000000 --- a/src/rabbit_definitions.erl +++ /dev/null @@ -1,767 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_definitions). --include_lib("rabbit_common/include/rabbit.hrl"). - --export([boot/0]). -%% automatic import on boot --export([maybe_load_definitions/0, maybe_load_definitions/2, maybe_load_definitions_from/2, - has_configured_definitions_to_load/0]). -%% import --export([import_raw/1, import_raw/2, import_parsed/1, import_parsed/2, - apply_defs/2, apply_defs/3, apply_defs/4, apply_defs/5]). - --export([all_definitions/0]). --export([ - list_users/0, list_vhosts/0, list_permissions/0, list_topic_permissions/0, - list_runtime_parameters/0, list_global_runtime_parameters/0, list_policies/0, - list_exchanges/0, list_queues/0, list_bindings/0, - is_internal_parameter/1 -]). --export([decode/1, decode/2, args/1]). - --import(rabbit_misc, [pget/2]). - -%% -%% API -%% - --type definition_category() :: 'users' | - 'vhosts' | - 'permissions' | - 'topic_permissions' | - 'parameters' | - 'global_parameters' | - 'policies' | - 'queues' | - 'bindings' | - 'exchanges'. - --type definition_object() :: #{binary() => any()}. --type definition_list() :: [definition_object()]. - --type definitions() :: #{ - definition_category() => definition_list() -}. - --export_type([definition_object/0, definition_list/0, definition_category/0, definitions/0]). - --define(IMPORT_WORK_POOL, definition_import_pool). - -boot() -> - PoolSize = application:get_env(rabbit, definition_import_work_pool_size, rabbit_runtime:guess_number_of_cpu_cores()), - rabbit_sup:start_supervisor_child(definition_import_pool_sup, worker_pool_sup, [PoolSize, ?IMPORT_WORK_POOL]). - -maybe_load_definitions() -> - %% Note that management.load_definitions is handled in the plugin for backwards compatibility. - %% This executes the "core" version of load_definitions. - maybe_load_definitions(rabbit, load_definitions). - --spec import_raw(Body :: binary() | iolist()) -> ok | {error, term()}. -import_raw(Body) -> - rabbit_log:info("Asked to import definitions. Acting user: ~s", [?INTERNAL_USER]), - case decode([], Body) of - {error, E} -> {error, E}; - {ok, _, Map} -> apply_defs(Map, ?INTERNAL_USER) - end. - --spec import_raw(Body :: binary() | iolist(), VHost :: vhost:name()) -> ok | {error, term()}. -import_raw(Body, VHost) -> - rabbit_log:info("Asked to import definitions. Acting user: ~s", [?INTERNAL_USER]), - case decode([], Body) of - {error, E} -> {error, E}; - {ok, _, Map} -> apply_defs(Map, ?INTERNAL_USER, fun() -> ok end, VHost) - end. - --spec import_parsed(Defs :: #{any() => any()} | list()) -> ok | {error, term()}. -import_parsed(Body0) when is_list(Body0) -> - import_parsed(maps:from_list(Body0)); -import_parsed(Body0) when is_map(Body0) -> - rabbit_log:info("Asked to import definitions. Acting user: ~s", [?INTERNAL_USER]), - Body = atomise_map_keys(Body0), - apply_defs(Body, ?INTERNAL_USER). - --spec import_parsed(Defs :: #{any() => any() | list()}, VHost :: vhost:name()) -> ok | {error, term()}. -import_parsed(Body0, VHost) when is_list(Body0) -> - import_parsed(maps:from_list(Body0), VHost); -import_parsed(Body0, VHost) -> - rabbit_log:info("Asked to import definitions. Acting user: ~s", [?INTERNAL_USER]), - Body = atomise_map_keys(Body0), - apply_defs(Body, ?INTERNAL_USER, fun() -> ok end, VHost). - --spec all_definitions() -> map(). -all_definitions() -> - Xs = list_exchanges(), - Qs = list_queues(), - Bs = list_bindings(), - - Users = list_users(), - VHosts = list_vhosts(), - Params = list_runtime_parameters(), - GParams = list_global_runtime_parameters(), - Pols = list_policies(), - - Perms = list_permissions(), - TPerms = list_topic_permissions(), - - {ok, Vsn} = application:get_key(rabbit, vsn), - #{ - rabbit_version => rabbit_data_coercion:to_binary(Vsn), - rabbitmq_version => rabbit_data_coercion:to_binary(Vsn), - users => Users, - vhosts => VHosts, - permissions => Perms, - topic_permissions => TPerms, - parameters => Params, - global_parameters => GParams, - policies => Pols, - queues => Qs, - bindings => Bs, - exchanges => Xs - }. - -%% -%% Implementation -%% - --spec has_configured_definitions_to_load() -> boolean(). -has_configured_definitions_to_load() -> - case application:get_env(rabbit, load_definitions) of - undefined -> false; - {ok, none} -> false; - {ok, _Path} -> true - end. - -maybe_load_definitions(App, Key) -> - case application:get_env(App, Key) of - undefined -> - rabbit_log:debug("No definition file configured to import via load_definitions"), - ok; - {ok, none} -> - rabbit_log:debug("No definition file configured to import via load_definitions"), - ok; - {ok, FileOrDir} -> - rabbit_log:debug("Will import definitions file from load_definitions"), - IsDir = filelib:is_dir(FileOrDir), - maybe_load_definitions_from(IsDir, FileOrDir) - end. - -maybe_load_definitions_from(true, Dir) -> - rabbit_log:info("Applying definitions from directory ~s", [Dir]), - load_definitions_from_files(file:list_dir(Dir), Dir); -maybe_load_definitions_from(false, File) -> - load_definitions_from_file(File). - -load_definitions_from_files({ok, Filenames0}, Dir) -> - Filenames1 = lists:sort(Filenames0), - Filenames2 = [filename:join(Dir, F) || F <- Filenames1], - load_definitions_from_filenames(Filenames2); -load_definitions_from_files({error, E}, Dir) -> - rabbit_log:error("Could not read definitions from directory ~s, Error: ~p", [Dir, E]), - {error, {could_not_read_defs, E}}. - -load_definitions_from_filenames([]) -> - ok; -load_definitions_from_filenames([File|Rest]) -> - case load_definitions_from_file(File) of - ok -> load_definitions_from_filenames(Rest); - {error, E} -> {error, {failed_to_import_definitions, File, E}} - end. - -load_definitions_from_file(File) -> - case file:read_file(File) of - {ok, Body} -> - rabbit_log:info("Applying definitions from file at '~s'", [File]), - import_raw(Body); - {error, E} -> - rabbit_log:error("Could not read definitions from file at '~s', error: ~p", [File, E]), - {error, {could_not_read_defs, {File, E}}} - end. - -decode(Keys, Body) -> - case decode(Body) of - {ok, J0} -> - J = maps:fold(fun(K, V, Acc) -> - Acc#{rabbit_data_coercion:to_atom(K, utf8) => V} - end, J0, J0), - Results = [get_or_missing(K, J) || K <- Keys], - case [E || E = {key_missing, _} <- Results] of - [] -> {ok, Results, J}; - Errors -> {error, Errors} - end; - Else -> Else - end. - -decode(<<"">>) -> - {ok, #{}}; -decode(Body) -> - try - Decoded = rabbit_json:decode(Body), - Normalised = atomise_map_keys(Decoded), - {ok, Normalised} - catch error:_ -> {error, not_json} - end. - -atomise_map_keys(Decoded) -> - maps:fold(fun(K, V, Acc) -> - Acc#{rabbit_data_coercion:to_atom(K, utf8) => V} - end, Decoded, Decoded). - --spec apply_defs(Map :: #{atom() => any()}, ActingUser :: rabbit_types:username()) -> 'ok' | {error, term()}. - -apply_defs(Map, ActingUser) -> - apply_defs(Map, ActingUser, fun () -> ok end). - --spec apply_defs(Map :: #{atom() => any()}, ActingUser :: rabbit_types:username(), - SuccessFun :: fun(() -> 'ok')) -> 'ok' | {error, term()}; - (Map :: #{atom() => any()}, ActingUser :: rabbit_types:username(), - VHost :: vhost:name()) -> 'ok' | {error, term()}. - -apply_defs(Map, ActingUser, VHost) when is_binary(VHost) -> - apply_defs(Map, ActingUser, fun () -> ok end, VHost); - -apply_defs(Map, ActingUser, SuccessFun) when is_function(SuccessFun) -> - Version = maps:get(rabbitmq_version, Map, maps:get(rabbit_version, Map, undefined)), - try - concurrent_for_all(users, ActingUser, Map, - fun(User, _Username) -> - rabbit_auth_backend_internal:put_user(User, Version, ActingUser) - end), - concurrent_for_all(vhosts, ActingUser, Map, fun add_vhost/2), - validate_limits(Map), - concurrent_for_all(permissions, ActingUser, Map, fun add_permission/2), - concurrent_for_all(topic_permissions, ActingUser, Map, fun add_topic_permission/2), - sequential_for_all(parameters, ActingUser, Map, fun add_parameter/2), - sequential_for_all(global_parameters, ActingUser, Map, fun add_global_parameter/2), - %% importing policies concurrently can be unsafe as queues will be getting - %% potentially out of order notifications of applicable policy changes - sequential_for_all(policies, ActingUser, Map, fun add_policy/2), - concurrent_for_all(queues, ActingUser, Map, fun add_queue/2), - concurrent_for_all(exchanges, ActingUser, Map, fun add_exchange/2), - concurrent_for_all(bindings, ActingUser, Map, fun add_binding/2), - SuccessFun(), - ok - catch {error, E} -> {error, E}; - exit:E -> {error, E} - end. - --spec apply_defs(Map :: #{atom() => any()}, - ActingUser :: rabbit_types:username(), - SuccessFun :: fun(() -> 'ok'), - VHost :: vhost:name()) -> 'ok' | {error, term()}. - -apply_defs(Map, ActingUser, SuccessFun, VHost) when is_binary(VHost) -> - rabbit_log:info("Asked to import definitions for a virtual host. Virtual host: ~p, acting user: ~p", - [VHost, ActingUser]), - try - validate_limits(Map, VHost), - sequential_for_all(parameters, ActingUser, Map, VHost, fun add_parameter/3), - %% importing policies concurrently can be unsafe as queues will be getting - %% potentially out of order notifications of applicable policy changes - sequential_for_all(policies, ActingUser, Map, VHost, fun add_policy/3), - concurrent_for_all(queues, ActingUser, Map, VHost, fun add_queue/3), - concurrent_for_all(exchanges, ActingUser, Map, VHost, fun add_exchange/3), - concurrent_for_all(bindings, ActingUser, Map, VHost, fun add_binding/3), - SuccessFun() - catch {error, E} -> {error, format(E)}; - exit:E -> {error, format(E)} - end. - --spec apply_defs(Map :: #{atom() => any()}, - ActingUser :: rabbit_types:username(), - SuccessFun :: fun(() -> 'ok'), - ErrorFun :: fun((any()) -> 'ok'), - VHost :: vhost:name()) -> 'ok' | {error, term()}. - -apply_defs(Map, ActingUser, SuccessFun, ErrorFun, VHost) -> - rabbit_log:info("Asked to import definitions for a virtual host. Virtual host: ~p, acting user: ~p", - [VHost, ActingUser]), - try - validate_limits(Map, VHost), - sequential_for_all(parameters, ActingUser, Map, VHost, fun add_parameter/3), - %% importing policies concurrently can be unsafe as queues will be getting - %% potentially out of order notifications of applicable policy changes - sequential_for_all(policies, ActingUser, Map, VHost, fun add_policy/3), - concurrent_for_all(queues, ActingUser, Map, VHost, fun add_queue/3), - concurrent_for_all(exchanges, ActingUser, Map, VHost, fun add_exchange/3), - concurrent_for_all(bindings, ActingUser, Map, VHost, fun add_binding/3), - SuccessFun() - catch {error, E} -> ErrorFun(format(E)); - exit:E -> ErrorFun(format(E)) - end. - -sequential_for_all(Category, ActingUser, Definitions, Fun) -> - case maps:get(rabbit_data_coercion:to_atom(Category), Definitions, undefined) of - undefined -> ok; - List -> - case length(List) of - 0 -> ok; - N -> rabbit_log:info("Importing sequentially ~p ~s...", [N, human_readable_category_name(Category)]) - end, - [begin - %% keys are expected to be atoms - Fun(atomize_keys(M), ActingUser) - end || M <- List, is_map(M)] - end. - -sequential_for_all(Name, ActingUser, Definitions, VHost, Fun) -> - case maps:get(rabbit_data_coercion:to_atom(Name), Definitions, undefined) of - undefined -> ok; - List -> [Fun(VHost, atomize_keys(M), ActingUser) || M <- List, is_map(M)] - end. - -concurrent_for_all(Category, ActingUser, Definitions, Fun) -> - case maps:get(rabbit_data_coercion:to_atom(Category), Definitions, undefined) of - undefined -> ok; - List -> - case length(List) of - 0 -> ok; - N -> rabbit_log:info("Importing concurrently ~p ~s...", [N, human_readable_category_name(Category)]) - end, - WorkPoolFun = fun(M) -> - Fun(atomize_keys(M), ActingUser) - end, - do_concurrent_for_all(List, WorkPoolFun) - end. - -concurrent_for_all(Name, ActingUser, Definitions, VHost, Fun) -> - case maps:get(rabbit_data_coercion:to_atom(Name), Definitions, undefined) of - undefined -> ok; - List -> - WorkPoolFun = fun(M) -> - Fun(VHost, atomize_keys(M), ActingUser) - end, - do_concurrent_for_all(List, WorkPoolFun) - end. - -do_concurrent_for_all(List, WorkPoolFun) -> - {ok, Gatherer} = gatherer:start_link(), - [begin - %% keys are expected to be atoms - ok = gatherer:fork(Gatherer), - worker_pool:submit_async( - ?IMPORT_WORK_POOL, - fun() -> - try - WorkPoolFun(M) - catch {error, E} -> gatherer:in(Gatherer, {error, E}); - _:E -> gatherer:in(Gatherer, {error, E}) - end, - gatherer:finish(Gatherer) - end) - end || M <- List, is_map(M)], - case gatherer:out(Gatherer) of - empty -> - ok = gatherer:stop(Gatherer); - {value, {error, E}} -> - ok = gatherer:stop(Gatherer), - throw({error, E}) - end. - --spec atomize_keys(#{any() => any()}) -> #{atom() => any()}. - -atomize_keys(M) -> - maps:fold(fun(K, V, Acc) -> - maps:put(rabbit_data_coercion:to_atom(K), V, Acc) - end, #{}, M). - --spec human_readable_category_name(definition_category()) -> string(). - -human_readable_category_name(topic_permissions) -> "topic permissions"; -human_readable_category_name(parameters) -> "runtime parameters"; -human_readable_category_name(global_parameters) -> "global runtime parameters"; -human_readable_category_name(Other) -> rabbit_data_coercion:to_list(Other). - - -format(#amqp_error{name = Name, explanation = Explanation}) -> - rabbit_data_coercion:to_binary(rabbit_misc:format("~s: ~s", [Name, Explanation])); -format({no_such_vhost, undefined}) -> - rabbit_data_coercion:to_binary( - "Virtual host does not exist and is not specified in definitions file."); -format({no_such_vhost, VHost}) -> - rabbit_data_coercion:to_binary( - rabbit_misc:format("Please create virtual host \"~s\" prior to importing definitions.", - [VHost])); -format({vhost_limit_exceeded, ErrMsg}) -> - rabbit_data_coercion:to_binary(ErrMsg); -format(E) -> - rabbit_data_coercion:to_binary(rabbit_misc:format("~p", [E])). - -add_parameter(Param, Username) -> - VHost = maps:get(vhost, Param, undefined), - add_parameter(VHost, Param, Username). - -add_parameter(VHost, Param, Username) -> - Comp = maps:get(component, Param, undefined), - Key = maps:get(name, Param, undefined), - Term = maps:get(value, Param, undefined), - Result = case is_map(Term) of - true -> - %% coerce maps to proplists for backwards compatibility. - %% See rabbitmq-management#528. - TermProplist = rabbit_data_coercion:to_proplist(Term), - rabbit_runtime_parameters:set(VHost, Comp, Key, TermProplist, Username); - _ -> - rabbit_runtime_parameters:set(VHost, Comp, Key, Term, Username) - end, - case Result of - ok -> ok; - {error_string, E} -> - S = rabbit_misc:format(" (~s/~s/~s)", [VHost, Comp, Key]), - exit(rabbit_data_coercion:to_binary(rabbit_misc:escape_html_tags(E ++ S))) - end. - -add_global_parameter(Param, Username) -> - Key = maps:get(name, Param, undefined), - Term = maps:get(value, Param, undefined), - case is_map(Term) of - true -> - %% coerce maps to proplists for backwards compatibility. - %% See rabbitmq-management#528. - TermProplist = rabbit_data_coercion:to_proplist(Term), - rabbit_runtime_parameters:set_global(Key, TermProplist, Username); - _ -> - rabbit_runtime_parameters:set_global(Key, Term, Username) - end. - -add_policy(Param, Username) -> - VHost = maps:get(vhost, Param, undefined), - add_policy(VHost, Param, Username). - -add_policy(VHost, Param, Username) -> - Key = maps:get(name, Param, undefined), - case rabbit_policy:set( - VHost, Key, maps:get(pattern, Param, undefined), - case maps:get(definition, Param, undefined) of - undefined -> undefined; - Def -> rabbit_data_coercion:to_proplist(Def) - end, - maps:get(priority, Param, undefined), - maps:get('apply-to', Param, <<"all">>), - Username) of - ok -> ok; - {error_string, E} -> S = rabbit_misc:format(" (~s/~s)", [VHost, Key]), - exit(rabbit_data_coercion:to_binary(rabbit_misc:escape_html_tags(E ++ S))) - end. - --spec add_vhost(map(), rabbit_types:username()) -> ok. - -add_vhost(VHost, ActingUser) -> - VHostName = maps:get(name, VHost, undefined), - VHostTrace = maps:get(tracing, VHost, undefined), - VHostDefinition = maps:get(definition, VHost, undefined), - VHostTags = maps:get(tags, VHost, undefined), - rabbit_vhost:put_vhost(VHostName, VHostDefinition, VHostTags, VHostTrace, ActingUser). - -add_permission(Permission, ActingUser) -> - rabbit_auth_backend_internal:set_permissions(maps:get(user, Permission, undefined), - maps:get(vhost, Permission, undefined), - maps:get(configure, Permission, undefined), - maps:get(write, Permission, undefined), - maps:get(read, Permission, undefined), - ActingUser). - -add_topic_permission(TopicPermission, ActingUser) -> - rabbit_auth_backend_internal:set_topic_permissions( - maps:get(user, TopicPermission, undefined), - maps:get(vhost, TopicPermission, undefined), - maps:get(exchange, TopicPermission, undefined), - maps:get(write, TopicPermission, undefined), - maps:get(read, TopicPermission, undefined), - ActingUser). - -add_queue(Queue, ActingUser) -> - add_queue_int(Queue, r(queue, Queue), ActingUser). - -add_queue(VHost, Queue, ActingUser) -> - add_queue_int(Queue, rv(VHost, queue, Queue), ActingUser). - -add_queue_int(_Queue, R = #resource{kind = queue, - name = <<"amq.", _/binary>>}, ActingUser) -> - Name = R#resource.name, - rabbit_log:warning("Skipping import of a queue whose name begins with 'amq.', " - "name: ~s, acting user: ~s", [Name, ActingUser]); -add_queue_int(Queue, Name, ActingUser) -> - rabbit_amqqueue:declare(Name, - maps:get(durable, Queue, undefined), - maps:get(auto_delete, Queue, undefined), - args(maps:get(arguments, Queue, undefined)), - none, - ActingUser). - -add_exchange(Exchange, ActingUser) -> - add_exchange_int(Exchange, r(exchange, Exchange), ActingUser). - -add_exchange(VHost, Exchange, ActingUser) -> - add_exchange_int(Exchange, rv(VHost, exchange, Exchange), ActingUser). - -add_exchange_int(_Exchange, #resource{kind = exchange, name = <<"">>}, ActingUser) -> - rabbit_log:warning("Not importing the default exchange, acting user: ~s", [ActingUser]); -add_exchange_int(_Exchange, R = #resource{kind = exchange, - name = <<"amq.", _/binary>>}, ActingUser) -> - Name = R#resource.name, - rabbit_log:warning("Skipping import of an exchange whose name begins with 'amq.', " - "name: ~s, acting user: ~s", [Name, ActingUser]); -add_exchange_int(Exchange, Name, ActingUser) -> - Internal = case maps:get(internal, Exchange, undefined) of - undefined -> false; %% =< 2.2.0 - I -> I - end, - rabbit_exchange:declare(Name, - rabbit_exchange:check_type(maps:get(type, Exchange, undefined)), - maps:get(durable, Exchange, undefined), - maps:get(auto_delete, Exchange, undefined), - Internal, - args(maps:get(arguments, Exchange, undefined)), - ActingUser). - -add_binding(Binding, ActingUser) -> - DestType = dest_type(Binding), - add_binding_int(Binding, r(exchange, source, Binding), - r(DestType, destination, Binding), ActingUser). - -add_binding(VHost, Binding, ActingUser) -> - DestType = dest_type(Binding), - add_binding_int(Binding, rv(VHost, exchange, source, Binding), - rv(VHost, DestType, destination, Binding), ActingUser). - -add_binding_int(Binding, Source, Destination, ActingUser) -> - rabbit_binding:add( - #binding{source = Source, - destination = Destination, - key = maps:get(routing_key, Binding, undefined), - args = args(maps:get(arguments, Binding, undefined))}, - ActingUser). - -dest_type(Binding) -> - rabbit_data_coercion:to_atom(maps:get(destination_type, Binding, undefined)). - -r(Type, Props) -> r(Type, name, Props). - -r(Type, Name, Props) -> - rabbit_misc:r(maps:get(vhost, Props, undefined), Type, maps:get(Name, Props, undefined)). - -rv(VHost, Type, Props) -> rv(VHost, Type, name, Props). - -rv(VHost, Type, Name, Props) -> - rabbit_misc:r(VHost, Type, maps:get(Name, Props, undefined)). - -%%-------------------------------------------------------------------- - -validate_limits(All) -> - case maps:get(queues, All, undefined) of - undefined -> ok; - Queues0 -> - {ok, VHostMap} = filter_out_existing_queues(Queues0), - maps:fold(fun validate_vhost_limit/3, ok, VHostMap) - end. - -validate_limits(All, VHost) -> - case maps:get(queues, All, undefined) of - undefined -> ok; - Queues0 -> - Queues1 = filter_out_existing_queues(VHost, Queues0), - AddCount = length(Queues1), - validate_vhost_limit(VHost, AddCount, ok) - end. - -filter_out_existing_queues(Queues) -> - build_filtered_map(Queues, maps:new()). - -filter_out_existing_queues(VHost, Queues) -> - Pred = fun(Queue) -> - Rec = rv(VHost, queue, <<"name">>, Queue), - case rabbit_amqqueue:lookup(Rec) of - {ok, _} -> false; - {error, not_found} -> true - end - end, - lists:filter(Pred, Queues). - -build_queue_data(Queue) -> - VHost = maps:get(<<"vhost">>, Queue, undefined), - Rec = rv(VHost, queue, <<"name">>, Queue), - {Rec, VHost}. - -build_filtered_map([], AccMap) -> - {ok, AccMap}; -build_filtered_map([Queue|Rest], AccMap0) -> - {Rec, VHost} = build_queue_data(Queue), - case rabbit_amqqueue:lookup(Rec) of - {error, not_found} -> - AccMap1 = maps:update_with(VHost, fun(V) -> V + 1 end, 1, AccMap0), - build_filtered_map(Rest, AccMap1); - {ok, _} -> - build_filtered_map(Rest, AccMap0) - end. - -validate_vhost_limit(VHost, AddCount, ok) -> - WouldExceed = rabbit_vhost_limit:would_exceed_queue_limit(AddCount, VHost), - validate_vhost_queue_limit(VHost, AddCount, WouldExceed). - -validate_vhost_queue_limit(_VHost, 0, _) -> - % Note: not adding any new queues so the upload - % must be update-only - ok; -validate_vhost_queue_limit(_VHost, _AddCount, false) -> - % Note: would not exceed queue limit - ok; -validate_vhost_queue_limit(VHost, AddCount, {true, Limit, QueueCount}) -> - ErrFmt = "Adding ~B queue(s) to virtual host \"~s\" would exceed the limit of ~B queue(s).~n~nThis virtual host currently has ~B queue(s) defined.~n~nImport aborted!", - ErrInfo = [AddCount, VHost, Limit, QueueCount], - ErrMsg = rabbit_misc:format(ErrFmt, ErrInfo), - exit({vhost_limit_exceeded, ErrMsg}). - -get_or_missing(K, L) -> - case maps:get(K, L, undefined) of - undefined -> {key_missing, K}; - V -> V - end. - -args([]) -> args(#{}); -args(L) -> rabbit_misc:to_amqp_table(L). - -%% -%% Export -%% - -list_exchanges() -> - %% exclude internal exchanges, they are not meant to be declared or used by - %% applications - [exchange_definition(X) || X <- lists:filter(fun(#exchange{internal = true}) -> false; - (#exchange{name = #resource{name = <<>>}}) -> false; - (X) -> not rabbit_exchange:is_amq_prefixed(X) - end, - rabbit_exchange:list())]. - -exchange_definition(#exchange{name = #resource{virtual_host = VHost, name = Name}, - type = Type, - durable = Durable, auto_delete = AD, arguments = Args}) -> - #{<<"vhost">> => VHost, - <<"name">> => Name, - <<"type">> => Type, - <<"durable">> => Durable, - <<"auto_delete">> => AD, - <<"arguments">> => rabbit_misc:amqp_table(Args)}. - -list_queues() -> - %% exclude exclusive queues, they cannot be restored - [queue_definition(Q) || Q <- lists:filter(fun(Q0) -> - amqqueue:get_exclusive_owner(Q0) =:= none - end, - rabbit_amqqueue:list())]. - -queue_definition(Q) -> - #resource{virtual_host = VHost, name = Name} = amqqueue:get_name(Q), - Type = case amqqueue:get_type(Q) of - rabbit_classic_queue -> classic; - rabbit_quorum_queue -> quorum; - rabbit_stream_queue -> stream; - T -> T - end, - #{ - <<"vhost">> => VHost, - <<"name">> => Name, - <<"type">> => Type, - <<"durable">> => amqqueue:is_durable(Q), - <<"auto_delete">> => amqqueue:is_auto_delete(Q), - <<"arguments">> => rabbit_misc:amqp_table(amqqueue:get_arguments(Q)) - }. - -list_bindings() -> - [binding_definition(B) || B <- rabbit_binding:list_explicit()]. - -binding_definition(#binding{source = S, - key = RoutingKey, - destination = D, - args = Args}) -> - #{ - <<"source">> => S#resource.name, - <<"vhost">> => S#resource.virtual_host, - <<"destination">> => D#resource.name, - <<"destination_type">> => D#resource.kind, - <<"routing_key">> => RoutingKey, - <<"arguments">> => rabbit_misc:amqp_table(Args) - }. - -list_vhosts() -> - [vhost_definition(V) || V <- rabbit_vhost:all()]. - -vhost_definition(VHost) -> - #{ - <<"name">> => vhost:get_name(VHost), - <<"limits">> => vhost:get_limits(VHost), - <<"metadata">> => vhost:get_metadata(VHost) - }. - -list_users() -> - [user_definition(U) || U <- rabbit_auth_backend_internal:all_users()]. - -user_definition(User) -> - #{<<"name">> => internal_user:get_username(User), - <<"password_hash">> => base64:encode(internal_user:get_password_hash(User)), - <<"hashing_algorithm">> => rabbit_auth_backend_internal:hashing_module_for_user(User), - <<"tags">> => tags_as_binaries(internal_user:get_tags(User)), - <<"limits">> => internal_user:get_limits(User) - }. - -list_runtime_parameters() -> - [runtime_parameter_definition(P) || P <- rabbit_runtime_parameters:list(), is_list(P)]. - -runtime_parameter_definition(Param) -> - #{ - <<"vhost">> => pget(vhost, Param), - <<"component">> => pget(component, Param), - <<"name">> => pget(name, Param), - <<"value">> => maps:from_list(pget(value, Param)) - }. - -list_global_runtime_parameters() -> - [global_runtime_parameter_definition(P) || P <- rabbit_runtime_parameters:list_global(), not is_internal_parameter(P)]. - -global_runtime_parameter_definition(P0) -> - P = [{rabbit_data_coercion:to_binary(K), V} || {K, V} <- P0], - maps:from_list(P). - --define(INTERNAL_GLOBAL_PARAM_PREFIX, "internal"). - -is_internal_parameter(Param) -> - Name = rabbit_data_coercion:to_list(pget(name, Param)), - %% if global parameter name starts with an "internal", consider it to be internal - %% and exclude it from definition export - string:left(Name, length(?INTERNAL_GLOBAL_PARAM_PREFIX)) =:= ?INTERNAL_GLOBAL_PARAM_PREFIX. - -list_policies() -> - [policy_definition(P) || P <- rabbit_policy:list()]. - -policy_definition(Policy) -> - #{ - <<"vhost">> => pget(vhost, Policy), - <<"name">> => pget(name, Policy), - <<"pattern">> => pget(pattern, Policy), - <<"apply-to">> => pget('apply-to', Policy), - <<"priority">> => pget(priority, Policy), - <<"definition">> => maps:from_list(pget(definition, Policy)) - }. - -list_permissions() -> - [permission_definition(P) || P <- rabbit_auth_backend_internal:list_permissions()]. - -permission_definition(P0) -> - P = [{rabbit_data_coercion:to_binary(K), V} || {K, V} <- P0], - maps:from_list(P). - -list_topic_permissions() -> - [topic_permission_definition(P) || P <- rabbit_auth_backend_internal:list_topic_permissions()]. - -topic_permission_definition(P0) -> - P = [{rabbit_data_coercion:to_binary(K), V} || {K, V} <- P0], - maps:from_list(P). - -tags_as_binaries(Tags) -> - list_to_binary(string:join([atom_to_list(T) || T <- Tags], ",")). diff --git a/src/rabbit_diagnostics.erl b/src/rabbit_diagnostics.erl deleted file mode 100644 index 999596cdc9..0000000000 --- a/src/rabbit_diagnostics.erl +++ /dev/null @@ -1,119 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_diagnostics). - --define(PROCESS_INFO, - [registered_name, current_stacktrace, initial_call, message_queue_len, - links, monitors, monitored_by, heap_size]). - --export([maybe_stuck/0, maybe_stuck/1, top_memory_use/0, top_memory_use/1, - top_binary_refs/0, top_binary_refs/1]). - -maybe_stuck() -> maybe_stuck(5000). - -maybe_stuck(Timeout) -> - Pids = processes(), - io:format("~s There are ~p processes.~n", [get_time(), length(Pids)]), - maybe_stuck(Pids, Timeout). - -maybe_stuck(Pids, Timeout) when Timeout =< 0 -> - io:format("~s Found ~p suspicious processes.~n", [get_time(), length(Pids)]), - [io:format("~s ~p~n", [get_time(), info(Pid)]) || Pid <- Pids], - ok; -maybe_stuck(Pids, Timeout) -> - Pids2 = [P || P <- Pids, looks_stuck(P)], - io:format("~s Investigated ~p processes this round, ~pms to go.~n", - [get_time(), length(Pids2), Timeout]), - timer:sleep(500), - maybe_stuck(Pids2, Timeout - 500). - -looks_stuck(Pid) -> - case info(Pid, status, gone) of - {status, waiting} -> - %% It's tempting to just check for message_queue_len > 0 - %% here rather than mess around with stack traces and - %% heuristics. But really, sometimes freshly stuck - %% processes can have 0 messages... - case info(Pid, current_stacktrace, gone) of - {current_stacktrace, [H|_]} -> - maybe_stuck_stacktrace(H); - _ -> - false - end; - _ -> - false - end. - -maybe_stuck_stacktrace({gen_server2, process_next_msg, _}) -> false; -maybe_stuck_stacktrace({gen_event, fetch_msg, _}) -> false; -maybe_stuck_stacktrace({prim_inet, accept0, _}) -> false; -maybe_stuck_stacktrace({prim_inet, recv0, _}) -> false; -maybe_stuck_stacktrace({rabbit_heartbeat, heartbeater, _}) -> false; -maybe_stuck_stacktrace({rabbit_net, recv, _}) -> false; -maybe_stuck_stacktrace({group, _, _}) -> false; -maybe_stuck_stacktrace({shell, _, _}) -> false; -maybe_stuck_stacktrace({io, _, _}) -> false; -maybe_stuck_stacktrace({M, F, A, _}) -> - maybe_stuck_stacktrace({M, F, A}); -maybe_stuck_stacktrace({_M, F, _A}) -> - case string:str(atom_to_list(F), "loop") of - 0 -> true; - _ -> false - end. - -top_memory_use() -> top_memory_use(30). - -top_memory_use(Count) -> - Pids = processes(), - io:format("~s Memory use: top ~p of ~p processes.~n", [get_time(), Count, length(Pids)]), - Procs = [{info(Pid, memory, 0), info(Pid)} || Pid <- Pids], - Sorted = lists:sublist(lists:reverse(lists:sort(Procs)), Count), - io:format("~s ~p~n", [get_time(), Sorted]). - -top_binary_refs() -> top_binary_refs(30). - -top_binary_refs(Count) -> - Pids = processes(), - io:format("~s Binary refs: top ~p of ~p processes.~n", [get_time(), Count, length(Pids)]), - Procs = [{{binary_refs, binary_refs(Pid)}, info(Pid)} || Pid <- Pids], - Sorted = lists:sublist(lists:reverse(lists:sort(Procs)), Count), - io:format("~s ~p~n", [get_time(), Sorted]). - -binary_refs(Pid) -> - case info(Pid, binary, []) of - {binary, Refs} -> - lists:sum([Sz || {_Ptr, Sz} <- lists:usort([{Ptr, Sz} || - {Ptr, Sz, _Cnt} <- Refs])]); - _ -> 0 - end. - -info(Pid) -> - [{pid, Pid} | info(Pid, ?PROCESS_INFO, [])]. - -info(Pid, Infos, Default) -> - try - process_info(Pid, Infos) - catch - _:_ -> case is_atom(Infos) of - true -> {Infos, Default}; - false -> Default - end - end. - -get_time() -> - {{Y,M,D}, {H,Min,Sec}} = calendar:local_time(), - [ integer_to_list(Y), "-", - prefix_zero(integer_to_list(M)), "-", - prefix_zero(integer_to_list(D)), " ", - prefix_zero(integer_to_list(H)), ":", - prefix_zero(integer_to_list(Min)), ":", - prefix_zero(integer_to_list(Sec)) - ]. - -prefix_zero([C]) -> [$0, C]; -prefix_zero([_,_] = Full) -> Full. diff --git a/src/rabbit_direct.erl b/src/rabbit_direct.erl deleted file mode 100644 index 3fc2d75908..0000000000 --- a/src/rabbit_direct.erl +++ /dev/null @@ -1,235 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_direct). - --export([boot/0, force_event_refresh/1, list/0, connect/5, - start_channel/10, disconnect/2]). - --deprecated([{force_event_refresh, 1, eventually}]). - -%% Internal --export([list_local/0]). - -%% For testing only --export([extract_extra_auth_props/4]). - --include("rabbit.hrl"). --include("rabbit_misc.hrl"). - -%%---------------------------------------------------------------------------- - --spec boot() -> 'ok'. - -boot() -> rabbit_sup:start_supervisor_child( - rabbit_direct_client_sup, rabbit_client_sup, - [{local, rabbit_direct_client_sup}, - {rabbit_channel_sup, start_link, []}]). - --spec force_event_refresh(reference()) -> 'ok'. - -force_event_refresh(Ref) -> - [Pid ! {force_event_refresh, Ref} || Pid <- list()], - ok. - --spec list_local() -> [pid()]. - -list_local() -> - pg_local:get_members(rabbit_direct). - --spec list() -> [pid()]. - -list() -> - Nodes = rabbit_nodes:all_running(), - rabbit_misc:append_rpc_all_nodes(Nodes, rabbit_direct, list_local, [], ?RPC_TIMEOUT). - -%%---------------------------------------------------------------------------- - -auth_fun({none, _}, _VHost, _ExtraAuthProps) -> - fun () -> {ok, rabbit_auth_backend_dummy:user()} end; - -auth_fun({Username, none}, _VHost, _ExtraAuthProps) -> - fun () -> rabbit_access_control:check_user_login(Username, []) end; - -auth_fun({Username, Password}, VHost, ExtraAuthProps) -> - fun () -> - rabbit_access_control:check_user_login( - Username, - [{password, Password}, {vhost, VHost}] ++ ExtraAuthProps) - end. - --spec connect - (({'none', 'none'} | {rabbit_types:username(), 'none'} | - {rabbit_types:username(), rabbit_types:password()}), - rabbit_types:vhost(), rabbit_types:protocol(), pid(), - rabbit_event:event_props()) -> - rabbit_types:ok_or_error2( - {rabbit_types:user(), rabbit_framing:amqp_table()}, - 'broker_not_found_on_node' | - {'auth_failure', string()} | 'access_refused'). - -connect(Creds, VHost, Protocol, Pid, Infos) -> - ExtraAuthProps = extract_extra_auth_props(Creds, VHost, Pid, Infos), - AuthFun = auth_fun(Creds, VHost, ExtraAuthProps), - case rabbit:is_running() of - true -> - case whereis(rabbit_direct_client_sup) of - undefined -> - {error, broker_is_booting}; - _ -> - case is_over_vhost_connection_limit(VHost, Creds, Pid) of - true -> - {error, not_allowed}; - false -> - case is_vhost_alive(VHost, Creds, Pid) of - false -> - {error, {internal_error, vhost_is_down}}; - true -> - case AuthFun() of - {ok, User = #user{username = Username}} -> - notify_auth_result(Username, - user_authentication_success, []), - connect1(User, VHost, Protocol, Pid, Infos); - {refused, Username, Msg, Args} -> - notify_auth_result(Username, - user_authentication_failure, - [{error, rabbit_misc:format(Msg, Args)}]), - {error, {auth_failure, "Refused"}} - end %% AuthFun() - end %% is_vhost_alive - end %% is_over_vhost_connection_limit - end; - false -> {error, broker_not_found_on_node} - end. - -extract_extra_auth_props(Creds, VHost, Pid, Infos) -> - case extract_protocol(Infos) of - undefined -> - []; - Protocol -> - maybe_call_connection_info_module(Protocol, Creds, VHost, Pid, Infos) - end. - -extract_protocol(Infos) -> - case proplists:get_value(protocol, Infos, undefined) of - {Protocol, _Version} -> - Protocol; - _ -> - undefined - end. - -maybe_call_connection_info_module(Protocol, Creds, VHost, Pid, Infos) -> - Module = rabbit_data_coercion:to_atom(string:to_lower( - "rabbit_" ++ - lists:flatten(string:replace(rabbit_data_coercion:to_list(Protocol), " ", "_", all)) ++ - "_connection_info") - ), - Args = [Creds, VHost, Pid, Infos], - code_server_cache:maybe_call_mfa(Module, additional_authn_params, Args, []). - -is_vhost_alive(VHost, {Username, _Password}, Pid) -> - PrintedUsername = case Username of - none -> ""; - _ -> Username - end, - case rabbit_vhost_sup_sup:is_vhost_alive(VHost) of - true -> true; - false -> - rabbit_log_connection:error( - "Error on Direct connection ~p~n" - "access to vhost '~s' refused for user '~s': " - "vhost '~s' is down", - [Pid, VHost, PrintedUsername, VHost]), - false - end. - -is_over_vhost_connection_limit(VHost, {Username, _Password}, Pid) -> - PrintedUsername = case Username of - none -> ""; - _ -> Username - end, - try rabbit_vhost_limit:is_over_connection_limit(VHost) of - false -> false; - {true, Limit} -> - rabbit_log_connection:error( - "Error on Direct connection ~p~n" - "access to vhost '~s' refused for user '~s': " - "vhost connection limit (~p) is reached", - [Pid, VHost, PrintedUsername, Limit]), - true - catch - throw:{error, {no_such_vhost, VHost}} -> - rabbit_log_connection:error( - "Error on Direct connection ~p~n" - "vhost ~s not found", [Pid, VHost]), - true - end. - -notify_auth_result(Username, AuthResult, ExtraProps) -> - EventProps = [{connection_type, direct}, - {name, case Username of none -> ''; _ -> Username end}] ++ - ExtraProps, - rabbit_event:notify(AuthResult, [P || {_, V} = P <- EventProps, V =/= '']). - -connect1(User = #user{username = Username}, VHost, Protocol, Pid, Infos) -> - case rabbit_auth_backend_internal:is_over_connection_limit(Username) of - false -> - % Note: peer_host can be either a tuple or - % a binary if reverse_dns_lookups is enabled - PeerHost = proplists:get_value(peer_host, Infos), - AuthzContext = proplists:get_value(variable_map, Infos, #{}), - try rabbit_access_control:check_vhost_access(User, VHost, - {ip, PeerHost}, AuthzContext) of - ok -> ok = pg_local:join(rabbit_direct, Pid), - rabbit_core_metrics:connection_created(Pid, Infos), - rabbit_event:notify(connection_created, Infos), - {ok, {User, rabbit_reader:server_properties(Protocol)}} - catch - exit:#amqp_error{name = Reason = not_allowed} -> - {error, Reason} - end; - {true, Limit} -> - rabbit_log_connection:error( - "Error on Direct connection ~p~n" - "access refused for user '~s': " - "user connection limit (~p) is reached", - [Pid, Username, Limit]), - {error, not_allowed} - end. - --spec start_channel - (rabbit_channel:channel_number(), pid(), pid(), string(), - rabbit_types:protocol(), rabbit_types:user(), rabbit_types:vhost(), - rabbit_framing:amqp_table(), pid(), any()) -> - {'ok', pid()}. - -start_channel(Number, ClientChannelPid, ConnPid, ConnName, Protocol, - User = #user{username = Username}, VHost, Capabilities, - Collector, AmqpParams) -> - case rabbit_auth_backend_internal:is_over_channel_limit(Username) of - false -> - {ok, _, {ChannelPid, _}} = - supervisor2:start_child( - rabbit_direct_client_sup, - [{direct, Number, ClientChannelPid, ConnPid, ConnName, Protocol, - User, VHost, Capabilities, Collector, AmqpParams}]), - {ok, ChannelPid}; - {true, Limit} -> - rabbit_log_connection:error( - "Error on direct connection ~p~n" - "number of channels opened for user '~s' has reached the " - "maximum allowed limit of (~w)", - [ConnPid, Username, Limit]), - {error, not_allowed} - end. - --spec disconnect(pid(), rabbit_event:event_props()) -> 'ok'. - -disconnect(Pid, Infos) -> - pg_local:leave(rabbit_direct, Pid), - rabbit_core_metrics:connection_closed(Pid), - rabbit_event:notify(connection_closed, Infos). diff --git a/src/rabbit_disk_monitor.erl b/src/rabbit_disk_monitor.erl deleted file mode 100644 index 8277794098..0000000000 --- a/src/rabbit_disk_monitor.erl +++ /dev/null @@ -1,317 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_disk_monitor). - -%% Disk monitoring server. Monitors free disk space -%% periodically and sets alarms when it is below a certain -%% watermark (configurable either as an absolute value or -%% relative to the memory limit). -%% -%% Disk monitoring is done by shelling out to /usr/bin/df -%% instead of related built-in OTP functions because currently -%% this is the most reliable way of determining free disk space -%% for the partition our internal database is on. -%% -%% Update interval is dynamically calculated assuming disk -%% space is being filled at FAST_RATE. - --behaviour(gen_server). - --export([start_link/1]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --export([get_disk_free_limit/0, set_disk_free_limit/1, - get_min_check_interval/0, set_min_check_interval/1, - get_max_check_interval/0, set_max_check_interval/1, - get_disk_free/0, set_enabled/1]). - --define(SERVER, ?MODULE). --define(DEFAULT_MIN_DISK_CHECK_INTERVAL, 100). --define(DEFAULT_MAX_DISK_CHECK_INTERVAL, 10000). --define(DEFAULT_DISK_FREE_LIMIT, 50000000). -%% 250MB/s i.e. 250kB/ms --define(FAST_RATE, (250 * 1000)). - --record(state, { - %% monitor partition on which this directory resides - dir, - %% configured limit in bytes - limit, - %% last known free disk space amount in bytes - actual, - %% minimum check interval - min_interval, - %% maximum check interval - max_interval, - %% timer that drives periodic checks - timer, - %% is free disk space alarm currently in effect? - alarmed, - %% is monitoring enabled? false on unsupported - %% platforms - enabled, - %% number of retries to enable monitoring if it fails - %% on start-up - retries, - %% Interval between retries - interval -}). - -%%---------------------------------------------------------------------------- - --type disk_free_limit() :: (integer() | string() | {'mem_relative', float() | integer()}). - -%%---------------------------------------------------------------------------- -%% Public API -%%---------------------------------------------------------------------------- - --spec get_disk_free_limit() -> integer(). - -get_disk_free_limit() -> - gen_server:call(?MODULE, get_disk_free_limit, infinity). - --spec set_disk_free_limit(disk_free_limit()) -> 'ok'. - -set_disk_free_limit(Limit) -> - gen_server:call(?MODULE, {set_disk_free_limit, Limit}, infinity). - --spec get_min_check_interval() -> integer(). - -get_min_check_interval() -> - gen_server:call(?MODULE, get_min_check_interval, infinity). - --spec set_min_check_interval(integer()) -> 'ok'. - -set_min_check_interval(Interval) -> - gen_server:call(?MODULE, {set_min_check_interval, Interval}, infinity). - --spec get_max_check_interval() -> integer(). - -get_max_check_interval() -> - gen_server:call(?MODULE, get_max_check_interval, infinity). - --spec set_max_check_interval(integer()) -> 'ok'. - -set_max_check_interval(Interval) -> - gen_server:call(?MODULE, {set_max_check_interval, Interval}, infinity). - --spec get_disk_free() -> (integer() | 'unknown'). --spec set_enabled(string()) -> 'ok'. - -get_disk_free() -> - gen_server:call(?MODULE, get_disk_free, infinity). - -set_enabled(Enabled) -> - gen_server:call(?MODULE, {set_enabled, Enabled}, infinity). - -%%---------------------------------------------------------------------------- -%% gen_server callbacks -%%---------------------------------------------------------------------------- - --spec start_link(disk_free_limit()) -> rabbit_types:ok_pid_or_error(). - -start_link(Args) -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [Args], []). - -init([Limit]) -> - Dir = dir(), - {ok, Retries} = application:get_env(rabbit, disk_monitor_failure_retries), - {ok, Interval} = application:get_env(rabbit, disk_monitor_failure_retry_interval), - State = #state{dir = Dir, - min_interval = ?DEFAULT_MIN_DISK_CHECK_INTERVAL, - max_interval = ?DEFAULT_MAX_DISK_CHECK_INTERVAL, - alarmed = false, - enabled = true, - limit = Limit, - retries = Retries, - interval = Interval}, - {ok, enable(State)}. - -handle_call(get_disk_free_limit, _From, State = #state{limit = Limit}) -> - {reply, Limit, State}; - -handle_call({set_disk_free_limit, _}, _From, #state{enabled = false} = State) -> - rabbit_log:info("Cannot set disk free limit: " - "disabled disk free space monitoring", []), - {reply, ok, State}; - -handle_call({set_disk_free_limit, Limit}, _From, State) -> - {reply, ok, set_disk_limits(State, Limit)}; - -handle_call(get_min_check_interval, _From, State) -> - {reply, State#state.min_interval, State}; - -handle_call(get_max_check_interval, _From, State) -> - {reply, State#state.max_interval, State}; - -handle_call({set_min_check_interval, MinInterval}, _From, State) -> - {reply, ok, State#state{min_interval = MinInterval}}; - -handle_call({set_max_check_interval, MaxInterval}, _From, State) -> - {reply, ok, State#state{max_interval = MaxInterval}}; - -handle_call(get_disk_free, _From, State = #state { actual = Actual }) -> - {reply, Actual, State}; - -handle_call({set_enabled, _Enabled = true}, _From, State) -> - start_timer(set_disk_limits(State, State#state.limit)), - rabbit_log:info("Free disk space monitor was enabled"), - {reply, ok, State#state{enabled = true}}; -handle_call({set_enabled, _Enabled = false}, _From, State) -> - erlang:cancel_timer(State#state.timer), - rabbit_log:info("Free disk space monitor was manually disabled"), - {reply, ok, State#state{enabled = false}}; - -handle_call(_Request, _From, State) -> - {noreply, State}. - -handle_cast(_Request, State) -> - {noreply, State}. - -handle_info(try_enable, #state{retries = Retries} = State) -> - {noreply, enable(State#state{retries = Retries - 1})}; -handle_info(update, State) -> - {noreply, start_timer(internal_update(State))}; - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%---------------------------------------------------------------------------- -%% Server Internals -%%---------------------------------------------------------------------------- - -% the partition / drive containing this directory will be monitored -dir() -> rabbit_mnesia:dir(). - -set_disk_limits(State, Limit0) -> - Limit = interpret_limit(Limit0), - State1 = State#state { limit = Limit }, - rabbit_log:info("Disk free limit set to ~pMB~n", - [trunc(Limit / 1000000)]), - internal_update(State1). - -internal_update(State = #state { limit = Limit, - dir = Dir, - alarmed = Alarmed}) -> - CurrentFree = get_disk_free(Dir), - NewAlarmed = CurrentFree < Limit, - case {Alarmed, NewAlarmed} of - {false, true} -> - emit_update_info("insufficient", CurrentFree, Limit), - rabbit_alarm:set_alarm({{resource_limit, disk, node()}, []}); - {true, false} -> - emit_update_info("sufficient", CurrentFree, Limit), - rabbit_alarm:clear_alarm({resource_limit, disk, node()}); - _ -> - ok - end, - State #state {alarmed = NewAlarmed, actual = CurrentFree}. - -get_disk_free(Dir) -> - get_disk_free(Dir, os:type()). - -get_disk_free(Dir, {unix, Sun}) - when Sun =:= sunos; Sun =:= sunos4; Sun =:= solaris -> - Df = os:find_executable("df"), - parse_free_unix(rabbit_misc:os_cmd(Df ++ " -k " ++ Dir)); -get_disk_free(Dir, {unix, _}) -> - Df = os:find_executable("df"), - parse_free_unix(rabbit_misc:os_cmd(Df ++ " -kP " ++ Dir)); -get_disk_free(Dir, {win32, _}) -> - %% On Windows, the Win32 API enforces a limit of 260 characters - %% (MAX_PATH). If we call `dir` with a path longer than that, it - %% fails with "File not found". Starting with Windows 10 version - %% 1607, this limit was removed, but the administrator has to - %% configure that. - %% - %% NTFS supports paths up to 32767 characters. Therefore, paths - %% longer than 260 characters exist but they are "inaccessible" to - %% `dir`. - %% - %% A workaround is to tell the Win32 API to not parse a path and - %% just pass it raw to the underlying filesystem. To do this, the - %% path must be prepended with "\\?\". That's what we do here. - %% - %% However, the underlying filesystem may not support forward - %% slashes transparently, as the Win32 API does. Therefore, we - %% convert all forward slashes to backslashes. - %% - %% See the following page to learn more about this: - %% https://ss64.com/nt/syntax-filenames.html - RawDir = "\\\\?\\" ++ string:replace(Dir, "/", "\\", all), - parse_free_win32(rabbit_misc:os_cmd("dir /-C /W \"" ++ RawDir ++ "\"")). - -parse_free_unix(Str) -> - case string:tokens(Str, "\n") of - [_, S | _] -> case string:tokens(S, " \t") of - [_, _, _, Free | _] -> list_to_integer(Free) * 1024; - _ -> exit({unparseable, Str}) - end; - _ -> exit({unparseable, Str}) - end. - -parse_free_win32(CommandResult) -> - LastLine = lists:last(string:tokens(CommandResult, "\r\n")), - {match, [Free]} = re:run(lists:reverse(LastLine), "(\\d+)", - [{capture, all_but_first, list}]), - list_to_integer(lists:reverse(Free)). - -interpret_limit({mem_relative, Relative}) - when is_number(Relative) -> - round(Relative * vm_memory_monitor:get_total_memory()); -interpret_limit(Absolute) -> - case rabbit_resource_monitor_misc:parse_information_unit(Absolute) of - {ok, ParsedAbsolute} -> ParsedAbsolute; - {error, parse_error} -> - rabbit_log:error("Unable to parse disk_free_limit value ~p", - [Absolute]), - ?DEFAULT_DISK_FREE_LIMIT - end. - -emit_update_info(StateStr, CurrentFree, Limit) -> - rabbit_log:info( - "Free disk space is ~s. Free bytes: ~p. Limit: ~p~n", - [StateStr, CurrentFree, Limit]). - -start_timer(State) -> - State#state{timer = erlang:send_after(interval(State), self(), update)}. - -interval(#state{alarmed = true, - max_interval = MaxInterval}) -> - MaxInterval; -interval(#state{limit = Limit, - actual = Actual, - min_interval = MinInterval, - max_interval = MaxInterval}) -> - IdealInterval = 2 * (Actual - Limit) / ?FAST_RATE, - trunc(erlang:max(MinInterval, erlang:min(MaxInterval, IdealInterval))). - -enable(#state{retries = 0} = State) -> - State; -enable(#state{dir = Dir, interval = Interval, limit = Limit, retries = Retries} - = State) -> - case {catch get_disk_free(Dir), - vm_memory_monitor:get_total_memory()} of - {N1, N2} when is_integer(N1), is_integer(N2) -> - rabbit_log:info("Enabling free disk space monitoring~n", []), - start_timer(set_disk_limits(State, Limit)); - Err -> - rabbit_log:info("Free disk space monitor encountered an error " - "(e.g. failed to parse output from OS tools): ~p, retries left: ~b~n", - [Err, Retries]), - erlang:send_after(Interval, self(), try_enable), - State#state{enabled = false} - end. diff --git a/src/rabbit_epmd_monitor.erl b/src/rabbit_epmd_monitor.erl deleted file mode 100644 index 938826dba6..0000000000 --- a/src/rabbit_epmd_monitor.erl +++ /dev/null @@ -1,104 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_epmd_monitor). - --behaviour(gen_server). - --export([start_link/0]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --record(state, {timer, mod, me, host, port}). - --define(SERVER, ?MODULE). --define(CHECK_FREQUENCY, 60000). - -%%---------------------------------------------------------------------------- -%% It's possible for epmd to be killed out from underneath us. If that -%% happens, then obviously clustering and rabbitmqctl stop -%% working. This process checks up on epmd and restarts it / -%% re-registers us with it if it has gone away. -%% -%% How could epmd be killed? -%% -%% 1) The most popular way for this to happen is when running as a -%% Windows service. The user starts rabbitmqctl first, and this starts -%% epmd under the user's account. When they log out epmd is killed. -%% -%% 2) Some packagings of (non-RabbitMQ?) Erlang apps might do "killall -%% epmd" as a shutdown or uninstall step. -%% ---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - -init([]) -> - {Me, Host} = rabbit_nodes:parts(node()), - Mod = net_kernel:epmd_module(), - {ok, Port} = handle_port_please(init, Mod:port_please(Me, Host), Me, undefined), - State = #state{mod = Mod, me = Me, host = Host, port = Port}, - {ok, ensure_timer(State)}. - -handle_call(_Request, _From, State) -> - {noreply, State}. - -handle_cast(check, State0) -> - {ok, State1} = check_epmd(State0), - {noreply, ensure_timer(State1#state{timer = undefined})}; -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(check, State0) -> - {ok, State1} = check_epmd(State0), - {noreply, ensure_timer(State1#state{timer = undefined})}; - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%---------------------------------------------------------------------------- - -ensure_timer(State) -> - rabbit_misc:ensure_timer(State, #state.timer, ?CHECK_FREQUENCY, check). - -check_epmd(State = #state{mod = Mod, - me = Me, - host = Host, - port = Port0}) -> - rabbit_log:debug("Asked to [re-]register this node (~s@~s) with epmd...", [Me, Host]), - {ok, Port1} = handle_port_please(check, Mod:port_please(Me, Host), Me, Port0), - rabbit_nodes:ensure_epmd(), - Mod:register_node(Me, Port1), - rabbit_log:debug("[Re-]registered this node (~s@~s) with epmd at port ~p", [Me, Host, Port1]), - {ok, State#state{port = Port1}}. - -handle_port_please(init, noport, Me, Port) -> - rabbit_log:info("epmd does not know us, re-registering as ~s~n", [Me]), - {ok, Port}; -handle_port_please(check, noport, Me, Port) -> - rabbit_log:warning("epmd does not know us, re-registering ~s at port ~b~n", [Me, Port]), - {ok, Port}; -handle_port_please(_, closed, _Me, Port) -> - rabbit_log:error("epmd monitor failed to retrieve our port from epmd: closed"), - {ok, Port}; -handle_port_please(init, {port, NewPort, _Version}, _Me, _Port) -> - rabbit_log:info("epmd monitor knows us, inter-node communication (distribution) port: ~p", [NewPort]), - {ok, NewPort}; -handle_port_please(check, {port, NewPort, _Version}, _Me, _Port) -> - {ok, NewPort}; -handle_port_please(_, {error, Error}, _Me, Port) -> - rabbit_log:error("epmd monitor failed to retrieve our port from epmd: ~p", [Error]), - {ok, Port}. diff --git a/src/rabbit_event_consumer.erl b/src/rabbit_event_consumer.erl deleted file mode 100644 index 489d39312e..0000000000 --- a/src/rabbit_event_consumer.erl +++ /dev/null @@ -1,197 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_event_consumer). - --include_lib("rabbit_common/include/rabbit.hrl"). - --export([register/4]). --export([init/1, handle_call/2, handle_event/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {pid, ref, monitor, pattern}). - -%%---------------------------------------------------------------------------- - -register(Pid, Ref, Duration, Pattern) -> - case gen_event:add_handler(rabbit_event, ?MODULE, [Pid, Ref, Duration, Pattern]) of - ok -> - {ok, Ref}; - Error -> - Error - end. - -%%---------------------------------------------------------------------------- - -init([Pid, Ref, Duration, Pattern]) -> - MRef = erlang:monitor(process, Pid), - case Duration of - infinity -> infinity; - _ -> erlang:send_after(Duration * 1000, self(), rabbit_event_consumer_timeout) - end, - {ok, #state{pid = Pid, ref = Ref, monitor = MRef, pattern = Pattern}}. - -handle_call(_Request, State) -> {ok, not_understood, State}. - -handle_event(#event{type = Type, - props = Props, - timestamp = TS, - reference = none}, #state{pid = Pid, - ref = Ref, - pattern = Pattern} = State) -> - case key(Type) of - ignore -> ok; - Key -> case re:run(Key, Pattern, [{capture, none}]) of - match -> - Data = [{'event', Key}] ++ - fmt_proplist([{'timestamp_in_ms', TS} | Props]), - Pid ! {Ref, Data, confinue}; - _ -> - ok - end - end, - {ok, State}; -handle_event(_Event, State) -> - {ok, State}. - -handle_info({'DOWN', MRef, _, _, _}, #state{monitor = MRef}) -> - remove_handler; -handle_info(rabbit_event_consumer_timeout, #state{pid = Pid, ref = Ref}) -> - Pid ! {Ref, <<>>, finished}, - remove_handler; -handle_info(_Info, State) -> - {ok, State}. - -terminate(_Arg, #state{monitor = MRef}) -> - erlang:demonitor(MRef), - ok. - -code_change(_OldVsn, State, _Extra) -> {ok, State}. - -%%---------------------------------------------------------------------------- - -%% pattern matching is way more efficient that the string operations, -%% let's use all the keys we're aware of to speed up the handler. -%% Any unknown or new one will be processed as before (see last function clause). -key(queue_deleted) -> - <<"queue.deleted">>; -key(queue_created) -> - <<"queue.created">>; -key(exchange_created) -> - <<"exchange.created">>; -key(exchange_deleted) -> - <<"exchange.deleted">>; -key(binding_created) -> - <<"binding.created">>; -key(connection_created) -> - <<"connection.created">>; -key(connection_closed) -> - <<"connection.closed">>; -key(channel_created) -> - <<"channel.created">>; -key(channel_closed) -> - <<"channel.closed">>; -key(consumer_created) -> - <<"consumer.created">>; -key(consumer_deleted) -> - <<"consumer.deleted">>; -key(queue_stats) -> - ignore; -key(connection_stats) -> - ignore; -key(policy_set) -> - <<"policy.set">>; -key(policy_cleared) -> - <<"policy.cleared">>; -key(parameter_set) -> - <<"parameter.set">>; -key(parameter_cleared) -> - <<"parameter.cleared">>; -key(vhost_created) -> - <<"vhost.created">>; -key(vhost_deleted) -> - <<"vhost.deleted">>; -key(vhost_limits_set) -> - <<"vhost.limits.set">>; -key(vhost_limits_cleared) -> - <<"vhost.limits.cleared">>; -key(user_authentication_success) -> - <<"user.authentication.success">>; -key(user_authentication_failure) -> - <<"user.authentication.failure">>; -key(user_created) -> - <<"user.created">>; -key(user_deleted) -> - <<"user.deleted">>; -key(user_password_changed) -> - <<"user.password.changed">>; -key(user_password_cleared) -> - <<"user.password.cleared">>; -key(user_tags_set) -> - <<"user.tags.set">>; -key(permission_created) -> - <<"permission.created">>; -key(permission_deleted) -> - <<"permission.deleted">>; -key(topic_permission_created) -> - <<"topic.permission.created">>; -key(topic_permission_deleted) -> - <<"topic.permission.deleted">>; -key(alarm_set) -> - <<"alarm.set">>; -key(alarm_cleared) -> - <<"alarm.cleared">>; -key(shovel_worker_status) -> - <<"shovel.worker.status">>; -key(shovel_worker_removed) -> - <<"shovel.worker.removed">>; -key(federation_link_status) -> - <<"federation.link.status">>; -key(federation_link_removed) -> - <<"federation.link.removed">>; -key(S) -> - case string:tokens(atom_to_list(S), "_") of - [_, "stats"] -> ignore; - Tokens -> list_to_binary(string:join(Tokens, ".")) - end. - -fmt_proplist(Props) -> - lists:foldl(fun({K, V}, Acc) -> - case fmt(K, V) of - L when is_list(L) -> lists:append(L, Acc); - T -> [T | Acc] - end - end, [], Props). - -fmt(K, #resource{virtual_host = VHost, - name = Name}) -> [{K, Name}, - {'vhost', VHost}]; -fmt(K, true) -> {K, true}; -fmt(K, false) -> {K, false}; -fmt(K, V) when is_atom(V) -> {K, atom_to_binary(V, utf8)}; -fmt(K, V) when is_integer(V) -> {K, V}; -fmt(K, V) when is_number(V) -> {K, V}; -fmt(K, V) when is_binary(V) -> {K, V}; -fmt(K, [{_, _}|_] = Vs) -> {K, fmt_proplist(Vs)}; -fmt(K, Vs) when is_list(Vs) -> {K, [fmt(V) || V <- Vs]}; -fmt(K, V) when is_pid(V) -> {K, list_to_binary(rabbit_misc:pid_to_string(V))}; -fmt(K, V) -> {K, - list_to_binary( - rabbit_misc:format("~1000000000p", [V]))}. - -%% Exactly the same as fmt/2, duplicated only for performance issues -fmt(true) -> true; -fmt(false) -> false; -fmt(V) when is_atom(V) -> atom_to_binary(V, utf8); -fmt(V) when is_integer(V) -> V; -fmt(V) when is_number(V) -> V; -fmt(V) when is_binary(V) -> V; -fmt([{_, _}|_] = Vs) -> fmt_proplist(Vs); -fmt(Vs) when is_list(Vs) -> [fmt(V) || V <- Vs]; -fmt(V) when is_pid(V) -> list_to_binary(rabbit_misc:pid_to_string(V)); -fmt(V) -> list_to_binary( - rabbit_misc:format("~1000000000p", [V])). diff --git a/src/rabbit_exchange.erl b/src/rabbit_exchange.erl deleted file mode 100644 index 129b2b868b..0000000000 --- a/src/rabbit_exchange.erl +++ /dev/null @@ -1,592 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_exchange). --include("rabbit.hrl"). --include("rabbit_framing.hrl"). - --export([recover/1, policy_changed/2, callback/4, declare/7, - assert_equivalence/6, assert_args_equivalence/2, check_type/1, - lookup/1, lookup_many/1, lookup_or_die/1, list/0, list/1, lookup_scratch/2, - update_scratch/3, update_decorators/1, immutable/1, - info_keys/0, info/1, info/2, info_all/1, info_all/2, info_all/4, - route/2, delete/3, validate_binding/2, count/0]). --export([list_names/0, is_amq_prefixed/1]). -%% these must be run inside a mnesia tx --export([maybe_auto_delete/2, serial/1, peek_serial/1, update/2]). - -%%---------------------------------------------------------------------------- - --export_type([name/0, type/0]). - --type name() :: rabbit_types:r('exchange'). --type type() :: atom(). --type fun_name() :: atom(). - -%%---------------------------------------------------------------------------- - --define(INFO_KEYS, [name, type, durable, auto_delete, internal, arguments, - policy, user_who_performed_action]). - --spec recover(rabbit_types:vhost()) -> [name()]. - -recover(VHost) -> - Xs = rabbit_misc:table_filter( - fun (#exchange{name = XName}) -> - XName#resource.virtual_host =:= VHost andalso - mnesia:read({rabbit_exchange, XName}) =:= [] - end, - fun (X, Tx) -> - X1 = case Tx of - true -> store_ram(X); - false -> rabbit_exchange_decorator:set(X) - end, - callback(X1, create, map_create_tx(Tx), [X1]) - end, - rabbit_durable_exchange), - [XName || #exchange{name = XName} <- Xs]. - --spec callback - (rabbit_types:exchange(), fun_name(), - fun((boolean()) -> non_neg_integer()) | atom(), [any()]) -> 'ok'. - -callback(X = #exchange{type = XType, - decorators = Decorators}, Fun, Serial0, Args) -> - Serial = if is_function(Serial0) -> Serial0; - is_atom(Serial0) -> fun (_Bool) -> Serial0 end - end, - [ok = apply(M, Fun, [Serial(M:serialise_events(X)) | Args]) || - M <- rabbit_exchange_decorator:select(all, Decorators)], - Module = type_to_module(XType), - apply(Module, Fun, [Serial(Module:serialise_events()) | Args]). - --spec policy_changed - (rabbit_types:exchange(), rabbit_types:exchange()) -> 'ok'. - -policy_changed(X = #exchange{type = XType, - decorators = Decorators}, - X1 = #exchange{decorators = Decorators1}) -> - D = rabbit_exchange_decorator:select(all, Decorators), - D1 = rabbit_exchange_decorator:select(all, Decorators1), - DAll = lists:usort(D ++ D1), - [ok = M:policy_changed(X, X1) || M <- [type_to_module(XType) | DAll]], - ok. - -serialise_events(X = #exchange{type = Type, decorators = Decorators}) -> - lists:any(fun (M) -> M:serialise_events(X) end, - rabbit_exchange_decorator:select(all, Decorators)) - orelse (type_to_module(Type)):serialise_events(). - --spec serial(rabbit_types:exchange()) -> - fun((boolean()) -> 'none' | pos_integer()). - -serial(#exchange{name = XName} = X) -> - Serial = case serialise_events(X) of - true -> next_serial(XName); - false -> none - end, - fun (true) -> Serial; - (false) -> none - end. - --spec is_amq_prefixed(rabbit_types:exchange() | binary()) -> boolean(). - -is_amq_prefixed(Name) when is_binary(Name) -> - case re:run(Name, <<"^amq\.">>) of - nomatch -> false; - {match, _} -> true - end; -is_amq_prefixed(#exchange{name = #resource{name = <<>>}}) -> - false; -is_amq_prefixed(#exchange{name = #resource{name = Name}}) -> - is_amq_prefixed(Name). - --spec declare - (name(), type(), boolean(), boolean(), boolean(), - rabbit_framing:amqp_table(), rabbit_types:username()) - -> rabbit_types:exchange(). - -declare(XName, Type, Durable, AutoDelete, Internal, Args, Username) -> - X = rabbit_exchange_decorator:set( - rabbit_policy:set(#exchange{name = XName, - type = Type, - durable = Durable, - auto_delete = AutoDelete, - internal = Internal, - arguments = Args, - options = #{user => Username}})), - XT = type_to_module(Type), - %% We want to upset things if it isn't ok - ok = XT:validate(X), - %% Avoid a channel exception if there's a race condition - %% with an exchange.delete operation. - %% - %% See rabbitmq/rabbitmq-federation#7. - case rabbit_runtime_parameters:lookup(XName#resource.virtual_host, - ?EXCHANGE_DELETE_IN_PROGRESS_COMPONENT, - XName#resource.name) of - not_found -> - rabbit_misc:execute_mnesia_transaction( - fun () -> - case mnesia:wread({rabbit_exchange, XName}) of - [] -> - {new, store(X)}; - [ExistingX] -> - {existing, ExistingX} - end - end, - fun ({new, Exchange}, Tx) -> - ok = callback(X, create, map_create_tx(Tx), [Exchange]), - rabbit_event:notify_if(not Tx, exchange_created, info(Exchange)), - Exchange; - ({existing, Exchange}, _Tx) -> - Exchange; - (Err, _Tx) -> - Err - end); - _ -> - rabbit_log:warning("ignoring exchange.declare for exchange ~p, - exchange.delete in progress~n.", [XName]), - X - end. - -map_create_tx(true) -> transaction; -map_create_tx(false) -> none. - - -store(X = #exchange{durable = true}) -> - mnesia:write(rabbit_durable_exchange, X#exchange{decorators = undefined}, - write), - store_ram(X); -store(X = #exchange{durable = false}) -> - store_ram(X). - -store_ram(X) -> - X1 = rabbit_exchange_decorator:set(X), - ok = mnesia:write(rabbit_exchange, rabbit_exchange_decorator:set(X1), - write), - X1. - -%% Used with binaries sent over the wire; the type may not exist. - --spec check_type - (binary()) -> atom() | rabbit_types:connection_exit(). - -check_type(TypeBin) -> - case rabbit_registry:binary_to_type(rabbit_data_coercion:to_binary(TypeBin)) of - {error, not_found} -> - rabbit_misc:protocol_error( - command_invalid, "unknown exchange type '~s'", [TypeBin]); - T -> - case rabbit_registry:lookup_module(exchange, T) of - {error, not_found} -> rabbit_misc:protocol_error( - command_invalid, - "invalid exchange type '~s'", [T]); - {ok, _Module} -> T - end - end. - --spec assert_equivalence - (rabbit_types:exchange(), atom(), boolean(), boolean(), boolean(), - rabbit_framing:amqp_table()) - -> 'ok' | rabbit_types:connection_exit(). - -assert_equivalence(X = #exchange{ name = XName, - durable = Durable, - auto_delete = AutoDelete, - internal = Internal, - type = Type}, - ReqType, ReqDurable, ReqAutoDelete, ReqInternal, ReqArgs) -> - AFE = fun rabbit_misc:assert_field_equivalence/4, - AFE(Type, ReqType, XName, type), - AFE(Durable, ReqDurable, XName, durable), - AFE(AutoDelete, ReqAutoDelete, XName, auto_delete), - AFE(Internal, ReqInternal, XName, internal), - (type_to_module(Type)):assert_args_equivalence(X, ReqArgs). - --spec assert_args_equivalence - (rabbit_types:exchange(), rabbit_framing:amqp_table()) - -> 'ok' | rabbit_types:connection_exit(). - -assert_args_equivalence(#exchange{ name = Name, arguments = Args }, - RequiredArgs) -> - %% The spec says "Arguments are compared for semantic - %% equivalence". The only arg we care about is - %% "alternate-exchange". - rabbit_misc:assert_args_equivalence(Args, RequiredArgs, Name, - [<<"alternate-exchange">>]). - --spec lookup - (name()) -> rabbit_types:ok(rabbit_types:exchange()) | - rabbit_types:error('not_found'). - -lookup(Name) -> - rabbit_misc:dirty_read({rabbit_exchange, Name}). - - --spec lookup_many([name()]) -> [rabbit_types:exchange()]. - -lookup_many([]) -> []; -lookup_many([Name]) -> ets:lookup(rabbit_exchange, Name); -lookup_many(Names) when is_list(Names) -> - %% Normally we'd call mnesia:dirty_read/1 here, but that is quite - %% expensive for reasons explained in rabbit_misc:dirty_read/1. - lists:append([ets:lookup(rabbit_exchange, Name) || Name <- Names]). - - --spec lookup_or_die - (name()) -> rabbit_types:exchange() | - rabbit_types:channel_exit(). - -lookup_or_die(Name) -> - case lookup(Name) of - {ok, X} -> X; - {error, not_found} -> rabbit_amqqueue:not_found(Name) - end. - --spec list() -> [rabbit_types:exchange()]. - -list() -> mnesia:dirty_match_object(rabbit_exchange, #exchange{_ = '_'}). - --spec count() -> non_neg_integer(). - -count() -> - mnesia:table_info(rabbit_exchange, size). - --spec list_names() -> [rabbit_exchange:name()]. - -list_names() -> mnesia:dirty_all_keys(rabbit_exchange). - -%% Not dirty_match_object since that would not be transactional when used in a -%% tx context - --spec list(rabbit_types:vhost()) -> [rabbit_types:exchange()]. - -list(VHostPath) -> - mnesia:async_dirty( - fun () -> - mnesia:match_object( - rabbit_exchange, - #exchange{name = rabbit_misc:r(VHostPath, exchange), _ = '_'}, - read) - end). - --spec lookup_scratch(name(), atom()) -> - rabbit_types:ok(term()) | - rabbit_types:error('not_found'). - -lookup_scratch(Name, App) -> - case lookup(Name) of - {ok, #exchange{scratches = undefined}} -> - {error, not_found}; - {ok, #exchange{scratches = Scratches}} -> - case orddict:find(App, Scratches) of - {ok, Value} -> {ok, Value}; - error -> {error, not_found} - end; - {error, not_found} -> - {error, not_found} - end. - --spec update_scratch(name(), atom(), fun((any()) -> any())) -> 'ok'. - -update_scratch(Name, App, Fun) -> - rabbit_misc:execute_mnesia_transaction( - fun() -> - update(Name, - fun(X = #exchange{scratches = Scratches0}) -> - Scratches1 = case Scratches0 of - undefined -> orddict:new(); - _ -> Scratches0 - end, - Scratch = case orddict:find(App, Scratches1) of - {ok, S} -> S; - error -> undefined - end, - Scratches2 = orddict:store( - App, Fun(Scratch), Scratches1), - X#exchange{scratches = Scratches2} - end), - ok - end). - --spec update_decorators(name()) -> 'ok'. - -update_decorators(Name) -> - rabbit_misc:execute_mnesia_transaction( - fun() -> - case mnesia:wread({rabbit_exchange, Name}) of - [X] -> store_ram(X), - ok; - [] -> ok - end - end). - --spec update - (name(), - fun((rabbit_types:exchange()) -> rabbit_types:exchange())) - -> not_found | rabbit_types:exchange(). - -update(Name, Fun) -> - case mnesia:wread({rabbit_exchange, Name}) of - [X] -> X1 = Fun(X), - store(X1); - [] -> not_found - end. - --spec immutable(rabbit_types:exchange()) -> rabbit_types:exchange(). - -immutable(X) -> X#exchange{scratches = none, - policy = none, - decorators = none}. - --spec info_keys() -> rabbit_types:info_keys(). - -info_keys() -> ?INFO_KEYS. - -map(VHostPath, F) -> - %% TODO: there is scope for optimisation here, e.g. using a - %% cursor, parallelising the function invocation - lists:map(F, list(VHostPath)). - -infos(Items, X) -> [{Item, i(Item, X)} || Item <- Items]. - -i(name, #exchange{name = Name}) -> Name; -i(type, #exchange{type = Type}) -> Type; -i(durable, #exchange{durable = Durable}) -> Durable; -i(auto_delete, #exchange{auto_delete = AutoDelete}) -> AutoDelete; -i(internal, #exchange{internal = Internal}) -> Internal; -i(arguments, #exchange{arguments = Arguments}) -> Arguments; -i(policy, X) -> case rabbit_policy:name(X) of - none -> ''; - Policy -> Policy - end; -i(user_who_performed_action, #exchange{options = Opts}) -> - maps:get(user, Opts, ?UNKNOWN_USER); -i(Item, #exchange{type = Type} = X) -> - case (type_to_module(Type)):info(X, [Item]) of - [{Item, I}] -> I; - [] -> throw({bad_argument, Item}) - end. - --spec info(rabbit_types:exchange()) -> rabbit_types:infos(). - -info(X = #exchange{type = Type}) -> - infos(?INFO_KEYS, X) ++ (type_to_module(Type)):info(X). - --spec info - (rabbit_types:exchange(), rabbit_types:info_keys()) - -> rabbit_types:infos(). - -info(X = #exchange{type = _Type}, Items) -> - infos(Items, X). - --spec info_all(rabbit_types:vhost()) -> [rabbit_types:infos()]. - -info_all(VHostPath) -> map(VHostPath, fun (X) -> info(X) end). - --spec info_all(rabbit_types:vhost(), rabbit_types:info_keys()) - -> [rabbit_types:infos()]. - -info_all(VHostPath, Items) -> map(VHostPath, fun (X) -> info(X, Items) end). - --spec info_all(rabbit_types:vhost(), rabbit_types:info_keys(), - reference(), pid()) - -> 'ok'. - -info_all(VHostPath, Items, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map( - AggregatorPid, Ref, fun(X) -> info(X, Items) end, list(VHostPath)). - --spec route(rabbit_types:exchange(), rabbit_types:delivery()) - -> [rabbit_amqqueue:name()]. - -route(#exchange{name = #resource{virtual_host = VHost, name = RName} = XName, - decorators = Decorators} = X, - #delivery{message = #basic_message{routing_keys = RKs}} = Delivery) -> - case RName of - <<>> -> - RKsSorted = lists:usort(RKs), - [rabbit_channel:deliver_reply(RK, Delivery) || - RK <- RKsSorted, virtual_reply_queue(RK)], - [rabbit_misc:r(VHost, queue, RK) || RK <- RKsSorted, - not virtual_reply_queue(RK)]; - _ -> - Decs = rabbit_exchange_decorator:select(route, Decorators), - lists:usort(route1(Delivery, Decs, {[X], XName, []})) - end. - -virtual_reply_queue(<<"amq.rabbitmq.reply-to.", _/binary>>) -> true; -virtual_reply_queue(_) -> false. - -route1(_, _, {[], _, QNames}) -> - QNames; -route1(Delivery, Decorators, - {[X = #exchange{type = Type} | WorkList], SeenXs, QNames}) -> - ExchangeDests = (type_to_module(Type)):route(X, Delivery), - DecorateDests = process_decorators(X, Decorators, Delivery), - AlternateDests = process_alternate(X, ExchangeDests), - route1(Delivery, Decorators, - lists:foldl(fun process_route/2, {WorkList, SeenXs, QNames}, - AlternateDests ++ DecorateDests ++ ExchangeDests)). - -process_alternate(X = #exchange{name = XName}, []) -> - case rabbit_policy:get_arg( - <<"alternate-exchange">>, <<"alternate-exchange">>, X) of - undefined -> []; - AName -> [rabbit_misc:r(XName, exchange, AName)] - end; -process_alternate(_X, _Results) -> - []. - -process_decorators(_, [], _) -> %% optimisation - []; -process_decorators(X, Decorators, Delivery) -> - lists:append([Decorator:route(X, Delivery) || Decorator <- Decorators]). - -process_route(#resource{kind = exchange} = XName, - {_WorkList, XName, _QNames} = Acc) -> - Acc; -process_route(#resource{kind = exchange} = XName, - {WorkList, #resource{kind = exchange} = SeenX, QNames}) -> - {cons_if_present(XName, WorkList), - gb_sets:from_list([SeenX, XName]), QNames}; -process_route(#resource{kind = exchange} = XName, - {WorkList, SeenXs, QNames} = Acc) -> - case gb_sets:is_element(XName, SeenXs) of - true -> Acc; - false -> {cons_if_present(XName, WorkList), - gb_sets:add_element(XName, SeenXs), QNames} - end; -process_route(#resource{kind = queue} = QName, - {WorkList, SeenXs, QNames}) -> - {WorkList, SeenXs, [QName | QNames]}. - -cons_if_present(XName, L) -> - case lookup(XName) of - {ok, X} -> [X | L]; - {error, not_found} -> L - end. - -call_with_exchange(XName, Fun) -> - rabbit_misc:execute_mnesia_tx_with_tail( - fun () -> case mnesia:read({rabbit_exchange, XName}) of - [] -> rabbit_misc:const({error, not_found}); - [X] -> Fun(X) - end - end). - --spec delete - (name(), 'true', rabbit_types:username()) -> - 'ok'| rabbit_types:error('not_found' | 'in_use'); - (name(), 'false', rabbit_types:username()) -> - 'ok' | rabbit_types:error('not_found'). - -delete(XName, IfUnused, Username) -> - Fun = case IfUnused of - true -> fun conditional_delete/2; - false -> fun unconditional_delete/2 - end, - try - %% guard exchange.declare operations from failing when there's - %% a race condition between it and an exchange.delete. - %% - %% see rabbitmq/rabbitmq-federation#7 - rabbit_runtime_parameters:set(XName#resource.virtual_host, - ?EXCHANGE_DELETE_IN_PROGRESS_COMPONENT, - XName#resource.name, true, Username), - call_with_exchange( - XName, - fun (X) -> - case Fun(X, false) of - {deleted, X, Bs, Deletions} -> - rabbit_binding:process_deletions( - rabbit_binding:add_deletion( - XName, {X, deleted, Bs}, Deletions), Username); - {error, _InUseOrNotFound} = E -> - rabbit_misc:const(E) - end - end) - after - rabbit_runtime_parameters:clear(XName#resource.virtual_host, - ?EXCHANGE_DELETE_IN_PROGRESS_COMPONENT, - XName#resource.name, Username) - end. - --spec validate_binding - (rabbit_types:exchange(), rabbit_types:binding()) - -> rabbit_types:ok_or_error({'binding_invalid', string(), [any()]}). - -validate_binding(X = #exchange{type = XType}, Binding) -> - Module = type_to_module(XType), - Module:validate_binding(X, Binding). - --spec maybe_auto_delete - (rabbit_types:exchange(), boolean()) - -> 'not_deleted' | {'deleted', rabbit_binding:deletions()}. - -maybe_auto_delete(#exchange{auto_delete = false}, _OnlyDurable) -> - not_deleted; -maybe_auto_delete(#exchange{auto_delete = true} = X, OnlyDurable) -> - case conditional_delete(X, OnlyDurable) of - {error, in_use} -> not_deleted; - {deleted, X, [], Deletions} -> {deleted, Deletions} - end. - -conditional_delete(X = #exchange{name = XName}, OnlyDurable) -> - case rabbit_binding:has_for_source(XName) of - false -> internal_delete(X, OnlyDurable, false); - true -> {error, in_use} - end. - -unconditional_delete(X, OnlyDurable) -> - internal_delete(X, OnlyDurable, true). - -internal_delete(X = #exchange{name = XName}, OnlyDurable, RemoveBindingsForSource) -> - ok = mnesia:delete({rabbit_exchange, XName}), - ok = mnesia:delete({rabbit_exchange_serial, XName}), - mnesia:delete({rabbit_durable_exchange, XName}), - Bindings = case RemoveBindingsForSource of - true -> rabbit_binding:remove_for_source(XName); - false -> [] - end, - {deleted, X, Bindings, rabbit_binding:remove_for_destination( - XName, OnlyDurable)}. - -next_serial(XName) -> - Serial = peek_serial(XName, write), - ok = mnesia:write(rabbit_exchange_serial, - #exchange_serial{name = XName, next = Serial + 1}, write), - Serial. - --spec peek_serial(name()) -> pos_integer() | 'undefined'. - -peek_serial(XName) -> peek_serial(XName, read). - -peek_serial(XName, LockType) -> - case mnesia:read(rabbit_exchange_serial, XName, LockType) of - [#exchange_serial{next = Serial}] -> Serial; - _ -> 1 - end. - -invalid_module(T) -> - rabbit_log:warning("Could not find exchange type ~s.~n", [T]), - put({xtype_to_module, T}, rabbit_exchange_type_invalid), - rabbit_exchange_type_invalid. - -%% Used with atoms from records; e.g., the type is expected to exist. -type_to_module(T) -> - case get({xtype_to_module, T}) of - undefined -> - case rabbit_registry:lookup_module(exchange, T) of - {ok, Module} -> put({xtype_to_module, T}, Module), - Module; - {error, not_found} -> invalid_module(T) - end; - Module -> - Module - end. diff --git a/src/rabbit_exchange_decorator.erl b/src/rabbit_exchange_decorator.erl deleted file mode 100644 index 02d0258d3c..0000000000 --- a/src/rabbit_exchange_decorator.erl +++ /dev/null @@ -1,105 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_exchange_decorator). - --include("rabbit.hrl"). - --export([select/2, set/1]). - --behaviour(rabbit_registry_class). - --export([added_to_rabbit_registry/2, removed_from_rabbit_registry/1]). - -%% This is like an exchange type except that: -%% -%% 1) It applies to all exchanges as soon as it is installed, therefore -%% 2) It is not allowed to affect validation, so no validate/1 or -%% assert_args_equivalence/2 -%% -%% It's possible in the future we might make decorators -%% able to manipulate messages as they are published. - --type(tx() :: 'transaction' | 'none'). --type(serial() :: pos_integer() | tx()). - --callback description() -> [proplists:property()]. - -%% Should Rabbit ensure that all binding events that are -%% delivered to an individual exchange can be serialised? (they -%% might still be delivered out of order, but there'll be a -%% serial number). --callback serialise_events(rabbit_types:exchange()) -> boolean(). - -%% called after declaration and recovery --callback create(tx(), rabbit_types:exchange()) -> 'ok'. - -%% called after exchange (auto)deletion. --callback delete(tx(), rabbit_types:exchange(), [rabbit_types:binding()]) -> - 'ok'. - -%% called when the policy attached to this exchange changes. --callback policy_changed(rabbit_types:exchange(), rabbit_types:exchange()) -> - 'ok'. - -%% called after a binding has been added or recovered --callback add_binding(serial(), rabbit_types:exchange(), - rabbit_types:binding()) -> 'ok'. - -%% called after bindings have been deleted. --callback remove_bindings(serial(), rabbit_types:exchange(), - [rabbit_types:binding()]) -> 'ok'. - -%% Allows additional destinations to be added to the routing decision. --callback route(rabbit_types:exchange(), rabbit_types:delivery()) -> - [rabbit_amqqueue:name() | rabbit_exchange:name()]. - -%% Whether the decorator wishes to receive callbacks for the exchange -%% none:no callbacks, noroute:all callbacks except route, all:all callbacks --callback active_for(rabbit_types:exchange()) -> 'none' | 'noroute' | 'all'. - -%%---------------------------------------------------------------------------- - -added_to_rabbit_registry(_Type, _ModuleName) -> - [maybe_recover(X) || X <- rabbit_exchange:list()], - ok. -removed_from_rabbit_registry(_Type) -> - [maybe_recover(X) || X <- rabbit_exchange:list()], - ok. - -%% select a subset of active decorators -select(all, {Route, NoRoute}) -> filter(Route ++ NoRoute); -select(route, {Route, _NoRoute}) -> filter(Route); -select(raw, {Route, NoRoute}) -> Route ++ NoRoute. - -filter(Modules) -> - [M || M <- Modules, code:which(M) =/= non_existing]. - -set(X) -> - Decs = lists:foldl(fun (D, {Route, NoRoute}) -> - ActiveFor = D:active_for(X), - {cons_if_eq(all, ActiveFor, D, Route), - cons_if_eq(noroute, ActiveFor, D, NoRoute)} - end, {[], []}, list()), - X#exchange{decorators = Decs}. - -list() -> [M || {_, M} <- rabbit_registry:lookup_all(exchange_decorator)]. - -cons_if_eq(Select, Select, Item, List) -> [Item | List]; -cons_if_eq(_Select, _Other, _Item, List) -> List. - -maybe_recover(X = #exchange{name = Name, - decorators = Decs}) -> - #exchange{decorators = Decs1} = set(X), - Old = lists:sort(select(all, Decs)), - New = lists:sort(select(all, Decs1)), - case New of - Old -> ok; - _ -> %% TODO create a tx here for non-federation decorators - [M:create(none, X) || M <- New -- Old], - rabbit_exchange:update_decorators(Name) - end. diff --git a/src/rabbit_exchange_parameters.erl b/src/rabbit_exchange_parameters.erl deleted file mode 100644 index f9de648cfa..0000000000 --- a/src/rabbit_exchange_parameters.erl +++ /dev/null @@ -1,39 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_exchange_parameters). - --behaviour(rabbit_runtime_parameter). - --include("rabbit.hrl"). - --export([register/0]). --export([validate/5, notify/5, notify_clear/4]). - --rabbit_boot_step({?MODULE, - [{description, "exchange parameters"}, - {mfa, {rabbit_exchange_parameters, register, []}}, - {requires, rabbit_registry}, - {enables, recovery}]}). - -register() -> - rabbit_registry:register(runtime_parameter, - ?EXCHANGE_DELETE_IN_PROGRESS_COMPONENT, ?MODULE), - %% ensure there are no leftovers from before node restart/crash - rabbit_runtime_parameters:clear_component( - ?EXCHANGE_DELETE_IN_PROGRESS_COMPONENT, - ?INTERNAL_USER), - ok. - -validate(_VHost, ?EXCHANGE_DELETE_IN_PROGRESS_COMPONENT, _Name, _Term, _User) -> - ok. - -notify(_VHost, ?EXCHANGE_DELETE_IN_PROGRESS_COMPONENT, _Name, _Term, _Username) -> - ok. - -notify_clear(_VHost, ?EXCHANGE_DELETE_IN_PROGRESS_COMPONENT, _Name, _Username) -> - ok. diff --git a/src/rabbit_exchange_type_direct.erl b/src/rabbit_exchange_type_direct.erl deleted file mode 100644 index 3f4350e7b0..0000000000 --- a/src/rabbit_exchange_type_direct.erl +++ /dev/null @@ -1,46 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_exchange_type_direct). --include("rabbit.hrl"). - --behaviour(rabbit_exchange_type). - --export([description/0, serialise_events/0, route/2]). --export([validate/1, validate_binding/2, - create/2, delete/3, policy_changed/2, add_binding/3, - remove_bindings/3, assert_args_equivalence/2]). --export([info/1, info/2]). - --rabbit_boot_step({?MODULE, - [{description, "exchange type direct"}, - {mfa, {rabbit_registry, register, - [exchange, <<"direct">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -info(_X) -> []. -info(_X, _) -> []. - -description() -> - [{description, <<"AMQP direct exchange, as per the AMQP specification">>}]. - -serialise_events() -> false. - -route(#exchange{name = Name}, - #delivery{message = #basic_message{routing_keys = Routes}}) -> - rabbit_router:match_routing_key(Name, Routes). - -validate(_X) -> ok. -validate_binding(_X, _B) -> ok. -create(_Tx, _X) -> ok. -delete(_Tx, _X, _Bs) -> ok. -policy_changed(_X1, _X2) -> ok. -add_binding(_Tx, _X, _B) -> ok. -remove_bindings(_Tx, _X, _Bs) -> ok. -assert_args_equivalence(X, Args) -> - rabbit_exchange:assert_args_equivalence(X, Args). diff --git a/src/rabbit_exchange_type_fanout.erl b/src/rabbit_exchange_type_fanout.erl deleted file mode 100644 index a8778cf0c7..0000000000 --- a/src/rabbit_exchange_type_fanout.erl +++ /dev/null @@ -1,45 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_exchange_type_fanout). --include("rabbit.hrl"). - --behaviour(rabbit_exchange_type). - --export([description/0, serialise_events/0, route/2]). --export([validate/1, validate_binding/2, - create/2, delete/3, policy_changed/2, add_binding/3, - remove_bindings/3, assert_args_equivalence/2]). --export([info/1, info/2]). - --rabbit_boot_step({?MODULE, - [{description, "exchange type fanout"}, - {mfa, {rabbit_registry, register, - [exchange, <<"fanout">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -info(_X) -> []. -info(_X, _) -> []. - -description() -> - [{description, <<"AMQP fanout exchange, as per the AMQP specification">>}]. - -serialise_events() -> false. - -route(#exchange{name = Name}, _Delivery) -> - rabbit_router:match_routing_key(Name, ['_']). - -validate(_X) -> ok. -validate_binding(_X, _B) -> ok. -create(_Tx, _X) -> ok. -delete(_Tx, _X, _Bs) -> ok. -policy_changed(_X1, _X2) -> ok. -add_binding(_Tx, _X, _B) -> ok. -remove_bindings(_Tx, _X, _Bs) -> ok. -assert_args_equivalence(X, Args) -> - rabbit_exchange:assert_args_equivalence(X, Args). diff --git a/src/rabbit_exchange_type_headers.erl b/src/rabbit_exchange_type_headers.erl deleted file mode 100644 index e40195de7a..0000000000 --- a/src/rabbit_exchange_type_headers.erl +++ /dev/null @@ -1,136 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_exchange_type_headers). --include("rabbit.hrl"). --include("rabbit_framing.hrl"). - --behaviour(rabbit_exchange_type). - --export([description/0, serialise_events/0, route/2]). --export([validate/1, validate_binding/2, - create/2, delete/3, policy_changed/2, add_binding/3, - remove_bindings/3, assert_args_equivalence/2]). --export([info/1, info/2]). - --rabbit_boot_step({?MODULE, - [{description, "exchange type headers"}, - {mfa, {rabbit_registry, register, - [exchange, <<"headers">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -info(_X) -> []. -info(_X, _) -> []. - -description() -> - [{description, <<"AMQP headers exchange, as per the AMQP specification">>}]. - -serialise_events() -> false. - -route(#exchange{name = Name}, - #delivery{message = #basic_message{content = Content}}) -> - Headers = case (Content#content.properties)#'P_basic'.headers of - undefined -> []; - H -> rabbit_misc:sort_field_table(H) - end, - rabbit_router:match_bindings( - Name, fun (#binding{args = Spec}) -> headers_match(Spec, Headers) end). - -validate_binding(_X, #binding{args = Args}) -> - case rabbit_misc:table_lookup(Args, <<"x-match">>) of - {longstr, <<"all">>} -> ok; - {longstr, <<"any">>} -> ok; - {longstr, Other} -> {error, - {binding_invalid, - "Invalid x-match field value ~p; " - "expected all or any", [Other]}}; - {Type, Other} -> {error, - {binding_invalid, - "Invalid x-match field type ~p (value ~p); " - "expected longstr", [Type, Other]}}; - undefined -> ok %% [0] - end. -%% [0] spec is vague on whether it can be omitted but in practice it's -%% useful to allow people to do this - -parse_x_match({longstr, <<"all">>}) -> all; -parse_x_match({longstr, <<"any">>}) -> any; -parse_x_match(_) -> all. %% legacy; we didn't validate - -%% Horrendous matching algorithm. Depends for its merge-like -%% (linear-time) behaviour on the lists:keysort -%% (rabbit_misc:sort_field_table) that route/1 and -%% rabbit_binding:{add,remove}/2 do. -%% -%% !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -%% In other words: REQUIRES BOTH PATTERN AND DATA TO BE SORTED ASCENDING BY KEY. -%% !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -%% - --spec headers_match - (rabbit_framing:amqp_table(), rabbit_framing:amqp_table()) -> - boolean(). - -headers_match(Args, Data) -> - MK = parse_x_match(rabbit_misc:table_lookup(Args, <<"x-match">>)), - headers_match(Args, Data, true, false, MK). - -% A bit less horrendous algorithm :) -headers_match(_, _, false, _, all) -> false; -headers_match(_, _, _, true, any) -> true; - -% No more bindings, return current state -headers_match([], _Data, AllMatch, _AnyMatch, all) -> AllMatch; -headers_match([], _Data, _AllMatch, AnyMatch, any) -> AnyMatch; - -% Delete bindings starting with x- -headers_match([{<<"x-", _/binary>>, _PT, _PV} | PRest], Data, - AllMatch, AnyMatch, MatchKind) -> - headers_match(PRest, Data, AllMatch, AnyMatch, MatchKind); - -% No more data, but still bindings, false with all -headers_match(_Pattern, [], _AllMatch, AnyMatch, MatchKind) -> - headers_match([], [], false, AnyMatch, MatchKind); - -% Data key header not in binding, go next data -headers_match(Pattern = [{PK, _PT, _PV} | _], [{DK, _DT, _DV} | DRest], - AllMatch, AnyMatch, MatchKind) when PK > DK -> - headers_match(Pattern, DRest, AllMatch, AnyMatch, MatchKind); - -% Binding key header not in data, false with all, go next binding -headers_match([{PK, _PT, _PV} | PRest], Data = [{DK, _DT, _DV} | _], - _AllMatch, AnyMatch, MatchKind) when PK < DK -> - headers_match(PRest, Data, false, AnyMatch, MatchKind); - -%% It's not properly specified, but a "no value" in a -%% pattern field is supposed to mean simple presence of -%% the corresponding data field. I've interpreted that to -%% mean a type of "void" for the pattern field. -headers_match([{PK, void, _PV} | PRest], [{DK, _DT, _DV} | DRest], - AllMatch, _AnyMatch, MatchKind) when PK == DK -> - headers_match(PRest, DRest, AllMatch, true, MatchKind); - -% Complete match, true with any, go next -headers_match([{PK, _PT, PV} | PRest], [{DK, _DT, DV} | DRest], - AllMatch, _AnyMatch, MatchKind) when PK == DK andalso PV == DV -> - headers_match(PRest, DRest, AllMatch, true, MatchKind); - -% Value does not match, false with all, go next -headers_match([{PK, _PT, _PV} | PRest], [{DK, _DT, _DV} | DRest], - _AllMatch, AnyMatch, MatchKind) when PK == DK -> - headers_match(PRest, DRest, false, AnyMatch, MatchKind). - - -validate(_X) -> ok. -create(_Tx, _X) -> ok. -delete(_Tx, _X, _Bs) -> ok. -policy_changed(_X1, _X2) -> ok. -add_binding(_Tx, _X, _B) -> ok. -remove_bindings(_Tx, _X, _Bs) -> ok. -assert_args_equivalence(X, Args) -> - rabbit_exchange:assert_args_equivalence(X, Args). diff --git a/src/rabbit_exchange_type_invalid.erl b/src/rabbit_exchange_type_invalid.erl deleted file mode 100644 index 3fa27d28e9..0000000000 --- a/src/rabbit_exchange_type_invalid.erl +++ /dev/null @@ -1,45 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_exchange_type_invalid). --include("rabbit.hrl"). - --behaviour(rabbit_exchange_type). - --export([description/0, serialise_events/0, route/2]). --export([validate/1, validate_binding/2, - create/2, delete/3, policy_changed/2, add_binding/3, - remove_bindings/3, assert_args_equivalence/2]). --export([info/1, info/2]). - -info(_X) -> []. -info(_X, _) -> []. - -description() -> - [{description, - <<"Dummy exchange type, to be used when the intended one is not found.">> - }]. - -serialise_events() -> false. - --spec route(rabbit_types:exchange(), rabbit_types:delivery()) -> no_return(). - -route(#exchange{name = Name, type = Type}, _) -> - rabbit_misc:protocol_error( - precondition_failed, - "Cannot route message through ~s: exchange type ~s not found", - [rabbit_misc:rs(Name), Type]). - -validate(_X) -> ok. -validate_binding(_X, _B) -> ok. -create(_Tx, _X) -> ok. -delete(_Tx, _X, _Bs) -> ok. -policy_changed(_X1, _X2) -> ok. -add_binding(_Tx, _X, _B) -> ok. -remove_bindings(_Tx, _X, _Bs) -> ok. -assert_args_equivalence(X, Args) -> - rabbit_exchange:assert_args_equivalence(X, Args). diff --git a/src/rabbit_exchange_type_topic.erl b/src/rabbit_exchange_type_topic.erl deleted file mode 100644 index 38b05895f2..0000000000 --- a/src/rabbit_exchange_type_topic.erl +++ /dev/null @@ -1,266 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_exchange_type_topic). - --include("rabbit.hrl"). - --behaviour(rabbit_exchange_type). - --export([description/0, serialise_events/0, route/2]). --export([validate/1, validate_binding/2, - create/2, delete/3, policy_changed/2, add_binding/3, - remove_bindings/3, assert_args_equivalence/2]). --export([info/1, info/2]). - --rabbit_boot_step({?MODULE, - [{description, "exchange type topic"}, - {mfa, {rabbit_registry, register, - [exchange, <<"topic">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -%%---------------------------------------------------------------------------- - -info(_X) -> []. -info(_X, _) -> []. - -description() -> - [{description, <<"AMQP topic exchange, as per the AMQP specification">>}]. - -serialise_events() -> false. - -%% NB: This may return duplicate results in some situations (that's ok) -route(#exchange{name = X}, - #delivery{message = #basic_message{routing_keys = Routes}}) -> - lists:append([begin - Words = split_topic_key(RKey), - mnesia:async_dirty(fun trie_match/2, [X, Words]) - end || RKey <- Routes]). - -validate(_X) -> ok. -validate_binding(_X, _B) -> ok. -create(_Tx, _X) -> ok. - -delete(transaction, #exchange{name = X}, _Bs) -> - trie_remove_all_nodes(X), - trie_remove_all_edges(X), - trie_remove_all_bindings(X), - ok; -delete(none, _Exchange, _Bs) -> - ok. - -policy_changed(_X1, _X2) -> ok. - -add_binding(transaction, _Exchange, Binding) -> - internal_add_binding(Binding); -add_binding(none, _Exchange, _Binding) -> - ok. - -remove_bindings(transaction, _X, Bs) -> - %% See rabbit_binding:lock_route_tables for the rationale for - %% taking table locks. - case Bs of - [_] -> ok; - _ -> [mnesia:lock({table, T}, write) || - T <- [rabbit_topic_trie_node, - rabbit_topic_trie_edge, - rabbit_topic_trie_binding]] - end, - [case follow_down_get_path(X, split_topic_key(K)) of - {ok, Path = [{FinalNode, _} | _]} -> - trie_remove_binding(X, FinalNode, D, Args), - remove_path_if_empty(X, Path); - {error, _Node, _RestW} -> - %% We're trying to remove a binding that no longer exists. - %% That's unexpected, but shouldn't be a problem. - ok - end || #binding{source = X, key = K, destination = D, args = Args} <- Bs], - ok; -remove_bindings(none, _X, _Bs) -> - ok. - -assert_args_equivalence(X, Args) -> - rabbit_exchange:assert_args_equivalence(X, Args). - -%%---------------------------------------------------------------------------- - -internal_add_binding(#binding{source = X, key = K, destination = D, - args = Args}) -> - FinalNode = follow_down_create(X, split_topic_key(K)), - trie_add_binding(X, FinalNode, D, Args), - ok. - -trie_match(X, Words) -> - trie_match(X, root, Words, []). - -trie_match(X, Node, [], ResAcc) -> - trie_match_part(X, Node, "#", fun trie_match_skip_any/4, [], - trie_bindings(X, Node) ++ ResAcc); -trie_match(X, Node, [W | RestW] = Words, ResAcc) -> - lists:foldl(fun ({WArg, MatchFun, RestWArg}, Acc) -> - trie_match_part(X, Node, WArg, MatchFun, RestWArg, Acc) - end, ResAcc, [{W, fun trie_match/4, RestW}, - {"*", fun trie_match/4, RestW}, - {"#", fun trie_match_skip_any/4, Words}]). - -trie_match_part(X, Node, Search, MatchFun, RestW, ResAcc) -> - case trie_child(X, Node, Search) of - {ok, NextNode} -> MatchFun(X, NextNode, RestW, ResAcc); - error -> ResAcc - end. - -trie_match_skip_any(X, Node, [], ResAcc) -> - trie_match(X, Node, [], ResAcc); -trie_match_skip_any(X, Node, [_ | RestW] = Words, ResAcc) -> - trie_match_skip_any(X, Node, RestW, - trie_match(X, Node, Words, ResAcc)). - -follow_down_create(X, Words) -> - case follow_down_last_node(X, Words) of - {ok, FinalNode} -> FinalNode; - {error, Node, RestW} -> lists:foldl( - fun (W, CurNode) -> - NewNode = new_node_id(), - trie_add_edge(X, CurNode, NewNode, W), - NewNode - end, Node, RestW) - end. - -follow_down_last_node(X, Words) -> - follow_down(X, fun (_, Node, _) -> Node end, root, Words). - -follow_down_get_path(X, Words) -> - follow_down(X, fun (W, Node, PathAcc) -> [{Node, W} | PathAcc] end, - [{root, none}], Words). - -follow_down(X, AccFun, Acc0, Words) -> - follow_down(X, root, AccFun, Acc0, Words). - -follow_down(_X, _CurNode, _AccFun, Acc, []) -> - {ok, Acc}; -follow_down(X, CurNode, AccFun, Acc, Words = [W | RestW]) -> - case trie_child(X, CurNode, W) of - {ok, NextNode} -> follow_down(X, NextNode, AccFun, - AccFun(W, NextNode, Acc), RestW); - error -> {error, Acc, Words} - end. - -remove_path_if_empty(_, [{root, none}]) -> - ok; -remove_path_if_empty(X, [{Node, W} | [{Parent, _} | _] = RestPath]) -> - case mnesia:read(rabbit_topic_trie_node, - #trie_node{exchange_name = X, node_id = Node}, write) of - [] -> trie_remove_edge(X, Parent, Node, W), - remove_path_if_empty(X, RestPath); - _ -> ok - end. - -trie_child(X, Node, Word) -> - case mnesia:read({rabbit_topic_trie_edge, - #trie_edge{exchange_name = X, - node_id = Node, - word = Word}}) of - [#topic_trie_edge{node_id = NextNode}] -> {ok, NextNode}; - [] -> error - end. - -trie_bindings(X, Node) -> - MatchHead = #topic_trie_binding{ - trie_binding = #trie_binding{exchange_name = X, - node_id = Node, - destination = '$1', - arguments = '_'}}, - mnesia:select(rabbit_topic_trie_binding, [{MatchHead, [], ['$1']}]). - -trie_update_node_counts(X, Node, Field, Delta) -> - E = case mnesia:read(rabbit_topic_trie_node, - #trie_node{exchange_name = X, - node_id = Node}, write) of - [] -> #topic_trie_node{trie_node = #trie_node{ - exchange_name = X, - node_id = Node}, - edge_count = 0, - binding_count = 0}; - [E0] -> E0 - end, - case setelement(Field, E, element(Field, E) + Delta) of - #topic_trie_node{edge_count = 0, binding_count = 0} -> - ok = mnesia:delete_object(rabbit_topic_trie_node, E, write); - EN -> - ok = mnesia:write(rabbit_topic_trie_node, EN, write) - end. - -trie_add_edge(X, FromNode, ToNode, W) -> - trie_update_node_counts(X, FromNode, #topic_trie_node.edge_count, +1), - trie_edge_op(X, FromNode, ToNode, W, fun mnesia:write/3). - -trie_remove_edge(X, FromNode, ToNode, W) -> - trie_update_node_counts(X, FromNode, #topic_trie_node.edge_count, -1), - trie_edge_op(X, FromNode, ToNode, W, fun mnesia:delete_object/3). - -trie_edge_op(X, FromNode, ToNode, W, Op) -> - ok = Op(rabbit_topic_trie_edge, - #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, - node_id = FromNode, - word = W}, - node_id = ToNode}, - write). - -trie_add_binding(X, Node, D, Args) -> - trie_update_node_counts(X, Node, #topic_trie_node.binding_count, +1), - trie_binding_op(X, Node, D, Args, fun mnesia:write/3). - -trie_remove_binding(X, Node, D, Args) -> - trie_update_node_counts(X, Node, #topic_trie_node.binding_count, -1), - trie_binding_op(X, Node, D, Args, fun mnesia:delete_object/3). - -trie_binding_op(X, Node, D, Args, Op) -> - ok = Op(rabbit_topic_trie_binding, - #topic_trie_binding{ - trie_binding = #trie_binding{exchange_name = X, - node_id = Node, - destination = D, - arguments = Args}}, - write). - -trie_remove_all_nodes(X) -> - remove_all(rabbit_topic_trie_node, - #topic_trie_node{trie_node = #trie_node{exchange_name = X, - _ = '_'}, - _ = '_'}). - -trie_remove_all_edges(X) -> - remove_all(rabbit_topic_trie_edge, - #topic_trie_edge{trie_edge = #trie_edge{exchange_name = X, - _ = '_'}, - _ = '_'}). - -trie_remove_all_bindings(X) -> - remove_all(rabbit_topic_trie_binding, - #topic_trie_binding{ - trie_binding = #trie_binding{exchange_name = X, _ = '_'}, - _ = '_'}). - -remove_all(Table, Pattern) -> - lists:foreach(fun (R) -> mnesia:delete_object(Table, R, write) end, - mnesia:match_object(Table, Pattern, write)). - -new_node_id() -> - rabbit_guid:gen(). - -split_topic_key(Key) -> - split_topic_key(Key, [], []). - -split_topic_key(<<>>, [], []) -> - []; -split_topic_key(<<>>, RevWordAcc, RevResAcc) -> - lists:reverse([lists:reverse(RevWordAcc) | RevResAcc]); -split_topic_key(<<$., Rest/binary>>, RevWordAcc, RevResAcc) -> - split_topic_key(Rest, [], [lists:reverse(RevWordAcc) | RevResAcc]); -split_topic_key(<<C:8, Rest/binary>>, RevWordAcc, RevResAcc) -> - split_topic_key(Rest, [C | RevWordAcc], RevResAcc). diff --git a/src/rabbit_feature_flags.erl b/src/rabbit_feature_flags.erl deleted file mode 100644 index 921ec9ab53..0000000000 --- a/src/rabbit_feature_flags.erl +++ /dev/null @@ -1,2470 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% @author The RabbitMQ team -%% @copyright 2018-2020 VMware, Inc. or its affiliates. -%% -%% @doc -%% This module offers a framework to declare capabilities a RabbitMQ node -%% supports and therefore a way to determine if multiple RabbitMQ nodes in -%% a cluster are compatible and can work together. -%% -%% == What a feature flag is == -%% -%% A <strong>feature flag</strong> is a name and several properties given -%% to a change in RabbitMQ which impacts its communication with other -%% RabbitMQ nodes. This kind of change can be: -%% <ul> -%% <li>an update to an Erlang record</li> -%% <li>a modification to a replicated Mnesia table schema</li> -%% <li>a modification to Erlang messages exchanged between Erlang processes -%% which might run on remote nodes</li> -%% </ul> -%% -%% A feature flag is qualified by: -%% <ul> -%% <li>a <strong>name</strong></li> -%% <li>a <strong>description</strong> (optional)</li> -%% <li>a list of other <strong>feature flags this feature flag depends on -%% </strong> (optional). This can be useful when the change builds up on -%% top of a previous change. For instance, it expands a record which was -%% already modified by a previous feature flag.</li> -%% <li>a <strong>migration function</strong> (optional). If provided, this -%% function is called when the feature flag is enabled. It is responsible -%% for doing all the data conversion, if any, and confirming the feature -%% flag can be enabled.</li> -%% <li>a level of stability (stable or experimental). For now, this is only -%% informational. But it might be used for specific purposes in the -%% future.</li> -%% </ul> -%% -%% == How to declare a feature flag == -%% -%% To define a new feature flag, you need to use the -%% `rabbit_feature_flag()' module attribute: -%% -%% ``` -%% -rabbit_feature_flag(FeatureFlag). -%% ''' -%% -%% `FeatureFlag' is a {@type feature_flag_modattr()}. -%% -%% == How to enable a feature flag == -%% -%% To enable a supported feature flag, you have the following solutions: -%% -%% <ul> -%% <li>Using this module API: -%% ``` -%% rabbit_feature_flags:enable(FeatureFlagName). -%% ''' -%% </li> -%% <li>Using the `rabbitmqctl' CLI: -%% ``` -%% rabbitmqctl enable_feature_flag "$feature_flag_name" -%% ''' -%% </li> -%% </ul> -%% -%% == How to disable a feature flag == -%% -%% Once enabled, there is <strong>currently no way to disable</strong> a -%% feature flag. - --module(rabbit_feature_flags). - --export([list/0, - list/1, - list/2, - enable/1, - enable_all/0, - disable/1, - disable_all/0, - is_supported/1, - is_supported/2, - is_supported_locally/1, - is_supported_remotely/1, - is_supported_remotely/2, - is_supported_remotely/3, - is_enabled/1, - is_enabled/2, - is_disabled/1, - is_disabled/2, - info/0, - info/1, - init/0, - get_state/1, - get_stability/1, - check_node_compatibility/1, - check_node_compatibility/2, - is_node_compatible/1, - is_node_compatible/2, - sync_feature_flags_with_cluster/2, - sync_feature_flags_with_cluster/3, - refresh_feature_flags_after_app_load/1, - enabled_feature_flags_list_file/0 - ]). - -%% RabbitMQ internal use only. --export([initialize_registry/0, - initialize_registry/1, - mark_as_enabled_locally/2, - remote_nodes/0, - running_remote_nodes/0, - does_node_support/3, - merge_feature_flags_from_unknown_apps/1, - do_sync_feature_flags_with_node/1]). - --ifdef(TEST). --export([inject_test_feature_flags/1, - initialize_registry/3, - query_supported_feature_flags/0, - mark_as_enabled_remotely/2, - mark_as_enabled_remotely/4, - registry_loading_lock/0]). --endif. - -%% Default timeout for operations on remote nodes. --define(TIMEOUT, 60000). - --define(FF_REGISTRY_LOADING_LOCK, {feature_flags_registry_loading, self()}). --define(FF_STATE_CHANGE_LOCK, {feature_flags_state_change, self()}). - --type feature_flag_modattr() :: {feature_name(), - feature_props()}. -%% The value of a `-rabbitmq_feature_flag()' module attribute used to -%% declare a new feature flag. - --type feature_name() :: atom(). -%% The feature flag's name. It is used in many places to identify a -%% specific feature flag. In particular, this is how an end-user (or -%% the CLI) can enable a feature flag. This is also the only bit which -%% is persisted so a node remember which feature flags are enabled. - --type feature_props() :: #{desc => string(), - doc_url => string(), - stability => stability(), - depends_on => [feature_name()], - migration_fun => migration_fun_name()}. -%% The feature flag properties. -%% -%% All properties are optional. -%% -%% The properties are: -%% <ul> -%% <li>`desc': a description of the feature flag</li> -%% <li>`doc_url': a URL pointing to more documentation about the feature -%% flag</li> -%% <li>`stability': the level of stability</li> -%% <li>`depends_on': a list of feature flags name which must be enabled -%% before this one</li> -%% <li>`migration_fun': a migration function specified by its module and -%% function names</li> -%% </ul> -%% -%% Note that the `migration_fun' is a {@type migration_fun_name()}, -%% not a {@type migration_fun()}. However, the function signature -%% must conform to the {@type migration_fun()} signature. The reason -%% is that we must be able to represent it as an Erlang term when -%% we regenerate the registry module source code (using {@link -%% erl_syntax:abstract/1}). - --type feature_flags() :: #{feature_name() => feature_props_extended()}. -%% The feature flags map as returned or accepted by several functions in -%% this module. In particular, this what the {@link list/0} function -%% returns. - --type feature_props_extended() :: #{desc => string(), - doc_url => string(), - stability => stability(), - migration_fun => migration_fun_name(), - depends_on => [feature_name()], - provided_by => atom()}. -%% The feature flag properties, once expanded by this module when feature -%% flags are discovered. -%% -%% The new properties compared to {@type feature_props()} are: -%% <ul> -%% <li>`provided_by': the name of the application providing the feature flag</li> -%% </ul> - --type feature_state() :: boolean() | state_changing. -%% The state of the feature flag: enabled if `true', disabled if `false' -%% or `state_changing'. - --type feature_states() :: #{feature_name() => feature_state()}. - --type stability() :: stable | experimental. -%% The level of stability of a feature flag. Currently, only informational. - --type migration_fun_name() :: {Module :: atom(), Function :: atom()}. -%% The name of the module and function to call when changing the state of -%% the feature flag. - --type migration_fun() :: fun((feature_name(), - feature_props_extended(), - migration_fun_context()) - -> ok | {error, any()} | % context = enable - boolean() | undefined). % context = is_enabled -%% The migration function signature. -%% -%% It is called with context `enable' when a feature flag is being enabled. -%% The function is responsible for this feature-flag-specific verification -%% and data conversion. It returns `ok' if RabbitMQ can mark the feature -%% flag as enabled an continue with the next one, if any. Otherwise, it -%% returns `{error, any()}' if there is an error and the feature flag should -%% remain disabled. The function must be idempotent: if the feature flag is -%% already enabled on another node and the local node is running this function -%% again because it is syncing its feature flags state, it should succeed. -%% -%% It is called with the context `is_enabled' to check if a feature flag -%% is actually enabled. It is useful on RabbitMQ startup, just in case -%% the previous instance failed to write the feature flags list file. - --type migration_fun_context() :: enable | is_enabled. - --type registry_vsn() :: term(). - --export_type([feature_flag_modattr/0, - feature_props/0, - feature_name/0, - feature_flags/0, - feature_props_extended/0, - feature_state/0, - feature_states/0, - stability/0, - migration_fun_name/0, - migration_fun/0, - migration_fun_context/0]). - --on_load(on_load/0). - --spec list() -> feature_flags(). -%% @doc -%% Lists all supported feature flags. -%% -%% @returns A map of all supported feature flags. - -list() -> list(all). - --spec list(Which :: all | enabled | disabled) -> feature_flags(). -%% @doc -%% Lists all, enabled or disabled feature flags, depending on the argument. -%% -%% @param Which The group of feature flags to return: `all', `enabled' or -%% `disabled'. -%% @returns A map of selected feature flags. - -list(all) -> rabbit_ff_registry:list(all); -list(enabled) -> rabbit_ff_registry:list(enabled); -list(disabled) -> maps:filter( - fun(FeatureName, _) -> is_disabled(FeatureName) end, - list(all)). - --spec list(all | enabled | disabled, stability()) -> feature_flags(). -%% @doc -%% Lists all, enabled or disabled feature flags, depending on the first -%% argument, only keeping those having the specified stability. -%% -%% @param Which The group of feature flags to return: `all', `enabled' or -%% `disabled'. -%% @param Stability The level of stability used to filter the map of feature -%% flags. -%% @returns A map of selected feature flags. - -list(Which, Stability) - when Stability =:= stable orelse Stability =:= experimental -> - maps:filter(fun(_, FeatureProps) -> - Stability =:= get_stability(FeatureProps) - end, list(Which)). - --spec enable(feature_name() | [feature_name()]) -> ok | - {error, Reason :: any()}. -%% @doc -%% Enables the specified feature flag or set of feature flags. -%% -%% @param FeatureName The name or the list of names of feature flags to -%% enable. -%% @returns `ok' if the feature flags (and all the feature flags they -%% depend on) were successfully enabled, or `{error, Reason}' if one -%% feature flag could not be enabled (subsequent feature flags in the -%% dependency tree are left unchanged). - -enable(FeatureName) when is_atom(FeatureName) -> - rabbit_log_feature_flags:debug( - "Feature flag `~s`: REQUEST TO ENABLE", - [FeatureName]), - case is_enabled(FeatureName) of - true -> - rabbit_log_feature_flags:debug( - "Feature flag `~s`: already enabled", - [FeatureName]), - ok; - false -> - rabbit_log_feature_flags:debug( - "Feature flag `~s`: not enabled, check if supported by cluster", - [FeatureName]), - %% The feature flag must be supported locally and remotely - %% (i.e. by all members of the cluster). - case is_supported(FeatureName) of - true -> - rabbit_log_feature_flags:info( - "Feature flag `~s`: supported, attempt to enable...", - [FeatureName]), - do_enable(FeatureName); - false -> - rabbit_log_feature_flags:error( - "Feature flag `~s`: not supported", - [FeatureName]), - {error, unsupported} - end - end; -enable(FeatureNames) when is_list(FeatureNames) -> - with_feature_flags(FeatureNames, fun enable/1). - --spec enable_all() -> ok | {error, any()}. -%% @doc -%% Enables all supported feature flags. -%% -%% @returns `ok' if the feature flags were successfully enabled, -%% or `{error, Reason}' if one feature flag could not be enabled -%% (subsequent feature flags in the dependency tree are left -%% unchanged). - -enable_all() -> - with_feature_flags(maps:keys(list(all)), fun enable/1). - --spec disable(feature_name() | [feature_name()]) -> ok | {error, any()}. -%% @doc -%% Disables the specified feature flag or set of feature flags. -%% -%% @param FeatureName The name or the list of names of feature flags to -%% disable. -%% @returns `ok' if the feature flags (and all the feature flags they -%% depend on) were successfully disabled, or `{error, Reason}' if one -%% feature flag could not be disabled (subsequent feature flags in the -%% dependency tree are left unchanged). - -disable(FeatureName) when is_atom(FeatureName) -> - {error, unsupported}; -disable(FeatureNames) when is_list(FeatureNames) -> - with_feature_flags(FeatureNames, fun disable/1). - --spec disable_all() -> ok | {error, any()}. -%% @doc -%% Disables all supported feature flags. -%% -%% @returns `ok' if the feature flags were successfully disabled, -%% or `{error, Reason}' if one feature flag could not be disabled -%% (subsequent feature flags in the dependency tree are left -%% unchanged). - -disable_all() -> - with_feature_flags(maps:keys(list(all)), fun disable/1). - --spec with_feature_flags([feature_name()], - fun((feature_name()) -> ok | {error, any()})) -> - ok | {error, any()}. -%% @private - -with_feature_flags([FeatureName | Rest], Fun) -> - case Fun(FeatureName) of - ok -> with_feature_flags(Rest, Fun); - Error -> Error - end; -with_feature_flags([], _) -> - ok. - --spec is_supported(feature_name() | [feature_name()]) -> boolean(). -%% @doc -%% Returns if a single feature flag or a set of feature flags is -%% supported by the entire cluster. -%% -%% This is the same as calling both {@link is_supported_locally/1} and -%% {@link is_supported_remotely/1} with a logical AND. -%% -%% @param FeatureNames The name or a list of names of the feature flag(s) -%% to be checked. -%% @returns `true' if the set of feature flags is entirely supported, or -%% `false' if one of them is not or the RPC timed out. - -is_supported(FeatureNames) -> - is_supported_locally(FeatureNames) andalso - is_supported_remotely(FeatureNames). - --spec is_supported(feature_name() | [feature_name()], timeout()) -> - boolean(). -%% @doc -%% Returns if a single feature flag or a set of feature flags is -%% supported by the entire cluster. -%% -%% This is the same as calling both {@link is_supported_locally/1} and -%% {@link is_supported_remotely/2} with a logical AND. -%% -%% @param FeatureNames The name or a list of names of the feature flag(s) -%% to be checked. -%% @param Timeout Time in milliseconds after which the RPC gives up. -%% @returns `true' if the set of feature flags is entirely supported, or -%% `false' if one of them is not or the RPC timed out. - -is_supported(FeatureNames, Timeout) -> - is_supported_locally(FeatureNames) andalso - is_supported_remotely(FeatureNames, Timeout). - --spec is_supported_locally(feature_name() | [feature_name()]) -> boolean(). -%% @doc -%% Returns if a single feature flag or a set of feature flags is -%% supported by the local node. -%% -%% @param FeatureNames The name or a list of names of the feature flag(s) -%% to be checked. -%% @returns `true' if the set of feature flags is entirely supported, or -%% `false' if one of them is not. - -is_supported_locally(FeatureName) when is_atom(FeatureName) -> - rabbit_ff_registry:is_supported(FeatureName); -is_supported_locally(FeatureNames) when is_list(FeatureNames) -> - lists:all(fun(F) -> rabbit_ff_registry:is_supported(F) end, FeatureNames). - --spec is_supported_remotely(feature_name() | [feature_name()]) -> boolean(). -%% @doc -%% Returns if a single feature flag or a set of feature flags is -%% supported by all remote nodes. -%% -%% @param FeatureNames The name or a list of names of the feature flag(s) -%% to be checked. -%% @returns `true' if the set of feature flags is entirely supported, or -%% `false' if one of them is not or the RPC timed out. - -is_supported_remotely(FeatureNames) -> - is_supported_remotely(FeatureNames, ?TIMEOUT). - --spec is_supported_remotely(feature_name() | [feature_name()], timeout()) -> boolean(). -%% @doc -%% Returns if a single feature flag or a set of feature flags is -%% supported by all remote nodes. -%% -%% @param FeatureNames The name or a list of names of the feature flag(s) -%% to be checked. -%% @param Timeout Time in milliseconds after which the RPC gives up. -%% @returns `true' if the set of feature flags is entirely supported, or -%% `false' if one of them is not or the RPC timed out. - -is_supported_remotely(FeatureName, Timeout) when is_atom(FeatureName) -> - is_supported_remotely([FeatureName], Timeout); -is_supported_remotely([], _) -> - rabbit_log_feature_flags:debug( - "Feature flags: skipping query for feature flags support as the " - "given list is empty"), - true; -is_supported_remotely(FeatureNames, Timeout) when is_list(FeatureNames) -> - case running_remote_nodes() of - [] -> - rabbit_log_feature_flags:debug( - "Feature flags: isolated node; skipping remote node query " - "=> consider `~p` supported", - [FeatureNames]), - true; - RemoteNodes -> - rabbit_log_feature_flags:debug( - "Feature flags: about to query these remote nodes about " - "support for `~p`: ~p", - [FeatureNames, RemoteNodes]), - is_supported_remotely(RemoteNodes, FeatureNames, Timeout) - end. - --spec is_supported_remotely([node()], - feature_name() | [feature_name()], - timeout()) -> boolean(). -%% @doc -%% Returns if a single feature flag or a set of feature flags is -%% supported by specified remote nodes. -%% -%% @param RemoteNodes The list of remote nodes to query. -%% @param FeatureNames The name or a list of names of the feature flag(s) -%% to be checked. -%% @param Timeout Time in milliseconds after which the RPC gives up. -%% @returns `true' if the set of feature flags is entirely supported by -%% all nodes, or `false' if one of them is not or the RPC timed out. - -is_supported_remotely(_, [], _) -> - rabbit_log_feature_flags:debug( - "Feature flags: skipping query for feature flags support as the " - "given list is empty"), - true; -is_supported_remotely([Node | Rest], FeatureNames, Timeout) -> - case does_node_support(Node, FeatureNames, Timeout) of - true -> - is_supported_remotely(Rest, FeatureNames, Timeout); - false -> - rabbit_log_feature_flags:debug( - "Feature flags: stopping query for support for `~p` here", - [FeatureNames]), - false - end; -is_supported_remotely([], FeatureNames, _) -> - rabbit_log_feature_flags:debug( - "Feature flags: all running remote nodes support `~p`", - [FeatureNames]), - true. - --spec is_enabled(feature_name() | [feature_name()]) -> boolean(). -%% @doc -%% Returns if a single feature flag or a set of feature flags is -%% enabled. -%% -%% This is the same as calling {@link is_enabled/2} as a `blocking' -%% call. -%% -%% @param FeatureNames The name or a list of names of the feature flag(s) -%% to be checked. -%% @returns `true' if the set of feature flags is enabled, or -%% `false' if one of them is not. - -is_enabled(FeatureNames) -> - is_enabled(FeatureNames, blocking). - --spec is_enabled -(feature_name() | [feature_name()], blocking) -> - boolean(); -(feature_name() | [feature_name()], non_blocking) -> - feature_state(). -%% @doc -%% Returns if a single feature flag or a set of feature flags is -%% enabled. -%% -%% When `blocking' is passed, the function waits (blocks) for the -%% state of a feature flag being disabled or enabled stabilizes before -%% returning its final state. -%% -%% When `non_blocking' is passed, the function returns immediately with -%% the state of the feature flag (`true' if enabled, `false' otherwise) -%% or `state_changing' is the state is being changed at the time of the -%% call. -%% -%% @param FeatureNames The name or a list of names of the feature flag(s) -%% to be checked. -%% @returns `true' if the set of feature flags is enabled, -%% `false' if one of them is not, or `state_changing' if one of them -%% is being worked on. Note that `state_changing' has precedence over -%% `false', so if one is `false' and another one is `state_changing', -%% `state_changing' is returned. - -is_enabled(FeatureNames, non_blocking) -> - is_enabled_nb(FeatureNames); -is_enabled(FeatureNames, blocking) -> - case is_enabled_nb(FeatureNames) of - state_changing -> - global:set_lock(?FF_STATE_CHANGE_LOCK), - global:del_lock(?FF_STATE_CHANGE_LOCK), - is_enabled(FeatureNames, blocking); - IsEnabled -> - IsEnabled - end. - -is_enabled_nb(FeatureName) when is_atom(FeatureName) -> - rabbit_ff_registry:is_enabled(FeatureName); -is_enabled_nb(FeatureNames) when is_list(FeatureNames) -> - lists:foldl( - fun - (_F, state_changing = Acc) -> - Acc; - (F, false = Acc) -> - case rabbit_ff_registry:is_enabled(F) of - state_changing -> state_changing; - _ -> Acc - end; - (F, _) -> - rabbit_ff_registry:is_enabled(F) - end, - true, FeatureNames). - --spec is_disabled(feature_name() | [feature_name()]) -> boolean(). -%% @doc -%% Returns if a single feature flag or one feature flag in a set of -%% feature flags is disabled. -%% -%% This is the same as negating the result of {@link is_enabled/1}. -%% -%% @param FeatureNames The name or a list of names of the feature flag(s) -%% to be checked. -%% @returns `true' if one of the feature flags is disabled, or -%% `false' if they are all enabled. - -is_disabled(FeatureNames) -> - is_disabled(FeatureNames, blocking). - --spec is_disabled -(feature_name() | [feature_name()], blocking) -> - boolean(); -(feature_name() | [feature_name()], non_blocking) -> - feature_state(). -%% @doc -%% Returns if a single feature flag or one feature flag in a set of -%% feature flags is disabled. -%% -%% This is the same as negating the result of {@link is_enabled/2}, -%% except that `state_changing' is returned as is. -%% -%% See {@link is_enabled/2} for a description of the `blocking' and -%% `non_blocking' modes. -%% -%% @param FeatureNames The name or a list of names of the feature flag(s) -%% to be checked. -%% @returns `true' if one feature flag in the set of feature flags is -%% disabled, `false' if they are all enabled, or `state_changing' if -%% one of them is being worked on. Note that `state_changing' has -%% precedence over `true', so if one is `true' (i.e. disabled) and -%% another one is `state_changing', `state_changing' is returned. -%% -%% @see is_enabled/2 - -is_disabled(FeatureName, Blocking) -> - case is_enabled(FeatureName, Blocking) of - state_changing -> state_changing; - IsEnabled -> not IsEnabled - end. - --spec info() -> ok. -%% @doc -%% Displays a table on stdout summing up the supported feature flags, -%% their state and various informations about them. - -info() -> - info(#{}). - --spec info(#{color => boolean(), - lines => boolean(), - verbose => non_neg_integer()}) -> ok. -%% @doc -%% Displays a table on stdout summing up the supported feature flags, -%% their state and various informations about them. -%% -%% Supported options are: -%% <ul> -%% <li>`color': a boolean to indicate if colors should be used to -%% highlight some elements.</li> -%% <li>`lines': a boolean to indicate if table borders should be drawn -%% using ASCII lines instead of regular characters.</li> -%% <li>`verbose': a non-negative integer to specify the level of -%% verbosity.</li> -%% </ul> -%% -%% @param Options A map of various options to tune the displayed table. - -info(Options) when is_map(Options) -> - rabbit_ff_extra:info(Options). - --spec get_state(feature_name()) -> enabled | disabled | unavailable. -%% @doc -%% Returns the state of a feature flag. -%% -%% The possible states are: -%% <ul> -%% <li>`enabled': the feature flag is enabled.</li> -%% <li>`disabled': the feature flag is supported by all nodes in the -%% cluster but currently disabled.</li> -%% <li>`unavailable': the feature flag is unsupported by at least one -%% node in the cluster and can not be enabled for now.</li> -%% </ul> -%% -%% @param FeatureName The name of the feature flag to check. -%% @returns `enabled', `disabled' or `unavailable'. - -get_state(FeatureName) when is_atom(FeatureName) -> - IsEnabled = is_enabled(FeatureName), - IsSupported = is_supported(FeatureName), - case IsEnabled of - true -> enabled; - false -> case IsSupported of - true -> disabled; - false -> unavailable - end - end. - --spec get_stability(feature_name() | feature_props_extended()) -> stability(). -%% @doc -%% Returns the stability of a feature flag. -%% -%% The possible stability levels are: -%% <ul> -%% <li>`stable': the feature flag is stable and will not change in future -%% releases: it can be enabled in production.</li> -%% <li>`experimental': the feature flag is experimental and may change in -%% the future (without a guaranteed upgrade path): enabling it in -%% production is not recommended.</li> -%% <li>`unavailable': the feature flag is unsupported by at least one -%% node in the cluster and can not be enabled for now.</li> -%% </ul> -%% -%% @param FeatureName The name of the feature flag to check. -%% @returns `stable' or `experimental'. - -get_stability(FeatureName) when is_atom(FeatureName) -> - case rabbit_ff_registry:get(FeatureName) of - undefined -> undefined; - FeatureProps -> get_stability(FeatureProps) - end; -get_stability(FeatureProps) when is_map(FeatureProps) -> - maps:get(stability, FeatureProps, stable). - -%% ------------------------------------------------------------------- -%% Feature flags registry. -%% ------------------------------------------------------------------- - --spec init() -> ok | no_return(). -%% @private - -init() -> - %% We want to make sure the `feature_flags` file exists once - %% RabbitMQ was started at least once. This is not required by - %% this module (it works fine if the file is missing) but it helps - %% external tools. - _ = ensure_enabled_feature_flags_list_file_exists(), - - %% We also "list" supported feature flags. We are not interested in - %% that list, however, it triggers the first initialization of the - %% registry. - _ = list(all), - ok. - --spec initialize_registry() -> ok | {error, any()} | no_return(). -%% @private -%% @doc -%% Initializes or reinitializes the registry. -%% -%% The registry is an Erlang module recompiled at runtime to hold the -%% state of all supported feature flags. -%% -%% That Erlang module is called {@link rabbit_ff_registry}. The initial -%% source code of this module simply calls this function so it is -%% replaced by a proper registry. -%% -%% Once replaced, the registry contains the map of all supported feature -%% flags and their state. This is makes it very efficient to query a -%% feature flag state or property. -%% -%% The registry is local to all RabbitMQ nodes. - -initialize_registry() -> - initialize_registry(#{}). - --spec initialize_registry(feature_flags()) -> - ok | {error, any()} | no_return(). -%% @private -%% @doc -%% Initializes or reinitializes the registry. -%% -%% See {@link initialize_registry/0} for a description of the registry. -%% -%% This function takes a map of new supported feature flags (so their -%% name and extended properties) to add to the existing known feature -%% flags. - -initialize_registry(NewSupportedFeatureFlags) -> - %% The first step is to get the feature flag states: if this is the - %% first time we initialize it, we read the list from disk (the - %% `feature_flags` file). Otherwise we query the existing registry - %% before it is replaced. - RegistryInitialized = rabbit_ff_registry:is_registry_initialized(), - FeatureStates = case RegistryInitialized of - true -> - rabbit_ff_registry:states(); - false -> - EnabledFeatureNames = - read_enabled_feature_flags_list(), - list_of_enabled_feature_flags_to_feature_states( - EnabledFeatureNames) - end, - - %% We also record if the feature flags state was correctly written - %% to disk. Currently we don't use this information, but in the - %% future, we might want to retry the write if it failed so far. - %% - %% TODO: Retry to write the feature flags state if the first try - %% failed. - WrittenToDisk = case RegistryInitialized of - true -> - rabbit_ff_registry:is_registry_written_to_disk(); - false -> - true - end, - initialize_registry(NewSupportedFeatureFlags, - FeatureStates, - WrittenToDisk). - --spec list_of_enabled_feature_flags_to_feature_states([feature_name()]) -> - feature_states(). - -list_of_enabled_feature_flags_to_feature_states(FeatureNames) -> - maps:from_list([{FeatureName, true} || FeatureName <- FeatureNames]). - --spec initialize_registry(feature_flags(), - feature_states(), - boolean()) -> - ok | {error, any()} | no_return(). -%% @private -%% @doc -%% Initializes or reinitializes the registry. -%% -%% See {@link initialize_registry/0} for a description of the registry. -%% -%% This function takes a map of new supported feature flags (so their -%% name and extended properties) to add to the existing known feature -%% flags, a map of the new feature flag states (whether they are -%% enabled, disabled or `state_changing'), and a flag to indicate if the -%% feature flag states was recorded to disk. -%% -%% The latter is used to block callers asking if a feature flag is -%% enabled or disabled while its state is changing. - -initialize_registry(NewSupportedFeatureFlags, - NewFeatureStates, - WrittenToDisk) -> - Ret = maybe_initialize_registry(NewSupportedFeatureFlags, - NewFeatureStates, - WrittenToDisk), - case Ret of - ok -> ok; - restart -> initialize_registry(NewSupportedFeatureFlags, - NewFeatureStates, - WrittenToDisk); - Error -> Error - end. - --spec maybe_initialize_registry(feature_flags(), - feature_states(), - boolean()) -> - ok | restart | {error, any()} | no_return(). - -maybe_initialize_registry(NewSupportedFeatureFlags, - NewFeatureStates, - WrittenToDisk) -> - %% We save the version of the current registry before computing - %% the new one. This is used when we do the actual reload: if the - %% current registry was reloaded in the meantime, we need to restart - %% the computation to make sure we don't loose data. - RegistryVsn = registry_vsn(), - - %% We take the feature flags already registered. - RegistryInitialized = rabbit_ff_registry:is_registry_initialized(), - KnownFeatureFlags1 = case RegistryInitialized of - true -> rabbit_ff_registry:list(all); - false -> #{} - end, - - %% Query the list (it's a map to be exact) of known - %% supported feature flags. That list comes from the - %% `-rabbitmq_feature_flag().` module attributes exposed by all - %% currently loaded Erlang modules. - KnownFeatureFlags2 = query_supported_feature_flags(), - - %% We merge the feature flags we already knew about - %% (KnownFeatureFlags1), those found in the loaded applications - %% (KnownFeatureFlags2) and those specified in arguments - %% (NewSupportedFeatureFlags). The latter come from remote nodes - %% usually: for example, they can come from plugins loaded on remote - %% node but the plugins are missing locally. In this case, we - %% consider those feature flags supported because there is no code - %% locally which would cause issues. - %% - %% It means that the list of feature flags only grows. we don't try - %% to clean it at some point because we want to remember about the - %% feature flags we saw (and their state). It should be fine because - %% that list should remain small. - KnownFeatureFlags = maps:merge(KnownFeatureFlags1, - KnownFeatureFlags2), - AllFeatureFlags = maps:merge(KnownFeatureFlags, - NewSupportedFeatureFlags), - - %% Next we want to update the feature states, based on the new - %% states passed as arguments. - FeatureStates0 = case RegistryInitialized of - true -> - maps:merge(rabbit_ff_registry:states(), - NewFeatureStates); - false -> - NewFeatureStates - end, - FeatureStates = maps:filter( - fun(_, true) -> true; - (_, state_changing) -> true; - (_, false) -> false - end, FeatureStates0), - - Proceed = does_registry_need_refresh(AllFeatureFlags, - FeatureStates, - WrittenToDisk), - - case Proceed of - true -> - rabbit_log_feature_flags:debug( - "Feature flags: (re)initialize registry (~p)", - [self()]), - T0 = erlang:timestamp(), - Ret = do_initialize_registry(RegistryVsn, - AllFeatureFlags, - FeatureStates, - WrittenToDisk), - T1 = erlang:timestamp(), - rabbit_log_feature_flags:debug( - "Feature flags: time to regen registry: ~p µs", - [timer:now_diff(T1, T0)]), - Ret; - false -> - rabbit_log_feature_flags:debug( - "Feature flags: registry already up-to-date, skipping init"), - ok - end. - --spec does_registry_need_refresh(feature_flags(), - feature_states(), - boolean()) -> - boolean(). - -does_registry_need_refresh(AllFeatureFlags, - FeatureStates, - WrittenToDisk) -> - case rabbit_ff_registry:is_registry_initialized() of - true -> - %% Before proceeding with the actual - %% (re)initialization, let's see if there are any - %% changes. - CurrentAllFeatureFlags = rabbit_ff_registry:list(all), - CurrentFeatureStates = rabbit_ff_registry:states(), - CurrentWrittenToDisk = - rabbit_ff_registry:is_registry_written_to_disk(), - - if - AllFeatureFlags =/= CurrentAllFeatureFlags -> - rabbit_log_feature_flags:debug( - "Feature flags: registry refresh needed: " - "yes, list of feature flags differs"), - true; - FeatureStates =/= CurrentFeatureStates -> - rabbit_log_feature_flags:debug( - "Feature flags: registry refresh needed: " - "yes, feature flag states differ"), - true; - WrittenToDisk =/= CurrentWrittenToDisk -> - rabbit_log_feature_flags:debug( - "Feature flags: registry refresh needed: " - "yes, \"written to disk\" state changed"), - true; - true -> - rabbit_log_feature_flags:debug( - "Feature flags: registry refresh needed: no"), - false - end; - false -> - rabbit_log_feature_flags:debug( - "Feature flags: registry refresh needed: " - "yes, first-time initialization"), - true - end. - --spec do_initialize_registry(registry_vsn(), - feature_flags(), - feature_states(), - boolean()) -> - ok | restart | {error, any()} | no_return(). -%% @private - -do_initialize_registry(RegistryVsn, - AllFeatureFlags, - FeatureStates, - WrittenToDisk) -> - %% We log the state of those feature flags. - rabbit_log_feature_flags:info( - "Feature flags: list of feature flags found:"), - lists:foreach( - fun(FeatureName) -> - rabbit_log_feature_flags:info( - "Feature flags: [~s] ~s", - [case maps:is_key(FeatureName, FeatureStates) of - true -> - case maps:get(FeatureName, FeatureStates) of - true -> "x"; - state_changing -> "~" - end; - false -> - " " - end, - FeatureName]) - end, lists:sort(maps:keys(AllFeatureFlags))), - rabbit_log_feature_flags:info( - "Feature flags: feature flag states written to disk: ~s", - [case WrittenToDisk of - true -> "yes"; - false -> "no" - end]), - - %% We request the registry to be regenerated and reloaded with the - %% new state. - regen_registry_mod(RegistryVsn, - AllFeatureFlags, - FeatureStates, - WrittenToDisk). - --spec query_supported_feature_flags() -> feature_flags(). -%% @private - --ifdef(TEST). --define(PT_TESTSUITE_ATTRS, {?MODULE, testsuite_feature_flags_attrs}). - -inject_test_feature_flags(AttributesFromTestsuite) -> - rabbit_log_feature_flags:debug( - "Feature flags: injecting feature flags from testsuite: ~p", - [AttributesFromTestsuite]), - ok = persistent_term:put(?PT_TESTSUITE_ATTRS, AttributesFromTestsuite), - initialize_registry(). - -module_attributes_from_testsuite() -> - persistent_term:get(?PT_TESTSUITE_ATTRS, []). - -query_supported_feature_flags() -> - rabbit_log_feature_flags:debug( - "Feature flags: query feature flags in loaded applications " - "+ testsuite"), - T0 = erlang:timestamp(), - AttributesPerApp = rabbit_misc:rabbitmq_related_module_attributes( - rabbit_feature_flag), - AttributesFromTestsuite = module_attributes_from_testsuite(), - T1 = erlang:timestamp(), - rabbit_log_feature_flags:debug( - "Feature flags: time to find supported feature flags: ~p µs", - [timer:now_diff(T1, T0)]), - AllAttributes = AttributesPerApp ++ AttributesFromTestsuite, - prepare_queried_feature_flags(AllAttributes, #{}). --else. -query_supported_feature_flags() -> - rabbit_log_feature_flags:debug( - "Feature flags: query feature flags in loaded applications"), - T0 = erlang:timestamp(), - AttributesPerApp = rabbit_misc:rabbitmq_related_module_attributes( - rabbit_feature_flag), - T1 = erlang:timestamp(), - rabbit_log_feature_flags:debug( - "Feature flags: time to find supported feature flags: ~p µs", - [timer:now_diff(T1, T0)]), - prepare_queried_feature_flags(AttributesPerApp, #{}). --endif. - -prepare_queried_feature_flags([{App, _Module, Attributes} | Rest], - AllFeatureFlags) -> - rabbit_log_feature_flags:debug( - "Feature flags: application `~s` has ~b feature flags", - [App, length(Attributes)]), - AllFeatureFlags1 = lists:foldl( - fun({FeatureName, FeatureProps}, AllFF) -> - merge_new_feature_flags(AllFF, - App, - FeatureName, - FeatureProps) - end, AllFeatureFlags, Attributes), - prepare_queried_feature_flags(Rest, AllFeatureFlags1); -prepare_queried_feature_flags([], AllFeatureFlags) -> - AllFeatureFlags. - --spec merge_new_feature_flags(feature_flags(), - atom(), - feature_name(), - feature_props()) -> feature_flags(). -%% @private - -merge_new_feature_flags(AllFeatureFlags, App, FeatureName, FeatureProps) - when is_atom(FeatureName) andalso is_map(FeatureProps) -> - %% We expand the feature flag properties map with: - %% - the name of the application providing it: only informational - %% for now, but can be handy to understand that a feature flag - %% comes from a plugin. - FeatureProps1 = maps:put(provided_by, App, FeatureProps), - maps:merge(AllFeatureFlags, - #{FeatureName => FeatureProps1}). - --spec regen_registry_mod(registry_vsn(), - feature_flags(), - feature_states(), - boolean()) -> - ok | restart | {error, any()} | no_return(). -%% @private - -regen_registry_mod(RegistryVsn, - AllFeatureFlags, - FeatureStates, - WrittenToDisk) -> - %% Here, we recreate the source code of the `rabbit_ff_registry` - %% module from scratch. - %% - %% IMPORTANT: We want both modules to have the exact same public - %% API in order to simplify the life of developers and their tools - %% (Dialyzer, completion, and so on). - - %% -module(rabbit_ff_registry). - ModuleAttr = erl_syntax:attribute( - erl_syntax:atom(module), - [erl_syntax:atom(rabbit_ff_registry)]), - ModuleForm = erl_syntax:revert(ModuleAttr), - %% -export([...]). - ExportAttr = erl_syntax:attribute( - erl_syntax:atom(export), - [erl_syntax:list( - [erl_syntax:arity_qualifier( - erl_syntax:atom(F), - erl_syntax:integer(A)) - || {F, A} <- [{get, 1}, - {list, 1}, - {states, 0}, - {is_supported, 1}, - {is_enabled, 1}, - {is_registry_initialized, 0}, - {is_registry_written_to_disk, 0}]] - ) - ] - ), - ExportForm = erl_syntax:revert(ExportAttr), - %% get(_) -> ... - GetClauses = [erl_syntax:clause( - [erl_syntax:atom(FeatureName)], - [], - [erl_syntax:abstract(maps:get(FeatureName, - AllFeatureFlags))]) - || FeatureName <- maps:keys(AllFeatureFlags) - ], - GetUnknownClause = erl_syntax:clause( - [erl_syntax:variable("_")], - [], - [erl_syntax:atom(undefined)]), - GetFun = erl_syntax:function( - erl_syntax:atom(get), - GetClauses ++ [GetUnknownClause]), - GetFunForm = erl_syntax:revert(GetFun), - %% list(_) -> ... - ListAllBody = erl_syntax:abstract(AllFeatureFlags), - ListAllClause = erl_syntax:clause([erl_syntax:atom(all)], - [], - [ListAllBody]), - EnabledFeatureFlags = maps:filter( - fun(FeatureName, _) -> - maps:is_key(FeatureName, - FeatureStates) - andalso - maps:get(FeatureName, FeatureStates) - =:= - true - end, AllFeatureFlags), - ListEnabledBody = erl_syntax:abstract(EnabledFeatureFlags), - ListEnabledClause = erl_syntax:clause( - [erl_syntax:atom(enabled)], - [], - [ListEnabledBody]), - DisabledFeatureFlags = maps:filter( - fun(FeatureName, _) -> - not maps:is_key(FeatureName, - FeatureStates) - end, AllFeatureFlags), - ListDisabledBody = erl_syntax:abstract(DisabledFeatureFlags), - ListDisabledClause = erl_syntax:clause( - [erl_syntax:atom(disabled)], - [], - [ListDisabledBody]), - StateChangingFeatureFlags = maps:filter( - fun(FeatureName, _) -> - maps:is_key(FeatureName, - FeatureStates) - andalso - maps:get(FeatureName, FeatureStates) - =:= - state_changing - end, AllFeatureFlags), - ListStateChangingBody = erl_syntax:abstract(StateChangingFeatureFlags), - ListStateChangingClause = erl_syntax:clause( - [erl_syntax:atom(state_changing)], - [], - [ListStateChangingBody]), - ListFun = erl_syntax:function( - erl_syntax:atom(list), - [ListAllClause, - ListEnabledClause, - ListDisabledClause, - ListStateChangingClause]), - ListFunForm = erl_syntax:revert(ListFun), - %% states() -> ... - StatesBody = erl_syntax:abstract(FeatureStates), - StatesClause = erl_syntax:clause([], [], [StatesBody]), - StatesFun = erl_syntax:function( - erl_syntax:atom(states), - [StatesClause]), - StatesFunForm = erl_syntax:revert(StatesFun), - %% is_supported(_) -> ... - IsSupportedClauses = [erl_syntax:clause( - [erl_syntax:atom(FeatureName)], - [], - [erl_syntax:atom(true)]) - || FeatureName <- maps:keys(AllFeatureFlags) - ], - NotSupportedClause = erl_syntax:clause( - [erl_syntax:variable("_")], - [], - [erl_syntax:atom(false)]), - IsSupportedFun = erl_syntax:function( - erl_syntax:atom(is_supported), - IsSupportedClauses ++ [NotSupportedClause]), - IsSupportedFunForm = erl_syntax:revert(IsSupportedFun), - %% is_enabled(_) -> ... - IsEnabledClauses = [erl_syntax:clause( - [erl_syntax:atom(FeatureName)], - [], - [case maps:is_key(FeatureName, FeatureStates) of - true -> - erl_syntax:atom( - maps:get(FeatureName, FeatureStates)); - false -> - erl_syntax:atom(false) - end]) - || FeatureName <- maps:keys(AllFeatureFlags) - ], - NotEnabledClause = erl_syntax:clause( - [erl_syntax:variable("_")], - [], - [erl_syntax:atom(false)]), - IsEnabledFun = erl_syntax:function( - erl_syntax:atom(is_enabled), - IsEnabledClauses ++ [NotEnabledClause]), - IsEnabledFunForm = erl_syntax:revert(IsEnabledFun), - %% is_registry_initialized() -> ... - IsInitializedClauses = [erl_syntax:clause( - [], - [], - [erl_syntax:atom(true)]) - ], - IsInitializedFun = erl_syntax:function( - erl_syntax:atom(is_registry_initialized), - IsInitializedClauses), - IsInitializedFunForm = erl_syntax:revert(IsInitializedFun), - %% is_registry_written_to_disk() -> ... - IsWrittenToDiskClauses = [erl_syntax:clause( - [], - [], - [erl_syntax:atom(WrittenToDisk)]) - ], - IsWrittenToDiskFun = erl_syntax:function( - erl_syntax:atom(is_registry_written_to_disk), - IsWrittenToDiskClauses), - IsWrittenToDiskFunForm = erl_syntax:revert(IsWrittenToDiskFun), - %% Compilation! - Forms = [ModuleForm, - ExportForm, - GetFunForm, - ListFunForm, - StatesFunForm, - IsSupportedFunForm, - IsEnabledFunForm, - IsInitializedFunForm, - IsWrittenToDiskFunForm], - maybe_log_registry_source_code(Forms), - CompileOpts = [return_errors, - return_warnings], - case compile:forms(Forms, CompileOpts) of - {ok, Mod, Bin, _} -> - load_registry_mod(RegistryVsn, Mod, Bin); - {error, Errors, Warnings} -> - rabbit_log_feature_flags:error( - "Feature flags: registry compilation:~n" - "Errors: ~p~n" - "Warnings: ~p", - [Errors, Warnings]), - {error, {compilation_failure, Errors, Warnings}} - end. - -maybe_log_registry_source_code(Forms) -> - case rabbit_prelaunch:get_context() of - #{log_feature_flags_registry := true} -> - rabbit_log_feature_flags:debug( - "== FEATURE FLAGS REGISTRY ==~n" - "~s~n" - "== END ==~n", - [erl_prettypr:format(erl_syntax:form_list(Forms))]); - _ -> - ok - end. - --ifdef(TEST). -registry_loading_lock() -> ?FF_REGISTRY_LOADING_LOCK. --endif. - --spec load_registry_mod(registry_vsn(), atom(), binary()) -> - ok | restart | no_return(). -%% @private - -load_registry_mod(RegistryVsn, Mod, Bin) -> - rabbit_log_feature_flags:debug( - "Feature flags: registry module ready, loading it (~p)...", - [self()]), - FakeFilename = "Compiled and loaded by " ?MODULE_STRING, - %% Time to load the new registry, replacing the old one. We use a - %% lock here to synchronize concurrent reloads. - global:set_lock(?FF_REGISTRY_LOADING_LOCK, [node()]), - rabbit_log_feature_flags:debug( - "Feature flags: acquired lock before reloading registry module (~p)", - [self()]), - %% We want to make sure that the old registry (not the one being - %% currently in use) is purged by the code server. It means no - %% process lingers on that old code. - %% - %% We use code:soft_purge() for that (meaning no process is killed) - %% and we wait in an infinite loop for that to succeed. - ok = purge_old_registry(Mod), - %% Now we can replace the currently loaded registry by the new one. - %% The code server takes care of marking the current registry as old - %% and load the new module in an atomic operation. - %% - %% Therefore there is no chance of a window where there is no - %% registry module available, causing the one on disk to be - %% reloaded. - Ret = case registry_vsn() of - RegistryVsn -> code:load_binary(Mod, FakeFilename, Bin); - OtherVsn -> {error, {restart, RegistryVsn, OtherVsn}} - end, - rabbit_log_feature_flags:debug( - "Feature flags: releasing lock after reloading registry module (~p)", - [self()]), - global:del_lock(?FF_REGISTRY_LOADING_LOCK, [node()]), - case Ret of - {module, _} -> - rabbit_log_feature_flags:debug( - "Feature flags: registry module loaded (vsn: ~p -> ~p)", - [RegistryVsn, registry_vsn()]), - ok; - {error, {restart, Expected, Current}} -> - rabbit_log_feature_flags:error( - "Feature flags: another registry module was loaded in the " - "meantime (expected old vsn: ~p, current vsn: ~p); " - "restarting the regen", - [Expected, Current]), - restart; - {error, Reason} -> - rabbit_log_feature_flags:error( - "Feature flags: failed to load registry module: ~p", - [Reason]), - throw({feature_flag_registry_reload_failure, Reason}) - end. - --spec registry_vsn() -> registry_vsn(). -%% @private - -registry_vsn() -> - Attrs = rabbit_ff_registry:module_info(attributes), - proplists:get_value(vsn, Attrs, undefined). - -purge_old_registry(Mod) -> - case code:is_loaded(Mod) of - {file, _} -> do_purge_old_registry(Mod); - false -> ok - end. - -do_purge_old_registry(Mod) -> - case code:soft_purge(Mod) of - true -> ok; - false -> do_purge_old_registry(Mod) - end. - -%% ------------------------------------------------------------------- -%% Feature flags state storage. -%% ------------------------------------------------------------------- - --spec ensure_enabled_feature_flags_list_file_exists() -> ok | {error, any()}. -%% @private - -ensure_enabled_feature_flags_list_file_exists() -> - File = enabled_feature_flags_list_file(), - case filelib:is_regular(File) of - true -> ok; - false -> write_enabled_feature_flags_list([]) - end. - --spec read_enabled_feature_flags_list() -> - [feature_name()] | no_return(). -%% @private - -read_enabled_feature_flags_list() -> - case try_to_read_enabled_feature_flags_list() of - {error, Reason} -> - File = enabled_feature_flags_list_file(), - throw({feature_flags_file_read_error, File, Reason}); - Ret -> - Ret - end. - --spec try_to_read_enabled_feature_flags_list() -> - [feature_name()] | {error, any()}. -%% @private - -try_to_read_enabled_feature_flags_list() -> - File = enabled_feature_flags_list_file(), - case file:consult(File) of - {ok, [List]} -> - List; - {error, enoent} -> - %% If the file is missing, we consider the list of enabled - %% feature flags to be empty. - []; - {error, Reason} = Error -> - rabbit_log_feature_flags:error( - "Feature flags: failed to read the `feature_flags` " - "file at `~s`: ~s", - [File, file:format_error(Reason)]), - Error - end. - --spec write_enabled_feature_flags_list([feature_name()]) -> - ok | no_return(). -%% @private - -write_enabled_feature_flags_list(FeatureNames) -> - case try_to_write_enabled_feature_flags_list(FeatureNames) of - {error, Reason} -> - File = enabled_feature_flags_list_file(), - throw({feature_flags_file_write_error, File, Reason}); - Ret -> - Ret - end. - --spec try_to_write_enabled_feature_flags_list([feature_name()]) -> - ok | {error, any()}. -%% @private - -try_to_write_enabled_feature_flags_list(FeatureNames) -> - %% Before writing the new file, we read the existing one. If there - %% are unknown feature flags in that file, we want to keep their - %% state, even though they are unsupported at this time. It could be - %% that a plugin was disabled in the meantime. - %% - %% FIXME: Lock this code to fix concurrent read/modify/write. - PreviouslyEnabled = case try_to_read_enabled_feature_flags_list() of - {error, _} -> []; - List -> List - end, - FeatureNames1 = lists:foldl( - fun(Name, Acc) -> - case is_supported_locally(Name) of - true -> Acc; - false -> [Name | Acc] - end - end, FeatureNames, PreviouslyEnabled), - FeatureNames2 = lists:sort(FeatureNames1), - - File = enabled_feature_flags_list_file(), - Content = io_lib:format("~p.~n", [FeatureNames2]), - %% TODO: If we fail to write the the file, we should spawn a process - %% to retry the operation. - case file:write_file(File, Content) of - ok -> - ok; - {error, Reason} = Error -> - rabbit_log_feature_flags:error( - "Feature flags: failed to write the `feature_flags` " - "file at `~s`: ~s", - [File, file:format_error(Reason)]), - Error - end. - --spec enabled_feature_flags_list_file() -> file:filename(). -%% @doc -%% Returns the path to the file where the state of feature flags is stored. -%% -%% @returns the path to the file. - -enabled_feature_flags_list_file() -> - case application:get_env(rabbit, feature_flags_file) of - {ok, Val} -> Val; - undefined -> throw(feature_flags_file_not_set) - end. - -%% ------------------------------------------------------------------- -%% Feature flags management: enabling. -%% ------------------------------------------------------------------- - --spec do_enable(feature_name()) -> ok | {error, any()} | no_return(). -%% @private - -do_enable(FeatureName) -> - %% We mark this feature flag as "state changing" before doing the - %% actual state change. We also take a global lock: this permits - %% to block callers asking about a feature flag changing state. - global:set_lock(?FF_STATE_CHANGE_LOCK), - Ret = case mark_as_enabled(FeatureName, state_changing) of - ok -> - case enable_dependencies(FeatureName, true) of - ok -> - case run_migration_fun(FeatureName, enable) of - ok -> - mark_as_enabled(FeatureName, true); - {error, no_migration_fun} -> - mark_as_enabled(FeatureName, true); - Error -> - Error - end; - Error -> - Error - end; - Error -> - Error - end, - case Ret of - ok -> ok; - _ -> mark_as_enabled(FeatureName, false) - end, - global:del_lock(?FF_STATE_CHANGE_LOCK), - Ret. - --spec enable_locally(feature_name()) -> ok | {error, any()} | no_return(). -%% @private - -enable_locally(FeatureName) when is_atom(FeatureName) -> - case is_enabled(FeatureName) of - true -> - ok; - false -> - rabbit_log_feature_flags:debug( - "Feature flag `~s`: enable locally (as part of feature " - "flag states synchronization)", - [FeatureName]), - do_enable_locally(FeatureName) - end. - --spec do_enable_locally(feature_name()) -> ok | {error, any()} | no_return(). -%% @private - -do_enable_locally(FeatureName) -> - case enable_dependencies(FeatureName, false) of - ok -> - case run_migration_fun(FeatureName, enable) of - ok -> - mark_as_enabled_locally(FeatureName, true); - {error, no_migration_fun} -> - mark_as_enabled_locally(FeatureName, true); - Error -> - Error - end; - Error -> - Error - end. - --spec enable_dependencies(feature_name(), boolean()) -> - ok | {error, any()} | no_return(). -%% @private - -enable_dependencies(FeatureName, Everywhere) -> - FeatureProps = rabbit_ff_registry:get(FeatureName), - DependsOn = maps:get(depends_on, FeatureProps, []), - rabbit_log_feature_flags:debug( - "Feature flag `~s`: enable dependencies: ~p", - [FeatureName, DependsOn]), - enable_dependencies(FeatureName, DependsOn, Everywhere). - --spec enable_dependencies(feature_name(), [feature_name()], boolean()) -> - ok | {error, any()} | no_return(). -%% @private - -enable_dependencies(TopLevelFeatureName, [FeatureName | Rest], Everywhere) -> - Ret = case Everywhere of - true -> enable(FeatureName); - false -> enable_locally(FeatureName) - end, - case Ret of - ok -> enable_dependencies(TopLevelFeatureName, Rest, Everywhere); - Error -> Error - end; -enable_dependencies(_, [], _) -> - ok. - --spec run_migration_fun(feature_name(), any()) -> - any() | {error, any()}. -%% @private - -run_migration_fun(FeatureName, Arg) -> - FeatureProps = rabbit_ff_registry:get(FeatureName), - run_migration_fun(FeatureName, FeatureProps, Arg). - -run_migration_fun(FeatureName, FeatureProps, Arg) -> - case maps:get(migration_fun, FeatureProps, none) of - {MigrationMod, MigrationFun} - when is_atom(MigrationMod) andalso is_atom(MigrationFun) -> - rabbit_log_feature_flags:debug( - "Feature flag `~s`: run migration function ~p with arg: ~p", - [FeatureName, MigrationFun, Arg]), - try - erlang:apply(MigrationMod, - MigrationFun, - [FeatureName, FeatureProps, Arg]) - catch - _:Reason:Stacktrace -> - rabbit_log_feature_flags:error( - "Feature flag `~s`: migration function crashed: ~p~n~p", - [FeatureName, Reason, Stacktrace]), - {error, {migration_fun_crash, Reason, Stacktrace}} - end; - none -> - {error, no_migration_fun}; - Invalid -> - rabbit_log_feature_flags:error( - "Feature flag `~s`: invalid migration function: ~p", - [FeatureName, Invalid]), - {error, {invalid_migration_fun, Invalid}} - end. - --spec mark_as_enabled(feature_name(), feature_state()) -> - any() | {error, any()} | no_return(). -%% @private - -mark_as_enabled(FeatureName, IsEnabled) -> - case mark_as_enabled_locally(FeatureName, IsEnabled) of - ok -> - mark_as_enabled_remotely(FeatureName, IsEnabled); - Error -> - Error - end. - --spec mark_as_enabled_locally(feature_name(), feature_state()) -> - any() | {error, any()} | no_return(). -%% @private - -mark_as_enabled_locally(FeatureName, IsEnabled) -> - rabbit_log_feature_flags:info( - "Feature flag `~s`: mark as enabled=~p", - [FeatureName, IsEnabled]), - EnabledFeatureNames = maps:keys(list(enabled)), - NewEnabledFeatureNames = case IsEnabled of - true -> - [FeatureName | EnabledFeatureNames]; - false -> - EnabledFeatureNames -- [FeatureName]; - state_changing -> - EnabledFeatureNames - end, - WrittenToDisk = case NewEnabledFeatureNames of - EnabledFeatureNames -> - rabbit_ff_registry:is_registry_written_to_disk(); - _ -> - ok =:= try_to_write_enabled_feature_flags_list( - NewEnabledFeatureNames) - end, - initialize_registry(#{}, - #{FeatureName => IsEnabled}, - WrittenToDisk). - --spec mark_as_enabled_remotely(feature_name(), feature_state()) -> - any() | {error, any()} | no_return(). -%% @private - -mark_as_enabled_remotely(FeatureName, IsEnabled) -> - Nodes = running_remote_nodes(), - mark_as_enabled_remotely(Nodes, FeatureName, IsEnabled, ?TIMEOUT). - --spec mark_as_enabled_remotely([node()], - feature_name(), - feature_state(), - timeout()) -> - any() | {error, any()} | no_return(). -%% @private - -mark_as_enabled_remotely([], _FeatureName, _IsEnabled, _Timeout) -> - ok; -mark_as_enabled_remotely(Nodes, FeatureName, IsEnabled, Timeout) -> - T0 = erlang:timestamp(), - Rets = [{Node, rpc:call(Node, - ?MODULE, - mark_as_enabled_locally, - [FeatureName, IsEnabled], - Timeout)} - || Node <- Nodes], - FailedNodes = [Node || {Node, Ret} <- Rets, Ret =/= ok], - case FailedNodes of - [] -> - rabbit_log_feature_flags:debug( - "Feature flags: `~s` successfully marked as enabled=~p on all " - "nodes", [FeatureName, IsEnabled]), - ok; - _ -> - rabbit_log_feature_flags:error( - "Feature flags: failed to mark feature flag `~s` as enabled=~p " - "on the following nodes:", [FeatureName, IsEnabled]), - [rabbit_log_feature_flags:error( - "Feature flags: - ~s: ~p", - [Node, Ret]) - || {Node, Ret} <- Rets, - Ret =/= ok], - Sleep = 1000, - T1 = erlang:timestamp(), - Duration = timer:now_diff(T1, T0), - NewTimeout = (Timeout * 1000 - Duration) div 1000 - Sleep, - if - NewTimeout > 0 -> - rabbit_log_feature_flags:debug( - "Feature flags: retrying with a timeout of ~b " - "ms after sleeping for ~b ms", - [NewTimeout, Sleep]), - timer:sleep(Sleep), - mark_as_enabled_remotely(FailedNodes, - FeatureName, - IsEnabled, - NewTimeout); - true -> - rabbit_log_feature_flags:debug( - "Feature flags: not retrying; RPC went over the " - "~b milliseconds timeout", [Timeout]), - %% FIXME: Is crashing the process the best solution here? - throw( - {failed_to_mark_feature_flag_as_enabled_on_remote_nodes, - FeatureName, IsEnabled, FailedNodes}) - end - end. - -%% ------------------------------------------------------------------- -%% Coordination with remote nodes. -%% ------------------------------------------------------------------- - --spec remote_nodes() -> [node()]. -%% @private - -remote_nodes() -> - mnesia:system_info(db_nodes) -- [node()]. - --spec running_remote_nodes() -> [node()]. -%% @private - -running_remote_nodes() -> - mnesia:system_info(running_db_nodes) -- [node()]. - -query_running_remote_nodes(Node, Timeout) -> - case rpc:call(Node, mnesia, system_info, [running_db_nodes], Timeout) of - {badrpc, _} = Error -> Error; - Nodes -> Nodes -- [node()] - end. - --spec does_node_support(node(), [feature_name()], timeout()) -> boolean(). -%% @private - -does_node_support(Node, FeatureNames, Timeout) -> - rabbit_log_feature_flags:debug( - "Feature flags: querying `~p` support on node ~s...", - [FeatureNames, Node]), - Ret = case node() of - Node -> - is_supported_locally(FeatureNames); - _ -> - run_feature_flags_mod_on_remote_node( - Node, is_supported_locally, [FeatureNames], Timeout) - end, - case Ret of - {error, pre_feature_flags_rabbitmq} -> - %% See run_feature_flags_mod_on_remote_node/4 for - %% an explanation why we consider this node a 3.7.x - %% pre-feature-flags node. - rabbit_log_feature_flags:debug( - "Feature flags: no feature flags support on node `~s`, " - "consider the feature flags unsupported: ~p", - [Node, FeatureNames]), - false; - {error, Reason} -> - rabbit_log_feature_flags:error( - "Feature flags: error while querying `~p` support on " - "node ~s: ~p", - [FeatureNames, Node, Reason]), - false; - true -> - rabbit_log_feature_flags:debug( - "Feature flags: node `~s` supports `~p`", - [Node, FeatureNames]), - true; - false -> - rabbit_log_feature_flags:debug( - "Feature flags: node `~s` does not support `~p`; " - "stopping query here", - [Node, FeatureNames]), - false - end. - --spec check_node_compatibility(node()) -> ok | {error, any()}. -%% @doc -%% Checks if a node is compatible with the local node. -%% -%% To be compatible, the following two conditions must be met: -%% <ol> -%% <li>feature flags enabled on the local node must be supported by the -%% remote node</li> -%% <li>feature flags enabled on the remote node must be supported by the -%% local node</li> -%% </ol> -%% -%% @param Node the name of the remote node to test. -%% @returns `ok' if they are compatible, `{error, Reason}' if they are not. - -check_node_compatibility(Node) -> - check_node_compatibility(Node, ?TIMEOUT). - --spec check_node_compatibility(node(), timeout()) -> ok | {error, any()}. -%% @doc -%% Checks if a node is compatible with the local node. -%% -%% See {@link check_node_compatibility/1} for the conditions required to -%% consider two nodes compatible. -%% -%% @param Node the name of the remote node to test. -%% @param Timeout Time in milliseconds after which the RPC gives up. -%% @returns `ok' if they are compatible, `{error, Reason}' if they are not. -%% -%% @see check_node_compatibility/1 - -check_node_compatibility(Node, Timeout) -> - %% Before checking compatibility, we exchange feature flags from - %% unknown Erlang applications. So we fetch remote feature flags - %% from applications which are not loaded locally, and the opposite. - %% - %% The goal is that such feature flags are not blocking the - %% communication between nodes because the code (which would - %% break) is missing on those nodes. Therefore they should not be - %% considered when determining compatibility. - exchange_feature_flags_from_unknown_apps(Node, Timeout), - - %% FIXME: - %% When we try to cluster two nodes, we get: - %% Feature flags: starting an unclustered node: all feature flags - %% will be enabled by default - %% It should probably not be the case... - - %% We can now proceed with the actual compatibility check. - rabbit_log_feature_flags:debug( - "Feature flags: node `~s` compatibility check, part 1/2", - [Node]), - Part1 = local_enabled_feature_flags_is_supported_remotely(Node, Timeout), - rabbit_log_feature_flags:debug( - "Feature flags: node `~s` compatibility check, part 2/2", - [Node]), - Part2 = remote_enabled_feature_flags_is_supported_locally(Node, Timeout), - case {Part1, Part2} of - {true, true} -> - rabbit_log_feature_flags:debug( - "Feature flags: node `~s` is compatible", - [Node]), - ok; - {false, _} -> - rabbit_log_feature_flags:error( - "Feature flags: node `~s` is INCOMPATIBLE: " - "feature flags enabled locally are not supported remotely", - [Node]), - {error, incompatible_feature_flags}; - {_, false} -> - rabbit_log_feature_flags:error( - "Feature flags: node `~s` is INCOMPATIBLE: " - "feature flags enabled remotely are not supported locally", - [Node]), - {error, incompatible_feature_flags} - end. - --spec is_node_compatible(node()) -> boolean(). -%% @doc -%% Returns if a node is compatible with the local node. -%% -%% This function calls {@link check_node_compatibility/2} and returns -%% `true' the latter returns `ok'. Therefore this is the same code, -%% except that this function returns a boolean, but not the reason of -%% the incompatibility if any. -%% -%% @param Node the name of the remote node to test. -%% @returns `true' if they are compatible, `false' otherwise. - -is_node_compatible(Node) -> - is_node_compatible(Node, ?TIMEOUT). - --spec is_node_compatible(node(), timeout()) -> boolean(). -%% @doc -%% Returns if a node is compatible with the local node. -%% -%% This function calls {@link check_node_compatibility/2} and returns -%% `true' the latter returns `ok'. Therefore this is the same code, -%% except that this function returns a boolean, but not the reason -%% of the incompatibility if any. If the RPC times out, nodes are -%% considered incompatible. -%% -%% @param Node the name of the remote node to test. -%% @param Timeout Time in milliseconds after which the RPC gives up. -%% @returns `true' if they are compatible, `false' otherwise. - -is_node_compatible(Node, Timeout) -> - check_node_compatibility(Node, Timeout) =:= ok. - --spec local_enabled_feature_flags_is_supported_remotely(node(), - timeout()) -> - boolean(). -%% @private - -local_enabled_feature_flags_is_supported_remotely(Node, Timeout) -> - LocalEnabledFeatureNames = maps:keys(list(enabled)), - is_supported_remotely([Node], LocalEnabledFeatureNames, Timeout). - --spec remote_enabled_feature_flags_is_supported_locally(node(), - timeout()) -> - boolean(). -%% @private - -remote_enabled_feature_flags_is_supported_locally(Node, Timeout) -> - case query_remote_feature_flags(Node, enabled, Timeout) of - {error, _} -> - false; - RemoteEnabledFeatureFlags when is_map(RemoteEnabledFeatureFlags) -> - RemoteEnabledFeatureNames = maps:keys(RemoteEnabledFeatureFlags), - is_supported_locally(RemoteEnabledFeatureNames) - end. - --spec run_feature_flags_mod_on_remote_node(node(), - atom(), - [term()], - timeout()) -> - term() | {error, term()}. -%% @private - -run_feature_flags_mod_on_remote_node(Node, Function, Args, Timeout) -> - case rpc:call(Node, ?MODULE, Function, Args, Timeout) of - {badrpc, {'EXIT', - {undef, - [{?MODULE, Function, Args, []} - | _]}}} -> - %% If rabbit_feature_flags:Function() is undefined - %% on the remote node, we consider it to be a 3.7.x - %% pre-feature-flags node. - %% - %% Theoretically, it could be an older version (3.6.x and - %% older). But the RabbitMQ version consistency check - %% (rabbit_misc:version_minor_equivalent/2) called from - %% rabbit_mnesia:check_rabbit_consistency/2 already blocked - %% this situation from happening before we reach this point. - rabbit_log_feature_flags:debug( - "Feature flags: ~s:~s~p unavailable on node `~s`: " - "assuming it is a RabbitMQ 3.7.x pre-feature-flags node", - [?MODULE, Function, Args, Node]), - {error, pre_feature_flags_rabbitmq}; - {badrpc, Reason} = Error -> - rabbit_log_feature_flags:error( - "Feature flags: error while running ~s:~s~p " - "on node `~s`: ~p", - [?MODULE, Function, Args, Node, Reason]), - {error, Error}; - Ret -> - Ret - end. - --spec query_remote_feature_flags(node(), - Which :: all | enabled | disabled, - timeout()) -> - feature_flags() | {error, any()}. -%% @private - -query_remote_feature_flags(Node, Which, Timeout) -> - rabbit_log_feature_flags:debug( - "Feature flags: querying ~s feature flags on node `~s`...", - [Which, Node]), - case run_feature_flags_mod_on_remote_node(Node, list, [Which], Timeout) of - {error, pre_feature_flags_rabbitmq} -> - %% See run_feature_flags_mod_on_remote_node/4 for - %% an explanation why we consider this node a 3.7.x - %% pre-feature-flags node. - rabbit_log_feature_flags:debug( - "Feature flags: no feature flags support on node `~s`, " - "consider the list of feature flags empty", [Node]), - #{}; - {error, Reason} = Error -> - rabbit_log_feature_flags:error( - "Feature flags: error while querying ~s feature flags " - "on node `~s`: ~p", - [Which, Node, Reason]), - Error; - RemoteFeatureFlags when is_map(RemoteFeatureFlags) -> - RemoteFeatureNames = maps:keys(RemoteFeatureFlags), - rabbit_log_feature_flags:debug( - "Feature flags: querying ~s feature flags on node `~s` " - "done; ~s features: ~p", - [Which, Node, Which, RemoteFeatureNames]), - RemoteFeatureFlags - end. - --spec merge_feature_flags_from_unknown_apps(feature_flags()) -> - ok | {error, any()}. -%% @private - -merge_feature_flags_from_unknown_apps(FeatureFlags) - when is_map(FeatureFlags) -> - LoadedApps = [App || {App, _, _} <- application:loaded_applications()], - FeatureFlagsFromUnknownApps = - maps:fold( - fun(FeatureName, FeatureProps, UnknownFF) -> - case is_supported_locally(FeatureName) of - true -> - UnknownFF; - false -> - FeatureProvider = maps:get(provided_by, FeatureProps), - case lists:member(FeatureProvider, LoadedApps) of - true -> UnknownFF; - false -> maps:put(FeatureName, FeatureProps, - UnknownFF) - end - end - end, - #{}, - FeatureFlags), - case maps:keys(FeatureFlagsFromUnknownApps) of - [] -> - ok; - _ -> - rabbit_log_feature_flags:debug( - "Feature flags: register feature flags provided by applications " - "unknown locally: ~p", - [maps:keys(FeatureFlagsFromUnknownApps)]), - initialize_registry(FeatureFlagsFromUnknownApps) - end. - -exchange_feature_flags_from_unknown_apps(Node, Timeout) -> - %% The first step is to fetch feature flags from Erlang applications - %% we don't know locally (they are loaded remotely, but not - %% locally). - fetch_remote_feature_flags_from_apps_unknown_locally(Node, Timeout), - - %% The next step is to do the opposite: push feature flags to remote - %% nodes so they can register those from applications they don't - %% know. - push_local_feature_flags_from_apps_unknown_remotely(Node, Timeout). - -fetch_remote_feature_flags_from_apps_unknown_locally(Node, Timeout) -> - RemoteFeatureFlags = query_remote_feature_flags(Node, all, Timeout), - merge_feature_flags_from_unknown_apps(RemoteFeatureFlags). - -push_local_feature_flags_from_apps_unknown_remotely(Node, Timeout) -> - LocalFeatureFlags = list(all), - push_local_feature_flags_from_apps_unknown_remotely( - Node, LocalFeatureFlags, Timeout). - -push_local_feature_flags_from_apps_unknown_remotely( - Node, FeatureFlags, Timeout) - when map_size(FeatureFlags) > 0 -> - case query_running_remote_nodes(Node, Timeout) of - {badrpc, Reason} -> - {error, Reason}; - Nodes -> - lists:foreach( - fun(N) -> - run_feature_flags_mod_on_remote_node( - N, - merge_feature_flags_from_unknown_apps, - [FeatureFlags], - Timeout) - end, Nodes) - end; -push_local_feature_flags_from_apps_unknown_remotely(_, _, _) -> - ok. - --spec sync_feature_flags_with_cluster([node()], boolean()) -> - ok | {error, any()} | no_return(). -%% @private - -sync_feature_flags_with_cluster(Nodes, NodeIsVirgin) -> - sync_feature_flags_with_cluster(Nodes, NodeIsVirgin, ?TIMEOUT). - --spec sync_feature_flags_with_cluster([node()], boolean(), timeout()) -> - ok | {error, any()} | no_return(). -%% @private - -sync_feature_flags_with_cluster([], NodeIsVirgin, _) -> - verify_which_feature_flags_are_actually_enabled(), - case NodeIsVirgin of - true -> - FeatureNames = get_forced_feature_flag_names(), - case remote_nodes() of - [] when FeatureNames =:= undefined -> - rabbit_log_feature_flags:debug( - "Feature flags: starting an unclustered node " - "for the first time: all feature flags will be " - "enabled by default"), - enable_all(); - [] -> - case FeatureNames of - [] -> - rabbit_log_feature_flags:debug( - "Feature flags: starting an unclustered " - "node for the first time: all feature " - "flags are forcibly left disabled from " - "the $RABBITMQ_FEATURE_FLAGS environment " - "variable"), - ok; - _ -> - rabbit_log_feature_flags:debug( - "Feature flags: starting an unclustered " - "node for the first time: only the " - "following feature flags specified in " - "the $RABBITMQ_FEATURE_FLAGS environment " - "variable will be enabled: ~p", - [FeatureNames]), - enable(FeatureNames) - end; - _ -> - ok - end; - false -> - rabbit_log_feature_flags:debug( - "Feature flags: starting an unclustered node which is " - "already initialized: all feature flags left in their " - "current state"), - ok - end; -sync_feature_flags_with_cluster(Nodes, _, Timeout) -> - verify_which_feature_flags_are_actually_enabled(), - RemoteNodes = Nodes -- [node()], - sync_feature_flags_with_cluster1(RemoteNodes, Timeout). - -sync_feature_flags_with_cluster1([], _) -> - ok; -sync_feature_flags_with_cluster1(RemoteNodes, Timeout) -> - RandomRemoteNode = pick_one_node(RemoteNodes), - rabbit_log_feature_flags:debug( - "Feature flags: SYNCING FEATURE FLAGS with node `~s`...", - [RandomRemoteNode]), - case query_remote_feature_flags(RandomRemoteNode, enabled, Timeout) of - {error, _} = Error -> - Error; - RemoteFeatureFlags -> - RemoteFeatureNames = maps:keys(RemoteFeatureFlags), - rabbit_log_feature_flags:debug( - "Feature flags: enabling locally feature flags already " - "enabled on node `~s`...", - [RandomRemoteNode]), - case do_sync_feature_flags_with_node(RemoteFeatureNames) of - ok -> - sync_feature_flags_with_cluster2( - RandomRemoteNode, Timeout); - Error -> - Error - end - end. - -sync_feature_flags_with_cluster2(RandomRemoteNode, Timeout) -> - LocalFeatureNames = maps:keys(list(enabled)), - rabbit_log_feature_flags:debug( - "Feature flags: enabling on node `~s` feature flags already " - "enabled locally...", - [RandomRemoteNode]), - Ret = run_feature_flags_mod_on_remote_node( - RandomRemoteNode, - do_sync_feature_flags_with_node, - [LocalFeatureNames], - Timeout), - case Ret of - {error, pre_feature_flags_rabbitmq} -> ok; - _ -> Ret - end. - -pick_one_node(Nodes) -> - RandomIndex = rand:uniform(length(Nodes)), - lists:nth(RandomIndex, Nodes). - -do_sync_feature_flags_with_node([FeatureFlag | Rest]) -> - case enable_locally(FeatureFlag) of - ok -> do_sync_feature_flags_with_node(Rest); - Error -> Error - end; -do_sync_feature_flags_with_node([]) -> - ok. - --spec get_forced_feature_flag_names() -> [feature_name()] | undefined. -%% @private -%% @doc -%% Returns the (possibly empty) list of feature flags the user want -%% to enable out-of-the-box when starting a node for the first time. -%% -%% Without this, the default is to enable all the supported feature -%% flags. -%% -%% There are two ways to specify that list: -%% <ol> -%% <li>Using the `$RABBITMQ_FEATURE_FLAGS' environment variable; for -%% instance `RABBITMQ_FEATURE_FLAGS=quorum_queue,mnevis'.</li> -%% <li>Using the `forced_feature_flags_on_init' configuration parameter; -%% for instance -%% `{rabbit, [{forced_feature_flags_on_init, [quorum_queue, mnevis]}]}'.</li> -%% </ol> -%% -%% The environment variable has precedence over the configuration -%% parameter. - -get_forced_feature_flag_names() -> - Ret = case get_forced_feature_flag_names_from_env() of - undefined -> get_forced_feature_flag_names_from_config(); - List -> List - end, - case Ret of - undefined -> ok; - [] -> rabbit_log_feature_flags:info( - "Feature flags: automatic enablement of feature " - "flags disabled (i.e. none will be enabled " - "automatically)"); - _ -> rabbit_log_feature_flags:info( - "Feature flags: automatic enablement of feature " - "flags limited to the following list: ~p", [Ret]) - end, - Ret. - --spec get_forced_feature_flag_names_from_env() -> [feature_name()] | undefined. -%% @private - -get_forced_feature_flag_names_from_env() -> - case rabbit_prelaunch:get_context() of - #{forced_feature_flags_on_init := ForcedFFs} - when is_list(ForcedFFs) -> - ForcedFFs; - _ -> - undefined - end. - --spec get_forced_feature_flag_names_from_config() -> [feature_name()] | undefined. -%% @private - -get_forced_feature_flag_names_from_config() -> - Value = application:get_env(rabbit, - forced_feature_flags_on_init, - undefined), - case Value of - undefined -> - Value; - _ when is_list(Value) -> - case lists:all(fun is_atom/1, Value) of - true -> Value; - false -> undefined - end; - _ -> - undefined - end. - --spec verify_which_feature_flags_are_actually_enabled() -> - ok | {error, any()} | no_return(). -%% @private - -verify_which_feature_flags_are_actually_enabled() -> - AllFeatureFlags = list(all), - EnabledFeatureNames = read_enabled_feature_flags_list(), - rabbit_log_feature_flags:debug( - "Feature flags: double-checking feature flag states..."), - %% In case the previous instance of the node failed to write the - %% feature flags list file, we want to double-check the list of - %% enabled feature flags read from disk. For each feature flag, - %% we call the migration function to query if the feature flag is - %% actually enabled. - %% - %% If a feature flag doesn't provide a migration function (or if the - %% function fails), we keep the current state of the feature flag. - List1 = maps:fold( - fun(Name, Props, Acc) -> - Ret = run_migration_fun(Name, Props, is_enabled), - case Ret of - true -> - [Name | Acc]; - false -> - Acc; - _ -> - MarkedAsEnabled = is_enabled(Name), - case MarkedAsEnabled of - true -> [Name | Acc]; - false -> Acc - end - end - end, - [], AllFeatureFlags), - RepairedEnabledFeatureNames = lists:sort(List1), - %% We log the list of feature flags for which the state changes - %% after the check above. - WereEnabled = RepairedEnabledFeatureNames -- EnabledFeatureNames, - WereDisabled = EnabledFeatureNames -- RepairedEnabledFeatureNames, - case {WereEnabled, WereDisabled} of - {[], []} -> ok; - _ -> rabbit_log_feature_flags:warning( - "Feature flags: the previous instance of this node " - "must have failed to write the `feature_flags` " - "file at `~s`:", - [enabled_feature_flags_list_file()]) - end, - case WereEnabled of - [] -> ok; - _ -> rabbit_log_feature_flags:warning( - "Feature flags: - list of previously enabled " - "feature flags now marked as such: ~p", [WereEnabled]) - end, - case WereDisabled of - [] -> ok; - _ -> rabbit_log_feature_flags:warning( - "Feature flags: - list of previously disabled " - "feature flags now marked as such: ~p", [WereDisabled]) - end, - %% Finally, if the new list of enabled feature flags is different - %% than the one on disk, we write the new list and re-initialize the - %% registry. - case RepairedEnabledFeatureNames of - EnabledFeatureNames -> - ok; - _ -> - rabbit_log_feature_flags:debug( - "Feature flags: write the repaired list of enabled feature " - "flags"), - WrittenToDisk = ok =:= try_to_write_enabled_feature_flags_list( - RepairedEnabledFeatureNames), - initialize_registry( - #{}, - list_of_enabled_feature_flags_to_feature_states( - RepairedEnabledFeatureNames), - WrittenToDisk) - end. - --spec refresh_feature_flags_after_app_load([atom()]) -> - ok | {error, any()} | no_return(). - -refresh_feature_flags_after_app_load([]) -> - ok; -refresh_feature_flags_after_app_load(Apps) -> - rabbit_log_feature_flags:debug( - "Feature flags: new apps loaded: ~p -> refreshing feature flags", - [Apps]), - - FeatureFlags0 = list(all), - FeatureFlags1 = query_supported_feature_flags(), - - %% The following list contains all the feature flags this node - %% learned about only because remote nodes have them. Now, the - %% applications providing them are loaded locally as well. - %% Therefore, we may run their migration function in case the state - %% of this node needs it. - AlreadySupportedFeatureNames = maps:keys( - maps:filter( - fun(_, #{provided_by := App}) -> - lists:member(App, Apps) - end, FeatureFlags0)), - case AlreadySupportedFeatureNames of - [] -> - ok; - _ -> - rabbit_log_feature_flags:debug( - "Feature flags: new apps loaded: feature flags already " - "supported: ~p", - [lists:sort(AlreadySupportedFeatureNames)]) - end, - - %% The following list contains all the feature flags no nodes in the - %% cluster knew about before: this is the first time we see them in - %% this instance of the cluster. We need to register them on all - %% nodes. - NewSupportedFeatureFlags = maps:filter( - fun(FeatureName, _) -> - not maps:is_key(FeatureName, - FeatureFlags0) - end, FeatureFlags1), - case maps:keys(NewSupportedFeatureFlags) of - [] -> - ok; - NewSupportedFeatureNames -> - rabbit_log_feature_flags:debug( - "Feature flags: new apps loaded: new feature flags (unseen so " - "far): ~p ", - [lists:sort(NewSupportedFeatureNames)]) - end, - - case initialize_registry() of - ok -> - Ret = maybe_enable_locally_after_app_load( - AlreadySupportedFeatureNames), - case Ret of - ok -> - share_new_feature_flags_after_app_load( - NewSupportedFeatureFlags, ?TIMEOUT); - Error -> - Error - end; - Error -> - Error - end. - -maybe_enable_locally_after_app_load([]) -> - ok; -maybe_enable_locally_after_app_load([FeatureName | Rest]) -> - case is_enabled(FeatureName) of - true -> - case do_enable_locally(FeatureName) of - ok -> maybe_enable_locally_after_app_load(Rest); - Error -> Error - end; - false -> - maybe_enable_locally_after_app_load(Rest) - end. - -share_new_feature_flags_after_app_load(FeatureFlags, Timeout) -> - push_local_feature_flags_from_apps_unknown_remotely( - node(), FeatureFlags, Timeout). - -on_load() -> - %% The goal of this `on_load()` code server hook is to prevent this - %% module from being loaded in an already running RabbitMQ node if - %% the running version does not have the feature flags subsystem. - %% - %% This situation happens when an upgrade overwrites RabbitMQ files - %% with the node still running. This is the case with many packages: - %% files are updated on disk, then a post-install step takes care of - %% restarting the service. - %% - %% The problem is that if many nodes in a cluster are updated at the - %% same time, one node running the newer version might query feature - %% flags on an old node where this module is already available - %% (because files were already overwritten). This causes the query - %% to report an unexpected answer and the newer node to refuse to - %% start. - %% - %% However, when the module is executed outside of RabbitMQ (for - %% debugging purpose or in the context of EUnit for instance), we - %% want to allow the load. That's why we first check if RabbitMQ is - %% actually running. - case rabbit:is_running() of - true -> - %% RabbitMQ is running. - %% - %% Now we want to differentiate a pre-feature-flags node - %% from one having the subsystem. - %% - %% To do that, we verify if the `feature_flags_file` - %% application environment variable is defined. With a - %% feature-flags-enabled node, this application environment - %% variable is defined by rabbitmq-server(8). - case application:get_env(rabbit, feature_flags_file) of - {ok, _} -> - %% This is a feature-flags-enabled version. Loading - %% the module is permitted. - ok; - _ -> - %% This is a pre-feature-flags version. We deny the - %% load and report why, possibly specifying the - %% version of RabbitMQ. - Vsn = case application:get_key(rabbit, vsn) of - {ok, V} -> V; - undefined -> "unknown version" - end, - "Refusing to load '" ?MODULE_STRING "' on this " - "node. It appears to be running a pre-feature-flags " - "version of RabbitMQ (" ++ Vsn ++ "). This is fine: " - "a newer version of RabbitMQ was deployed on this " - "node, but it was not restarted yet. This warning " - "is probably caused by a remote node querying this " - "node for its feature flags." - end; - false -> - %% RabbitMQ is not running. Loading the module is permitted - %% because this Erlang node will never be queried for its - %% feature flags. - ok - end. diff --git a/src/rabbit_ff_extra.erl b/src/rabbit_ff_extra.erl deleted file mode 100644 index f0728d491e..0000000000 --- a/src/rabbit_ff_extra.erl +++ /dev/null @@ -1,244 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% @copyright 2018-2020 VMware, Inc. or its affiliates. -%% -%% @doc -%% This module provides extra functions unused by the feature flags -%% subsystem core functionality. - --module(rabbit_ff_extra). - --include_lib("stdout_formatter/include/stdout_formatter.hrl"). - --export([cli_info/0, - info/1, - info/2, - format_error/1]). - --type cli_info() :: [cli_info_entry()]. -%% A list of feature flags properties, formatted for the RabbitMQ CLI. - --type cli_info_entry() :: [{name, rabbit_feature_flags:feature_name()} | - {state, enabled | disabled | unavailable} | - {stability, rabbit_feature_flags:stability()} | - {provided_by, atom()} | - {desc, string()} | - {doc_url, string()}]. -%% A list of properties for a single feature flag, formatted for the -%% RabbitMQ CLI. - --type info_options() :: #{colors => boolean(), - lines => boolean(), - verbose => non_neg_integer()}. -%% Options accepted by {@link info/1} and {@link info/2}. - --export_type([info_options/0]). - --spec cli_info() -> cli_info(). -%% @doc -%% Returns a list of all feature flags properties. -%% -%% @returns the list of all feature flags properties. - -cli_info() -> - cli_info(rabbit_feature_flags:list(all)). - --spec cli_info(rabbit_feature_flags:feature_flags()) -> cli_info(). -%% @doc -%% Formats a map of feature flags and their properties into a list of -%% feature flags properties as expected by the RabbitMQ CLI. -%% -%% @param FeatureFlags A map of feature flags. -%% @returns the list of feature flags properties, created from the map -%% specified in arguments. - -cli_info(FeatureFlags) -> - lists:foldr( - fun(FeatureName, Acc) -> - FeatureProps = maps:get(FeatureName, FeatureFlags), - State = rabbit_feature_flags:get_state(FeatureName), - Stability = rabbit_feature_flags:get_stability(FeatureProps), - App = maps:get(provided_by, FeatureProps), - Desc = maps:get(desc, FeatureProps, ""), - DocUrl = maps:get(doc_url, FeatureProps, ""), - FFInfo = [{name, FeatureName}, - {desc, unicode:characters_to_binary(Desc)}, - {doc_url, unicode:characters_to_binary(DocUrl)}, - {state, State}, - {stability, Stability}, - {provided_by, App}], - [FFInfo | Acc] - end, [], lists:sort(maps:keys(FeatureFlags))). - --spec info(info_options()) -> ok. -%% @doc -%% Displays an array of all supported feature flags and their properties -%% on `stdout'. -%% -%% @param Options Options to tune what is displayed and how. - -info(Options) -> - %% Two tables: one for stable feature flags, one for experimental ones. - StableFF = rabbit_feature_flags:list(all, stable), - case maps:size(StableFF) of - 0 -> - ok; - _ -> - stdout_formatter:display( - #paragraph{content = "\n## Stable feature flags:", - props = #{bold => true}}), - info(StableFF, Options) - end, - ExpFF = rabbit_feature_flags:list(all, experimental), - case maps:size(ExpFF) of - 0 -> - ok; - _ -> - stdout_formatter:display( - #paragraph{content = "\n## Experimental feature flags:", - props = #{bold => true}}), - info(ExpFF, Options) - end, - case maps:size(StableFF) + maps:size(ExpFF) of - 0 -> ok; - _ -> state_legend(Options) - end. - --spec info(rabbit_feature_flags:feature_flags(), info_options()) -> ok. -%% @doc -%% Displays an array of feature flags and their properties on `stdout', -%% based on the specified feature flags map. -%% -%% @param FeatureFlags Map of the feature flags to display. -%% @param Options Options to tune what is displayed and how. - -info(FeatureFlags, Options) -> - Verbose = maps:get(verbose, Options, 0), - UseColors = use_colors(Options), - UseLines = use_lines(Options), - Title = case UseColors of - true -> #{title => true}; - false -> #{} - end, - Bold = case UseColors of - true -> #{bold => true}; - false -> #{} - end, - {Green, Yellow, Red} = case UseColors of - true -> - {#{fg => green}, - #{fg => yellow}, - #{bold => true, - bg => red}}; - false -> - {#{}, #{}, #{}} - end, - Border = case UseLines of - true -> #{border_drawing => ansi}; - false -> #{border_drawing => ascii} - end, - %% Table columns: - %% | Name | State | Provided by | Description - %% - %% where: - %% State = Enabled | Disabled | Unavailable (if a node doesn't - %% support it). - TableHeader = #row{cells = ["Name", - "State", - "Provided", - "Description"], - props = Title}, - Nodes = lists:sort([node() | rabbit_feature_flags:remote_nodes()]), - Rows = lists:map( - fun(FeatureName) -> - FeatureProps = maps:get(FeatureName, FeatureFlags), - State0 = rabbit_feature_flags:get_state(FeatureName), - {State, Color} = case State0 of - enabled -> - {"Enabled", Green}; - disabled -> - {"Disabled", Yellow}; - unavailable -> - {"Unavailable", Red} - end, - App = maps:get(provided_by, FeatureProps), - Desc = maps:get(desc, FeatureProps, ""), - VFun = fun(Node) -> - Supported = - rabbit_feature_flags:does_node_support( - Node, [FeatureName], 60000), - {Label, LabelColor} = - case Supported of - true -> {"supported", #{}}; - false -> {"unsupported", Red} - end, - #paragraph{content = - [rabbit_misc:format(" ~s: ", - [Node]), - #paragraph{content = Label, - props = LabelColor}]} - end, - ExtraLines = if - Verbose > 0 -> - NodesList = lists:join( - "\n", - lists:map( - VFun, Nodes)), - ["\n\n", - "Per-node support level:\n" - | NodesList]; - true -> - [] - end, - [#paragraph{content = FeatureName, - props = Bold}, - #paragraph{content = State, - props = Color}, - #paragraph{content = App}, - #paragraph{content = [Desc | ExtraLines]}] - end, lists:sort(maps:keys(FeatureFlags))), - io:format("~n", []), - stdout_formatter:display(#table{rows = [TableHeader | Rows], - props = Border#{cell_padding => {0, 1}}}). - -use_colors(Options) -> - maps:get(colors, Options, true). - -use_lines(Options) -> - maps:get(lines, Options, true). - -state_legend(Options) -> - UseColors = use_colors(Options), - {Green, Yellow, Red} = case UseColors of - true -> - {#{fg => green}, - #{fg => yellow}, - #{bold => true, - bg => red}}; - false -> - {#{}, #{}, #{}} - end, - Enabled = #paragraph{content = "Enabled", props = Green}, - Disabled = #paragraph{content = "Disabled", props = Yellow}, - Unavailable = #paragraph{content = "Unavailable", props = Red}, - stdout_formatter:display( - #paragraph{ - content = - ["\n", - "Possible states:\n", - " ", Enabled, ": The feature flag is enabled on all nodes\n", - " ", Disabled, ": The feature flag is disabled on all nodes\n", - " ", Unavailable, ": The feature flag cannot be enabled because" - " one or more nodes do not support it\n"]}). - --spec format_error(any()) -> string(). -%% @doc -%% Formats the error reason term so it can be presented to human beings. -%% -%% @param Reason The term in the `{error, Reason}' tuple. -%% @returns the formatted error reason. - -format_error(Reason) -> - rabbit_misc:format("~p", [Reason]). diff --git a/src/rabbit_ff_registry.erl b/src/rabbit_ff_registry.erl deleted file mode 100644 index 372971f949..0000000000 --- a/src/rabbit_ff_registry.erl +++ /dev/null @@ -1,189 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% @author The RabbitMQ team -%% @copyright 2018-2020 VMware, Inc. or its affiliates. -%% -%% @doc -%% This module exposes the API of the {@link rabbit_feature_flags} -%% registry. The feature flags registry is an Erlang module, compiled at -%% runtime, storing all the informations about feature flags: which are -%% supported, which are enabled, etc. -%% -%% Because it is compiled at runtime, the initial source code is mostly -%% an API reference. What the initial module does is merely ask {@link -%% rabbit_feature_flags} to generate the real registry. - --module(rabbit_ff_registry). - --export([get/1, - list/1, - states/0, - is_supported/1, - is_enabled/1, - is_registry_initialized/0, - is_registry_written_to_disk/0]). - --ifdef(TEST). --on_load(on_load/0). --endif. - --spec get(rabbit_feature_flags:feature_name()) -> - rabbit_feature_flags:feature_props() | undefined. -%% @doc -%% Returns the properties of a feature flag. -%% -%% Only the informations stored in the local registry is used to answer -%% this call. -%% -%% @param FeatureName The name of the feature flag. -%% @returns the properties of the specified feature flag. - -get(FeatureName) -> - rabbit_feature_flags:initialize_registry(), - %% Initially, is_registry_initialized/0 always returns `false` - %% and this ?MODULE:get(FeatureName) is always called. The case - %% statement is here to please Dialyzer. - case is_registry_initialized() of - false -> ?MODULE:get(FeatureName); - true -> undefined - end. - --spec list(all | enabled | disabled) -> rabbit_feature_flags:feature_flags(). -%% @doc -%% Lists all, enabled or disabled feature flags, depending on the argument. -%% -%% Only the informations stored in the local registry is used to answer -%% this call. -%% -%% @param Which The group of feature flags to return: `all', `enabled' or -%% `disabled'. -%% @returns A map of selected feature flags. - -list(Which) -> - rabbit_feature_flags:initialize_registry(), - %% See get/1 for an explanation of the case statement below. - case is_registry_initialized() of - false -> ?MODULE:list(Which); - true -> #{} - end. - --spec states() -> rabbit_feature_flags:feature_states(). -%% @doc -%% Returns the states of supported feature flags. -%% -%% Only the informations stored in the local registry is used to answer -%% this call. -%% -%% @returns A map of feature flag states. - -states() -> - rabbit_feature_flags:initialize_registry(), - %% See get/1 for an explanation of the case statement below. - case is_registry_initialized() of - false -> ?MODULE:states(); - true -> #{} - end. - --spec is_supported(rabbit_feature_flags:feature_name()) -> boolean(). -%% @doc -%% Returns if a feature flag is supported. -%% -%% Only the informations stored in the local registry is used to answer -%% this call. -%% -%% @param FeatureName The name of the feature flag to be checked. -%% @returns `true' if the feature flag is supported, or `false' -%% otherwise. - -is_supported(FeatureName) -> - rabbit_feature_flags:initialize_registry(), - %% See get/1 for an explanation of the case statement below. - case is_registry_initialized() of - false -> ?MODULE:is_supported(FeatureName); - true -> false - end. - --spec is_enabled(rabbit_feature_flags:feature_name()) -> boolean() | state_changing. -%% @doc -%% Returns if a feature flag is supported or if its state is changing. -%% -%% Only the informations stored in the local registry is used to answer -%% this call. -%% -%% @param FeatureName The name of the feature flag to be checked. -%% @returns `true' if the feature flag is supported, `state_changing' if -%% its state is transient, or `false' otherwise. - -is_enabled(FeatureName) -> - rabbit_feature_flags:initialize_registry(), - %% See get/1 for an explanation of the case statement below. - case is_registry_initialized() of - false -> ?MODULE:is_enabled(FeatureName); - true -> false - end. - --spec is_registry_initialized() -> boolean(). -%% @doc -%% Indicates if the registry is initialized. -%% -%% The registry is considered initialized once the initial Erlang module -%% was replaced by the copy compiled at runtime. -%% -%% @returns `true' when the module is the one compiled at runtime, -%% `false' when the module is the initial one compiled from RabbitMQ -%% source code. - -is_registry_initialized() -> - always_return_false(). - --spec is_registry_written_to_disk() -> boolean(). -%% @doc -%% Indicates if the feature flags state was successfully persisted to disk. -%% -%% Note that on startup, {@link rabbit_feature_flags} tries to determine -%% the state of each supported feature flag, regardless of the -%% information on disk, to ensure maximum consistency. However, this can -%% be done for feature flags supporting it only. -%% -%% @returns `true' if the state was successfully written to disk and -%% the registry can be initialized from that during the next RabbitMQ -%% startup, `false' if the write failed and the node might loose feature -%% flags state on restart. - -is_registry_written_to_disk() -> - always_return_true(). - -always_return_true() -> - %% This function is here to trick Dialyzer. We want some functions - %% in this initial on-disk registry to always return `true` or - %% `false`. However the generated registry will return actual - %% booleans. The `-spec()` correctly advertises a return type of - %% `boolean()`. But in the meantime, Dialyzer only knows about this - %% copy which, without the trick below, would always return either - %% `true` (e.g. in is_registry_written_to_disk/0) or `false` (e.g. - %% is_registry_initialized/0). This obviously causes some warnings - %% where the registry functions are used: Dialyzer believes that - %% e.g. matching the return value of is_registry_initialized/0 - %% against `true` will never succeed. - %% - %% That's why this function makes a call which we know the result, - %% but not Dialyzer, to "create" that hard-coded `true` return - %% value. - erlang:get({?MODULE, always_undefined}) =:= undefined. - -always_return_false() -> - not always_return_true(). - --ifdef(TEST). -on_load() -> - _ = (catch rabbit_log_feature_flags:debug( - "Feature flags: Loading initial (uninitialized) registry " - "module (~p)", - [self()])), - ok. --endif. diff --git a/src/rabbit_fhc_helpers.erl b/src/rabbit_fhc_helpers.erl deleted file mode 100644 index d310e84008..0000000000 --- a/src/rabbit_fhc_helpers.erl +++ /dev/null @@ -1,45 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_fhc_helpers). - --export([clear_read_cache/0]). - --include("amqqueue.hrl"). - -clear_read_cache() -> - case application:get_env(rabbit, fhc_read_buffering) of - {ok, true} -> - file_handle_cache:clear_read_cache(), - clear_vhost_read_cache(rabbit_vhost:list_names()); - _ -> %% undefined or {ok, false} - ok - end. - -clear_vhost_read_cache([]) -> - ok; -clear_vhost_read_cache([VHost | Rest]) -> - clear_queue_read_cache(rabbit_amqqueue:list(VHost)), - clear_vhost_read_cache(Rest). - -clear_queue_read_cache([]) -> - ok; -clear_queue_read_cache([Q | Rest]) when ?is_amqqueue(Q) -> - MPid = amqqueue:get_pid(Q), - SPids = amqqueue:get_slave_pids(Q), - %% Limit the action to the current node. - Pids = [P || P <- [MPid | SPids], node(P) =:= node()], - %% This function is executed in the context of the backing queue - %% process because the read buffer is stored in the process - %% dictionary. - Fun = fun(_, State) -> - _ = file_handle_cache:clear_process_read_cache(), - State - end, - [rabbit_amqqueue:run_backing_queue(Pid, rabbit_variable_queue, Fun) - || Pid <- Pids], - clear_queue_read_cache(Rest). diff --git a/src/rabbit_fifo.erl b/src/rabbit_fifo.erl deleted file mode 100644 index 51acfffd0d..0000000000 --- a/src/rabbit_fifo.erl +++ /dev/null @@ -1,2124 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_fifo). - --behaviour(ra_machine). - --compile(inline_list_funcs). --compile(inline). --compile({no_auto_import, [apply/3]}). - --include("rabbit_fifo.hrl"). --include_lib("rabbit_common/include/rabbit.hrl"). - --export([ - init/1, - apply/3, - state_enter/2, - tick/2, - overview/1, - get_checked_out/4, - %% versioning - version/0, - which_module/1, - %% aux - init_aux/1, - handle_aux/6, - % queries - query_messages_ready/1, - query_messages_checked_out/1, - query_messages_total/1, - query_processes/1, - query_ra_indexes/1, - query_consumer_count/1, - query_consumers/1, - query_stat/1, - query_single_active_consumer/1, - query_in_memory_usage/1, - query_peek/2, - usage/1, - - zero/1, - - %% misc - dehydrate_state/1, - normalize/1, - - %% protocol helpers - make_enqueue/3, - make_register_enqueuer/1, - make_checkout/3, - make_settle/2, - make_return/2, - make_discard/2, - make_credit/4, - make_purge/0, - make_purge_nodes/1, - make_update_config/1, - make_garbage_collection/0 - ]). - -%% command records representing all the protocol actions that are supported --record(enqueue, {pid :: option(pid()), - seq :: option(msg_seqno()), - msg :: raw_msg()}). --record(register_enqueuer, {pid :: pid()}). --record(checkout, {consumer_id :: consumer_id(), - spec :: checkout_spec(), - meta :: consumer_meta()}). --record(settle, {consumer_id :: consumer_id(), - msg_ids :: [msg_id()]}). --record(return, {consumer_id :: consumer_id(), - msg_ids :: [msg_id()]}). --record(discard, {consumer_id :: consumer_id(), - msg_ids :: [msg_id()]}). --record(credit, {consumer_id :: consumer_id(), - credit :: non_neg_integer(), - delivery_count :: non_neg_integer(), - drain :: boolean()}). --record(purge, {}). --record(purge_nodes, {nodes :: [node()]}). --record(update_config, {config :: config()}). --record(garbage_collection, {}). - --opaque protocol() :: - #enqueue{} | - #register_enqueuer{} | - #checkout{} | - #settle{} | - #return{} | - #discard{} | - #credit{} | - #purge{} | - #purge_nodes{} | - #update_config{} | - #garbage_collection{}. - --type command() :: protocol() | ra_machine:builtin_command(). -%% all the command types supported by ra fifo - --type client_msg() :: delivery(). -%% the messages `rabbit_fifo' can send to consumers. - --opaque state() :: #?MODULE{}. - --export_type([protocol/0, - delivery/0, - command/0, - credit_mode/0, - consumer_tag/0, - consumer_meta/0, - consumer_id/0, - client_msg/0, - msg/0, - msg_id/0, - msg_seqno/0, - delivery_msg/0, - state/0, - config/0]). - --spec init(config()) -> state(). -init(#{name := Name, - queue_resource := Resource} = Conf) -> - update_config(Conf, #?MODULE{cfg = #cfg{name = Name, - resource = Resource}}). - -update_config(Conf, State) -> - DLH = maps:get(dead_letter_handler, Conf, undefined), - BLH = maps:get(become_leader_handler, Conf, undefined), - RCI = maps:get(release_cursor_interval, Conf, ?RELEASE_CURSOR_EVERY), - Overflow = maps:get(overflow_strategy, Conf, drop_head), - MaxLength = maps:get(max_length, Conf, undefined), - MaxBytes = maps:get(max_bytes, Conf, undefined), - MaxMemoryLength = maps:get(max_in_memory_length, Conf, undefined), - MaxMemoryBytes = maps:get(max_in_memory_bytes, Conf, undefined), - DeliveryLimit = maps:get(delivery_limit, Conf, undefined), - Expires = maps:get(expires, Conf, undefined), - ConsumerStrategy = case maps:get(single_active_consumer_on, Conf, false) of - true -> - single_active; - false -> - competing - end, - Cfg = State#?MODULE.cfg, - RCISpec = {RCI, RCI}, - - LastActive = maps:get(created, Conf, undefined), - State#?MODULE{cfg = Cfg#cfg{release_cursor_interval = RCISpec, - dead_letter_handler = DLH, - become_leader_handler = BLH, - overflow_strategy = Overflow, - max_length = MaxLength, - max_bytes = MaxBytes, - max_in_memory_length = MaxMemoryLength, - max_in_memory_bytes = MaxMemoryBytes, - consumer_strategy = ConsumerStrategy, - delivery_limit = DeliveryLimit, - expires = Expires}, - last_active = LastActive}. - -zero(_) -> - 0. - -% msg_ids are scoped per consumer -% ra_indexes holds all raft indexes for enqueues currently on queue --spec apply(ra_machine:command_meta_data(), command(), state()) -> - {state(), Reply :: term(), ra_machine:effects()} | - {state(), Reply :: term()}. -apply(Meta, #enqueue{pid = From, seq = Seq, - msg = RawMsg}, State00) -> - apply_enqueue(Meta, From, Seq, RawMsg, State00); -apply(_Meta, #register_enqueuer{pid = Pid}, - #?MODULE{enqueuers = Enqueuers0, - cfg = #cfg{overflow_strategy = Overflow}} = State0) -> - - State = case maps:is_key(Pid, Enqueuers0) of - true -> - %% if the enqueuer exits just echo the overflow state - State0; - false -> - State0#?MODULE{enqueuers = Enqueuers0#{Pid => #enqueuer{}}} - end, - Res = case is_over_limit(State) of - true when Overflow == reject_publish -> - reject_publish; - _ -> - ok - end, - {State, Res, [{monitor, process, Pid}]}; -apply(Meta, - #settle{msg_ids = MsgIds, consumer_id = ConsumerId}, - #?MODULE{consumers = Cons0} = State) -> - case Cons0 of - #{ConsumerId := Con0} -> - % need to increment metrics before completing as any snapshot - % states taken need to include them - complete_and_checkout(Meta, MsgIds, ConsumerId, - Con0, [], State); - _ -> - {State, ok} - - end; -apply(Meta, #discard{msg_ids = MsgIds, consumer_id = ConsumerId}, - #?MODULE{consumers = Cons0} = State0) -> - case Cons0 of - #{ConsumerId := Con0} -> - Discarded = maps:with(MsgIds, Con0#consumer.checked_out), - Effects = dead_letter_effects(rejected, Discarded, State0, []), - complete_and_checkout(Meta, MsgIds, ConsumerId, Con0, - Effects, State0); - _ -> - {State0, ok} - end; -apply(Meta, #return{msg_ids = MsgIds, consumer_id = ConsumerId}, - #?MODULE{consumers = Cons0} = State) -> - case Cons0 of - #{ConsumerId := #consumer{checked_out = Checked0}} -> - Returned = maps:with(MsgIds, Checked0), - return(Meta, ConsumerId, Returned, [], State); - _ -> - {State, ok} - end; -apply(Meta, #credit{credit = NewCredit, delivery_count = RemoteDelCnt, - drain = Drain, consumer_id = ConsumerId}, - #?MODULE{consumers = Cons0, - service_queue = ServiceQueue0, - waiting_consumers = Waiting0} = State0) -> - case Cons0 of - #{ConsumerId := #consumer{delivery_count = DelCnt} = Con0} -> - %% this can go below 0 when credit is reduced - C = max(0, RemoteDelCnt + NewCredit - DelCnt), - %% grant the credit - Con1 = Con0#consumer{credit = C}, - ServiceQueue = maybe_queue_consumer(ConsumerId, Con1, - ServiceQueue0), - Cons = maps:put(ConsumerId, Con1, Cons0), - {State1, ok, Effects} = - checkout(Meta, State0, - State0#?MODULE{service_queue = ServiceQueue, - consumers = Cons}, []), - Response = {send_credit_reply, messages_ready(State1)}, - %% by this point all checkouts for the updated credit value - %% should be processed so we can evaluate the drain - case Drain of - false -> - %% just return the result of the checkout - {State1, Response, Effects}; - true -> - Con = #consumer{credit = PostCred} = - maps:get(ConsumerId, State1#?MODULE.consumers), - %% add the outstanding credit to the delivery count - DeliveryCount = Con#consumer.delivery_count + PostCred, - Consumers = maps:put(ConsumerId, - Con#consumer{delivery_count = DeliveryCount, - credit = 0}, - State1#?MODULE.consumers), - Drained = Con#consumer.credit, - {CTag, _} = ConsumerId, - {State1#?MODULE{consumers = Consumers}, - %% returning a multi response with two client actions - %% for the channel to execute - {multi, [Response, {send_drained, {CTag, Drained}}]}, - Effects} - end; - _ when Waiting0 /= [] -> - %% there are waiting consuemrs - case lists:keytake(ConsumerId, 1, Waiting0) of - {value, {_, Con0 = #consumer{delivery_count = DelCnt}}, Waiting} -> - %% the consumer is a waiting one - %% grant the credit - C = max(0, RemoteDelCnt + NewCredit - DelCnt), - Con = Con0#consumer{credit = C}, - State = State0#?MODULE{waiting_consumers = - [{ConsumerId, Con} | Waiting]}, - {State, {send_credit_reply, messages_ready(State)}}; - false -> - {State0, ok} - end; - _ -> - %% credit for unknown consumer - just ignore - {State0, ok} - end; -apply(_, #checkout{spec = {dequeue, _}}, - #?MODULE{cfg = #cfg{consumer_strategy = single_active}} = State0) -> - {State0, {error, {unsupported, single_active_consumer}}}; -apply(#{index := Index, - system_time := Ts, - from := From} = Meta, #checkout{spec = {dequeue, Settlement}, - meta = ConsumerMeta, - consumer_id = ConsumerId}, - #?MODULE{consumers = Consumers} = State00) -> - %% dequeue always updates last_active - State0 = State00#?MODULE{last_active = Ts}, - %% all dequeue operations result in keeping the queue from expiring - Exists = maps:is_key(ConsumerId, Consumers), - case messages_ready(State0) of - 0 -> - {State0, {dequeue, empty}}; - _ when Exists -> - %% a dequeue using the same consumer_id isn't possible at this point - {State0, {dequeue, empty}}; - Ready -> - State1 = update_consumer(ConsumerId, ConsumerMeta, - {once, 1, simple_prefetch}, 0, - State0), - {success, _, MsgId, Msg, State2} = checkout_one(Meta, State1), - {State4, Effects1} = case Settlement of - unsettled -> - {_, Pid} = ConsumerId, - {State2, [{monitor, process, Pid}]}; - settled -> - %% immediately settle the checkout - {State3, _, Effects0} = - apply(Meta, make_settle(ConsumerId, [MsgId]), - State2), - {State3, Effects0} - end, - {Reply, Effects2} = - case Msg of - {RaftIdx, {Header, empty}} -> - %% TODO add here new log effect with reply - {'$ra_no_reply', - [reply_log_effect(RaftIdx, MsgId, Header, Ready - 1, From) | - Effects1]}; - _ -> - {{dequeue, {MsgId, Msg}, Ready-1}, Effects1} - - end, - - case evaluate_limit(Index, false, State0, State4, Effects2) of - {State, true, Effects} -> - update_smallest_raft_index(Index, Reply, State, Effects); - {State, false, Effects} -> - {State, Reply, Effects} - end - end; -apply(Meta, #checkout{spec = cancel, consumer_id = ConsumerId}, State0) -> - {State, Effects} = cancel_consumer(Meta, ConsumerId, State0, [], - consumer_cancel), - checkout(Meta, State0, State, Effects); -apply(Meta, #checkout{spec = Spec, meta = ConsumerMeta, - consumer_id = {_, Pid} = ConsumerId}, - State0) -> - Priority = get_priority_from_args(ConsumerMeta), - State1 = update_consumer(ConsumerId, ConsumerMeta, Spec, Priority, State0), - checkout(Meta, State0, State1, [{monitor, process, Pid}]); -apply(#{index := Index}, #purge{}, - #?MODULE{ra_indexes = Indexes0, - returns = Returns, - messages = Messages} = State0) -> - Total = messages_ready(State0), - Indexes1 = lists:foldl(fun rabbit_fifo_index:delete/2, Indexes0, - [I || {_, {I, _}} <- lqueue:to_list(Messages)]), - Indexes = lists:foldl(fun rabbit_fifo_index:delete/2, Indexes1, - [I || {_, {I, _}} <- lqueue:to_list(Returns)]), - - State1 = State0#?MODULE{ra_indexes = Indexes, - messages = lqueue:new(), - returns = lqueue:new(), - msg_bytes_enqueue = 0, - prefix_msgs = {0, [], 0, []}, - msg_bytes_in_memory = 0, - msgs_ready_in_memory = 0}, - Effects0 = [garbage_collection], - Reply = {purge, Total}, - {State, _, Effects} = evaluate_limit(Index, false, State0, - State1, Effects0), - update_smallest_raft_index(Index, Reply, State, Effects); -apply(_Meta, #garbage_collection{}, State) -> - {State, ok, [{aux, garbage_collection}]}; -apply(#{system_time := Ts} = Meta, {down, Pid, noconnection}, - #?MODULE{consumers = Cons0, - cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = Waiting0, - enqueuers = Enqs0} = State0) -> - Node = node(Pid), - %% if the pid refers to an active or cancelled consumer, - %% mark it as suspected and return it to the waiting queue - {State1, Effects0} = - maps:fold(fun({_, P} = Cid, C0, {S0, E0}) - when node(P) =:= Node -> - %% the consumer should be returned to waiting - %% and checked out messages should be returned - Effs = consumer_update_active_effects( - S0, Cid, C0, false, suspected_down, E0), - Checked = C0#consumer.checked_out, - Credit = increase_credit(C0, maps:size(Checked)), - {St, Effs1} = return_all(Meta, S0, Effs, - Cid, C0#consumer{credit = Credit}), - %% if the consumer was cancelled there is a chance it got - %% removed when returning hence we need to be defensive here - Waiting = case St#?MODULE.consumers of - #{Cid := C} -> - Waiting0 ++ [{Cid, C}]; - _ -> - Waiting0 - end, - {St#?MODULE{consumers = maps:remove(Cid, St#?MODULE.consumers), - waiting_consumers = Waiting, - last_active = Ts}, - Effs1}; - (_, _, S) -> - S - end, {State0, []}, Cons0), - WaitingConsumers = update_waiting_consumer_status(Node, State1, - suspected_down), - - %% select a new consumer from the waiting queue and run a checkout - State2 = State1#?MODULE{waiting_consumers = WaitingConsumers}, - {State, Effects1} = activate_next_consumer(State2, Effects0), - - %% mark any enquers as suspected - Enqs = maps:map(fun(P, E) when node(P) =:= Node -> - E#enqueuer{status = suspected_down}; - (_, E) -> E - end, Enqs0), - Effects = [{monitor, node, Node} | Effects1], - checkout(Meta, State0, State#?MODULE{enqueuers = Enqs}, Effects); -apply(#{system_time := Ts} = Meta, {down, Pid, noconnection}, - #?MODULE{consumers = Cons0, - enqueuers = Enqs0} = State0) -> - %% A node has been disconnected. This doesn't necessarily mean that - %% any processes on this node are down, they _may_ come back so here - %% we just mark them as suspected (effectively deactivated) - %% and return all checked out messages to the main queue for delivery to any - %% live consumers - %% - %% all pids for the disconnected node will be marked as suspected not just - %% the one we got the `down' command for - Node = node(Pid), - - {State, Effects1} = - maps:fold( - fun({_, P} = Cid, #consumer{checked_out = Checked0, - status = up} = C0, - {St0, Eff}) when node(P) =:= Node -> - Credit = increase_credit(C0, map_size(Checked0)), - C = C0#consumer{status = suspected_down, - credit = Credit}, - {St, Eff0} = return_all(Meta, St0, Eff, Cid, C), - Eff1 = consumer_update_active_effects(St, Cid, C, false, - suspected_down, Eff0), - {St, Eff1}; - (_, _, {St, Eff}) -> - {St, Eff} - end, {State0, []}, Cons0), - Enqs = maps:map(fun(P, E) when node(P) =:= Node -> - E#enqueuer{status = suspected_down}; - (_, E) -> E - end, Enqs0), - - % Monitor the node so that we can "unsuspect" these processes when the node - % comes back, then re-issue all monitors and discover the final fate of - % these processes - Effects = case maps:size(State#?MODULE.consumers) of - 0 -> - [{aux, inactive}, {monitor, node, Node}]; - _ -> - [{monitor, node, Node}] - end ++ Effects1, - checkout(Meta, State0, State#?MODULE{enqueuers = Enqs, - last_active = Ts}, Effects); -apply(Meta, {down, Pid, _Info}, State0) -> - {State, Effects} = handle_down(Meta, Pid, State0), - checkout(Meta, State0, State, Effects); -apply(Meta, {nodeup, Node}, #?MODULE{consumers = Cons0, - enqueuers = Enqs0, - service_queue = _SQ0} = State0) -> - %% A node we are monitoring has come back. - %% If we have suspected any processes of being - %% down we should now re-issue the monitors for them to detect if they're - %% actually down or not - Monitors = [{monitor, process, P} - || P <- suspected_pids_for(Node, State0)], - - Enqs1 = maps:map(fun(P, E) when node(P) =:= Node -> - E#enqueuer{status = up}; - (_, E) -> E - end, Enqs0), - ConsumerUpdateActiveFun = consumer_active_flag_update_function(State0), - %% mark all consumers as up - {State1, Effects1} = - maps:fold(fun({_, P} = ConsumerId, C, {SAcc, EAcc}) - when (node(P) =:= Node) and - (C#consumer.status =/= cancelled) -> - EAcc1 = ConsumerUpdateActiveFun(SAcc, ConsumerId, - C, true, up, EAcc), - {update_or_remove_sub(Meta, ConsumerId, - C#consumer{status = up}, - SAcc), EAcc1}; - (_, _, Acc) -> - Acc - end, {State0, Monitors}, Cons0), - Waiting = update_waiting_consumer_status(Node, State1, up), - State2 = State1#?MODULE{ - enqueuers = Enqs1, - waiting_consumers = Waiting}, - {State, Effects} = activate_next_consumer(State2, Effects1), - checkout(Meta, State0, State, Effects); -apply(_, {nodedown, _Node}, State) -> - {State, ok}; -apply(Meta, #purge_nodes{nodes = Nodes}, State0) -> - {State, Effects} = lists:foldl(fun(Node, {S, E}) -> - purge_node(Meta, Node, S, E) - end, {State0, []}, Nodes), - {State, ok, Effects}; -apply(Meta, #update_config{config = Conf}, State) -> - checkout(Meta, State, update_config(Conf, State), []); -apply(_Meta, {machine_version, 0, 1}, V0State) -> - State = convert_v0_to_v1(V0State), - {State, ok, []}. - -convert_v0_to_v1(V0State0) -> - V0State = rabbit_fifo_v0:normalize_for_v1(V0State0), - V0Msgs = rabbit_fifo_v0:get_field(messages, V0State), - V1Msgs = lqueue:from_list(lists:sort(maps:to_list(V0Msgs))), - V0Enqs = rabbit_fifo_v0:get_field(enqueuers, V0State), - V1Enqs = maps:map( - fun (_EPid, E) -> - #enqueuer{next_seqno = element(2, E), - pending = element(3, E), - status = element(4, E)} - end, V0Enqs), - V0Cons = rabbit_fifo_v0:get_field(consumers, V0State), - V1Cons = maps:map( - fun (_CId, C0) -> - %% add the priority field - list_to_tuple(tuple_to_list(C0) ++ [0]) - end, V0Cons), - V0SQ = rabbit_fifo_v0:get_field(service_queue, V0State), - V1SQ = priority_queue:from_list(queue:to_list(V0SQ)), - Cfg = #cfg{name = rabbit_fifo_v0:get_cfg_field(name, V0State), - resource = rabbit_fifo_v0:get_cfg_field(resource, V0State), - release_cursor_interval = rabbit_fifo_v0:get_cfg_field(release_cursor_interval, V0State), - dead_letter_handler = rabbit_fifo_v0:get_cfg_field(dead_letter_handler, V0State), - become_leader_handler = rabbit_fifo_v0:get_cfg_field(become_leader_handler, V0State), - %% TODO: what if policy enabling reject_publish was applied before conversion? - overflow_strategy = drop_head, - max_length = rabbit_fifo_v0:get_cfg_field(max_length, V0State), - max_bytes = rabbit_fifo_v0:get_cfg_field(max_bytes, V0State), - consumer_strategy = rabbit_fifo_v0:get_cfg_field(consumer_strategy, V0State), - delivery_limit = rabbit_fifo_v0:get_cfg_field(delivery_limit, V0State), - max_in_memory_length = rabbit_fifo_v0:get_cfg_field(max_in_memory_length, V0State), - max_in_memory_bytes = rabbit_fifo_v0:get_cfg_field(max_in_memory_bytes, V0State) - }, - - #?MODULE{cfg = Cfg, - messages = V1Msgs, - next_msg_num = rabbit_fifo_v0:get_field(next_msg_num, V0State), - returns = rabbit_fifo_v0:get_field(returns, V0State), - enqueue_count = rabbit_fifo_v0:get_field(enqueue_count, V0State), - enqueuers = V1Enqs, - ra_indexes = rabbit_fifo_v0:get_field(ra_indexes, V0State), - release_cursors = rabbit_fifo_v0:get_field(release_cursors, V0State), - consumers = V1Cons, - service_queue = V1SQ, - prefix_msgs = rabbit_fifo_v0:get_field(prefix_msgs, V0State), - msg_bytes_enqueue = rabbit_fifo_v0:get_field(msg_bytes_enqueue, V0State), - msg_bytes_checkout = rabbit_fifo_v0:get_field(msg_bytes_checkout, V0State), - waiting_consumers = rabbit_fifo_v0:get_field(waiting_consumers, V0State), - msg_bytes_in_memory = rabbit_fifo_v0:get_field(msg_bytes_in_memory, V0State), - msgs_ready_in_memory = rabbit_fifo_v0:get_field(msgs_ready_in_memory, V0State) - }. - -purge_node(Meta, Node, State, Effects) -> - lists:foldl(fun(Pid, {S0, E0}) -> - {S, E} = handle_down(Meta, Pid, S0), - {S, E0 ++ E} - end, {State, Effects}, all_pids_for(Node, State)). - -%% any downs that re not noconnection -handle_down(Meta, Pid, #?MODULE{consumers = Cons0, - enqueuers = Enqs0} = State0) -> - % Remove any enqueuer for the same pid and enqueue any pending messages - % This should be ok as we won't see any more enqueues from this pid - State1 = case maps:take(Pid, Enqs0) of - {#enqueuer{pending = Pend}, Enqs} -> - lists:foldl(fun ({_, RIdx, RawMsg}, S) -> - enqueue(RIdx, RawMsg, S) - end, State0#?MODULE{enqueuers = Enqs}, Pend); - error -> - State0 - end, - {Effects1, State2} = handle_waiting_consumer_down(Pid, State1), - % return checked out messages to main queue - % Find the consumers for the down pid - DownConsumers = maps:keys( - maps:filter(fun({_, P}, _) -> P =:= Pid end, Cons0)), - lists:foldl(fun(ConsumerId, {S, E}) -> - cancel_consumer(Meta, ConsumerId, S, E, down) - end, {State2, Effects1}, DownConsumers). - -consumer_active_flag_update_function(#?MODULE{cfg = #cfg{consumer_strategy = competing}}) -> - fun(State, ConsumerId, Consumer, Active, ActivityStatus, Effects) -> - consumer_update_active_effects(State, ConsumerId, Consumer, Active, - ActivityStatus, Effects) - end; -consumer_active_flag_update_function(#?MODULE{cfg = #cfg{consumer_strategy = single_active}}) -> - fun(_, _, _, _, _, Effects) -> - Effects - end. - -handle_waiting_consumer_down(_Pid, - #?MODULE{cfg = #cfg{consumer_strategy = competing}} = State) -> - {[], State}; -handle_waiting_consumer_down(_Pid, - #?MODULE{cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = []} = State) -> - {[], State}; -handle_waiting_consumer_down(Pid, - #?MODULE{cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = WaitingConsumers0} = State0) -> - % get cancel effects for down waiting consumers - Down = lists:filter(fun({{_, P}, _}) -> P =:= Pid end, - WaitingConsumers0), - Effects = lists:foldl(fun ({ConsumerId, _}, Effects) -> - cancel_consumer_effects(ConsumerId, State0, - Effects) - end, [], Down), - % update state to have only up waiting consumers - StillUp = lists:filter(fun({{_, P}, _}) -> P =/= Pid end, - WaitingConsumers0), - State = State0#?MODULE{waiting_consumers = StillUp}, - {Effects, State}. - -update_waiting_consumer_status(Node, - #?MODULE{waiting_consumers = WaitingConsumers}, - Status) -> - [begin - case node(Pid) of - Node -> - {ConsumerId, Consumer#consumer{status = Status}}; - _ -> - {ConsumerId, Consumer} - end - end || {{_, Pid} = ConsumerId, Consumer} <- WaitingConsumers, - Consumer#consumer.status =/= cancelled]. - --spec state_enter(ra_server:ra_state(), state()) -> ra_machine:effects(). -state_enter(leader, #?MODULE{consumers = Cons, - enqueuers = Enqs, - waiting_consumers = WaitingConsumers, - cfg = #cfg{name = Name, - resource = Resource, - become_leader_handler = BLH}, - prefix_msgs = {0, [], 0, []} - }) -> - % return effects to monitor all current consumers and enqueuers - Pids = lists:usort(maps:keys(Enqs) - ++ [P || {_, P} <- maps:keys(Cons)] - ++ [P || {{_, P}, _} <- WaitingConsumers]), - Mons = [{monitor, process, P} || P <- Pids], - Nots = [{send_msg, P, leader_change, ra_event} || P <- Pids], - NodeMons = lists:usort([{monitor, node, node(P)} || P <- Pids]), - FHReservation = [{mod_call, rabbit_quorum_queue, file_handle_leader_reservation, [Resource]}], - Effects = Mons ++ Nots ++ NodeMons ++ FHReservation, - case BLH of - undefined -> - Effects; - {Mod, Fun, Args} -> - [{mod_call, Mod, Fun, Args ++ [Name]} | Effects] - end; -state_enter(eol, #?MODULE{enqueuers = Enqs, - consumers = Custs0, - waiting_consumers = WaitingConsumers0}) -> - Custs = maps:fold(fun({_, P}, V, S) -> S#{P => V} end, #{}, Custs0), - WaitingConsumers1 = lists:foldl(fun({{_, P}, V}, Acc) -> Acc#{P => V} end, - #{}, WaitingConsumers0), - AllConsumers = maps:merge(Custs, WaitingConsumers1), - [{send_msg, P, eol, ra_event} - || P <- maps:keys(maps:merge(Enqs, AllConsumers))] ++ - [{mod_call, rabbit_quorum_queue, file_handle_release_reservation, []}]; -state_enter(State, #?MODULE{cfg = #cfg{resource = _Resource}}) when State =/= leader -> - FHReservation = {mod_call, rabbit_quorum_queue, file_handle_other_reservation, []}, - [FHReservation]; - state_enter(_, _) -> - %% catch all as not handling all states - []. - - --spec tick(non_neg_integer(), state()) -> ra_machine:effects(). -tick(Ts, #?MODULE{cfg = #cfg{name = Name, - resource = QName}, - msg_bytes_enqueue = EnqueueBytes, - msg_bytes_checkout = CheckoutBytes} = State) -> - case is_expired(Ts, State) of - true -> - [{mod_call, rabbit_quorum_queue, spawn_deleter, [QName]}]; - false -> - Metrics = {Name, - messages_ready(State), - num_checked_out(State), % checked out - messages_total(State), - query_consumer_count(State), % Consumers - EnqueueBytes, - CheckoutBytes}, - [{mod_call, rabbit_quorum_queue, - handle_tick, [QName, Metrics, all_nodes(State)]}] - end. - --spec overview(state()) -> map(). -overview(#?MODULE{consumers = Cons, - enqueuers = Enqs, - release_cursors = Cursors, - enqueue_count = EnqCount, - msg_bytes_enqueue = EnqueueBytes, - msg_bytes_checkout = CheckoutBytes, - cfg = Cfg} = State) -> - Conf = #{name => Cfg#cfg.name, - resource => Cfg#cfg.resource, - release_cursor_interval => Cfg#cfg.release_cursor_interval, - dead_lettering_enabled => undefined =/= Cfg#cfg.dead_letter_handler, - max_length => Cfg#cfg.max_length, - max_bytes => Cfg#cfg.max_bytes, - consumer_strategy => Cfg#cfg.consumer_strategy, - max_in_memory_length => Cfg#cfg.max_in_memory_length, - max_in_memory_bytes => Cfg#cfg.max_in_memory_bytes, - expires => Cfg#cfg.expires, - delivery_limit => Cfg#cfg.delivery_limit - }, - #{type => ?MODULE, - config => Conf, - num_consumers => maps:size(Cons), - num_checked_out => num_checked_out(State), - num_enqueuers => maps:size(Enqs), - num_ready_messages => messages_ready(State), - num_messages => messages_total(State), - num_release_cursors => lqueue:len(Cursors), - release_cursors => [I || {_, I, _} <- lqueue:to_list(Cursors)], - release_cursor_enqueue_counter => EnqCount, - enqueue_message_bytes => EnqueueBytes, - checkout_message_bytes => CheckoutBytes}. - --spec get_checked_out(consumer_id(), msg_id(), msg_id(), state()) -> - [delivery_msg()]. -get_checked_out(Cid, From, To, #?MODULE{consumers = Consumers}) -> - case Consumers of - #{Cid := #consumer{checked_out = Checked}} -> - [{K, snd(snd(maps:get(K, Checked)))} - || K <- lists:seq(From, To), - maps:is_key(K, Checked)]; - _ -> - [] - end. - --spec version() -> pos_integer(). -version() -> 1. - -which_module(0) -> rabbit_fifo_v0; -which_module(1) -> ?MODULE. - --record(aux_gc, {last_raft_idx = 0 :: ra:index()}). --record(aux, {name :: atom(), - utilisation :: term(), - gc = #aux_gc{} :: #aux_gc{}}). - -init_aux(Name) when is_atom(Name) -> - %% TODO: catch specific exception throw if table already exists - ok = ra_machine_ets:create_table(rabbit_fifo_usage, - [named_table, set, public, - {write_concurrency, true}]), - Now = erlang:monotonic_time(micro_seconds), - #aux{name = Name, - utilisation = {inactive, Now, 1, 1.0}}. - -handle_aux(leader, _, garbage_collection, State, Log, _MacState) -> - ra_log_wal:force_roll_over(ra_log_wal), - {no_reply, State, Log}; -handle_aux(follower, _, garbage_collection, State, Log, MacState) -> - ra_log_wal:force_roll_over(ra_log_wal), - {no_reply, force_eval_gc(Log, MacState, State), Log}; -handle_aux(_RaState, cast, eval, Aux0, Log, _MacState) -> - {no_reply, Aux0, Log}; -handle_aux(_RaState, cast, Cmd, #aux{utilisation = Use0} = Aux0, - Log, _MacState) - when Cmd == active orelse Cmd == inactive -> - {no_reply, Aux0#aux{utilisation = update_use(Use0, Cmd)}, Log}; -handle_aux(_RaState, cast, tick, #aux{name = Name, - utilisation = Use0} = State0, - Log, MacState) -> - true = ets:insert(rabbit_fifo_usage, - {Name, utilisation(Use0)}), - Aux = eval_gc(Log, MacState, State0), - {no_reply, Aux, Log}; -handle_aux(_RaState, {call, _From}, {peek, Pos}, Aux0, - Log0, MacState) -> - case rabbit_fifo:query_peek(Pos, MacState) of - {ok, {Idx, {Header, empty}}} -> - %% need to re-hydrate from the log - {{_, _, {_, _, Cmd, _}}, Log} = ra_log:fetch(Idx, Log0), - #enqueue{msg = Msg} = Cmd, - {reply, {ok, {Header, Msg}}, Aux0, Log}; - {ok, {_Idx, {Header, Msg}}} -> - {reply, {ok, {Header, Msg}}, Aux0, Log0}; - Err -> - {reply, Err, Aux0, Log0} - end. - - -eval_gc(Log, #?MODULE{cfg = #cfg{resource = QR}} = MacState, - #aux{gc = #aux_gc{last_raft_idx = LastGcIdx} = Gc} = AuxState) -> - {Idx, _} = ra_log:last_index_term(Log), - {memory, Mem} = erlang:process_info(self(), memory), - case messages_total(MacState) of - 0 when Idx > LastGcIdx andalso - Mem > ?GC_MEM_LIMIT_B -> - garbage_collect(), - {memory, MemAfter} = erlang:process_info(self(), memory), - rabbit_log:debug("~s: full GC sweep complete. " - "Process memory changed from ~.2fMB to ~.2fMB.", - [rabbit_misc:rs(QR), Mem/?MB, MemAfter/?MB]), - AuxState#aux{gc = Gc#aux_gc{last_raft_idx = Idx}}; - _ -> - AuxState - end. - -force_eval_gc(Log, #?MODULE{cfg = #cfg{resource = QR}}, - #aux{gc = #aux_gc{last_raft_idx = LastGcIdx} = Gc} = AuxState) -> - {Idx, _} = ra_log:last_index_term(Log), - {memory, Mem} = erlang:process_info(self(), memory), - case Idx > LastGcIdx of - true -> - garbage_collect(), - {memory, MemAfter} = erlang:process_info(self(), memory), - rabbit_log:debug("~s: full GC sweep complete. " - "Process memory changed from ~.2fMB to ~.2fMB.", - [rabbit_misc:rs(QR), Mem/?MB, MemAfter/?MB]), - AuxState#aux{gc = Gc#aux_gc{last_raft_idx = Idx}}; - false -> - AuxState - end. - -%%% Queries - -query_messages_ready(State) -> - messages_ready(State). - -query_messages_checked_out(#?MODULE{consumers = Consumers}) -> - maps:fold(fun (_, #consumer{checked_out = C}, S) -> - maps:size(C) + S - end, 0, Consumers). - -query_messages_total(State) -> - messages_total(State). - -query_processes(#?MODULE{enqueuers = Enqs, consumers = Cons0}) -> - Cons = maps:fold(fun({_, P}, V, S) -> S#{P => V} end, #{}, Cons0), - maps:keys(maps:merge(Enqs, Cons)). - - -query_ra_indexes(#?MODULE{ra_indexes = RaIndexes}) -> - RaIndexes. - -query_consumer_count(#?MODULE{consumers = Consumers, - waiting_consumers = WaitingConsumers}) -> - Up = maps:filter(fun(_ConsumerId, #consumer{status = Status}) -> - Status =/= suspected_down - end, Consumers), - maps:size(Up) + length(WaitingConsumers). - -query_consumers(#?MODULE{consumers = Consumers, - waiting_consumers = WaitingConsumers, - cfg = #cfg{consumer_strategy = ConsumerStrategy}} = State) -> - ActiveActivityStatusFun = - case ConsumerStrategy of - competing -> - fun(_ConsumerId, - #consumer{status = Status}) -> - case Status of - suspected_down -> - {false, Status}; - _ -> - {true, Status} - end - end; - single_active -> - SingleActiveConsumer = query_single_active_consumer(State), - fun({Tag, Pid} = _Consumer, _) -> - case SingleActiveConsumer of - {value, {Tag, Pid}} -> - {true, single_active}; - _ -> - {false, waiting} - end - end - end, - FromConsumers = - maps:fold(fun (_, #consumer{status = cancelled}, Acc) -> - Acc; - ({Tag, Pid}, #consumer{meta = Meta} = Consumer, Acc) -> - {Active, ActivityStatus} = - ActiveActivityStatusFun({Tag, Pid}, Consumer), - maps:put({Tag, Pid}, - {Pid, Tag, - maps:get(ack, Meta, undefined), - maps:get(prefetch, Meta, undefined), - Active, - ActivityStatus, - maps:get(args, Meta, []), - maps:get(username, Meta, undefined)}, - Acc) - end, #{}, Consumers), - FromWaitingConsumers = - lists:foldl(fun ({_, #consumer{status = cancelled}}, Acc) -> - Acc; - ({{Tag, Pid}, #consumer{meta = Meta} = Consumer}, Acc) -> - {Active, ActivityStatus} = - ActiveActivityStatusFun({Tag, Pid}, Consumer), - maps:put({Tag, Pid}, - {Pid, Tag, - maps:get(ack, Meta, undefined), - maps:get(prefetch, Meta, undefined), - Active, - ActivityStatus, - maps:get(args, Meta, []), - maps:get(username, Meta, undefined)}, - Acc) - end, #{}, WaitingConsumers), - maps:merge(FromConsumers, FromWaitingConsumers). - - -query_single_active_consumer(#?MODULE{cfg = #cfg{consumer_strategy = single_active}, - consumers = Consumers}) -> - case maps:size(Consumers) of - 0 -> - {error, no_value}; - 1 -> - {value, lists:nth(1, maps:keys(Consumers))}; - _ - -> - {error, illegal_size} - end ; -query_single_active_consumer(_) -> - disabled. - -query_stat(#?MODULE{consumers = Consumers} = State) -> - {messages_ready(State), maps:size(Consumers)}. - -query_in_memory_usage(#?MODULE{msg_bytes_in_memory = Bytes, - msgs_ready_in_memory = Length}) -> - {Length, Bytes}. - -query_peek(Pos, State0) when Pos > 0 -> - case take_next_msg(State0) of - empty -> - {error, no_message_at_pos}; - {{_Seq, IdxMsg}, _State} - when Pos == 1 -> - {ok, IdxMsg}; - {_Msg, State} -> - query_peek(Pos-1, State) - end. - - --spec usage(atom()) -> float(). -usage(Name) when is_atom(Name) -> - case ets:lookup(rabbit_fifo_usage, Name) of - [] -> 0.0; - [{_, Use}] -> Use - end. - -%%% Internal - -messages_ready(#?MODULE{messages = M, - prefix_msgs = {RCnt, _R, PCnt, _P}, - returns = R}) -> - %% prefix messages will rarely have anything in them during normal - %% operations so length/1 is fine here - lqueue:len(M) + lqueue:len(R) + RCnt + PCnt. - -messages_total(#?MODULE{ra_indexes = I, - prefix_msgs = {RCnt, _R, PCnt, _P}}) -> - rabbit_fifo_index:size(I) + RCnt + PCnt. - -update_use({inactive, _, _, _} = CUInfo, inactive) -> - CUInfo; -update_use({active, _, _} = CUInfo, active) -> - CUInfo; -update_use({active, Since, Avg}, inactive) -> - Now = erlang:monotonic_time(micro_seconds), - {inactive, Now, Now - Since, Avg}; -update_use({inactive, Since, Active, Avg}, active) -> - Now = erlang:monotonic_time(micro_seconds), - {active, Now, use_avg(Active, Now - Since, Avg)}. - -utilisation({active, Since, Avg}) -> - use_avg(erlang:monotonic_time(micro_seconds) - Since, 0, Avg); -utilisation({inactive, Since, Active, Avg}) -> - use_avg(Active, erlang:monotonic_time(micro_seconds) - Since, Avg). - -use_avg(0, 0, Avg) -> - Avg; -use_avg(Active, Inactive, Avg) -> - Time = Inactive + Active, - moving_average(Time, ?USE_AVG_HALF_LIFE, Active / Time, Avg). - -moving_average(_Time, _, Next, undefined) -> - Next; -moving_average(Time, HalfLife, Next, Current) -> - Weight = math:exp(Time * math:log(0.5) / HalfLife), - Next * (1 - Weight) + Current * Weight. - -num_checked_out(#?MODULE{consumers = Cons}) -> - maps:fold(fun (_, #consumer{checked_out = C}, Acc) -> - maps:size(C) + Acc - end, 0, Cons). - -cancel_consumer(Meta, ConsumerId, - #?MODULE{cfg = #cfg{consumer_strategy = competing}} = State, - Effects, Reason) -> - cancel_consumer0(Meta, ConsumerId, State, Effects, Reason); -cancel_consumer(Meta, ConsumerId, - #?MODULE{cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = []} = State, - Effects, Reason) -> - %% single active consumer on, no consumers are waiting - cancel_consumer0(Meta, ConsumerId, State, Effects, Reason); -cancel_consumer(Meta, ConsumerId, - #?MODULE{consumers = Cons0, - cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = Waiting0} = State0, - Effects0, Reason) -> - %% single active consumer on, consumers are waiting - case maps:is_key(ConsumerId, Cons0) of - true -> - % The active consumer is to be removed - {State1, Effects1} = cancel_consumer0(Meta, ConsumerId, State0, - Effects0, Reason), - activate_next_consumer(State1, Effects1); - false -> - % The cancelled consumer is not active or cancelled - % Just remove it from idle_consumers - Waiting = lists:keydelete(ConsumerId, 1, Waiting0), - Effects = cancel_consumer_effects(ConsumerId, State0, Effects0), - % A waiting consumer isn't supposed to have any checked out messages, - % so nothing special to do here - {State0#?MODULE{waiting_consumers = Waiting}, Effects} - end. - -consumer_update_active_effects(#?MODULE{cfg = #cfg{resource = QName}}, - ConsumerId, #consumer{meta = Meta}, - Active, ActivityStatus, - Effects) -> - Ack = maps:get(ack, Meta, undefined), - Prefetch = maps:get(prefetch, Meta, undefined), - Args = maps:get(args, Meta, []), - [{mod_call, rabbit_quorum_queue, update_consumer_handler, - [QName, ConsumerId, false, Ack, Prefetch, Active, ActivityStatus, Args]} - | Effects]. - -cancel_consumer0(Meta, ConsumerId, - #?MODULE{consumers = C0} = S0, Effects0, Reason) -> - case C0 of - #{ConsumerId := Consumer} -> - {S, Effects2} = maybe_return_all(Meta, ConsumerId, Consumer, - S0, Effects0, Reason), - %% The effects are emitted before the consumer is actually removed - %% if the consumer has unacked messages. This is a bit weird but - %% in line with what classic queues do (from an external point of - %% view) - Effects = cancel_consumer_effects(ConsumerId, S, Effects2), - case maps:size(S#?MODULE.consumers) of - 0 -> - {S, [{aux, inactive} | Effects]}; - _ -> - {S, Effects} - end; - _ -> - %% already removed: do nothing - {S0, Effects0} - end. - -activate_next_consumer(#?MODULE{consumers = Cons, - waiting_consumers = Waiting0} = State0, - Effects0) -> - case maps:filter(fun (_, #consumer{status = S}) -> S == up end, Cons) of - Up when map_size(Up) == 0 -> - %% there are no active consumer in the consumer map - case lists:filter(fun ({_, #consumer{status = Status}}) -> - Status == up - end, Waiting0) of - [{NextConsumerId, NextConsumer} | _] -> - %% there is a potential next active consumer - Remaining = lists:keydelete(NextConsumerId, 1, Waiting0), - #?MODULE{service_queue = ServiceQueue} = State0, - ServiceQueue1 = maybe_queue_consumer(NextConsumerId, - NextConsumer, - ServiceQueue), - State = State0#?MODULE{consumers = Cons#{NextConsumerId => NextConsumer}, - service_queue = ServiceQueue1, - waiting_consumers = Remaining}, - Effects = consumer_update_active_effects(State, NextConsumerId, - NextConsumer, true, - single_active, Effects0), - {State, Effects}; - [] -> - {State0, [{aux, inactive} | Effects0]} - end; - _ -> - {State0, Effects0} - end. - - - -maybe_return_all(#{system_time := Ts} = Meta, ConsumerId, Consumer, S0, Effects0, Reason) -> - case Reason of - consumer_cancel -> - {update_or_remove_sub(Meta, ConsumerId, - Consumer#consumer{lifetime = once, - credit = 0, - status = cancelled}, - S0), Effects0}; - down -> - {S1, Effects1} = return_all(Meta, S0, Effects0, ConsumerId, Consumer), - {S1#?MODULE{consumers = maps:remove(ConsumerId, S1#?MODULE.consumers), - last_active = Ts}, - Effects1} - end. - -apply_enqueue(#{index := RaftIdx} = Meta, From, Seq, RawMsg, State0) -> - case maybe_enqueue(RaftIdx, From, Seq, RawMsg, [], State0) of - {ok, State1, Effects1} -> - State2 = append_to_master_index(RaftIdx, State1), - {State, ok, Effects} = checkout(Meta, State0, State2, Effects1), - {maybe_store_dehydrated_state(RaftIdx, State), ok, Effects}; - {duplicate, State, Effects} -> - {State, ok, Effects} - end. - -drop_head(#?MODULE{ra_indexes = Indexes0} = State0, Effects0) -> - case take_next_msg(State0) of - {FullMsg = {_MsgId, {RaftIdxToDrop, {Header, Msg}}}, - State1} -> - Indexes = rabbit_fifo_index:delete(RaftIdxToDrop, Indexes0), - State2 = add_bytes_drop(Header, State1#?MODULE{ra_indexes = Indexes}), - State = case Msg of - 'empty' -> State2; - _ -> subtract_in_memory_counts(Header, State2) - end, - Effects = dead_letter_effects(maxlen, #{none => FullMsg}, - State, Effects0), - {State, Effects}; - {{'$prefix_msg', Header}, State1} -> - State2 = subtract_in_memory_counts(Header, add_bytes_drop(Header, State1)), - {State2, Effects0}; - {{'$empty_msg', Header}, State1} -> - State2 = add_bytes_drop(Header, State1), - {State2, Effects0}; - empty -> - {State0, Effects0} - end. - -enqueue(RaftIdx, RawMsg, #?MODULE{messages = Messages, - next_msg_num = NextMsgNum} = State0) -> - %% the initial header is an integer only - it will get expanded to a map - %% when the next required key is added - Header = message_size(RawMsg), - {State1, Msg} = - case evaluate_memory_limit(Header, State0) of - true -> - % indexed message with header map - {State0, {RaftIdx, {Header, 'empty'}}}; - false -> - {add_in_memory_counts(Header, State0), - {RaftIdx, {Header, RawMsg}}} % indexed message with header map - end, - State = add_bytes_enqueue(Header, State1), - State#?MODULE{messages = lqueue:in({NextMsgNum, Msg}, Messages), - next_msg_num = NextMsgNum + 1}. - -append_to_master_index(RaftIdx, - #?MODULE{ra_indexes = Indexes0} = State0) -> - State = incr_enqueue_count(State0), - Indexes = rabbit_fifo_index:append(RaftIdx, Indexes0), - State#?MODULE{ra_indexes = Indexes}. - - -incr_enqueue_count(#?MODULE{enqueue_count = EC, - cfg = #cfg{release_cursor_interval = {_Base, C}} - } = State0) when EC >= C-> - %% this will trigger a dehydrated version of the state to be stored - %% at this raft index for potential future snapshot generation - %% Q: Why don't we just stash the release cursor here? - %% A: Because it needs to be the very last thing we do and we - %% first needs to run the checkout logic. - State0#?MODULE{enqueue_count = 0}; -incr_enqueue_count(#?MODULE{enqueue_count = C} = State) -> - State#?MODULE{enqueue_count = C + 1}. - -maybe_store_dehydrated_state(RaftIdx, - #?MODULE{cfg = - #cfg{release_cursor_interval = {Base, _}} - = Cfg, - ra_indexes = Indexes, - enqueue_count = 0, - release_cursors = Cursors0} = State0) -> - case rabbit_fifo_index:exists(RaftIdx, Indexes) of - false -> - %% the incoming enqueue must already have been dropped - State0; - true -> - Interval = case Base of - 0 -> 0; - _ -> - Total = messages_total(State0), - min(max(Total, Base), ?RELEASE_CURSOR_EVERY_MAX) - end, - State = State0#?MODULE{cfg = Cfg#cfg{release_cursor_interval = - {Base, Interval}}}, - Dehydrated = dehydrate_state(State), - Cursor = {release_cursor, RaftIdx, Dehydrated}, - Cursors = lqueue:in(Cursor, Cursors0), - State#?MODULE{release_cursors = Cursors} - end; -maybe_store_dehydrated_state(_RaftIdx, State) -> - State. - -enqueue_pending(From, - #enqueuer{next_seqno = Next, - pending = [{Next, RaftIdx, RawMsg} | Pending]} = Enq0, - State0) -> - State = enqueue(RaftIdx, RawMsg, State0), - Enq = Enq0#enqueuer{next_seqno = Next + 1, pending = Pending}, - enqueue_pending(From, Enq, State); -enqueue_pending(From, Enq, #?MODULE{enqueuers = Enqueuers0} = State) -> - State#?MODULE{enqueuers = Enqueuers0#{From => Enq}}. - -maybe_enqueue(RaftIdx, undefined, undefined, RawMsg, Effects, State0) -> - % direct enqueue without tracking - State = enqueue(RaftIdx, RawMsg, State0), - {ok, State, Effects}; -maybe_enqueue(RaftIdx, From, MsgSeqNo, RawMsg, Effects0, - #?MODULE{enqueuers = Enqueuers0} = State0) -> - case maps:get(From, Enqueuers0, undefined) of - undefined -> - State1 = State0#?MODULE{enqueuers = Enqueuers0#{From => #enqueuer{}}}, - {ok, State, Effects} = maybe_enqueue(RaftIdx, From, MsgSeqNo, - RawMsg, Effects0, State1), - {ok, State, [{monitor, process, From} | Effects]}; - #enqueuer{next_seqno = MsgSeqNo} = Enq0 -> - % it is the next expected seqno - State1 = enqueue(RaftIdx, RawMsg, State0), - Enq = Enq0#enqueuer{next_seqno = MsgSeqNo + 1}, - State = enqueue_pending(From, Enq, State1), - {ok, State, Effects0}; - #enqueuer{next_seqno = Next, - pending = Pending0} = Enq0 - when MsgSeqNo > Next -> - % out of order delivery - Pending = [{MsgSeqNo, RaftIdx, RawMsg} | Pending0], - Enq = Enq0#enqueuer{pending = lists:sort(Pending)}, - {ok, State0#?MODULE{enqueuers = Enqueuers0#{From => Enq}}, Effects0}; - #enqueuer{next_seqno = Next} when MsgSeqNo =< Next -> - % duplicate delivery - remove the raft index from the ra_indexes - % map as it was added earlier - {duplicate, State0, Effects0} - end. - -snd(T) -> - element(2, T). - -return(#{index := IncomingRaftIdx} = Meta, ConsumerId, Returned, - Effects0, State0) -> - {State1, Effects1} = maps:fold( - fun(MsgId, {Tag, _} = Msg, {S0, E0}) - when Tag == '$prefix_msg'; - Tag == '$empty_msg'-> - return_one(Meta, MsgId, 0, Msg, S0, E0, ConsumerId); - (MsgId, {MsgNum, Msg}, {S0, E0}) -> - return_one(Meta, MsgId, MsgNum, Msg, S0, E0, - ConsumerId) - end, {State0, Effects0}, Returned), - State2 = - case State1#?MODULE.consumers of - #{ConsumerId := Con0} -> - Con = Con0#consumer{credit = increase_credit(Con0, - map_size(Returned))}, - update_or_remove_sub(Meta, ConsumerId, Con, State1); - _ -> - State1 - end, - {State, ok, Effects} = checkout(Meta, State0, State2, Effects1), - update_smallest_raft_index(IncomingRaftIdx, State, Effects). - -% used to processes messages that are finished -complete(Meta, ConsumerId, Discarded, - #consumer{checked_out = Checked} = Con0, Effects, - #?MODULE{ra_indexes = Indexes0} = State0) -> - %% TODO optimise use of Discarded map here - MsgRaftIdxs = [RIdx || {_, {RIdx, _}} <- maps:values(Discarded)], - %% credit_mode = simple_prefetch should automatically top-up credit - %% as messages are simple_prefetch or otherwise returned - Con = Con0#consumer{checked_out = maps:without(maps:keys(Discarded), Checked), - credit = increase_credit(Con0, map_size(Discarded))}, - State1 = update_or_remove_sub(Meta, ConsumerId, Con, State0), - Indexes = lists:foldl(fun rabbit_fifo_index:delete/2, Indexes0, - MsgRaftIdxs), - %% TODO: use maps:fold instead - State2 = lists:foldl(fun({_, {_, {Header, _}}}, Acc) -> - add_bytes_settle(Header, Acc); - ({'$prefix_msg', Header}, Acc) -> - add_bytes_settle(Header, Acc); - ({'$empty_msg', Header}, Acc) -> - add_bytes_settle(Header, Acc) - end, State1, maps:values(Discarded)), - {State2#?MODULE{ra_indexes = Indexes}, Effects}. - -increase_credit(#consumer{lifetime = once, - credit = Credit}, _) -> - %% once consumers cannot increment credit - Credit; -increase_credit(#consumer{lifetime = auto, - credit_mode = credited, - credit = Credit}, _) -> - %% credit_mode: credit also doesn't automatically increment credit - Credit; -increase_credit(#consumer{credit = Current}, Credit) -> - Current + Credit. - -complete_and_checkout(#{index := IncomingRaftIdx} = Meta, MsgIds, ConsumerId, - #consumer{checked_out = Checked0} = Con0, - Effects0, State0) -> - Discarded = maps:with(MsgIds, Checked0), - {State2, Effects1} = complete(Meta, ConsumerId, Discarded, Con0, - Effects0, State0), - {State, ok, Effects} = checkout(Meta, State0, State2, Effects1), - update_smallest_raft_index(IncomingRaftIdx, State, Effects). - -dead_letter_effects(_Reason, _Discarded, - #?MODULE{cfg = #cfg{dead_letter_handler = undefined}}, - Effects) -> - Effects; -dead_letter_effects(Reason, Discarded, - #?MODULE{cfg = #cfg{dead_letter_handler = {Mod, Fun, Args}}}, - Effects) -> - RaftIdxs = maps:fold( - fun (_, {_, {RaftIdx, {_Header, 'empty'}}}, Acc) -> - [RaftIdx | Acc]; - (_, _, Acc) -> - Acc - end, [], Discarded), - [{log, RaftIdxs, - fun (Log) -> - Lookup = maps:from_list(lists:zip(RaftIdxs, Log)), - DeadLetters = maps:fold( - fun (_, {_, {RaftIdx, {_Header, 'empty'}}}, Acc) -> - {enqueue, _, _, Msg} = maps:get(RaftIdx, Lookup), - [{Reason, Msg} | Acc]; - (_, {_, {_, {_Header, Msg}}}, Acc) -> - [{Reason, Msg} | Acc]; - (_, _, Acc) -> - Acc - end, [], Discarded), - [{mod_call, Mod, Fun, Args ++ [DeadLetters]}] - end} | Effects]. - -cancel_consumer_effects(ConsumerId, - #?MODULE{cfg = #cfg{resource = QName}}, Effects) -> - [{mod_call, rabbit_quorum_queue, - cancel_consumer_handler, [QName, ConsumerId]} | Effects]. - -update_smallest_raft_index(Idx, State, Effects) -> - update_smallest_raft_index(Idx, ok, State, Effects). - -update_smallest_raft_index(IncomingRaftIdx, Reply, - #?MODULE{cfg = Cfg, - ra_indexes = Indexes, - release_cursors = Cursors0} = State0, - Effects) -> - case rabbit_fifo_index:size(Indexes) of - 0 -> - % there are no messages on queue anymore and no pending enqueues - % we can forward release_cursor all the way until - % the last received command, hooray - %% reset the release cursor interval - #cfg{release_cursor_interval = {Base, _}} = Cfg, - RCI = {Base, Base}, - State = State0#?MODULE{cfg = Cfg#cfg{release_cursor_interval = RCI}, - release_cursors = lqueue:new(), - enqueue_count = 0}, - {State, Reply, Effects ++ [{release_cursor, IncomingRaftIdx, State}]}; - _ -> - Smallest = rabbit_fifo_index:smallest(Indexes), - case find_next_cursor(Smallest, Cursors0) of - {empty, Cursors} -> - {State0#?MODULE{release_cursors = Cursors}, Reply, Effects}; - {Cursor, Cursors} -> - %% we can emit a release cursor when we've passed the smallest - %% release cursor available. - {State0#?MODULE{release_cursors = Cursors}, Reply, - Effects ++ [Cursor]} - end - end. - -find_next_cursor(Idx, Cursors) -> - find_next_cursor(Idx, Cursors, empty). - -find_next_cursor(Smallest, Cursors0, Potential) -> - case lqueue:out(Cursors0) of - {{value, {_, Idx, _} = Cursor}, Cursors} when Idx < Smallest -> - %% we found one but it may not be the largest one - find_next_cursor(Smallest, Cursors, Cursor); - _ -> - {Potential, Cursors0} - end. - -update_header(Key, UpdateFun, Default, Header) - when is_integer(Header) -> - update_header(Key, UpdateFun, Default, #{size => Header}); -update_header(Key, UpdateFun, Default, Header) -> - maps:update_with(Key, UpdateFun, Default, Header). - - -return_one(Meta, MsgId, 0, {Tag, Header0}, - #?MODULE{returns = Returns, - consumers = Consumers, - cfg = #cfg{delivery_limit = DeliveryLimit}} = State0, - Effects0, ConsumerId) - when Tag == '$prefix_msg'; Tag == '$empty_msg' -> - #consumer{checked_out = Checked} = Con0 = maps:get(ConsumerId, Consumers), - Header = update_header(delivery_count, fun (C) -> C+1 end, 1, Header0), - Msg0 = {Tag, Header}, - case maps:get(delivery_count, Header) of - DeliveryCount when DeliveryCount > DeliveryLimit -> - complete(Meta, ConsumerId, #{MsgId => Msg0}, Con0, Effects0, State0); - _ -> - %% this should not affect the release cursor in any way - Con = Con0#consumer{checked_out = maps:remove(MsgId, Checked)}, - {Msg, State1} = case Tag of - '$empty_msg' -> - {Msg0, State0}; - _ -> case evaluate_memory_limit(Header, State0) of - true -> - {{'$empty_msg', Header}, State0}; - false -> - {Msg0, add_in_memory_counts(Header, State0)} - end - end, - {add_bytes_return( - Header, - State1#?MODULE{consumers = Consumers#{ConsumerId => Con}, - returns = lqueue:in(Msg, Returns)}), - Effects0} - end; -return_one(Meta, MsgId, MsgNum, {RaftId, {Header0, RawMsg}}, - #?MODULE{returns = Returns, - consumers = Consumers, - cfg = #cfg{delivery_limit = DeliveryLimit}} = State0, - Effects0, ConsumerId) -> - #consumer{checked_out = Checked} = Con0 = maps:get(ConsumerId, Consumers), - Header = update_header(delivery_count, fun (C) -> C+1 end, 1, Header0), - Msg0 = {RaftId, {Header, RawMsg}}, - case maps:get(delivery_count, Header) of - DeliveryCount when DeliveryCount > DeliveryLimit -> - DlMsg = {MsgNum, Msg0}, - Effects = dead_letter_effects(delivery_limit, #{none => DlMsg}, - State0, Effects0), - complete(Meta, ConsumerId, #{MsgId => DlMsg}, Con0, Effects, State0); - _ -> - Con = Con0#consumer{checked_out = maps:remove(MsgId, Checked)}, - %% this should not affect the release cursor in any way - {Msg, State1} = case RawMsg of - 'empty' -> - {Msg0, State0}; - _ -> - case evaluate_memory_limit(Header, State0) of - true -> - {{RaftId, {Header, 'empty'}}, State0}; - false -> - {Msg0, add_in_memory_counts(Header, State0)} - end - end, - {add_bytes_return( - Header, - State1#?MODULE{consumers = Consumers#{ConsumerId => Con}, - returns = lqueue:in({MsgNum, Msg}, Returns)}), - Effects0} - end. - -return_all(Meta, #?MODULE{consumers = Cons} = State0, Effects0, ConsumerId, - #consumer{checked_out = Checked0} = Con) -> - %% need to sort the list so that we return messages in the order - %% they were checked out - Checked = lists:sort(maps:to_list(Checked0)), - State = State0#?MODULE{consumers = Cons#{ConsumerId => Con}}, - lists:foldl(fun ({MsgId, {'$prefix_msg', _} = Msg}, {S, E}) -> - return_one(Meta, MsgId, 0, Msg, S, E, ConsumerId); - ({MsgId, {'$empty_msg', _} = Msg}, {S, E}) -> - return_one(Meta, MsgId, 0, Msg, S, E, ConsumerId); - ({MsgId, {MsgNum, Msg}}, {S, E}) -> - return_one(Meta, MsgId, MsgNum, Msg, S, E, ConsumerId) - end, {State, Effects0}, Checked). - -%% checkout new messages to consumers -checkout(#{index := Index} = Meta, OldState, State0, Effects0) -> - {State1, _Result, Effects1} = checkout0(Meta, checkout_one(Meta, State0), - Effects0, {#{}, #{}}), - case evaluate_limit(Index, false, OldState, State1, Effects1) of - {State, true, Effects} -> - update_smallest_raft_index(Index, State, Effects); - {State, false, Effects} -> - {State, ok, Effects} - end. - -checkout0(Meta, {success, ConsumerId, MsgId, {RaftIdx, {Header, 'empty'}}, State}, - Effects, {SendAcc, LogAcc0}) -> - DelMsg = {RaftIdx, {MsgId, Header}}, - LogAcc = maps:update_with(ConsumerId, - fun (M) -> [DelMsg | M] end, - [DelMsg], LogAcc0), - checkout0(Meta, checkout_one(Meta, State), Effects, {SendAcc, LogAcc}); -checkout0(Meta, {success, ConsumerId, MsgId, Msg, State}, Effects, - {SendAcc0, LogAcc}) -> - DelMsg = {MsgId, Msg}, - SendAcc = maps:update_with(ConsumerId, - fun (M) -> [DelMsg | M] end, - [DelMsg], SendAcc0), - checkout0(Meta, checkout_one(Meta, State), Effects, {SendAcc, LogAcc}); -checkout0(_Meta, {Activity, State0}, Effects0, {SendAcc, LogAcc}) -> - Effects1 = case Activity of - nochange -> - append_send_msg_effects( - append_log_effects(Effects0, LogAcc), SendAcc); - inactive -> - [{aux, inactive} - | append_send_msg_effects( - append_log_effects(Effects0, LogAcc), SendAcc)] - end, - {State0, ok, lists:reverse(Effects1)}. - -evaluate_limit(_Index, Result, _BeforeState, - #?MODULE{cfg = #cfg{max_length = undefined, - max_bytes = undefined}} = State, - Effects) -> - {State, Result, Effects}; -evaluate_limit(Index, Result, BeforeState, - #?MODULE{cfg = #cfg{overflow_strategy = Strategy}, - enqueuers = Enqs0} = State0, - Effects0) -> - case is_over_limit(State0) of - true when Strategy == drop_head -> - {State, Effects} = drop_head(State0, Effects0), - evaluate_limit(Index, true, BeforeState, State, Effects); - true when Strategy == reject_publish -> - %% generate send_msg effect for each enqueuer to let them know - %% they need to block - {Enqs, Effects} = - maps:fold( - fun (P, #enqueuer{blocked = undefined} = E0, {Enqs, Acc}) -> - E = E0#enqueuer{blocked = Index}, - {Enqs#{P => E}, - [{send_msg, P, {queue_status, reject_publish}, - [ra_event]} | Acc]}; - (_P, _E, Acc) -> - Acc - end, {Enqs0, Effects0}, Enqs0), - {State0#?MODULE{enqueuers = Enqs}, Result, Effects}; - false when Strategy == reject_publish -> - %% TODO: optimise as this case gets called for every command - %% pretty much - Before = is_below_soft_limit(BeforeState), - case {Before, is_below_soft_limit(State0)} of - {false, true} -> - %% we have moved below the lower limit which - {Enqs, Effects} = - maps:fold( - fun (P, #enqueuer{} = E0, {Enqs, Acc}) -> - E = E0#enqueuer{blocked = undefined}, - {Enqs#{P => E}, - [{send_msg, P, {queue_status, go}, [ra_event]} - | Acc]}; - (_P, _E, Acc) -> - Acc - end, {Enqs0, Effects0}, Enqs0), - {State0#?MODULE{enqueuers = Enqs}, Result, Effects}; - _ -> - {State0, Result, Effects0} - end; - false -> - {State0, Result, Effects0} - end. - -evaluate_memory_limit(_Header, - #?MODULE{cfg = #cfg{max_in_memory_length = undefined, - max_in_memory_bytes = undefined}}) -> - false; -evaluate_memory_limit(#{size := Size}, State) -> - evaluate_memory_limit(Size, State); -evaluate_memory_limit(Size, - #?MODULE{cfg = #cfg{max_in_memory_length = MaxLength, - max_in_memory_bytes = MaxBytes}, - msg_bytes_in_memory = Bytes, - msgs_ready_in_memory = Length}) - when is_integer(Size) -> - (Length >= MaxLength) orelse ((Bytes + Size) > MaxBytes). - -append_send_msg_effects(Effects, AccMap) when map_size(AccMap) == 0 -> - Effects; -append_send_msg_effects(Effects0, AccMap) -> - Effects = maps:fold(fun (C, Msgs, Ef) -> - [send_msg_effect(C, lists:reverse(Msgs)) | Ef] - end, Effects0, AccMap), - [{aux, active} | Effects]. - -append_log_effects(Effects0, AccMap) -> - maps:fold(fun (C, Msgs, Ef) -> - [send_log_effect(C, lists:reverse(Msgs)) | Ef] - end, Effects0, AccMap). - -%% next message is determined as follows: -%% First we check if there are are prefex returns -%% Then we check if there are current returns -%% then we check prefix msgs -%% then we check current messages -%% -%% When we return it is always done to the current return queue -%% for both prefix messages and current messages -take_next_msg(#?MODULE{prefix_msgs = {R, P}} = State) -> - %% conversion - take_next_msg(State#?MODULE{prefix_msgs = {length(R), R, length(P), P}}); -take_next_msg(#?MODULE{prefix_msgs = {NumR, [{'$empty_msg', _} = Msg | Rem], - NumP, P}} = State) -> - %% there are prefix returns, these should be served first - {Msg, State#?MODULE{prefix_msgs = {NumR-1, Rem, NumP, P}}}; -take_next_msg(#?MODULE{prefix_msgs = {NumR, [Header | Rem], NumP, P}} = State) -> - %% there are prefix returns, these should be served first - {{'$prefix_msg', Header}, - State#?MODULE{prefix_msgs = {NumR-1, Rem, NumP, P}}}; -take_next_msg(#?MODULE{returns = Returns, - messages = Messages0, - prefix_msgs = {NumR, R, NumP, P}} = State) -> - %% use peek rather than out there as the most likely case is an empty - %% queue - case lqueue:peek(Returns) of - {value, NextMsg} -> - {NextMsg, - State#?MODULE{returns = lqueue:drop(Returns)}}; - empty when P == [] -> - case lqueue:out(Messages0) of - {empty, _} -> - empty; - {{value, {_, _} = SeqMsg}, Messages} -> - {SeqMsg, State#?MODULE{messages = Messages }} - end; - empty -> - [Msg | Rem] = P, - case Msg of - {Header, 'empty'} -> - %% There are prefix msgs - {{'$empty_msg', Header}, - State#?MODULE{prefix_msgs = {NumR, R, NumP-1, Rem}}}; - Header -> - {{'$prefix_msg', Header}, - State#?MODULE{prefix_msgs = {NumR, R, NumP-1, Rem}}} - end - end. - -send_msg_effect({CTag, CPid}, Msgs) -> - {send_msg, CPid, {delivery, CTag, Msgs}, [local, ra_event]}. - -send_log_effect({CTag, CPid}, IdxMsgs) -> - {RaftIdxs, Data} = lists:unzip(IdxMsgs), - {log, RaftIdxs, - fun(Log) -> - Msgs = lists:zipwith(fun ({enqueue, _, _, Msg}, {MsgId, Header}) -> - {MsgId, {Header, Msg}} - end, Log, Data), - [{send_msg, CPid, {delivery, CTag, Msgs}, [local, ra_event]}] - end, - {local, node(CPid)}}. - -reply_log_effect(RaftIdx, MsgId, Header, Ready, From) -> - {log, [RaftIdx], - fun([{enqueue, _, _, Msg}]) -> - [{reply, From, {wrap_reply, - {dequeue, {MsgId, {Header, Msg}}, Ready}}}] - end}. - -checkout_one(Meta, #?MODULE{service_queue = SQ0, - messages = Messages0, - consumers = Cons0} = InitState) -> - case priority_queue:out(SQ0) of - {{value, ConsumerId}, SQ1} -> - case take_next_msg(InitState) of - {ConsumerMsg, State0} -> - %% there are consumers waiting to be serviced - %% process consumer checkout - case maps:find(ConsumerId, Cons0) of - {ok, #consumer{credit = 0}} -> - %% no credit but was still on queue - %% can happen when draining - %% recurse without consumer on queue - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); - {ok, #consumer{status = cancelled}} -> - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); - {ok, #consumer{status = suspected_down}} -> - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}); - {ok, #consumer{checked_out = Checked0, - next_msg_id = Next, - credit = Credit, - delivery_count = DelCnt} = Con0} -> - Checked = maps:put(Next, ConsumerMsg, Checked0), - Con = Con0#consumer{checked_out = Checked, - next_msg_id = Next + 1, - credit = Credit - 1, - delivery_count = DelCnt + 1}, - State1 = update_or_remove_sub(Meta, - ConsumerId, Con, - State0#?MODULE{service_queue = SQ1}), - {State, Msg} = - case ConsumerMsg of - {'$prefix_msg', Header} -> - {subtract_in_memory_counts( - Header, add_bytes_checkout(Header, State1)), - ConsumerMsg}; - {'$empty_msg', Header} -> - {add_bytes_checkout(Header, State1), - ConsumerMsg}; - {_, {_, {Header, 'empty'}} = M} -> - {add_bytes_checkout(Header, State1), - M}; - {_, {_, {Header, _} = M}} -> - {subtract_in_memory_counts( - Header, - add_bytes_checkout(Header, State1)), - M} - end, - {success, ConsumerId, Next, Msg, State}; - error -> - %% consumer did not exist but was queued, recurse - checkout_one(Meta, InitState#?MODULE{service_queue = SQ1}) - end; - empty -> - {nochange, InitState} - end; - {empty, _} -> - case lqueue:len(Messages0) of - 0 -> {nochange, InitState}; - _ -> {inactive, InitState} - end - end. - -update_or_remove_sub(_Meta, ConsumerId, #consumer{lifetime = auto, - credit = 0} = Con, - #?MODULE{consumers = Cons} = State) -> - State#?MODULE{consumers = maps:put(ConsumerId, Con, Cons)}; -update_or_remove_sub(_Meta, ConsumerId, #consumer{lifetime = auto} = Con, - #?MODULE{consumers = Cons, - service_queue = ServiceQueue} = State) -> - State#?MODULE{consumers = maps:put(ConsumerId, Con, Cons), - service_queue = uniq_queue_in(ConsumerId, Con, ServiceQueue)}; -update_or_remove_sub(#{system_time := Ts}, - ConsumerId, #consumer{lifetime = once, - checked_out = Checked, - credit = 0} = Con, - #?MODULE{consumers = Cons} = State) -> - case maps:size(Checked) of - 0 -> - % we're done with this consumer - State#?MODULE{consumers = maps:remove(ConsumerId, Cons), - last_active = Ts}; - _ -> - % there are unsettled items so need to keep around - State#?MODULE{consumers = maps:put(ConsumerId, Con, Cons)} - end; -update_or_remove_sub(_Meta, ConsumerId, #consumer{lifetime = once} = Con, - #?MODULE{consumers = Cons, - service_queue = ServiceQueue} = State) -> - State#?MODULE{consumers = maps:put(ConsumerId, Con, Cons), - service_queue = uniq_queue_in(ConsumerId, Con, ServiceQueue)}. - -uniq_queue_in(Key, #consumer{priority = P}, Queue) -> - % TODO: queue:member could surely be quite expensive, however the practical - % number of unique consumers may not be large enough for it to matter - case priority_queue:member(Key, Queue) of - true -> - Queue; - false -> - priority_queue:in(Key, P, Queue) - end. - -update_consumer(ConsumerId, Meta, Spec, Priority, - #?MODULE{cfg = #cfg{consumer_strategy = competing}} = State0) -> - %% general case, single active consumer off - update_consumer0(ConsumerId, Meta, Spec, Priority, State0); -update_consumer(ConsumerId, Meta, Spec, Priority, - #?MODULE{consumers = Cons0, - cfg = #cfg{consumer_strategy = single_active}} = State0) - when map_size(Cons0) == 0 -> - %% single active consumer on, no one is consuming yet - update_consumer0(ConsumerId, Meta, Spec, Priority, State0); -update_consumer(ConsumerId, Meta, {Life, Credit, Mode}, Priority, - #?MODULE{cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = WaitingConsumers0} = State0) -> - %% single active consumer on and one active consumer already - %% adding the new consumer to the waiting list - Consumer = #consumer{lifetime = Life, meta = Meta, - priority = Priority, - credit = Credit, credit_mode = Mode}, - WaitingConsumers1 = WaitingConsumers0 ++ [{ConsumerId, Consumer}], - State0#?MODULE{waiting_consumers = WaitingConsumers1}. - -update_consumer0(ConsumerId, Meta, {Life, Credit, Mode}, Priority, - #?MODULE{consumers = Cons0, - service_queue = ServiceQueue0} = State0) -> - %% TODO: this logic may not be correct for updating a pre-existing consumer - Init = #consumer{lifetime = Life, meta = Meta, - priority = Priority, - credit = Credit, credit_mode = Mode}, - Cons = maps:update_with(ConsumerId, - fun(S) -> - %% remove any in-flight messages from - %% the credit update - N = maps:size(S#consumer.checked_out), - C = max(0, Credit - N), - S#consumer{lifetime = Life, credit = C} - end, Init, Cons0), - ServiceQueue = maybe_queue_consumer(ConsumerId, maps:get(ConsumerId, Cons), - ServiceQueue0), - State0#?MODULE{consumers = Cons, service_queue = ServiceQueue}. - -maybe_queue_consumer(ConsumerId, #consumer{credit = Credit} = Con, - ServiceQueue0) -> - case Credit > 0 of - true -> - % consumerect needs service - check if already on service queue - uniq_queue_in(ConsumerId, Con, ServiceQueue0); - false -> - ServiceQueue0 - end. - -%% creates a dehydrated version of the current state to be cached and -%% potentially used to for a snaphot at a later point -dehydrate_state(#?MODULE{messages = Messages, - consumers = Consumers, - returns = Returns, - prefix_msgs = {PRCnt, PrefRet0, PPCnt, PrefMsg0}, - waiting_consumers = Waiting0} = State) -> - RCnt = lqueue:len(Returns), - %% TODO: optimise this function as far as possible - PrefRet1 = lists:foldr(fun ({'$prefix_msg', Header}, Acc) -> - [Header | Acc]; - ({'$empty_msg', _} = Msg, Acc) -> - [Msg | Acc]; - ({_, {_, {Header, 'empty'}}}, Acc) -> - [{'$empty_msg', Header} | Acc]; - ({_, {_, {Header, _}}}, Acc) -> - [Header | Acc] - end, - [], - lqueue:to_list(Returns)), - PrefRet = PrefRet0 ++ PrefRet1, - PrefMsgsSuff = dehydrate_messages(Messages, []), - %% prefix messages are not populated in normal operation only after - %% recovering from a snapshot - PrefMsgs = PrefMsg0 ++ PrefMsgsSuff, - Waiting = [{Cid, dehydrate_consumer(C)} || {Cid, C} <- Waiting0], - State#?MODULE{messages = lqueue:new(), - ra_indexes = rabbit_fifo_index:empty(), - release_cursors = lqueue:new(), - consumers = maps:map(fun (_, C) -> - dehydrate_consumer(C) - end, Consumers), - returns = lqueue:new(), - prefix_msgs = {PRCnt + RCnt, PrefRet, - PPCnt + lqueue:len(Messages), PrefMsgs}, - waiting_consumers = Waiting}. - -%% TODO make body recursive to avoid allocating lists:reverse call -dehydrate_messages(Msgs0, Acc0) -> - {OutRes, Msgs} = lqueue:out(Msgs0), - case OutRes of - {value, {_MsgId, {_RaftId, {_, 'empty'} = Msg}}} -> - dehydrate_messages(Msgs, [Msg | Acc0]); - {value, {_MsgId, {_RaftId, {Header, _}}}} -> - dehydrate_messages(Msgs, [Header | Acc0]); - empty -> - lists:reverse(Acc0) - end. - -dehydrate_consumer(#consumer{checked_out = Checked0} = Con) -> - Checked = maps:map(fun (_, {'$prefix_msg', _} = M) -> - M; - (_, {'$empty_msg', _} = M) -> - M; - (_, {_, {_, {Header, 'empty'}}}) -> - {'$empty_msg', Header}; - (_, {_, {_, {Header, _}}}) -> - {'$prefix_msg', Header} - end, Checked0), - Con#consumer{checked_out = Checked}. - -%% make the state suitable for equality comparison -normalize(#?MODULE{messages = Messages, - release_cursors = Cursors} = State) -> - State#?MODULE{messages = lqueue:from_list(lqueue:to_list(Messages)), - release_cursors = lqueue:from_list(lqueue:to_list(Cursors))}. - -is_over_limit(#?MODULE{cfg = #cfg{max_length = undefined, - max_bytes = undefined}}) -> - false; -is_over_limit(#?MODULE{cfg = #cfg{max_length = MaxLength, - max_bytes = MaxBytes}, - msg_bytes_enqueue = BytesEnq} = State) -> - messages_ready(State) > MaxLength orelse (BytesEnq > MaxBytes). - -is_below_soft_limit(#?MODULE{cfg = #cfg{max_length = undefined, - max_bytes = undefined}}) -> - false; -is_below_soft_limit(#?MODULE{cfg = #cfg{max_length = MaxLength, - max_bytes = MaxBytes}, - msg_bytes_enqueue = BytesEnq} = State) -> - is_below(MaxLength, messages_ready(State)) andalso - is_below(MaxBytes, BytesEnq). - -is_below(undefined, _Num) -> - true; -is_below(Val, Num) when is_integer(Val) andalso is_integer(Num) -> - Num =< trunc(Val * ?LOW_LIMIT). - --spec make_enqueue(option(pid()), option(msg_seqno()), raw_msg()) -> protocol(). -make_enqueue(Pid, Seq, Msg) -> - #enqueue{pid = Pid, seq = Seq, msg = Msg}. - --spec make_register_enqueuer(pid()) -> protocol(). -make_register_enqueuer(Pid) -> - #register_enqueuer{pid = Pid}. - --spec make_checkout(consumer_id(), - checkout_spec(), consumer_meta()) -> protocol(). -make_checkout(ConsumerId, Spec, Meta) -> - #checkout{consumer_id = ConsumerId, - spec = Spec, meta = Meta}. - --spec make_settle(consumer_id(), [msg_id()]) -> protocol(). -make_settle(ConsumerId, MsgIds) when is_list(MsgIds) -> - #settle{consumer_id = ConsumerId, msg_ids = MsgIds}. - --spec make_return(consumer_id(), [msg_id()]) -> protocol(). -make_return(ConsumerId, MsgIds) -> - #return{consumer_id = ConsumerId, msg_ids = MsgIds}. - --spec make_discard(consumer_id(), [msg_id()]) -> protocol(). -make_discard(ConsumerId, MsgIds) -> - #discard{consumer_id = ConsumerId, msg_ids = MsgIds}. - --spec make_credit(consumer_id(), non_neg_integer(), non_neg_integer(), - boolean()) -> protocol(). -make_credit(ConsumerId, Credit, DeliveryCount, Drain) -> - #credit{consumer_id = ConsumerId, - credit = Credit, - delivery_count = DeliveryCount, - drain = Drain}. - --spec make_purge() -> protocol(). -make_purge() -> #purge{}. - --spec make_garbage_collection() -> protocol(). -make_garbage_collection() -> #garbage_collection{}. - --spec make_purge_nodes([node()]) -> protocol(). -make_purge_nodes(Nodes) -> - #purge_nodes{nodes = Nodes}. - --spec make_update_config(config()) -> protocol(). -make_update_config(Config) -> - #update_config{config = Config}. - -add_bytes_enqueue(Bytes, - #?MODULE{msg_bytes_enqueue = Enqueue} = State) - when is_integer(Bytes) -> - State#?MODULE{msg_bytes_enqueue = Enqueue + Bytes}; -add_bytes_enqueue(#{size := Bytes}, State) -> - add_bytes_enqueue(Bytes, State). - -add_bytes_drop(Bytes, - #?MODULE{msg_bytes_enqueue = Enqueue} = State) - when is_integer(Bytes) -> - State#?MODULE{msg_bytes_enqueue = Enqueue - Bytes}; -add_bytes_drop(#{size := Bytes}, State) -> - add_bytes_drop(Bytes, State). - -add_bytes_checkout(Bytes, - #?MODULE{msg_bytes_checkout = Checkout, - msg_bytes_enqueue = Enqueue } = State) - when is_integer(Bytes) -> - State#?MODULE{msg_bytes_checkout = Checkout + Bytes, - msg_bytes_enqueue = Enqueue - Bytes}; -add_bytes_checkout(#{size := Bytes}, State) -> - add_bytes_checkout(Bytes, State). - -add_bytes_settle(Bytes, - #?MODULE{msg_bytes_checkout = Checkout} = State) - when is_integer(Bytes) -> - State#?MODULE{msg_bytes_checkout = Checkout - Bytes}; -add_bytes_settle(#{size := Bytes}, State) -> - add_bytes_settle(Bytes, State). - -add_bytes_return(Bytes, - #?MODULE{msg_bytes_checkout = Checkout, - msg_bytes_enqueue = Enqueue} = State) - when is_integer(Bytes) -> - State#?MODULE{msg_bytes_checkout = Checkout - Bytes, - msg_bytes_enqueue = Enqueue + Bytes}; -add_bytes_return(#{size := Bytes}, State) -> - add_bytes_return(Bytes, State). - -add_in_memory_counts(Bytes, - #?MODULE{msg_bytes_in_memory = InMemoryBytes, - msgs_ready_in_memory = InMemoryCount} = State) - when is_integer(Bytes) -> - State#?MODULE{msg_bytes_in_memory = InMemoryBytes + Bytes, - msgs_ready_in_memory = InMemoryCount + 1}; -add_in_memory_counts(#{size := Bytes}, State) -> - add_in_memory_counts(Bytes, State). - -subtract_in_memory_counts(Bytes, - #?MODULE{msg_bytes_in_memory = InMemoryBytes, - msgs_ready_in_memory = InMemoryCount} = State) - when is_integer(Bytes) -> - State#?MODULE{msg_bytes_in_memory = InMemoryBytes - Bytes, - msgs_ready_in_memory = InMemoryCount - 1}; -subtract_in_memory_counts(#{size := Bytes}, State) -> - subtract_in_memory_counts(Bytes, State). - -message_size(#basic_message{content = Content}) -> - #content{payload_fragments_rev = PFR} = Content, - iolist_size(PFR); -message_size({'$prefix_msg', H}) -> - get_size_from_header(H); -message_size({'$empty_msg', H}) -> - get_size_from_header(H); -message_size(B) when is_binary(B) -> - byte_size(B); -message_size(Msg) -> - %% probably only hit this for testing so ok to use erts_debug - erts_debug:size(Msg). - -get_size_from_header(Size) when is_integer(Size) -> - Size; -get_size_from_header(#{size := B}) -> - B. - - -all_nodes(#?MODULE{consumers = Cons0, - enqueuers = Enqs0, - waiting_consumers = WaitingConsumers0}) -> - Nodes0 = maps:fold(fun({_, P}, _, Acc) -> - Acc#{node(P) => ok} - end, #{}, Cons0), - Nodes1 = maps:fold(fun(P, _, Acc) -> - Acc#{node(P) => ok} - end, Nodes0, Enqs0), - maps:keys( - lists:foldl(fun({{_, P}, _}, Acc) -> - Acc#{node(P) => ok} - end, Nodes1, WaitingConsumers0)). - -all_pids_for(Node, #?MODULE{consumers = Cons0, - enqueuers = Enqs0, - waiting_consumers = WaitingConsumers0}) -> - Cons = maps:fold(fun({_, P}, _, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, _, Acc) -> Acc - end, [], Cons0), - Enqs = maps:fold(fun(P, _, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, _, Acc) -> Acc - end, Cons, Enqs0), - lists:foldl(fun({{_, P}, _}, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, Acc) -> Acc - end, Enqs, WaitingConsumers0). - -suspected_pids_for(Node, #?MODULE{consumers = Cons0, - enqueuers = Enqs0, - waiting_consumers = WaitingConsumers0}) -> - Cons = maps:fold(fun({_, P}, #consumer{status = suspected_down}, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, _, Acc) -> Acc - end, [], Cons0), - Enqs = maps:fold(fun(P, #enqueuer{status = suspected_down}, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, _, Acc) -> Acc - end, Cons, Enqs0), - lists:foldl(fun({{_, P}, - #consumer{status = suspected_down}}, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, Acc) -> Acc - end, Enqs, WaitingConsumers0). - -is_expired(Ts, #?MODULE{cfg = #cfg{expires = Expires}, - last_active = LastActive, - consumers = Consumers}) - when is_number(LastActive) andalso is_number(Expires) -> - %% TODO: should it be active consumers? - Active = maps:filter(fun (_, #consumer{status = suspected_down}) -> - false; - (_, _) -> - true - end, Consumers), - - Ts > (LastActive + Expires) andalso maps:size(Active) == 0; -is_expired(_Ts, _State) -> - false. - -get_priority_from_args(#{args := Args}) -> - case rabbit_misc:table_lookup(Args, <<"x-priority">>) of - {_Key, Value} -> - Value; - _ -> 0 - end; -get_priority_from_args(_) -> - 0. diff --git a/src/rabbit_fifo.hrl b/src/rabbit_fifo.hrl deleted file mode 100644 index a63483becd..0000000000 --- a/src/rabbit_fifo.hrl +++ /dev/null @@ -1,210 +0,0 @@ - --type option(T) :: undefined | T. - --type raw_msg() :: term(). -%% The raw message. It is opaque to rabbit_fifo. - --type msg_in_id() :: non_neg_integer(). -% a queue scoped monotonically incrementing integer used to enforce order -% in the unassigned messages map - --type msg_id() :: non_neg_integer(). -%% A consumer-scoped monotonically incrementing integer included with a -%% {@link delivery/0.}. Used to settle deliveries using -%% {@link rabbit_fifo_client:settle/3.} - --type msg_seqno() :: non_neg_integer(). -%% A sender process scoped monotonically incrementing integer included -%% in enqueue messages. Used to ensure ordering of messages send from the -%% same process - --type msg_header() :: msg_size() | - #{size := msg_size(), - delivery_count => non_neg_integer()}. -%% The message header: -%% delivery_count: the number of unsuccessful delivery attempts. -%% A non-zero value indicates a previous attempt. -%% If it only contains the size it can be condensed to an integer only - --type msg() :: {msg_header(), raw_msg()}. -%% message with a header map. - --type msg_size() :: non_neg_integer(). -%% the size in bytes of the msg payload - --type indexed_msg() :: {ra:index(), msg()}. - --type prefix_msg() :: {'$prefix_msg', msg_header()}. - --type delivery_msg() :: {msg_id(), msg()}. -%% A tuple consisting of the message id and the headered message. - --type consumer_tag() :: binary(). -%% An arbitrary binary tag used to distinguish between different consumers -%% set up by the same process. See: {@link rabbit_fifo_client:checkout/3.} - --type delivery() :: {delivery, consumer_tag(), [delivery_msg()]}. -%% Represents the delivery of one or more rabbit_fifo messages. - --type consumer_id() :: {consumer_tag(), pid()}. -%% The entity that receives messages. Uniquely identifies a consumer. - --type credit_mode() :: simple_prefetch | credited. -%% determines how credit is replenished - --type checkout_spec() :: {once | auto, Num :: non_neg_integer(), - credit_mode()} | - {dequeue, settled | unsettled} | - cancel. - --type consumer_meta() :: #{ack => boolean(), - username => binary(), - prefetch => non_neg_integer(), - args => list()}. -%% static meta data associated with a consumer - - --type applied_mfa() :: {module(), atom(), list()}. -% represents a partially applied module call - --define(RELEASE_CURSOR_EVERY, 2048). --define(RELEASE_CURSOR_EVERY_MAX, 3200000). --define(USE_AVG_HALF_LIFE, 10000.0). -%% an average QQ without any message uses about 100KB so setting this limit -%% to ~10 times that should be relatively safe. --define(GC_MEM_LIMIT_B, 2000000). - --define(MB, 1048576). --define(LOW_LIMIT, 0.8). - --record(consumer, - {meta = #{} :: consumer_meta(), - checked_out = #{} :: #{msg_id() => {msg_in_id(), indexed_msg()}}, - next_msg_id = 0 :: msg_id(), % part of snapshot data - %% max number of messages that can be sent - %% decremented for each delivery - credit = 0 : non_neg_integer(), - %% total number of checked out messages - ever - %% incremented for each delivery - delivery_count = 0 :: non_neg_integer(), - %% the mode of how credit is incremented - %% simple_prefetch: credit is re-filled as deliveries are settled - %% or returned. - %% credited: credit can only be changed by receiving a consumer_credit - %% command: `{consumer_credit, ReceiverDeliveryCount, Credit}' - credit_mode = simple_prefetch :: credit_mode(), % part of snapshot data - lifetime = once :: once | auto, - status = up :: up | suspected_down | cancelled, - priority = 0 :: non_neg_integer() - }). - --type consumer() :: #consumer{}. - --type consumer_strategy() :: competing | single_active. - --type milliseconds() :: non_neg_integer(). - --record(enqueuer, - {next_seqno = 1 :: msg_seqno(), - % out of order enqueues - sorted list - pending = [] :: [{msg_seqno(), ra:index(), raw_msg()}], - status = up :: up | - suspected_down, - %% it is useful to have a record of when this was blocked - %% so that we can retry sending the block effect if - %% the publisher did not receive the initial one - blocked :: undefined | ra:index(), - unused_1, - unused_2 - }). - --record(cfg, - {name :: atom(), - resource :: rabbit_types:r('queue'), - release_cursor_interval :: option({non_neg_integer(), non_neg_integer()}), - dead_letter_handler :: option(applied_mfa()), - become_leader_handler :: option(applied_mfa()), - overflow_strategy = drop_head :: drop_head | reject_publish, - max_length :: option(non_neg_integer()), - max_bytes :: option(non_neg_integer()), - %% whether single active consumer is on or not for this queue - consumer_strategy = competing :: consumer_strategy(), - %% the maximum number of unsuccessful delivery attempts permitted - delivery_limit :: option(non_neg_integer()), - max_in_memory_length :: option(non_neg_integer()), - max_in_memory_bytes :: option(non_neg_integer()), - expires :: undefined | milliseconds(), - unused_1, - unused_2 - }). - --type prefix_msgs() :: {list(), list()} | - {non_neg_integer(), list(), - non_neg_integer(), list()}. - --record(rabbit_fifo, - {cfg :: #cfg{}, - % unassigned messages - messages = lqueue:new() :: lqueue:lqueue({msg_in_id(), indexed_msg()}), - % defines the next message id - next_msg_num = 1 :: msg_in_id(), - % queue of returned msg_in_ids - when checking out it picks from - returns = lqueue:new() :: lqueue:lqueue(prefix_msg() | - {msg_in_id(), indexed_msg()}), - % a counter of enqueues - used to trigger shadow copy points - enqueue_count = 0 :: non_neg_integer(), - % a map containing all the live processes that have ever enqueued - % a message to this queue as well as a cached value of the smallest - % ra_index of all pending enqueues - enqueuers = #{} :: #{pid() => #enqueuer{}}, - % master index of all enqueue raft indexes including pending - % enqueues - % rabbit_fifo_index can be slow when calculating the smallest - % index when there are large gaps but should be faster than gb_trees - % for normal appending operations as it's backed by a map - ra_indexes = rabbit_fifo_index:empty() :: rabbit_fifo_index:state(), - release_cursors = lqueue:new() :: lqueue:lqueue({release_cursor, - ra:index(), #rabbit_fifo{}}), - % consumers need to reflect consumer state at time of snapshot - % needs to be part of snapshot - consumers = #{} :: #{consumer_id() => #consumer{}}, - % consumers that require further service are queued here - % needs to be part of snapshot - service_queue = priority_queue:new() :: priority_queue:q(), - %% This is a special field that is only used for snapshots - %% It represents the queued messages at the time the - %% dehydrated snapshot state was cached. - %% As release_cursors are only emitted for raft indexes where all - %% prior messages no longer contribute to the current state we can - %% replace all message payloads with their sizes (to be used for - %% overflow calculations). - %% This is done so that consumers are still served in a deterministic - %% order on recovery. - prefix_msgs = {0, [], 0, []} :: prefix_msgs(), - msg_bytes_enqueue = 0 :: non_neg_integer(), - msg_bytes_checkout = 0 :: non_neg_integer(), - %% waiting consumers, one is picked active consumer is cancelled or dies - %% used only when single active consumer is on - waiting_consumers = [] :: [{consumer_id(), consumer()}], - msg_bytes_in_memory = 0 :: non_neg_integer(), - msgs_ready_in_memory = 0 :: non_neg_integer(), - last_active :: undefined | non_neg_integer(), - unused_1, - unused_2 - }). - --type config() :: #{name := atom(), - queue_resource := rabbit_types:r('queue'), - dead_letter_handler => applied_mfa(), - become_leader_handler => applied_mfa(), - release_cursor_interval => non_neg_integer(), - max_length => non_neg_integer(), - max_bytes => non_neg_integer(), - max_in_memory_length => non_neg_integer(), - max_in_memory_bytes => non_neg_integer(), - overflow_strategy => drop_head | reject_publish, - single_active_consumer_on => boolean(), - delivery_limit => non_neg_integer(), - expires => non_neg_integer(), - created => non_neg_integer() - }. diff --git a/src/rabbit_fifo_client.erl b/src/rabbit_fifo_client.erl deleted file mode 100644 index 6673cadc93..0000000000 --- a/src/rabbit_fifo_client.erl +++ /dev/null @@ -1,920 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% @doc Provides an easy to consume API for interacting with the {@link rabbit_fifo.} -%% state machine implementation running inside a `ra' raft system. -%% -%% Handles command tracking and other non-functional concerns. --module(rabbit_fifo_client). - --export([ - init/2, - init/3, - init/5, - checkout/4, - checkout/5, - cancel_checkout/2, - enqueue/2, - enqueue/3, - dequeue/3, - settle/3, - return/3, - discard/3, - credit/4, - handle_ra_event/3, - untracked_enqueue/2, - purge/1, - cluster_name/1, - update_machine_state/2, - pending_size/1, - stat/1, - stat/2 - ]). - --include_lib("rabbit_common/include/rabbit.hrl"). - --define(SOFT_LIMIT, 32). --define(TIMER_TIME, 10000). - --type seq() :: non_neg_integer(). -%% last_applied is initialised to -1 --type maybe_seq() :: integer(). --type action() :: {send_credit_reply, Available :: non_neg_integer()} | - {send_drained, CTagCredit :: - {rabbit_fifo:consumer_tag(), non_neg_integer()}}. --type actions() :: [action()]. - --type cluster_name() :: rabbit_types:r(queue). - --record(consumer, {last_msg_id :: seq() | -1, - ack = false :: boolean(), - delivery_count = 0 :: non_neg_integer()}). - --record(cfg, {cluster_name :: cluster_name(), - servers = [] :: [ra:server_id()], - soft_limit = ?SOFT_LIMIT :: non_neg_integer(), - block_handler = fun() -> ok end :: fun(() -> term()), - unblock_handler = fun() -> ok end :: fun(() -> ok), - timeout :: non_neg_integer(), - version = 0 :: non_neg_integer()}). - --record(state, {cfg :: #cfg{}, - leader :: undefined | ra:server_id(), - queue_status :: undefined | go | reject_publish, - next_seq = 0 :: seq(), - %% Last applied is initialise to -1 to note that no command has yet been - %% applied, but allowing to resend messages if the first ones on the sequence - %% are lost (messages are sent from last_applied + 1) - last_applied = -1 :: maybe_seq(), - next_enqueue_seq = 1 :: seq(), - %% indicates that we've exceeded the soft limit - slow = false :: boolean(), - unsent_commands = #{} :: #{rabbit_fifo:consumer_id() => - {[seq()], [seq()], [seq()]}}, - pending = #{} :: #{seq() => - {term(), rabbit_fifo:command()}}, - consumer_deliveries = #{} :: #{rabbit_fifo:consumer_tag() => - #consumer{}}, - timer_state :: term() - }). - --opaque state() :: #state{}. - --export_type([ - state/0, - actions/0 - ]). - - -%% @doc Create the initial state for a new rabbit_fifo sessions. A state is needed -%% to interact with a rabbit_fifo queue using @module. -%% @param ClusterName the id of the cluster to interact with -%% @param Servers The known servers of the queue. If the current leader is known -%% ensure the leader node is at the head of the list. --spec init(cluster_name(), [ra:server_id()]) -> state(). -init(ClusterName, Servers) -> - init(ClusterName, Servers, ?SOFT_LIMIT). - -%% @doc Create the initial state for a new rabbit_fifo sessions. A state is needed -%% to interact with a rabbit_fifo queue using @module. -%% @param ClusterName the id of the cluster to interact with -%% @param Servers The known servers of the queue. If the current leader is known -%% ensure the leader node is at the head of the list. -%% @param MaxPending size defining the max number of pending commands. --spec init(cluster_name(), [ra:server_id()], non_neg_integer()) -> state(). -init(ClusterName = #resource{}, Servers, SoftLimit) -> - Timeout = application:get_env(kernel, net_ticktime, 60) + 5, - #state{cfg = #cfg{cluster_name = ClusterName, - servers = Servers, - soft_limit = SoftLimit, - timeout = Timeout * 1000}}. - --spec init(cluster_name(), [ra:server_id()], non_neg_integer(), fun(() -> ok), - fun(() -> ok)) -> state(). -init(ClusterName = #resource{}, Servers, SoftLimit, BlockFun, UnblockFun) -> - %% net ticktime is in seconds - Timeout = application:get_env(kernel, net_ticktime, 60) + 5, - #state{cfg = #cfg{cluster_name = ClusterName, - servers = Servers, - block_handler = BlockFun, - unblock_handler = UnblockFun, - soft_limit = SoftLimit, - timeout = Timeout * 1000}}. - - -%% @doc Enqueues a message. -%% @param Correlation an arbitrary erlang term used to correlate this -%% command when it has been applied. -%% @param Msg an arbitrary erlang term representing the message. -%% @param State the current {@module} state. -%% @returns -%% `{ok | slow, State}' if the command was successfully sent. If the return -%% tag is `slow' it means the limit is approaching and it is time to slow down -%% the sending rate. -%% {@module} assigns a sequence number to every raft command it issues. The -%% SequenceNumber can be correlated to the applied sequence numbers returned -%% by the {@link handle_ra_event/2. handle_ra_event/2} function. --spec enqueue(Correlation :: term(), Msg :: term(), State :: state()) -> - {ok | slow | reject_publish, state()}. -enqueue(Correlation, Msg, - #state{queue_status = undefined, - next_enqueue_seq = 1, - cfg = #cfg{timeout = Timeout}} = State0) -> - %% it is the first enqueue, check the version - {_, Node} = Server = pick_server(State0), - case rpc:call(Node, ra_machine, version, [{machine, rabbit_fifo, #{}}]) of - 0 -> - %% the leader is running the old version - %% so we can't initialize the enqueuer session safely - %% fall back on old behavour - enqueue(Correlation, Msg, State0#state{queue_status = go}); - 1 -> - %% were running the new version on the leader do sync initialisation - %% of enqueuer session - Reg = rabbit_fifo:make_register_enqueuer(self()), - case ra:process_command(Server, Reg, Timeout) of - {ok, reject_publish, _} -> - {reject_publish, State0#state{queue_status = reject_publish}}; - {ok, ok, _} -> - enqueue(Correlation, Msg, State0#state{queue_status = go}); - {timeout, _} -> - %% if we timeout it is probably better to reject - %% the message than being uncertain - {reject_publish, State0}; - Err -> - exit(Err) - end; - {badrpc, nodedown} -> - {reject_publish, State0} - end; -enqueue(_Correlation, _Msg, - #state{queue_status = reject_publish, - cfg = #cfg{}} = State) -> - {reject_publish, State}; -enqueue(Correlation, Msg, - #state{slow = Slow, - queue_status = go, - cfg = #cfg{block_handler = BlockFun}} = State0) -> - Node = pick_server(State0), - {Next, State1} = next_enqueue_seq(State0), - % by default there is no correlation id - Cmd = rabbit_fifo:make_enqueue(self(), Next, Msg), - case send_command(Node, Correlation, Cmd, low, State1) of - {slow, State} when not Slow -> - BlockFun(), - {slow, set_timer(State)}; - Any -> - Any - end. - -%% @doc Enqueues a message. -%% @param Msg an arbitrary erlang term representing the message. -%% @param State the current {@module} state. -%% @returns -%% `{ok | slow, State}' if the command was successfully sent. If the return -%% tag is `slow' it means the limit is approaching and it is time to slow down -%% the sending rate. -%% {@module} assigns a sequence number to every raft command it issues. The -%% SequenceNumber can be correlated to the applied sequence numbers returned -%% by the {@link handle_ra_event/2. handle_ra_event/2} function. -%% --spec enqueue(Msg :: term(), State :: state()) -> - {ok | slow | reject_publish, state()}. -enqueue(Msg, State) -> - enqueue(undefined, Msg, State). - -%% @doc Dequeue a message from the queue. -%% -%% This is a synchronous call. I.e. the call will block until the command -%% has been accepted by the ra process or it times out. -%% -%% @param ConsumerTag a unique tag to identify this particular consumer. -%% @param Settlement either `settled' or `unsettled'. When `settled' no -%% further settlement needs to be done. -%% @param State The {@module} state. -%% -%% @returns `{ok, IdMsg, State}' or `{error | timeout, term()}' --spec dequeue(rabbit_fifo:consumer_tag(), - Settlement :: settled | unsettled, state()) -> - {ok, non_neg_integer(), term(), non_neg_integer()} - | {empty, state()} | {error | timeout, term()}. -dequeue(ConsumerTag, Settlement, - #state{cfg = #cfg{timeout = Timeout, - cluster_name = QName}} = State0) -> - Node = pick_server(State0), - ConsumerId = consumer_id(ConsumerTag), - case ra:process_command(Node, - rabbit_fifo:make_checkout(ConsumerId, - {dequeue, Settlement}, - #{}), - Timeout) of - {ok, {dequeue, empty}, Leader} -> - {empty, State0#state{leader = Leader}}; - {ok, {dequeue, {MsgId, {MsgHeader, Msg0}}, MsgsReady}, Leader} -> - Count = case MsgHeader of - #{delivery_count := C} -> C; - _ -> 0 - end, - IsDelivered = Count > 0, - Msg = add_delivery_count_header(Msg0, Count), - {ok, MsgsReady, - {QName, qref(Leader), MsgId, IsDelivered, Msg}, - State0#state{leader = Leader}}; - {ok, {error, _} = Err, _Leader} -> - Err; - Err -> - Err - end. - -add_delivery_count_header(#basic_message{} = Msg0, Count) - when is_integer(Count) -> - rabbit_basic:add_header(<<"x-delivery-count">>, long, Count, Msg0); -add_delivery_count_header(Msg, _Count) -> - Msg. - - -%% @doc Settle a message. Permanently removes message from the queue. -%% @param ConsumerTag the tag uniquely identifying the consumer. -%% @param MsgIds the message ids received with the {@link rabbit_fifo:delivery/0.} -%% @param State the {@module} state -%% @returns -%% `{ok | slow, State}' if the command was successfully sent. If the return -%% tag is `slow' it means the limit is approaching and it is time to slow down -%% the sending rate. -%% --spec settle(rabbit_fifo:consumer_tag(), [rabbit_fifo:msg_id()], state()) -> - {state(), list()}. -settle(ConsumerTag, [_|_] = MsgIds, #state{slow = false} = State0) -> - Node = pick_server(State0), - Cmd = rabbit_fifo:make_settle(consumer_id(ConsumerTag), MsgIds), - case send_command(Node, undefined, Cmd, normal, State0) of - {_, S} -> - % turn slow into ok for this function - {S, []} - end; -settle(ConsumerTag, [_|_] = MsgIds, - #state{unsent_commands = Unsent0} = State0) -> - ConsumerId = consumer_id(ConsumerTag), - %% we've reached the soft limit so will stash the command to be - %% sent once we have seen enough notifications - Unsent = maps:update_with(ConsumerId, - fun ({Settles, Returns, Discards}) -> - {Settles ++ MsgIds, Returns, Discards} - end, {MsgIds, [], []}, Unsent0), - {State0#state{unsent_commands = Unsent}, []}. - -%% @doc Return a message to the queue. -%% @param ConsumerTag the tag uniquely identifying the consumer. -%% @param MsgIds the message ids to return received -%% from {@link rabbit_fifo:delivery/0.} -%% @param State the {@module} state -%% @returns -%% `{ok | slow, State}' if the command was successfully sent. If the return -%% tag is `slow' it means the limit is approaching and it is time to slow down -%% the sending rate. -%% --spec return(rabbit_fifo:consumer_tag(), [rabbit_fifo:msg_id()], state()) -> - {state(), list()}. -return(ConsumerTag, [_|_] = MsgIds, #state{slow = false} = State0) -> - Node = pick_server(State0), - % TODO: make rabbit_fifo return support lists of message ids - Cmd = rabbit_fifo:make_return(consumer_id(ConsumerTag), MsgIds), - case send_command(Node, undefined, Cmd, normal, State0) of - {_, S} -> - {S, []} - end; -return(ConsumerTag, [_|_] = MsgIds, - #state{unsent_commands = Unsent0} = State0) -> - ConsumerId = consumer_id(ConsumerTag), - %% we've reached the soft limit so will stash the command to be - %% sent once we have seen enough notifications - Unsent = maps:update_with(ConsumerId, - fun ({Settles, Returns, Discards}) -> - {Settles, Returns ++ MsgIds, Discards} - end, {[], MsgIds, []}, Unsent0), - {State0#state{unsent_commands = Unsent}, []}. - -%% @doc Discards a checked out message. -%% If the queue has a dead_letter_handler configured this will be called. -%% @param ConsumerTag the tag uniquely identifying the consumer. -%% @param MsgIds the message ids to discard -%% from {@link rabbit_fifo:delivery/0.} -%% @param State the {@module} state -%% @returns -%% `{ok | slow, State}' if the command was successfully sent. If the return -%% tag is `slow' it means the limit is approaching and it is time to slow down -%% the sending rate. --spec discard(rabbit_fifo:consumer_tag(), [rabbit_fifo:msg_id()], state()) -> - {state(), list()}. -discard(ConsumerTag, [_|_] = MsgIds, #state{slow = false} = State0) -> - Node = pick_server(State0), - Cmd = rabbit_fifo:make_discard(consumer_id(ConsumerTag), MsgIds), - case send_command(Node, undefined, Cmd, normal, State0) of - {_, S} -> - % turn slow into ok for this function - {S, []} - end; -discard(ConsumerTag, [_|_] = MsgIds, - #state{unsent_commands = Unsent0} = State0) -> - ConsumerId = consumer_id(ConsumerTag), - %% we've reached the soft limit so will stash the command to be - %% sent once we have seen enough notifications - Unsent = maps:update_with(ConsumerId, - fun ({Settles, Returns, Discards}) -> - {Settles, Returns, Discards ++ MsgIds} - end, {[], [], MsgIds}, Unsent0), - {State0#state{unsent_commands = Unsent}, []}. - - -%% @doc Register with the rabbit_fifo queue to "checkout" messages as they -%% become available. -%% -%% This is a synchronous call. I.e. the call will block until the command -%% has been accepted by the ra process or it times out. -%% -%% @param ConsumerTag a unique tag to identify this particular consumer. -%% @param NumUnsettled the maximum number of in-flight messages. Once this -%% number of messages has been received but not settled no further messages -%% will be delivered to the consumer. -%% @param State The {@module} state. -%% -%% @returns `{ok, State}' or `{error | timeout, term()}' --spec checkout(rabbit_fifo:consumer_tag(), NumUnsettled :: non_neg_integer(), - rabbit_fifo:consumer_meta(), - state()) -> {ok, state()} | {error | timeout, term()}. -checkout(ConsumerTag, NumUnsettled, ConsumerInfo, State0) - when is_map(ConsumerInfo) -> - checkout(ConsumerTag, NumUnsettled, get_credit_mode(ConsumerInfo), ConsumerInfo, State0). - -%% @doc Register with the rabbit_fifo queue to "checkout" messages as they -%% become available. -%% -%% This is a synchronous call. I.e. the call will block until the command -%% has been accepted by the ra process or it times out. -%% -%% @param ConsumerTag a unique tag to identify this particular consumer. -%% @param NumUnsettled the maximum number of in-flight messages. Once this -%% number of messages has been received but not settled no further messages -%% will be delivered to the consumer. -%% @param CreditMode The credit mode to use for the checkout. -%% simple_prefetch: credit is auto topped up as deliveries are settled -%% credited: credit is only increased by sending credit to the queue -%% @param State The {@module} state. -%% -%% @returns `{ok, State}' or `{error | timeout, term()}' --spec checkout(rabbit_fifo:consumer_tag(), - NumUnsettled :: non_neg_integer(), - CreditMode :: rabbit_fifo:credit_mode(), - Meta :: rabbit_fifo:consumer_meta(), - state()) -> {ok, state()} | {error | timeout, term()}. -checkout(ConsumerTag, NumUnsettled, CreditMode, Meta, - #state{consumer_deliveries = CDels0} = State0) -> - Servers = sorted_servers(State0), - ConsumerId = {ConsumerTag, self()}, - Cmd = rabbit_fifo:make_checkout(ConsumerId, - {auto, NumUnsettled, CreditMode}, - Meta), - %% ??? - Ack = maps:get(ack, Meta, true), - - SDels = maps:update_with(ConsumerTag, - fun (V) -> - V#consumer{ack = Ack} - end, - #consumer{last_msg_id = -1, - ack = Ack}, CDels0), - try_process_command(Servers, Cmd, State0#state{consumer_deliveries = SDels}). - -%% @doc Provide credit to the queue -%% -%% This only has an effect if the consumer uses credit mode: credited -%% @param ConsumerTag a unique tag to identify this particular consumer. -%% @param Credit the amount of credit to provide to theq queue -%% @param Drain tells the queue to use up any credit that cannot be immediately -%% fulfilled. (i.e. there are not enough messages on queue to use up all the -%% provided credit). --spec credit(rabbit_fifo:consumer_tag(), - Credit :: non_neg_integer(), - Drain :: boolean(), - state()) -> - {state(), actions()}. -credit(ConsumerTag, Credit, Drain, - #state{consumer_deliveries = CDels} = State0) -> - ConsumerId = consumer_id(ConsumerTag), - %% the last received msgid provides us with the delivery count if we - %% add one as it is 0 indexed - C = maps:get(ConsumerTag, CDels, #consumer{last_msg_id = -1}), - Node = pick_server(State0), - Cmd = rabbit_fifo:make_credit(ConsumerId, Credit, - C#consumer.last_msg_id + 1, Drain), - case send_command(Node, undefined, Cmd, normal, State0) of - {_, S} -> - % turn slow into ok for this function - {S, []} - end. - -%% @doc Cancels a checkout with the rabbit_fifo queue for the consumer tag -%% -%% This is a synchronous call. I.e. the call will block until the command -%% has been accepted by the ra process or it times out. -%% -%% @param ConsumerTag a unique tag to identify this particular consumer. -%% @param State The {@module} state. -%% -%% @returns `{ok, State}' or `{error | timeout, term()}' --spec cancel_checkout(rabbit_fifo:consumer_tag(), state()) -> - {ok, state()} | {error | timeout, term()}. -cancel_checkout(ConsumerTag, #state{consumer_deliveries = CDels} = State0) -> - Servers = sorted_servers(State0), - ConsumerId = {ConsumerTag, self()}, - Cmd = rabbit_fifo:make_checkout(ConsumerId, cancel, #{}), - State = State0#state{consumer_deliveries = maps:remove(ConsumerTag, CDels)}, - try_process_command(Servers, Cmd, State). - -%% @doc Purges all the messages from a rabbit_fifo queue and returns the number -%% of messages purged. --spec purge(ra:server_id()) -> {ok, non_neg_integer()} | {error | timeout, term()}. -purge(Node) -> - case ra:process_command(Node, rabbit_fifo:make_purge()) of - {ok, {purge, Reply}, _} -> - {ok, Reply}; - Err -> - Err - end. - --spec pending_size(state()) -> non_neg_integer(). -pending_size(#state{pending = Pend}) -> - maps:size(Pend). - --spec stat(ra:server_id()) -> - {ok, non_neg_integer(), non_neg_integer()} - | {error | timeout, term()}. -stat(Leader) -> - %% short timeout as we don't want to spend too long if it is going to - %% fail anyway - stat(Leader, 250). - --spec stat(ra:server_id(), non_neg_integer()) -> - {ok, non_neg_integer(), non_neg_integer()} - | {error | timeout, term()}. -stat(Leader, Timeout) -> - %% short timeout as we don't want to spend too long if it is going to - %% fail anyway - case ra:local_query(Leader, fun rabbit_fifo:query_stat/1, Timeout) of - {ok, {_, {R, C}}, _} -> {ok, R, C}; - {error, _} = Error -> Error; - {timeout, _} = Error -> Error - end. - -%% @doc returns the cluster name --spec cluster_name(state()) -> cluster_name(). -cluster_name(#state{cfg = #cfg{cluster_name = ClusterName}}) -> - ClusterName. - -update_machine_state(Server, Conf) -> - case ra:process_command(Server, rabbit_fifo:make_update_config(Conf)) of - {ok, ok, _} -> - ok; - Err -> - Err - end. - -%% @doc Handles incoming `ra_events'. Events carry both internal "bookeeping" -%% events emitted by the `ra' leader as well as `rabbit_fifo' emitted events such -%% as message deliveries. All ra events need to be handled by {@module} -%% to ensure bookeeping, resends and flow control is correctly handled. -%% -%% If the `ra_event' contains a `rabbit_fifo' generated message it will be returned -%% for further processing. -%% -%% Example: -%% -%% ``` -%% receive -%% {ra_event, From, Evt} -> -%% case rabbit_fifo_client:handle_ra_event(From, Evt, State0) of -%% {internal, _Seq, State} -> State; -%% {{delivery, _ConsumerTag, Msgs}, State} -> -%% handle_messages(Msgs), -%% ... -%% end -%% end -%% ''' -%% -%% @param From the {@link ra:server_id().} of the sending process. -%% @param Event the body of the `ra_event'. -%% @param State the current {@module} state. -%% -%% @returns -%% `{internal, AppliedCorrelations, State}' if the event contained an internally -%% handled event such as a notification and a correlation was included with -%% the command (e.g. in a call to `enqueue/3' the correlation terms are returned -%% here. -%% -%% `{RaFifoEvent, State}' if the event contained a client message generated by -%% the `rabbit_fifo' state machine such as a delivery. -%% -%% The type of `rabbit_fifo' client messages that can be received are: -%% -%% `{delivery, ConsumerTag, [{MsgId, {MsgHeader, Msg}}]}' -%% -%% <li>`ConsumerTag' the binary tag passed to {@link checkout/3.}</li> -%% <li>`MsgId' is a consumer scoped monotonically incrementing id that can be -%% used to {@link settle/3.} (roughly: AMQP 0.9.1 ack) message once finished -%% with them.</li> --spec handle_ra_event(ra:server_id(), ra_server_proc:ra_event_body(), state()) -> - {internal, Correlators :: [term()], actions(), state()} | - {rabbit_fifo:client_msg(), state()} | eol. -handle_ra_event(From, {applied, Seqs}, - #state{cfg = #cfg{cluster_name = QRef, - soft_limit = SftLmt, - unblock_handler = UnblockFun}} = State0) -> - - {Corrs, Actions0, State1} = lists:foldl(fun seq_applied/2, - {[], [], State0#state{leader = From}}, - Seqs), - Actions = case Corrs of - [] -> - lists:reverse(Actions0); - _ -> - [{settled, QRef, Corrs} - | lists:reverse(Actions0)] - end, - case maps:size(State1#state.pending) < SftLmt of - true when State1#state.slow == true -> - % we have exited soft limit state - % send any unsent commands and cancel the time as - % TODO: really the timer should only be cancelled when the channel - % exits flow state (which depends on the state of all queues the - % channel is interacting with) - % but the fact the queue has just applied suggests - % it's ok to cancel here anyway - State2 = cancel_timer(State1#state{slow = false, - unsent_commands = #{}}), - % build up a list of commands to issue - Commands = maps:fold( - fun (Cid, {Settled, Returns, Discards}, Acc) -> - add_command(Cid, settle, Settled, - add_command(Cid, return, Returns, - add_command(Cid, discard, - Discards, Acc))) - end, [], State1#state.unsent_commands), - Node = pick_server(State2), - %% send all the settlements and returns - State = lists:foldl(fun (C, S0) -> - case send_command(Node, undefined, - C, normal, S0) of - {T, S} when T =/= error -> - S - end - end, State2, Commands), - UnblockFun(), - {ok, State, Actions}; - _ -> - {ok, State1, Actions} - end; -handle_ra_event(From, {machine, {delivery, _ConsumerTag, _} = Del}, State0) -> - handle_delivery(From, Del, State0); -handle_ra_event(_, {machine, {queue_status, Status}}, - #state{} = State) -> - %% just set the queue status - {ok, State#state{queue_status = Status}, []}; -handle_ra_event(Leader, {machine, leader_change}, - #state{leader = Leader} = State) -> - %% leader already known - {ok, State, []}; -handle_ra_event(Leader, {machine, leader_change}, State0) -> - %% we need to update leader - %% and resend any pending commands - State = resend_all_pending(State0#state{leader = Leader}), - {ok, State, []}; -handle_ra_event(_From, {rejected, {not_leader, undefined, _Seq}}, State0) -> - % TODO: how should these be handled? re-sent on timer or try random - {ok, State0, []}; -handle_ra_event(_From, {rejected, {not_leader, Leader, Seq}}, State0) -> - State1 = State0#state{leader = Leader}, - State = resend(Seq, State1), - {ok, State, []}; -handle_ra_event(_, timeout, #state{cfg = #cfg{servers = Servers}} = State0) -> - case find_leader(Servers) of - undefined -> - %% still no leader, set the timer again - {ok, set_timer(State0), []}; - Leader -> - State = resend_all_pending(State0#state{leader = Leader}), - {ok, State, []} - end; -handle_ra_event(_Leader, {machine, eol}, _State0) -> - eol. - -%% @doc Attempts to enqueue a message using cast semantics. This provides no -%% guarantees or retries if the message fails to achieve consensus or if the -%% servers sent to happens not to be available. If the message is sent to a -%% follower it will attempt the deliver it to the leader, if known. Else it will -%% drop the messages. -%% -%% NB: only use this for non-critical enqueues where a full rabbit_fifo_client state -%% cannot be maintained. -%% -%% @param CusterId the cluster id. -%% @param Servers the known servers in the cluster. -%% @param Msg the message to enqueue. -%% -%% @returns `ok' --spec untracked_enqueue([ra:server_id()], term()) -> - ok. -untracked_enqueue([Node | _], Msg) -> - Cmd = rabbit_fifo:make_enqueue(undefined, undefined, Msg), - ok = ra:pipeline_command(Node, Cmd), - ok. - -%% Internal - -try_process_command([Server | Rem], Cmd, State) -> - case ra:process_command(Server, Cmd, 30000) of - {ok, _, Leader} -> - {ok, State#state{leader = Leader}}; - Err when length(Rem) =:= 0 -> - Err; - _ -> - try_process_command(Rem, Cmd, State) - end. - -seq_applied({Seq, MaybeAction}, - {Corrs, Actions0, #state{last_applied = Last} = State0}) - when Seq > Last -> - State1 = do_resends(Last+1, Seq-1, State0), - {Actions, State} = maybe_add_action(MaybeAction, Actions0, State1), - case maps:take(Seq, State#state.pending) of - {{undefined, _}, Pending} -> - {Corrs, Actions, State#state{pending = Pending, - last_applied = Seq}}; - {{Corr, _}, Pending} -> - {[Corr | Corrs], Actions, State#state{pending = Pending, - last_applied = Seq}}; - error -> - % must have already been resent or removed for some other reason - % still need to update last_applied or we may inadvertently resend - % stuff later - {Corrs, Actions, State#state{last_applied = Seq}} - end; -seq_applied(_Seq, Acc) -> - Acc. - -maybe_add_action(ok, Acc, State) -> - {Acc, State}; -maybe_add_action({multi, Actions}, Acc0, State0) -> - lists:foldl(fun (Act, {Acc, State}) -> - maybe_add_action(Act, Acc, State) - end, {Acc0, State0}, Actions); -maybe_add_action({send_drained, {Tag, Credit}} = Action, Acc, - #state{consumer_deliveries = CDels} = State) -> - %% add credit to consumer delivery_count - C = maps:get(Tag, CDels), - {[Action | Acc], - State#state{consumer_deliveries = - update_consumer(Tag, C#consumer.last_msg_id, - Credit, C, CDels)}}; -maybe_add_action(Action, Acc, State) -> - %% anything else is assumed to be an action - {[Action | Acc], State}. - -do_resends(From, To, State) when From =< To -> - % ?INFO("rabbit_fifo_client: doing resends From ~w To ~w~n", [From, To]), - lists:foldl(fun resend/2, State, lists:seq(From, To)); -do_resends(_, _, State) -> - State. - -% resends a command with a new sequence number -resend(OldSeq, #state{pending = Pending0, leader = Leader} = State) -> - case maps:take(OldSeq, Pending0) of - {{Corr, Cmd}, Pending} -> - %% resends aren't subject to flow control here - resend_command(Leader, Corr, Cmd, State#state{pending = Pending}); - error -> - State - end. - -resend_all_pending(#state{pending = Pend} = State) -> - Seqs = lists:sort(maps:keys(Pend)), - lists:foldl(fun resend/2, State, Seqs). - -maybe_auto_ack(true, Deliver, State0) -> - %% manual ack is enabled - {ok, State0, [Deliver]}; -maybe_auto_ack(false, {deliver, Tag, _Ack, Msgs} = Deliver, State0) -> - %% we have to auto ack these deliveries - MsgIds = [I || {_, _, I, _, _} <- Msgs], - {State, Actions} = settle(Tag, MsgIds, State0), - {ok, State, [Deliver] ++ Actions}. - - -handle_delivery(Leader, {delivery, Tag, [{FstId, _} | _] = IdMsgs}, - #state{cfg = #cfg{cluster_name = QName}, - consumer_deliveries = CDels0} = State0) -> - QRef = qref(Leader), - {LastId, _} = lists:last(IdMsgs), - Consumer = #consumer{ack = Ack} = maps:get(Tag, CDels0), - %% format as a deliver action - Del = {deliver, Tag, Ack, transform_msgs(QName, QRef, IdMsgs)}, - %% TODO: remove potential default allocation - case Consumer of - #consumer{last_msg_id = Prev} = C - when FstId =:= Prev+1 -> - maybe_auto_ack(Ack, Del, - State0#state{consumer_deliveries = - update_consumer(Tag, LastId, - length(IdMsgs), C, - CDels0)}); - #consumer{last_msg_id = Prev} = C - when FstId > Prev+1 -> - NumMissing = FstId - Prev + 1, - %% there may actually be fewer missing messages returned than expected - %% This can happen when a node the channel is on gets disconnected - %% from the node the leader is on and then reconnected afterwards. - %% When the node is disconnected the leader will return all checked - %% out messages to the main queue to ensure they don't get stuck in - %% case the node never comes back. - case get_missing_deliveries(Leader, Prev+1, FstId-1, Tag) of - {protocol_error, _, _, _} = Err -> - Err; - Missing -> - XDel = {deliver, Tag, Ack, transform_msgs(QName, QRef, - Missing ++ IdMsgs)}, - maybe_auto_ack(Ack, XDel, - State0#state{consumer_deliveries = - update_consumer(Tag, LastId, - length(IdMsgs) + NumMissing, - C, CDels0)}) - end; - #consumer{last_msg_id = Prev} - when FstId =< Prev -> - case lists:dropwhile(fun({Id, _}) -> Id =< Prev end, IdMsgs) of - [] -> - {ok, State0, []}; - IdMsgs2 -> - handle_delivery(Leader, {delivery, Tag, IdMsgs2}, State0) - end; - C when FstId =:= 0 -> - % the very first delivery - maybe_auto_ack(Ack, Del, - State0#state{consumer_deliveries = - update_consumer(Tag, LastId, - length(IdMsgs), - C#consumer{last_msg_id = LastId}, - CDels0)}) - end. - -transform_msgs(QName, QRef, Msgs) -> - lists:map( - fun({MsgId, {MsgHeader, Msg0}}) -> - {Msg, Redelivered} = case MsgHeader of - #{delivery_count := C} -> - {add_delivery_count_header(Msg0, C), true}; - _ -> - {Msg0, false} - end, - {QName, QRef, MsgId, Redelivered, Msg} - end, Msgs). - -update_consumer(Tag, LastId, DelCntIncr, - #consumer{delivery_count = D} = C, Consumers) -> - maps:put(Tag, - C#consumer{last_msg_id = LastId, - delivery_count = D + DelCntIncr}, - Consumers). - - -get_missing_deliveries(Leader, From, To, ConsumerTag) -> - ConsumerId = consumer_id(ConsumerTag), - % ?INFO("get_missing_deliveries for ~w from ~b to ~b", - % [ConsumerId, From, To]), - Query = fun (State) -> - rabbit_fifo:get_checked_out(ConsumerId, From, To, State) - end, - case ra:local_query(Leader, Query) of - {ok, {_, Missing}, _} -> - Missing; - {error, Error} -> - {protocol_error, internal_error, "Cannot query missing deliveries from ~p: ~p", - [Leader, Error]}; - {timeout, _} -> - {protocol_error, internal_error, "Cannot query missing deliveries from ~p: timeout", - [Leader]} - end. - -pick_server(#state{leader = undefined, - cfg = #cfg{servers = [N | _]}}) -> - %% TODO: pick random rather that first? - N; -pick_server(#state{leader = Leader}) -> - Leader. - -% servers sorted by last known leader -sorted_servers(#state{leader = undefined, - cfg = #cfg{servers = Servers}}) -> - Servers; -sorted_servers(#state{leader = Leader, - cfg = #cfg{servers = Servers}}) -> - [Leader | lists:delete(Leader, Servers)]. - -next_seq(#state{next_seq = Seq} = State) -> - {Seq, State#state{next_seq = Seq + 1}}. - -next_enqueue_seq(#state{next_enqueue_seq = Seq} = State) -> - {Seq, State#state{next_enqueue_seq = Seq + 1}}. - -consumer_id(ConsumerTag) -> - {ConsumerTag, self()}. - -send_command(Server, Correlation, Command, Priority, - #state{pending = Pending, - cfg = #cfg{soft_limit = SftLmt}} = State0) -> - {Seq, State} = next_seq(State0), - ok = ra:pipeline_command(Server, Command, Seq, Priority), - Tag = case maps:size(Pending) >= SftLmt of - true -> slow; - false -> ok - end, - {Tag, State#state{pending = Pending#{Seq => {Correlation, Command}}, - slow = Tag == slow}}. - -resend_command(Node, Correlation, Command, - #state{pending = Pending} = State0) -> - {Seq, State} = next_seq(State0), - ok = ra:pipeline_command(Node, Command, Seq), - State#state{pending = Pending#{Seq => {Correlation, Command}}}. - -add_command(_, _, [], Acc) -> - Acc; -add_command(Cid, settle, MsgIds, Acc) -> - [rabbit_fifo:make_settle(Cid, MsgIds) | Acc]; -add_command(Cid, return, MsgIds, Acc) -> - [rabbit_fifo:make_return(Cid, MsgIds) | Acc]; -add_command(Cid, discard, MsgIds, Acc) -> - [rabbit_fifo:make_discard(Cid, MsgIds) | Acc]. - -set_timer(#state{leader = Leader0, - cfg = #cfg{servers = [Server | _], - cluster_name = QName}} = State) -> - Leader = case Leader0 of - undefined -> Server; - _ -> - Leader0 - end, - Ref = erlang:send_after(?TIMER_TIME, self(), - {'$gen_cast', - {queue_event, QName, {Leader, timeout}}}), - State#state{timer_state = Ref}. - -cancel_timer(#state{timer_state = undefined} = State) -> - State; -cancel_timer(#state{timer_state = Ref} = State) -> - erlang:cancel_timer(Ref, [{async, true}, {info, false}]), - State#state{timer_state = undefined}. - -find_leader([]) -> - undefined; -find_leader([Server | Servers]) -> - case ra:members(Server, 500) of - {ok, _, Leader} -> Leader; - _ -> - find_leader(Servers) - end. - -qref({Ref, _}) -> Ref; -qref(Ref) -> Ref. - -get_credit_mode(#{args := Args}) -> - case rabbit_misc:table_lookup(Args, <<"x-credit">>) of - {_Key, Value} -> - Value; - _ -> - simple_prefetch - end; -get_credit_mode(_) -> - simple_prefetch. diff --git a/src/rabbit_fifo_index.erl b/src/rabbit_fifo_index.erl deleted file mode 100644 index 14ac89faff..0000000000 --- a/src/rabbit_fifo_index.erl +++ /dev/null @@ -1,119 +0,0 @@ --module(rabbit_fifo_index). - --export([ - empty/0, - exists/2, - append/2, - delete/2, - size/1, - smallest/1, - map/2 - ]). - --compile({no_auto_import, [size/1]}). - -%% the empty atom is a lot smaller (4 bytes) than e.g. `undefined` (13 bytes). -%% This matters as the data map gets persisted as part of the snapshot --define(NIL, ''). - --record(?MODULE, {data = #{} :: #{integer() => ?NIL}, - smallest :: undefined | non_neg_integer(), - largest :: undefined | non_neg_integer() - }). - - --opaque state() :: #?MODULE{}. - --export_type([state/0]). - --spec empty() -> state(). -empty() -> - #?MODULE{}. - --spec exists(integer(), state()) -> boolean(). -exists(Key, #?MODULE{data = Data}) -> - maps:is_key(Key, Data). - -% only integer keys are supported --spec append(integer(), state()) -> state(). -append(Key, - #?MODULE{data = Data, - smallest = Smallest, - largest = Largest} = State) - when Key > Largest orelse Largest =:= undefined -> - State#?MODULE{data = maps:put(Key, ?NIL, Data), - smallest = ra_lib:default(Smallest, Key), - largest = Key}. - --spec delete(Index :: integer(), state()) -> state(). -delete(Smallest, #?MODULE{data = Data0, - largest = Largest, - smallest = Smallest} = State) -> - Data = maps:remove(Smallest, Data0), - case find_next(Smallest + 1, Largest, Data) of - undefined -> - State#?MODULE{data = Data, - smallest = undefined, - largest = undefined}; - Next -> - State#?MODULE{data = Data, smallest = Next} - end; -delete(Key, #?MODULE{data = Data} = State) -> - State#?MODULE{data = maps:remove(Key, Data)}. - --spec size(state()) -> non_neg_integer(). -size(#?MODULE{data = Data}) -> - maps:size(Data). - --spec smallest(state()) -> undefined | integer(). -smallest(#?MODULE{smallest = Smallest}) -> - Smallest. - - --spec map(fun(), state()) -> state(). -map(F, #?MODULE{data = Data} = State) -> - State#?MODULE{data = maps:map(F, Data)}. - - -%% internal - -find_next(Next, Last, _Map) when Next > Last -> - undefined; -find_next(Next, Last, Map) -> - case Map of - #{Next := _} -> - Next; - _ -> - % in degenerate cases the range here could be very large - % and hence this could be very slow - % the typical case should ideally be better - % assuming fifo-ish deletion of entries - find_next(Next+1, Last, Map) - end. - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -append_test() -> - S0 = empty(), - false = exists(99, S0), - undefined = smallest(S0), - 0 = size(S0), - S1 = append(1, S0), - false = exists(99, S1), - true = exists(1, S1), - 1 = size(S1), - 1 = smallest(S1), - S2 = append(2, S1), - true = exists(2, S2), - 2 = size(S2), - 1 = smallest(S2), - S3 = delete(1, S2), - 2 = smallest(S3), - 1 = size(S3), - S5 = delete(2, S3), - undefined = smallest(S5), - 0 = size(S0), - ok. - --endif. diff --git a/src/rabbit_fifo_v0.erl b/src/rabbit_fifo_v0.erl deleted file mode 100644 index a61f42616d..0000000000 --- a/src/rabbit_fifo_v0.erl +++ /dev/null @@ -1,1961 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License -%% at https://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and -%% limitations under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% The Initial Developer of the Original Code is GoPivotal, Inc. -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_fifo_v0). - --behaviour(ra_machine). - --compile(inline_list_funcs). --compile(inline). --compile({no_auto_import, [apply/3]}). - --include("rabbit_fifo_v0.hrl"). --include_lib("rabbit_common/include/rabbit.hrl"). - --export([ - init/1, - apply/3, - state_enter/2, - tick/2, - overview/1, - get_checked_out/4, - %% aux - init_aux/1, - handle_aux/6, - % queries - query_messages_ready/1, - query_messages_checked_out/1, - query_messages_total/1, - query_processes/1, - query_ra_indexes/1, - query_consumer_count/1, - query_consumers/1, - query_stat/1, - query_single_active_consumer/1, - query_in_memory_usage/1, - usage/1, - - zero/1, - - %% misc - dehydrate_state/1, - normalize/1, - normalize_for_v1/1, - %% getters for coversions - get_field/2, - get_cfg_field/2, - - %% protocol helpers - make_enqueue/3, - make_checkout/3, - make_settle/2, - make_return/2, - make_discard/2, - make_credit/4, - make_purge/0, - make_purge_nodes/1, - make_update_config/1 - ]). - -%% command records representing all the protocol actions that are supported --record(enqueue, {pid :: option(pid()), - seq :: option(msg_seqno()), - msg :: raw_msg()}). --record(checkout, {consumer_id :: consumer_id(), - spec :: checkout_spec(), - meta :: consumer_meta()}). --record(settle, {consumer_id :: consumer_id(), - msg_ids :: [msg_id()]}). --record(return, {consumer_id :: consumer_id(), - msg_ids :: [msg_id()]}). --record(discard, {consumer_id :: consumer_id(), - msg_ids :: [msg_id()]}). --record(credit, {consumer_id :: consumer_id(), - credit :: non_neg_integer(), - delivery_count :: non_neg_integer(), - drain :: boolean()}). --record(purge, {}). --record(purge_nodes, {nodes :: [node()]}). --record(update_config, {config :: config()}). - --opaque protocol() :: - #enqueue{} | - #checkout{} | - #settle{} | - #return{} | - #discard{} | - #credit{} | - #purge{} | - #purge_nodes{} | - #update_config{}. - --type command() :: protocol() | ra_machine:builtin_command(). -%% all the command types supported by ra fifo - --type client_msg() :: delivery(). -%% the messages `rabbit_fifo' can send to consumers. - --opaque state() :: #?STATE{}. - --export_type([protocol/0, - delivery/0, - command/0, - credit_mode/0, - consumer_tag/0, - consumer_meta/0, - consumer_id/0, - client_msg/0, - msg/0, - msg_id/0, - msg_seqno/0, - delivery_msg/0, - state/0, - config/0]). - --spec init(config()) -> state(). -init(#{name := Name, - queue_resource := Resource} = Conf) -> - update_config(Conf, #?STATE{cfg = #cfg{name = Name, - resource = Resource}}). - -update_config(Conf, State) -> - DLH = maps:get(dead_letter_handler, Conf, undefined), - BLH = maps:get(become_leader_handler, Conf, undefined), - SHI = maps:get(release_cursor_interval, Conf, ?RELEASE_CURSOR_EVERY), - MaxLength = maps:get(max_length, Conf, undefined), - MaxBytes = maps:get(max_bytes, Conf, undefined), - MaxMemoryLength = maps:get(max_in_memory_length, Conf, undefined), - MaxMemoryBytes = maps:get(max_in_memory_bytes, Conf, undefined), - DeliveryLimit = maps:get(delivery_limit, Conf, undefined), - ConsumerStrategy = case maps:get(single_active_consumer_on, Conf, false) of - true -> - single_active; - false -> - competing - end, - Cfg = State#?STATE.cfg, - SHICur = case State#?STATE.cfg of - #cfg{release_cursor_interval = {_, C}} -> - C; - #cfg{release_cursor_interval = undefined} -> - SHI; - #cfg{release_cursor_interval = C} -> - C - end, - - State#?STATE{cfg = Cfg#cfg{release_cursor_interval = {SHI, SHICur}, - dead_letter_handler = DLH, - become_leader_handler = BLH, - max_length = MaxLength, - max_bytes = MaxBytes, - max_in_memory_length = MaxMemoryLength, - max_in_memory_bytes = MaxMemoryBytes, - consumer_strategy = ConsumerStrategy, - delivery_limit = DeliveryLimit}}. - -zero(_) -> - 0. - -% msg_ids are scoped per consumer -% ra_indexes holds all raft indexes for enqueues currently on queue --spec apply(ra_machine:command_meta_data(), command(), state()) -> - {state(), Reply :: term(), ra_machine:effects()} | - {state(), Reply :: term()}. -apply(Metadata, #enqueue{pid = From, seq = Seq, - msg = RawMsg}, State00) -> - apply_enqueue(Metadata, From, Seq, RawMsg, State00); -apply(Meta, - #settle{msg_ids = MsgIds, consumer_id = ConsumerId}, - #?STATE{consumers = Cons0} = State) -> - case Cons0 of - #{ConsumerId := Con0} -> - % need to increment metrics before completing as any snapshot - % states taken need to include them - complete_and_checkout(Meta, MsgIds, ConsumerId, - Con0, [], State); - _ -> - {State, ok} - - end; -apply(Meta, #discard{msg_ids = MsgIds, consumer_id = ConsumerId}, - #?STATE{consumers = Cons0} = State0) -> - case Cons0 of - #{ConsumerId := Con0} -> - Discarded = maps:with(MsgIds, Con0#consumer.checked_out), - Effects = dead_letter_effects(rejected, Discarded, State0, []), - complete_and_checkout(Meta, MsgIds, ConsumerId, Con0, - Effects, State0); - _ -> - {State0, ok} - end; -apply(Meta, #return{msg_ids = MsgIds, consumer_id = ConsumerId}, - #?STATE{consumers = Cons0} = State) -> - case Cons0 of - #{ConsumerId := #consumer{checked_out = Checked0}} -> - Returned = maps:with(MsgIds, Checked0), - return(Meta, ConsumerId, Returned, [], State); - _ -> - {State, ok} - end; -apply(Meta, #credit{credit = NewCredit, delivery_count = RemoteDelCnt, - drain = Drain, consumer_id = ConsumerId}, - #?STATE{consumers = Cons0, - service_queue = ServiceQueue0, - waiting_consumers = Waiting0} = State0) -> - case Cons0 of - #{ConsumerId := #consumer{delivery_count = DelCnt} = Con0} -> - %% this can go below 0 when credit is reduced - C = max(0, RemoteDelCnt + NewCredit - DelCnt), - %% grant the credit - Con1 = Con0#consumer{credit = C}, - ServiceQueue = maybe_queue_consumer(ConsumerId, Con1, - ServiceQueue0), - Cons = maps:put(ConsumerId, Con1, Cons0), - {State1, ok, Effects} = - checkout(Meta, State0#?STATE{service_queue = ServiceQueue, - consumers = Cons}, []), - Response = {send_credit_reply, messages_ready(State1)}, - %% by this point all checkouts for the updated credit value - %% should be processed so we can evaluate the drain - case Drain of - false -> - %% just return the result of the checkout - {State1, Response, Effects}; - true -> - Con = #consumer{credit = PostCred} = - maps:get(ConsumerId, State1#?STATE.consumers), - %% add the outstanding credit to the delivery count - DeliveryCount = Con#consumer.delivery_count + PostCred, - Consumers = maps:put(ConsumerId, - Con#consumer{delivery_count = DeliveryCount, - credit = 0}, - State1#?STATE.consumers), - Drained = Con#consumer.credit, - {CTag, _} = ConsumerId, - {State1#?STATE{consumers = Consumers}, - %% returning a multi response with two client actions - %% for the channel to execute - {multi, [Response, {send_drained, {CTag, Drained}}]}, - Effects} - end; - _ when Waiting0 /= [] -> - %% there are waiting consuemrs - case lists:keytake(ConsumerId, 1, Waiting0) of - {value, {_, Con0 = #consumer{delivery_count = DelCnt}}, Waiting} -> - %% the consumer is a waiting one - %% grant the credit - C = max(0, RemoteDelCnt + NewCredit - DelCnt), - Con = Con0#consumer{credit = C}, - State = State0#?STATE{waiting_consumers = - [{ConsumerId, Con} | Waiting]}, - {State, {send_credit_reply, messages_ready(State)}}; - false -> - {State0, ok} - end; - _ -> - %% credit for unknown consumer - just ignore - {State0, ok} - end; -apply(_, #checkout{spec = {dequeue, _}}, - #?STATE{cfg = #cfg{consumer_strategy = single_active}} = State0) -> - {State0, {error, unsupported}}; -apply(#{from := From} = Meta, #checkout{spec = {dequeue, Settlement}, - meta = ConsumerMeta, - consumer_id = ConsumerId}, - #?STATE{consumers = Consumers} = State0) -> - Exists = maps:is_key(ConsumerId, Consumers), - case messages_ready(State0) of - 0 -> - {State0, {dequeue, empty}}; - _ when Exists -> - %% a dequeue using the same consumer_id isn't possible at this point - {State0, {dequeue, empty}}; - Ready -> - State1 = update_consumer(ConsumerId, ConsumerMeta, - {once, 1, simple_prefetch}, - State0), - {success, _, MsgId, Msg, State2} = checkout_one(State1), - {State, Effects} = case Settlement of - unsettled -> - {_, Pid} = ConsumerId, - {State2, [{monitor, process, Pid}]}; - settled -> - %% immediately settle the checkout - {State3, _, Effects0} = - apply(Meta, make_settle(ConsumerId, [MsgId]), - State2), - {State3, Effects0} - end, - case Msg of - {RaftIdx, {Header, 'empty'}} -> - %% TODO add here new log effect with reply - {State, '$ra_no_reply', - reply_log_effect(RaftIdx, MsgId, Header, Ready - 1, From)}; - _ -> - {State, {dequeue, {MsgId, Msg}, Ready-1}, Effects} - end - end; -apply(Meta, #checkout{spec = cancel, consumer_id = ConsumerId}, State0) -> - {State, Effects} = cancel_consumer(ConsumerId, State0, [], consumer_cancel), - checkout(Meta, State, Effects); -apply(Meta, #checkout{spec = Spec, meta = ConsumerMeta, - consumer_id = {_, Pid} = ConsumerId}, - State0) -> - State1 = update_consumer(ConsumerId, ConsumerMeta, Spec, State0), - checkout(Meta, State1, [{monitor, process, Pid}]); -apply(#{index := RaftIdx}, #purge{}, - #?STATE{ra_indexes = Indexes0, - returns = Returns, - messages = Messages} = State0) -> - Total = messages_ready(State0), - Indexes1 = lists:foldl(fun rabbit_fifo_index:delete/2, Indexes0, - [I || {I, _} <- lists:sort(maps:values(Messages))]), - Indexes = lists:foldl(fun rabbit_fifo_index:delete/2, Indexes1, - [I || {_, {I, _}} <- lqueue:to_list(Returns)]), - {State, _, Effects} = - update_smallest_raft_index(RaftIdx, - State0#?STATE{ra_indexes = Indexes, - messages = #{}, - returns = lqueue:new(), - msg_bytes_enqueue = 0, - prefix_msgs = {0, [], 0, []}, - low_msg_num = undefined, - msg_bytes_in_memory = 0, - msgs_ready_in_memory = 0}, - []), - %% as we're not checking out after a purge (no point) we have to - %% reverse the effects ourselves - {State, {purge, Total}, - lists:reverse([garbage_collection | Effects])}; -apply(Meta, {down, Pid, noconnection}, - #?STATE{consumers = Cons0, - cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = Waiting0, - enqueuers = Enqs0} = State0) -> - Node = node(Pid), - %% if the pid refers to an active or cancelled consumer, - %% mark it as suspected and return it to the waiting queue - {State1, Effects0} = - maps:fold(fun({_, P} = Cid, C0, {S0, E0}) - when node(P) =:= Node -> - %% the consumer should be returned to waiting - %% and checked out messages should be returned - Effs = consumer_update_active_effects( - S0, Cid, C0, false, suspected_down, E0), - Checked = C0#consumer.checked_out, - Credit = increase_credit(C0, maps:size(Checked)), - {St, Effs1} = return_all(S0, Effs, - Cid, C0#consumer{credit = Credit}), - %% if the consumer was cancelled there is a chance it got - %% removed when returning hence we need to be defensive here - Waiting = case St#?STATE.consumers of - #{Cid := C} -> - Waiting0 ++ [{Cid, C}]; - _ -> - Waiting0 - end, - {St#?STATE{consumers = maps:remove(Cid, St#?STATE.consumers), - waiting_consumers = Waiting}, - Effs1}; - (_, _, S) -> - S - end, {State0, []}, Cons0), - WaitingConsumers = update_waiting_consumer_status(Node, State1, - suspected_down), - - %% select a new consumer from the waiting queue and run a checkout - State2 = State1#?STATE{waiting_consumers = WaitingConsumers}, - {State, Effects1} = activate_next_consumer(State2, Effects0), - - %% mark any enquers as suspected - Enqs = maps:map(fun(P, E) when node(P) =:= Node -> - E#enqueuer{status = suspected_down}; - (_, E) -> E - end, Enqs0), - Effects = [{monitor, node, Node} | Effects1], - checkout(Meta, State#?STATE{enqueuers = Enqs}, Effects); -apply(Meta, {down, Pid, noconnection}, - #?STATE{consumers = Cons0, - enqueuers = Enqs0} = State0) -> - %% A node has been disconnected. This doesn't necessarily mean that - %% any processes on this node are down, they _may_ come back so here - %% we just mark them as suspected (effectively deactivated) - %% and return all checked out messages to the main queue for delivery to any - %% live consumers - %% - %% all pids for the disconnected node will be marked as suspected not just - %% the one we got the `down' command for - Node = node(Pid), - - {State, Effects1} = - maps:fold( - fun({_, P} = Cid, #consumer{checked_out = Checked0, - status = up} = C0, - {St0, Eff}) when node(P) =:= Node -> - Credit = increase_credit(C0, map_size(Checked0)), - C = C0#consumer{status = suspected_down, - credit = Credit}, - {St, Eff0} = return_all(St0, Eff, Cid, C), - Eff1 = consumer_update_active_effects(St, Cid, C, false, - suspected_down, Eff0), - {St, Eff1}; - (_, _, {St, Eff}) -> - {St, Eff} - end, {State0, []}, Cons0), - Enqs = maps:map(fun(P, E) when node(P) =:= Node -> - E#enqueuer{status = suspected_down}; - (_, E) -> E - end, Enqs0), - - % Monitor the node so that we can "unsuspect" these processes when the node - % comes back, then re-issue all monitors and discover the final fate of - % these processes - Effects = case maps:size(State#?STATE.consumers) of - 0 -> - [{aux, inactive}, {monitor, node, Node}]; - _ -> - [{monitor, node, Node}] - end ++ Effects1, - checkout(Meta, State#?STATE{enqueuers = Enqs}, Effects); -apply(Meta, {down, Pid, _Info}, State0) -> - {State, Effects} = handle_down(Pid, State0), - checkout(Meta, State, Effects); -apply(Meta, {nodeup, Node}, #?STATE{consumers = Cons0, - enqueuers = Enqs0, - service_queue = SQ0} = State0) -> - %% A node we are monitoring has come back. - %% If we have suspected any processes of being - %% down we should now re-issue the monitors for them to detect if they're - %% actually down or not - Monitors = [{monitor, process, P} - || P <- suspected_pids_for(Node, State0)], - - Enqs1 = maps:map(fun(P, E) when node(P) =:= Node -> - E#enqueuer{status = up}; - (_, E) -> E - end, Enqs0), - ConsumerUpdateActiveFun = consumer_active_flag_update_function(State0), - %% mark all consumers as up - {Cons1, SQ, Effects1} = - maps:fold(fun({_, P} = ConsumerId, C, {CAcc, SQAcc, EAcc}) - when (node(P) =:= Node) and - (C#consumer.status =/= cancelled) -> - EAcc1 = ConsumerUpdateActiveFun(State0, ConsumerId, - C, true, up, EAcc), - update_or_remove_sub(ConsumerId, - C#consumer{status = up}, CAcc, - SQAcc, EAcc1); - (_, _, Acc) -> - Acc - end, {Cons0, SQ0, Monitors}, Cons0), - Waiting = update_waiting_consumer_status(Node, State0, up), - State1 = State0#?STATE{consumers = Cons1, - enqueuers = Enqs1, - service_queue = SQ, - waiting_consumers = Waiting}, - {State, Effects} = activate_next_consumer(State1, Effects1), - checkout(Meta, State, Effects); -apply(_, {nodedown, _Node}, State) -> - {State, ok}; -apply(_, #purge_nodes{nodes = Nodes}, State0) -> - {State, Effects} = lists:foldl(fun(Node, {S, E}) -> - purge_node(Node, S, E) - end, {State0, []}, Nodes), - {State, ok, Effects}; -apply(Meta, #update_config{config = Conf}, State) -> - checkout(Meta, update_config(Conf, State), []). - -purge_node(Node, State, Effects) -> - lists:foldl(fun(Pid, {S0, E0}) -> - {S, E} = handle_down(Pid, S0), - {S, E0 ++ E} - end, {State, Effects}, all_pids_for(Node, State)). - -%% any downs that re not noconnection -handle_down(Pid, #?STATE{consumers = Cons0, - enqueuers = Enqs0} = State0) -> - % Remove any enqueuer for the same pid and enqueue any pending messages - % This should be ok as we won't see any more enqueues from this pid - State1 = case maps:take(Pid, Enqs0) of - {#enqueuer{pending = Pend}, Enqs} -> - lists:foldl(fun ({_, RIdx, RawMsg}, S) -> - enqueue(RIdx, RawMsg, S) - end, State0#?STATE{enqueuers = Enqs}, Pend); - error -> - State0 - end, - {Effects1, State2} = handle_waiting_consumer_down(Pid, State1), - % return checked out messages to main queue - % Find the consumers for the down pid - DownConsumers = maps:keys( - maps:filter(fun({_, P}, _) -> P =:= Pid end, Cons0)), - lists:foldl(fun(ConsumerId, {S, E}) -> - cancel_consumer(ConsumerId, S, E, down) - end, {State2, Effects1}, DownConsumers). - -consumer_active_flag_update_function(#?STATE{cfg = #cfg{consumer_strategy = competing}}) -> - fun(State, ConsumerId, Consumer, Active, ActivityStatus, Effects) -> - consumer_update_active_effects(State, ConsumerId, Consumer, Active, - ActivityStatus, Effects) - end; -consumer_active_flag_update_function(#?STATE{cfg = #cfg{consumer_strategy = single_active}}) -> - fun(_, _, _, _, _, Effects) -> - Effects - end. - -handle_waiting_consumer_down(_Pid, - #?STATE{cfg = #cfg{consumer_strategy = competing}} = State) -> - {[], State}; -handle_waiting_consumer_down(_Pid, - #?STATE{cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = []} = State) -> - {[], State}; -handle_waiting_consumer_down(Pid, - #?STATE{cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = WaitingConsumers0} = State0) -> - % get cancel effects for down waiting consumers - Down = lists:filter(fun({{_, P}, _}) -> P =:= Pid end, - WaitingConsumers0), - Effects = lists:foldl(fun ({ConsumerId, _}, Effects) -> - cancel_consumer_effects(ConsumerId, State0, - Effects) - end, [], Down), - % update state to have only up waiting consumers - StillUp = lists:filter(fun({{_, P}, _}) -> P =/= Pid end, - WaitingConsumers0), - State = State0#?STATE{waiting_consumers = StillUp}, - {Effects, State}. - -update_waiting_consumer_status(Node, - #?STATE{waiting_consumers = WaitingConsumers}, - Status) -> - [begin - case node(Pid) of - Node -> - {ConsumerId, Consumer#consumer{status = Status}}; - _ -> - {ConsumerId, Consumer} - end - end || {{_, Pid} = ConsumerId, Consumer} <- WaitingConsumers, - Consumer#consumer.status =/= cancelled]. - --spec state_enter(ra_server:ra_state(), state()) -> ra_machine:effects(). -state_enter(leader, #?STATE{consumers = Cons, - enqueuers = Enqs, - waiting_consumers = WaitingConsumers, - cfg = #cfg{name = Name, - resource = Resource, - become_leader_handler = BLH}, - prefix_msgs = {0, [], 0, []} - }) -> - % return effects to monitor all current consumers and enqueuers - Pids = lists:usort(maps:keys(Enqs) - ++ [P || {_, P} <- maps:keys(Cons)] - ++ [P || {{_, P}, _} <- WaitingConsumers]), - Mons = [{monitor, process, P} || P <- Pids], - Nots = [{send_msg, P, leader_change, ra_event} || P <- Pids], - NodeMons = lists:usort([{monitor, node, node(P)} || P <- Pids]), - FHReservation = [{mod_call, rabbit_quorum_queue, file_handle_leader_reservation, [Resource]}], - Effects = Mons ++ Nots ++ NodeMons ++ FHReservation, - case BLH of - undefined -> - Effects; - {Mod, Fun, Args} -> - [{mod_call, Mod, Fun, Args ++ [Name]} | Effects] - end; -state_enter(eol, #?STATE{enqueuers = Enqs, - consumers = Custs0, - waiting_consumers = WaitingConsumers0}) -> - Custs = maps:fold(fun({_, P}, V, S) -> S#{P => V} end, #{}, Custs0), - WaitingConsumers1 = lists:foldl(fun({{_, P}, V}, Acc) -> Acc#{P => V} end, - #{}, WaitingConsumers0), - AllConsumers = maps:merge(Custs, WaitingConsumers1), - [{send_msg, P, eol, ra_event} - || P <- maps:keys(maps:merge(Enqs, AllConsumers))] ++ - [{mod_call, rabbit_quorum_queue, file_handle_release_reservation, []}]; -state_enter(State, #?STATE{cfg = #cfg{resource = _Resource}}) when State =/= leader -> - FHReservation = {mod_call, rabbit_quorum_queue, file_handle_other_reservation, []}, - [FHReservation]; - state_enter(_, _) -> - %% catch all as not handling all states - []. - - --spec tick(non_neg_integer(), state()) -> ra_machine:effects(). -tick(_Ts, #?STATE{cfg = #cfg{name = Name, - resource = QName}, - msg_bytes_enqueue = EnqueueBytes, - msg_bytes_checkout = CheckoutBytes} = State) -> - Metrics = {Name, - messages_ready(State), - num_checked_out(State), % checked out - messages_total(State), - query_consumer_count(State), % Consumers - EnqueueBytes, - CheckoutBytes}, - [{mod_call, rabbit_quorum_queue, - handle_tick, [QName, Metrics, all_nodes(State)]}]. - --spec overview(state()) -> map(). -overview(#?STATE{consumers = Cons, - enqueuers = Enqs, - release_cursors = Cursors, - enqueue_count = EnqCount, - msg_bytes_enqueue = EnqueueBytes, - msg_bytes_checkout = CheckoutBytes, - cfg = Cfg} = State) -> - Conf = #{name => Cfg#cfg.name, - resource => Cfg#cfg.resource, - release_cursor_interval => Cfg#cfg.release_cursor_interval, - dead_lettering_enabled => undefined =/= Cfg#cfg.dead_letter_handler, - max_length => Cfg#cfg.max_length, - max_bytes => Cfg#cfg.max_bytes, - consumer_strategy => Cfg#cfg.consumer_strategy, - max_in_memory_length => Cfg#cfg.max_in_memory_length, - max_in_memory_bytes => Cfg#cfg.max_in_memory_bytes}, - #{type => ?MODULE, - config => Conf, - num_consumers => maps:size(Cons), - num_checked_out => num_checked_out(State), - num_enqueuers => maps:size(Enqs), - num_ready_messages => messages_ready(State), - num_messages => messages_total(State), - num_release_cursors => lqueue:len(Cursors), - release_crusor_enqueue_counter => EnqCount, - enqueue_message_bytes => EnqueueBytes, - checkout_message_bytes => CheckoutBytes}. - --spec get_checked_out(consumer_id(), msg_id(), msg_id(), state()) -> - [delivery_msg()]. -get_checked_out(Cid, From, To, #?STATE{consumers = Consumers}) -> - case Consumers of - #{Cid := #consumer{checked_out = Checked}} -> - [{K, snd(snd(maps:get(K, Checked)))} - || K <- lists:seq(From, To), - maps:is_key(K, Checked)]; - _ -> - [] - end. - --record(aux_gc, {last_raft_idx = 0 :: ra:index()}). --record(aux, {name :: atom(), - utilisation :: term(), - gc = #aux_gc{} :: #aux_gc{}}). - -init_aux(Name) when is_atom(Name) -> - %% TODO: catch specific exception throw if table already exists - ok = ra_machine_ets:create_table(rabbit_fifo_usage, - [named_table, set, public, - {write_concurrency, true}]), - Now = erlang:monotonic_time(micro_seconds), - #aux{name = Name, - utilisation = {inactive, Now, 1, 1.0}}. - -handle_aux(_RaState, cast, Cmd, #aux{name = Name, - utilisation = Use0} = State0, - Log, MacState) -> - State = case Cmd of - _ when Cmd == active orelse Cmd == inactive -> - State0#aux{utilisation = update_use(Use0, Cmd)}; - tick -> - true = ets:insert(rabbit_fifo_usage, - {Name, utilisation(Use0)}), - eval_gc(Log, MacState, State0); - eval -> - State0 - end, - {no_reply, State, Log}. - -eval_gc(Log, #?STATE{cfg = #cfg{resource = QR}} = MacState, - #aux{gc = #aux_gc{last_raft_idx = LastGcIdx} = Gc} = AuxState) -> - {Idx, _} = ra_log:last_index_term(Log), - {memory, Mem} = erlang:process_info(self(), memory), - case messages_total(MacState) of - 0 when Idx > LastGcIdx andalso - Mem > ?GC_MEM_LIMIT_B -> - garbage_collect(), - {memory, MemAfter} = erlang:process_info(self(), memory), - rabbit_log:debug("~s: full GC sweep complete. " - "Process memory changed from ~.2fMB to ~.2fMB.", - [rabbit_misc:rs(QR), Mem/?MB, MemAfter/?MB]), - AuxState#aux{gc = Gc#aux_gc{last_raft_idx = Idx}}; - _ -> - AuxState - end. - -%%% Queries - -query_messages_ready(State) -> - messages_ready(State). - -query_messages_checked_out(#?STATE{consumers = Consumers}) -> - maps:fold(fun (_, #consumer{checked_out = C}, S) -> - maps:size(C) + S - end, 0, Consumers). - -query_messages_total(State) -> - messages_total(State). - -query_processes(#?STATE{enqueuers = Enqs, consumers = Cons0}) -> - Cons = maps:fold(fun({_, P}, V, S) -> S#{P => V} end, #{}, Cons0), - maps:keys(maps:merge(Enqs, Cons)). - - -query_ra_indexes(#?STATE{ra_indexes = RaIndexes}) -> - RaIndexes. - -query_consumer_count(#?STATE{consumers = Consumers, - waiting_consumers = WaitingConsumers}) -> - maps:size(Consumers) + length(WaitingConsumers). - -query_consumers(#?STATE{consumers = Consumers, - waiting_consumers = WaitingConsumers, - cfg = #cfg{consumer_strategy = ConsumerStrategy}} = State) -> - ActiveActivityStatusFun = - case ConsumerStrategy of - competing -> - fun(_ConsumerId, - #consumer{status = Status}) -> - case Status of - suspected_down -> - {false, Status}; - _ -> - {true, Status} - end - end; - single_active -> - SingleActiveConsumer = query_single_active_consumer(State), - fun({Tag, Pid} = _Consumer, _) -> - case SingleActiveConsumer of - {value, {Tag, Pid}} -> - {true, single_active}; - _ -> - {false, waiting} - end - end - end, - FromConsumers = - maps:fold(fun (_, #consumer{status = cancelled}, Acc) -> - Acc; - ({Tag, Pid}, #consumer{meta = Meta} = Consumer, Acc) -> - {Active, ActivityStatus} = - ActiveActivityStatusFun({Tag, Pid}, Consumer), - maps:put({Tag, Pid}, - {Pid, Tag, - maps:get(ack, Meta, undefined), - maps:get(prefetch, Meta, undefined), - Active, - ActivityStatus, - maps:get(args, Meta, []), - maps:get(username, Meta, undefined)}, - Acc) - end, #{}, Consumers), - FromWaitingConsumers = - lists:foldl(fun ({_, #consumer{status = cancelled}}, Acc) -> - Acc; - ({{Tag, Pid}, #consumer{meta = Meta} = Consumer}, Acc) -> - {Active, ActivityStatus} = - ActiveActivityStatusFun({Tag, Pid}, Consumer), - maps:put({Tag, Pid}, - {Pid, Tag, - maps:get(ack, Meta, undefined), - maps:get(prefetch, Meta, undefined), - Active, - ActivityStatus, - maps:get(args, Meta, []), - maps:get(username, Meta, undefined)}, - Acc) - end, #{}, WaitingConsumers), - maps:merge(FromConsumers, FromWaitingConsumers). - -query_single_active_consumer(#?STATE{cfg = #cfg{consumer_strategy = single_active}, - consumers = Consumers}) -> - case maps:size(Consumers) of - 0 -> - {error, no_value}; - 1 -> - {value, lists:nth(1, maps:keys(Consumers))}; - _ - -> - {error, illegal_size} - end ; -query_single_active_consumer(_) -> - disabled. - -query_stat(#?STATE{consumers = Consumers} = State) -> - {messages_ready(State), maps:size(Consumers)}. - -query_in_memory_usage(#?STATE{msg_bytes_in_memory = Bytes, - msgs_ready_in_memory = Length}) -> - {Length, Bytes}. - --spec usage(atom()) -> float(). -usage(Name) when is_atom(Name) -> - case ets:lookup(rabbit_fifo_usage, Name) of - [] -> 0.0; - [{_, Use}] -> Use - end. - -%%% Internal - -messages_ready(#?STATE{messages = M, - prefix_msgs = {RCnt, _R, PCnt, _P}, - returns = R}) -> - - %% prefix messages will rarely have anything in them during normal - %% operations so length/1 is fine here - maps:size(M) + lqueue:len(R) + RCnt + PCnt. - -messages_total(#?STATE{ra_indexes = I, - prefix_msgs = {RCnt, _R, PCnt, _P}}) -> - rabbit_fifo_index:size(I) + RCnt + PCnt. - -update_use({inactive, _, _, _} = CUInfo, inactive) -> - CUInfo; -update_use({active, _, _} = CUInfo, active) -> - CUInfo; -update_use({active, Since, Avg}, inactive) -> - Now = erlang:monotonic_time(micro_seconds), - {inactive, Now, Now - Since, Avg}; -update_use({inactive, Since, Active, Avg}, active) -> - Now = erlang:monotonic_time(micro_seconds), - {active, Now, use_avg(Active, Now - Since, Avg)}. - -utilisation({active, Since, Avg}) -> - use_avg(erlang:monotonic_time(micro_seconds) - Since, 0, Avg); -utilisation({inactive, Since, Active, Avg}) -> - use_avg(Active, erlang:monotonic_time(micro_seconds) - Since, Avg). - -use_avg(0, 0, Avg) -> - Avg; -use_avg(Active, Inactive, Avg) -> - Time = Inactive + Active, - moving_average(Time, ?USE_AVG_HALF_LIFE, Active / Time, Avg). - -moving_average(_Time, _, Next, undefined) -> - Next; -moving_average(Time, HalfLife, Next, Current) -> - Weight = math:exp(Time * math:log(0.5) / HalfLife), - Next * (1 - Weight) + Current * Weight. - -num_checked_out(#?STATE{consumers = Cons}) -> - maps:fold(fun (_, #consumer{checked_out = C}, Acc) -> - maps:size(C) + Acc - end, 0, Cons). - -cancel_consumer(ConsumerId, - #?STATE{cfg = #cfg{consumer_strategy = competing}} = State, - Effects, Reason) -> - cancel_consumer0(ConsumerId, State, Effects, Reason); -cancel_consumer(ConsumerId, - #?STATE{cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = []} = State, - Effects, Reason) -> - %% single active consumer on, no consumers are waiting - cancel_consumer0(ConsumerId, State, Effects, Reason); -cancel_consumer(ConsumerId, - #?STATE{consumers = Cons0, - cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = Waiting0} = State0, - Effects0, Reason) -> - %% single active consumer on, consumers are waiting - case maps:is_key(ConsumerId, Cons0) of - true -> - % The active consumer is to be removed - {State1, Effects1} = cancel_consumer0(ConsumerId, State0, - Effects0, Reason), - activate_next_consumer(State1, Effects1); - false -> - % The cancelled consumer is not active or cancelled - % Just remove it from idle_consumers - Waiting = lists:keydelete(ConsumerId, 1, Waiting0), - Effects = cancel_consumer_effects(ConsumerId, State0, Effects0), - % A waiting consumer isn't supposed to have any checked out messages, - % so nothing special to do here - {State0#?STATE{waiting_consumers = Waiting}, Effects} - end. - -consumer_update_active_effects(#?STATE{cfg = #cfg{resource = QName}}, - ConsumerId, #consumer{meta = Meta}, - Active, ActivityStatus, - Effects) -> - Ack = maps:get(ack, Meta, undefined), - Prefetch = maps:get(prefetch, Meta, undefined), - Args = maps:get(args, Meta, []), - [{mod_call, rabbit_quorum_queue, update_consumer_handler, - [QName, ConsumerId, false, Ack, Prefetch, Active, ActivityStatus, Args]} - | Effects]. - -cancel_consumer0(ConsumerId, #?STATE{consumers = C0} = S0, Effects0, Reason) -> - case C0 of - #{ConsumerId := Consumer} -> - {S, Effects2} = maybe_return_all(ConsumerId, Consumer, S0, - Effects0, Reason), - %% The effects are emitted before the consumer is actually removed - %% if the consumer has unacked messages. This is a bit weird but - %% in line with what classic queues do (from an external point of - %% view) - Effects = cancel_consumer_effects(ConsumerId, S, Effects2), - case maps:size(S#?STATE.consumers) of - 0 -> - {S, [{aux, inactive} | Effects]}; - _ -> - {S, Effects} - end; - _ -> - %% already removed: do nothing - {S0, Effects0} - end. - -activate_next_consumer(#?STATE{consumers = Cons, - waiting_consumers = Waiting0} = State0, - Effects0) -> - case maps:filter(fun (_, #consumer{status = S}) -> S == up end, Cons) of - Up when map_size(Up) == 0 -> - %% there are no active consumer in the consumer map - case lists:filter(fun ({_, #consumer{status = Status}}) -> - Status == up - end, Waiting0) of - [{NextConsumerId, NextConsumer} | _] -> - %% there is a potential next active consumer - Remaining = lists:keydelete(NextConsumerId, 1, Waiting0), - #?STATE{service_queue = ServiceQueue} = State0, - ServiceQueue1 = maybe_queue_consumer(NextConsumerId, - NextConsumer, - ServiceQueue), - State = State0#?STATE{consumers = Cons#{NextConsumerId => NextConsumer}, - service_queue = ServiceQueue1, - waiting_consumers = Remaining}, - Effects = consumer_update_active_effects(State, NextConsumerId, - NextConsumer, true, - single_active, Effects0), - {State, Effects}; - [] -> - {State0, [{aux, inactive} | Effects0]} - end; - _ -> - {State0, Effects0} - end. - - - -maybe_return_all(ConsumerId, Consumer, - #?STATE{consumers = C0, - service_queue = SQ0} = S0, - Effects0, Reason) -> - case Reason of - consumer_cancel -> - {Cons, SQ, Effects1} = - update_or_remove_sub(ConsumerId, - Consumer#consumer{lifetime = once, - credit = 0, - status = cancelled}, - C0, SQ0, Effects0), - {S0#?STATE{consumers = Cons, - service_queue = SQ}, Effects1}; - down -> - {S1, Effects1} = return_all(S0, Effects0, ConsumerId, Consumer), - {S1#?STATE{consumers = maps:remove(ConsumerId, S1#?STATE.consumers)}, - Effects1} - end. - -apply_enqueue(#{index := RaftIdx} = Meta, From, Seq, RawMsg, State0) -> - case maybe_enqueue(RaftIdx, From, Seq, RawMsg, [], State0) of - {ok, State1, Effects1} -> - State2 = append_to_master_index(RaftIdx, State1), - {State, ok, Effects} = checkout(Meta, State2, Effects1), - {maybe_store_dehydrated_state(RaftIdx, State), ok, Effects}; - {duplicate, State, Effects} -> - {State, ok, Effects} - end. - -drop_head(#?STATE{ra_indexes = Indexes0} = State0, Effects0) -> - case take_next_msg(State0) of - {FullMsg = {_MsgId, {RaftIdxToDrop, {Header, Msg}}}, - State1} -> - Indexes = rabbit_fifo_index:delete(RaftIdxToDrop, Indexes0), - State2 = add_bytes_drop(Header, State1#?STATE{ra_indexes = Indexes}), - State = case Msg of - 'empty' -> State2; - _ -> subtract_in_memory_counts(Header, State2) - end, - Effects = dead_letter_effects(maxlen, #{none => FullMsg}, - State, Effects0), - {State, Effects}; - {{'$prefix_msg', Header}, State1} -> - State2 = subtract_in_memory_counts(Header, add_bytes_drop(Header, State1)), - {State2, Effects0}; - {{'$empty_msg', Header}, State1} -> - State2 = add_bytes_drop(Header, State1), - {State2, Effects0}; - empty -> - {State0, Effects0} - end. - -enqueue(RaftIdx, RawMsg, #?STATE{messages = Messages, - low_msg_num = LowMsgNum, - next_msg_num = NextMsgNum} = State0) -> - %% the initial header is an integer only - it will get expanded to a map - %% when the next required key is added - Header = message_size(RawMsg), - {State1, Msg} = - case evaluate_memory_limit(Header, State0) of - true -> - % indexed message with header map - {State0, {RaftIdx, {Header, 'empty'}}}; - false -> - {add_in_memory_counts(Header, State0), - {RaftIdx, {Header, RawMsg}}} % indexed message with header map - end, - State = add_bytes_enqueue(Header, State1), - State#?STATE{messages = Messages#{NextMsgNum => Msg}, - %% this is probably only done to record it when low_msg_num - %% is undefined - low_msg_num = min(LowMsgNum, NextMsgNum), - next_msg_num = NextMsgNum + 1}. - -append_to_master_index(RaftIdx, - #?STATE{ra_indexes = Indexes0} = State0) -> - State = incr_enqueue_count(State0), - Indexes = rabbit_fifo_index:append(RaftIdx, Indexes0), - State#?STATE{ra_indexes = Indexes}. - - -incr_enqueue_count(#?STATE{enqueue_count = C, - cfg = #cfg{release_cursor_interval = {_Base, C}} - } = State0) -> - %% this will trigger a dehydrated version of the state to be stored - %% at this raft index for potential future snapshot generation - %% Q: Why don't we just stash the release cursor here? - %% A: Because it needs to be the very last thing we do and we - %% first needs to run the checkout logic. - State0#?STATE{enqueue_count = 0}; -incr_enqueue_count(#?STATE{cfg = #cfg{release_cursor_interval = C} = Cfg} - = State0) - when is_integer(C) -> - %% conversion to new release cursor interval format - State = State0#?STATE{cfg = Cfg#cfg{release_cursor_interval = {C, C}}}, - incr_enqueue_count(State); -incr_enqueue_count(#?STATE{enqueue_count = C} = State) -> - State#?STATE{enqueue_count = C + 1}. - -maybe_store_dehydrated_state(RaftIdx, - #?STATE{cfg = - #cfg{release_cursor_interval = {Base, _}} - = Cfg, - ra_indexes = Indexes, - enqueue_count = 0, - release_cursors = Cursors0} = State0) -> - case rabbit_fifo_index:exists(RaftIdx, Indexes) of - false -> - %% the incoming enqueue must already have been dropped - State0; - true -> - Interval = case Base of - 0 -> 0; - _ -> - Total = messages_total(State0), - min(max(Total, Base), - ?RELEASE_CURSOR_EVERY_MAX) - end, - State = convert_prefix_msgs( - State0#?STATE{cfg = Cfg#cfg{release_cursor_interval = - {Base, Interval}}}), - Dehydrated = dehydrate_state(State), - Cursor = {release_cursor, RaftIdx, Dehydrated}, - Cursors = lqueue:in(Cursor, Cursors0), - State#?STATE{release_cursors = Cursors} - end; -maybe_store_dehydrated_state(RaftIdx, - #?STATE{cfg = - #cfg{release_cursor_interval = C} = Cfg} - = State0) - when is_integer(C) -> - %% convert to new format - State = State0#?STATE{cfg = Cfg#cfg{release_cursor_interval = {C, C}}}, - maybe_store_dehydrated_state(RaftIdx, State); -maybe_store_dehydrated_state(_RaftIdx, State) -> - State. - -enqueue_pending(From, - #enqueuer{next_seqno = Next, - pending = [{Next, RaftIdx, RawMsg} | Pending]} = Enq0, - State0) -> - State = enqueue(RaftIdx, RawMsg, State0), - Enq = Enq0#enqueuer{next_seqno = Next + 1, pending = Pending}, - enqueue_pending(From, Enq, State); -enqueue_pending(From, Enq, #?STATE{enqueuers = Enqueuers0} = State) -> - State#?STATE{enqueuers = Enqueuers0#{From => Enq}}. - -maybe_enqueue(RaftIdx, undefined, undefined, RawMsg, Effects, State0) -> - % direct enqueue without tracking - State = enqueue(RaftIdx, RawMsg, State0), - {ok, State, Effects}; -maybe_enqueue(RaftIdx, From, MsgSeqNo, RawMsg, Effects0, - #?STATE{enqueuers = Enqueuers0} = State0) -> - case maps:get(From, Enqueuers0, undefined) of - undefined -> - State1 = State0#?STATE{enqueuers = Enqueuers0#{From => #enqueuer{}}}, - {ok, State, Effects} = maybe_enqueue(RaftIdx, From, MsgSeqNo, - RawMsg, Effects0, State1), - {ok, State, [{monitor, process, From} | Effects]}; - #enqueuer{next_seqno = MsgSeqNo} = Enq0 -> - % it is the next expected seqno - State1 = enqueue(RaftIdx, RawMsg, State0), - Enq = Enq0#enqueuer{next_seqno = MsgSeqNo + 1}, - State = enqueue_pending(From, Enq, State1), - {ok, State, Effects0}; - #enqueuer{next_seqno = Next, - pending = Pending0} = Enq0 - when MsgSeqNo > Next -> - % out of order delivery - Pending = [{MsgSeqNo, RaftIdx, RawMsg} | Pending0], - Enq = Enq0#enqueuer{pending = lists:sort(Pending)}, - {ok, State0#?STATE{enqueuers = Enqueuers0#{From => Enq}}, Effects0}; - #enqueuer{next_seqno = Next} when MsgSeqNo =< Next -> - % duplicate delivery - remove the raft index from the ra_indexes - % map as it was added earlier - {duplicate, State0, Effects0} - end. - -snd(T) -> - element(2, T). - -return(#{index := IncomingRaftIdx} = Meta, ConsumerId, Returned, - Effects0, #?STATE{service_queue = SQ0} = State0) -> - {State1, Effects1} = maps:fold( - fun(MsgId, {Tag, _} = Msg, {S0, E0}) - when Tag == '$prefix_msg'; - Tag == '$empty_msg'-> - return_one(MsgId, 0, Msg, S0, E0, ConsumerId); - (MsgId, {MsgNum, Msg}, {S0, E0}) -> - return_one(MsgId, MsgNum, Msg, S0, E0, - ConsumerId) - end, {State0, Effects0}, Returned), - {State2, Effects3} = - case State1#?STATE.consumers of - #{ConsumerId := Con0} = Cons0 -> - Con = Con0#consumer{credit = increase_credit(Con0, - map_size(Returned))}, - {Cons, SQ, Effects2} = update_or_remove_sub(ConsumerId, Con, - Cons0, SQ0, Effects1), - {State1#?STATE{consumers = Cons, - service_queue = SQ}, Effects2}; - _ -> - {State1, Effects1} - end, - {State, ok, Effects} = checkout(Meta, State2, Effects3), - update_smallest_raft_index(IncomingRaftIdx, State, Effects). - -% used to processes messages that are finished -complete(ConsumerId, Discarded, - #consumer{checked_out = Checked} = Con0, Effects0, - #?STATE{consumers = Cons0, service_queue = SQ0, - ra_indexes = Indexes0} = State0) -> - %% TODO optimise use of Discarded map here - MsgRaftIdxs = [RIdx || {_, {RIdx, _}} <- maps:values(Discarded)], - %% credit_mode = simple_prefetch should automatically top-up credit - %% as messages are simple_prefetch or otherwise returned - Con = Con0#consumer{checked_out = maps:without(maps:keys(Discarded), Checked), - credit = increase_credit(Con0, map_size(Discarded))}, - {Cons, SQ, Effects} = update_or_remove_sub(ConsumerId, Con, Cons0, - SQ0, Effects0), - Indexes = lists:foldl(fun rabbit_fifo_index:delete/2, Indexes0, - MsgRaftIdxs), - %% TODO: use maps:fold instead - State1 = lists:foldl(fun({_, {_, {Header, _}}}, Acc) -> - add_bytes_settle(Header, Acc); - ({'$prefix_msg', Header}, Acc) -> - add_bytes_settle(Header, Acc); - ({'$empty_msg', Header}, Acc) -> - add_bytes_settle(Header, Acc) - end, State0, maps:values(Discarded)), - {State1#?STATE{consumers = Cons, - ra_indexes = Indexes, - service_queue = SQ}, Effects}. - -increase_credit(#consumer{lifetime = once, - credit = Credit}, _) -> - %% once consumers cannot increment credit - Credit; -increase_credit(#consumer{lifetime = auto, - credit_mode = credited, - credit = Credit}, _) -> - %% credit_mode: credit also doesn't automatically increment credit - Credit; -increase_credit(#consumer{credit = Current}, Credit) -> - Current + Credit. - -complete_and_checkout(#{index := IncomingRaftIdx} = Meta, MsgIds, ConsumerId, - #consumer{checked_out = Checked0} = Con0, - Effects0, State0) -> - Discarded = maps:with(MsgIds, Checked0), - {State2, Effects1} = complete(ConsumerId, Discarded, Con0, - Effects0, State0), - {State, ok, Effects} = checkout(Meta, State2, Effects1), - update_smallest_raft_index(IncomingRaftIdx, State, Effects). - -dead_letter_effects(_Reason, _Discarded, - #?STATE{cfg = #cfg{dead_letter_handler = undefined}}, - Effects) -> - Effects; -dead_letter_effects(Reason, Discarded, - #?STATE{cfg = #cfg{dead_letter_handler = {Mod, Fun, Args}}}, - Effects) -> - RaftIdxs = maps:fold( - fun (_, {_, {RaftIdx, {_Header, 'empty'}}}, Acc) -> - [RaftIdx | Acc]; - (_, _, Acc) -> - Acc - end, [], Discarded), - [{log, RaftIdxs, - fun (Log) -> - Lookup = maps:from_list(lists:zip(RaftIdxs, Log)), - DeadLetters = maps:fold( - fun (_, {_, {RaftIdx, {_Header, 'empty'}}}, Acc) -> - {enqueue, _, _, Msg} = maps:get(RaftIdx, Lookup), - [{Reason, Msg} | Acc]; - (_, {_, {_, {_Header, Msg}}}, Acc) -> - [{Reason, Msg} | Acc]; - (_, _, Acc) -> - Acc - end, [], Discarded), - [{mod_call, Mod, Fun, Args ++ [DeadLetters]}] - end} | Effects]. - -cancel_consumer_effects(ConsumerId, - #?STATE{cfg = #cfg{resource = QName}}, Effects) -> - [{mod_call, rabbit_quorum_queue, - cancel_consumer_handler, [QName, ConsumerId]} | Effects]. - -update_smallest_raft_index(IncomingRaftIdx, - #?STATE{ra_indexes = Indexes, - release_cursors = Cursors0} = State0, - Effects) -> - case rabbit_fifo_index:size(Indexes) of - 0 -> - % there are no messages on queue anymore and no pending enqueues - % we can forward release_cursor all the way until - % the last received command, hooray - State = State0#?STATE{release_cursors = lqueue:new()}, - {State, ok, Effects ++ [{release_cursor, IncomingRaftIdx, State}]}; - _ -> - Smallest = rabbit_fifo_index:smallest(Indexes), - case find_next_cursor(Smallest, Cursors0) of - {empty, Cursors} -> - {State0#?STATE{release_cursors = Cursors}, - ok, Effects}; - {Cursor, Cursors} -> - %% we can emit a release cursor we've passed the smallest - %% release cursor available. - {State0#?STATE{release_cursors = Cursors}, ok, - Effects ++ [Cursor]} - end - end. - -find_next_cursor(Idx, Cursors) -> - find_next_cursor(Idx, Cursors, empty). - -find_next_cursor(Smallest, Cursors0, Potential) -> - case lqueue:out(Cursors0) of - {{value, {_, Idx, _} = Cursor}, Cursors} when Idx < Smallest -> - %% we found one but it may not be the largest one - find_next_cursor(Smallest, Cursors, Cursor); - _ -> - {Potential, Cursors0} - end. - -update_header(Key, UpdateFun, Default, Header) - when is_integer(Header) -> - update_header(Key, UpdateFun, Default, #{size => Header}); -update_header(Key, UpdateFun, Default, Header) -> - maps:update_with(Key, UpdateFun, Default, Header). - - -return_one(MsgId, 0, {Tag, Header0}, - #?STATE{returns = Returns, - consumers = Consumers, - cfg = #cfg{delivery_limit = DeliveryLimit}} = State0, - Effects0, ConsumerId) - when Tag == '$prefix_msg'; Tag == '$empty_msg' -> - #consumer{checked_out = Checked} = Con0 = maps:get(ConsumerId, Consumers), - Header = update_header(delivery_count, fun (C) -> C+1 end, 1, Header0), - Msg0 = {Tag, Header}, - case maps:get(delivery_count, Header) of - DeliveryCount when DeliveryCount > DeliveryLimit -> - complete(ConsumerId, #{MsgId => Msg0}, Con0, Effects0, State0); - _ -> - %% this should not affect the release cursor in any way - Con = Con0#consumer{checked_out = maps:remove(MsgId, Checked)}, - {Msg, State1} = case Tag of - '$empty_msg' -> - {Msg0, State0}; - _ -> case evaluate_memory_limit(Header, State0) of - true -> - {{'$empty_msg', Header}, State0}; - false -> - {Msg0, add_in_memory_counts(Header, State0)} - end - end, - {add_bytes_return( - Header, - State1#?STATE{consumers = Consumers#{ConsumerId => Con}, - returns = lqueue:in(Msg, Returns)}), - Effects0} - end; -return_one(MsgId, MsgNum, {RaftId, {Header0, RawMsg}}, - #?STATE{returns = Returns, - consumers = Consumers, - cfg = #cfg{delivery_limit = DeliveryLimit}} = State0, - Effects0, ConsumerId) -> - #consumer{checked_out = Checked} = Con0 = maps:get(ConsumerId, Consumers), - Header = update_header(delivery_count, fun (C) -> C+1 end, 1, Header0), - Msg0 = {RaftId, {Header, RawMsg}}, - case maps:get(delivery_count, Header) of - DeliveryCount when DeliveryCount > DeliveryLimit -> - DlMsg = {MsgNum, Msg0}, - Effects = dead_letter_effects(delivery_limit, #{none => DlMsg}, - State0, Effects0), - complete(ConsumerId, #{MsgId => DlMsg}, Con0, Effects, State0); - _ -> - Con = Con0#consumer{checked_out = maps:remove(MsgId, Checked)}, - %% this should not affect the release cursor in any way - {Msg, State1} = case RawMsg of - 'empty' -> - {Msg0, State0}; - _ -> - case evaluate_memory_limit(Header, State0) of - true -> - {{RaftId, {Header, 'empty'}}, State0}; - false -> - {Msg0, add_in_memory_counts(Header, State0)} - end - end, - {add_bytes_return( - Header, - State1#?STATE{consumers = Consumers#{ConsumerId => Con}, - returns = lqueue:in({MsgNum, Msg}, Returns)}), - Effects0} - end. - -return_all(#?STATE{consumers = Cons} = State0, Effects0, ConsumerId, - #consumer{checked_out = Checked0} = Con) -> - %% need to sort the list so that we return messages in the order - %% they were checked out - Checked = lists:sort(maps:to_list(Checked0)), - State = State0#?STATE{consumers = Cons#{ConsumerId => Con}}, - lists:foldl(fun ({MsgId, {'$prefix_msg', _} = Msg}, {S, E}) -> - return_one(MsgId, 0, Msg, S, E, ConsumerId); - ({MsgId, {'$empty_msg', _} = Msg}, {S, E}) -> - return_one(MsgId, 0, Msg, S, E, ConsumerId); - ({MsgId, {MsgNum, Msg}}, {S, E}) -> - return_one(MsgId, MsgNum, Msg, S, E, ConsumerId) - end, {State, Effects0}, Checked). - -%% checkout new messages to consumers -checkout(#{index := Index}, State0, Effects0) -> - {State1, _Result, Effects1} = checkout0(checkout_one(State0), - Effects0, {#{}, #{}}), - case evaluate_limit(false, State1, Effects1) of - {State, true, Effects} -> - update_smallest_raft_index(Index, State, Effects); - {State, false, Effects} -> - {State, ok, Effects} - end. - -checkout0({success, ConsumerId, MsgId, {RaftIdx, {Header, 'empty'}}, State}, - Effects, {SendAcc, LogAcc0}) -> - DelMsg = {RaftIdx, {MsgId, Header}}, - LogAcc = maps:update_with(ConsumerId, - fun (M) -> [DelMsg | M] end, - [DelMsg], LogAcc0), - checkout0(checkout_one(State), Effects, {SendAcc, LogAcc}); -checkout0({success, ConsumerId, MsgId, Msg, State}, Effects, - {SendAcc0, LogAcc}) -> - DelMsg = {MsgId, Msg}, - SendAcc = maps:update_with(ConsumerId, - fun (M) -> [DelMsg | M] end, - [DelMsg], SendAcc0), - checkout0(checkout_one(State), Effects, {SendAcc, LogAcc}); -checkout0({Activity, State0}, Effects0, {SendAcc, LogAcc}) -> - Effects1 = case Activity of - nochange -> - append_send_msg_effects( - append_log_effects(Effects0, LogAcc), SendAcc); - inactive -> - [{aux, inactive} - | append_send_msg_effects( - append_log_effects(Effects0, LogAcc), SendAcc)] - end, - {State0, ok, lists:reverse(Effects1)}. - -evaluate_limit(Result, - #?STATE{cfg = #cfg{max_length = undefined, - max_bytes = undefined}} = State, - Effects) -> - {State, Result, Effects}; -evaluate_limit(Result, State00, Effects0) -> - State0 = convert_prefix_msgs(State00), - case is_over_limit(State0) of - true -> - {State, Effects} = drop_head(State0, Effects0), - evaluate_limit(true, State, Effects); - false -> - {State0, Result, Effects0} - end. - -evaluate_memory_limit(_Header, - #?STATE{cfg = #cfg{max_in_memory_length = undefined, - max_in_memory_bytes = undefined}}) -> - false; -evaluate_memory_limit(#{size := Size}, State) -> - evaluate_memory_limit(Size, State); -evaluate_memory_limit(Size, - #?STATE{cfg = #cfg{max_in_memory_length = MaxLength, - max_in_memory_bytes = MaxBytes}, - msg_bytes_in_memory = Bytes, - msgs_ready_in_memory = Length}) - when is_integer(Size) -> - (Length >= MaxLength) orelse ((Bytes + Size) > MaxBytes). - -append_send_msg_effects(Effects, AccMap) when map_size(AccMap) == 0 -> - Effects; -append_send_msg_effects(Effects0, AccMap) -> - Effects = maps:fold(fun (C, Msgs, Ef) -> - [send_msg_effect(C, lists:reverse(Msgs)) | Ef] - end, Effects0, AccMap), - [{aux, active} | Effects]. - -append_log_effects(Effects0, AccMap) -> - maps:fold(fun (C, Msgs, Ef) -> - [send_log_effect(C, lists:reverse(Msgs)) | Ef] - end, Effects0, AccMap). - -%% next message is determined as follows: -%% First we check if there are are prefex returns -%% Then we check if there are current returns -%% then we check prefix msgs -%% then we check current messages -%% -%% When we return it is always done to the current return queue -%% for both prefix messages and current messages -take_next_msg(#?STATE{prefix_msgs = {R, P}} = State) -> - %% conversion - take_next_msg(State#?STATE{prefix_msgs = {length(R), R, length(P), P}}); -take_next_msg(#?STATE{prefix_msgs = {NumR, [{'$empty_msg', _} = Msg | Rem], - NumP, P}} = State) -> - %% there are prefix returns, these should be served first - {Msg, State#?STATE{prefix_msgs = {NumR-1, Rem, NumP, P}}}; -take_next_msg(#?STATE{prefix_msgs = {NumR, [Header | Rem], NumP, P}} = State) -> - %% there are prefix returns, these should be served first - {{'$prefix_msg', Header}, - State#?STATE{prefix_msgs = {NumR-1, Rem, NumP, P}}}; -take_next_msg(#?STATE{returns = Returns, - low_msg_num = Low0, - messages = Messages0, - prefix_msgs = {NumR, R, NumP, P}} = State) -> - %% use peek rather than out there as the most likely case is an empty - %% queue - case lqueue:peek(Returns) of - {value, NextMsg} -> - {NextMsg, - State#?STATE{returns = lqueue:drop(Returns)}}; - empty when P == [] -> - case Low0 of - undefined -> - empty; - _ -> - {Msg, Messages} = maps:take(Low0, Messages0), - case maps:size(Messages) of - 0 -> - {{Low0, Msg}, - State#?STATE{messages = Messages, - low_msg_num = undefined}}; - _ -> - {{Low0, Msg}, - State#?STATE{messages = Messages, - low_msg_num = Low0 + 1}} - end - end; - empty -> - [Msg | Rem] = P, - case Msg of - {Header, 'empty'} -> - %% There are prefix msgs - {{'$empty_msg', Header}, - State#?STATE{prefix_msgs = {NumR, R, NumP-1, Rem}}}; - Header -> - {{'$prefix_msg', Header}, - State#?STATE{prefix_msgs = {NumR, R, NumP-1, Rem}}} - end - end. - -send_msg_effect({CTag, CPid}, Msgs) -> - {send_msg, CPid, {delivery, CTag, Msgs}, [local, ra_event]}. - -send_log_effect({CTag, CPid}, IdxMsgs) -> - {RaftIdxs, Data} = lists:unzip(IdxMsgs), - {log, RaftIdxs, - fun(Log) -> - Msgs = lists:zipwith(fun ({enqueue, _, _, Msg}, {MsgId, Header}) -> - {MsgId, {Header, Msg}} - end, Log, Data), - [{send_msg, CPid, {delivery, CTag, Msgs}, [local, ra_event]}] - end, - {local, node(CPid)}}. - -reply_log_effect(RaftIdx, MsgId, Header, Ready, From) -> - {log, [RaftIdx], - fun([{enqueue, _, _, Msg}]) -> - [{reply, From, {wrap_reply, - {dequeue, {MsgId, {Header, Msg}}, Ready}}}] - end}. - -checkout_one(#?STATE{service_queue = SQ0, - messages = Messages0, - consumers = Cons0} = InitState) -> - case queue:peek(SQ0) of - {value, ConsumerId} -> - case take_next_msg(InitState) of - {ConsumerMsg, State0} -> - SQ1 = queue:drop(SQ0), - %% there are consumers waiting to be serviced - %% process consumer checkout - case maps:find(ConsumerId, Cons0) of - {ok, #consumer{credit = 0}} -> - %% no credit but was still on queue - %% can happen when draining - %% recurse without consumer on queue - checkout_one(InitState#?STATE{service_queue = SQ1}); - {ok, #consumer{status = cancelled}} -> - checkout_one(InitState#?STATE{service_queue = SQ1}); - {ok, #consumer{status = suspected_down}} -> - checkout_one(InitState#?STATE{service_queue = SQ1}); - {ok, #consumer{checked_out = Checked0, - next_msg_id = Next, - credit = Credit, - delivery_count = DelCnt} = Con0} -> - Checked = maps:put(Next, ConsumerMsg, Checked0), - Con = Con0#consumer{checked_out = Checked, - next_msg_id = Next + 1, - credit = Credit - 1, - delivery_count = DelCnt + 1}, - {Cons, SQ, []} = % we expect no effects - update_or_remove_sub(ConsumerId, Con, - Cons0, SQ1, []), - State1 = State0#?STATE{service_queue = SQ, - consumers = Cons}, - {State, Msg} = - case ConsumerMsg of - {'$prefix_msg', Header} -> - {subtract_in_memory_counts( - Header, add_bytes_checkout(Header, State1)), - ConsumerMsg}; - {'$empty_msg', Header} -> - {add_bytes_checkout(Header, State1), - ConsumerMsg}; - {_, {_, {Header, 'empty'}} = M} -> - {add_bytes_checkout(Header, State1), - M}; - {_, {_, {Header, _} = M}} -> - {subtract_in_memory_counts( - Header, - add_bytes_checkout(Header, State1)), - M} - end, - {success, ConsumerId, Next, Msg, State}; - error -> - %% consumer did not exist but was queued, recurse - checkout_one(InitState#?STATE{service_queue = SQ1}) - end; - empty -> - {nochange, InitState} - end; - empty -> - case maps:size(Messages0) of - 0 -> {nochange, InitState}; - _ -> {inactive, InitState} - end - end. - -update_or_remove_sub(ConsumerId, #consumer{lifetime = auto, - credit = 0} = Con, - Cons, ServiceQueue, Effects) -> - {maps:put(ConsumerId, Con, Cons), ServiceQueue, Effects}; -update_or_remove_sub(ConsumerId, #consumer{lifetime = auto} = Con, - Cons, ServiceQueue, Effects) -> - {maps:put(ConsumerId, Con, Cons), - uniq_queue_in(ConsumerId, ServiceQueue), Effects}; -update_or_remove_sub(ConsumerId, #consumer{lifetime = once, - checked_out = Checked, - credit = 0} = Con, - Cons, ServiceQueue, Effects) -> - case maps:size(Checked) of - 0 -> - % we're done with this consumer - % TODO: demonitor consumer pid but _only_ if there are no other - % monitors for this pid - {maps:remove(ConsumerId, Cons), ServiceQueue, Effects}; - _ -> - % there are unsettled items so need to keep around - {maps:put(ConsumerId, Con, Cons), ServiceQueue, Effects} - end; -update_or_remove_sub(ConsumerId, #consumer{lifetime = once} = Con, - Cons, ServiceQueue, Effects) -> - {maps:put(ConsumerId, Con, Cons), - uniq_queue_in(ConsumerId, ServiceQueue), Effects}. - -uniq_queue_in(Key, Queue) -> - % TODO: queue:member could surely be quite expensive, however the practical - % number of unique consumers may not be large enough for it to matter - case queue:member(Key, Queue) of - true -> - Queue; - false -> - queue:in(Key, Queue) - end. - -update_consumer(ConsumerId, Meta, Spec, - #?STATE{cfg = #cfg{consumer_strategy = competing}} = State0) -> - %% general case, single active consumer off - update_consumer0(ConsumerId, Meta, Spec, State0); -update_consumer(ConsumerId, Meta, Spec, - #?STATE{consumers = Cons0, - cfg = #cfg{consumer_strategy = single_active}} = State0) - when map_size(Cons0) == 0 -> - %% single active consumer on, no one is consuming yet - update_consumer0(ConsumerId, Meta, Spec, State0); -update_consumer(ConsumerId, Meta, {Life, Credit, Mode}, - #?STATE{cfg = #cfg{consumer_strategy = single_active}, - waiting_consumers = WaitingConsumers0} = State0) -> - %% single active consumer on and one active consumer already - %% adding the new consumer to the waiting list - Consumer = #consumer{lifetime = Life, meta = Meta, - credit = Credit, credit_mode = Mode}, - WaitingConsumers1 = WaitingConsumers0 ++ [{ConsumerId, Consumer}], - State0#?STATE{waiting_consumers = WaitingConsumers1}. - -update_consumer0(ConsumerId, Meta, {Life, Credit, Mode}, - #?STATE{consumers = Cons0, - service_queue = ServiceQueue0} = State0) -> - %% TODO: this logic may not be correct for updating a pre-existing consumer - Init = #consumer{lifetime = Life, meta = Meta, - credit = Credit, credit_mode = Mode}, - Cons = maps:update_with(ConsumerId, - fun(S) -> - %% remove any in-flight messages from - %% the credit update - N = maps:size(S#consumer.checked_out), - C = max(0, Credit - N), - S#consumer{lifetime = Life, credit = C} - end, Init, Cons0), - ServiceQueue = maybe_queue_consumer(ConsumerId, maps:get(ConsumerId, Cons), - ServiceQueue0), - State0#?STATE{consumers = Cons, service_queue = ServiceQueue}. - -maybe_queue_consumer(ConsumerId, #consumer{credit = Credit}, - ServiceQueue0) -> - case Credit > 0 of - true -> - % consumerect needs service - check if already on service queue - uniq_queue_in(ConsumerId, ServiceQueue0); - false -> - ServiceQueue0 - end. - -convert_prefix_msgs(#?STATE{prefix_msgs = {R, P}} = State) -> - State#?STATE{prefix_msgs = {length(R), R, length(P), P}}; -convert_prefix_msgs(State) -> - State. - -%% creates a dehydrated version of the current state to be cached and -%% potentially used to for a snaphot at a later point -dehydrate_state(#?STATE{messages = Messages, - consumers = Consumers, - returns = Returns, - low_msg_num = Low, - next_msg_num = Next, - prefix_msgs = {PRCnt, PrefRet0, PPCnt, PrefMsg0}, - waiting_consumers = Waiting0} = State) -> - RCnt = lqueue:len(Returns), - %% TODO: optimise this function as far as possible - PrefRet1 = lists:foldr(fun ({'$prefix_msg', Header}, Acc) -> - [Header | Acc]; - ({'$empty_msg', _} = Msg, Acc) -> - [Msg | Acc]; - ({_, {_, {Header, 'empty'}}}, Acc) -> - [{'$empty_msg', Header} | Acc]; - ({_, {_, {Header, _}}}, Acc) -> - [Header | Acc] - end, - [], - lqueue:to_list(Returns)), - PrefRet = PrefRet0 ++ PrefRet1, - PrefMsgsSuff = dehydrate_messages(Low, Next - 1, Messages, []), - %% prefix messages are not populated in normal operation only after - %% recovering from a snapshot - PrefMsgs = PrefMsg0 ++ PrefMsgsSuff, - Waiting = [{Cid, dehydrate_consumer(C)} || {Cid, C} <- Waiting0], - State#?STATE{messages = #{}, - ra_indexes = rabbit_fifo_index:empty(), - release_cursors = lqueue:new(), - low_msg_num = undefined, - consumers = maps:map(fun (_, C) -> - dehydrate_consumer(C) - end, Consumers), - returns = lqueue:new(), - prefix_msgs = {PRCnt + RCnt, PrefRet, - PPCnt + maps:size(Messages), PrefMsgs}, - waiting_consumers = Waiting}. - -dehydrate_messages(Low, Next, _Msgs, Acc) - when Next < Low -> - Acc; -dehydrate_messages(Low, Next, Msgs, Acc0) -> - Acc = case maps:get(Next, Msgs) of - {_RaftIdx, {_, 'empty'} = Msg} -> - [Msg | Acc0]; - {_RaftIdx, {Header, _}} -> - [Header | Acc0] - end, - dehydrate_messages(Low, Next - 1, Msgs, Acc). - -dehydrate_consumer(#consumer{checked_out = Checked0} = Con) -> - Checked = maps:map(fun (_, {'$prefix_msg', _} = M) -> - M; - (_, {'$empty_msg', _} = M) -> - M; - (_, {_, {_, {Header, 'empty'}}}) -> - {'$empty_msg', Header}; - (_, {_, {_, {Header, _}}}) -> - {'$prefix_msg', Header} - end, Checked0), - Con#consumer{checked_out = Checked}. - -%% make the state suitable for equality comparison -normalize(#?STATE{release_cursors = Cursors} = State) -> - State#?STATE{release_cursors = lqueue:from_list(lqueue:to_list(Cursors))}. - -is_over_limit(#?STATE{cfg = #cfg{max_length = undefined, - max_bytes = undefined}}) -> - false; -is_over_limit(#?STATE{cfg = #cfg{max_length = MaxLength, - max_bytes = MaxBytes}, - msg_bytes_enqueue = BytesEnq} = State) -> - - messages_ready(State) > MaxLength orelse (BytesEnq > MaxBytes). - -normalize_for_v1(#?STATE{cfg = Cfg} = State) -> - %% run all v0 conversions so that v1 does not have to have this code - RCI = case Cfg of - #cfg{release_cursor_interval = {_, _} = R} -> - R; - #cfg{release_cursor_interval = undefined} -> - {?RELEASE_CURSOR_EVERY, ?RELEASE_CURSOR_EVERY}; - #cfg{release_cursor_interval = C} -> - {?RELEASE_CURSOR_EVERY, C} - end, - convert_prefix_msgs( - State#?STATE{cfg = Cfg#cfg{release_cursor_interval = RCI}}). - -get_field(Field, State) -> - Fields = record_info(fields, ?STATE), - Index = record_index_of(Field, Fields), - element(Index, State). - -get_cfg_field(Field, #?STATE{cfg = Cfg} ) -> - Fields = record_info(fields, cfg), - Index = record_index_of(Field, Fields), - element(Index, Cfg). - -record_index_of(F, Fields) -> - index_of(2, F, Fields). - -index_of(_, F, []) -> - exit({field_not_found, F}); -index_of(N, F, [F | _]) -> - N; -index_of(N, F, [_ | T]) -> - index_of(N+1, F, T). - --spec make_enqueue(option(pid()), option(msg_seqno()), raw_msg()) -> protocol(). -make_enqueue(Pid, Seq, Msg) -> - #enqueue{pid = Pid, seq = Seq, msg = Msg}. --spec make_checkout(consumer_id(), - checkout_spec(), consumer_meta()) -> protocol(). -make_checkout(ConsumerId, Spec, Meta) -> - #checkout{consumer_id = ConsumerId, - spec = Spec, meta = Meta}. - --spec make_settle(consumer_id(), [msg_id()]) -> protocol(). -make_settle(ConsumerId, MsgIds) -> - #settle{consumer_id = ConsumerId, msg_ids = MsgIds}. - --spec make_return(consumer_id(), [msg_id()]) -> protocol(). -make_return(ConsumerId, MsgIds) -> - #return{consumer_id = ConsumerId, msg_ids = MsgIds}. - --spec make_discard(consumer_id(), [msg_id()]) -> protocol(). -make_discard(ConsumerId, MsgIds) -> - #discard{consumer_id = ConsumerId, msg_ids = MsgIds}. - --spec make_credit(consumer_id(), non_neg_integer(), non_neg_integer(), - boolean()) -> protocol(). -make_credit(ConsumerId, Credit, DeliveryCount, Drain) -> - #credit{consumer_id = ConsumerId, - credit = Credit, - delivery_count = DeliveryCount, - drain = Drain}. - --spec make_purge() -> protocol(). -make_purge() -> #purge{}. - --spec make_purge_nodes([node()]) -> protocol(). -make_purge_nodes(Nodes) -> - #purge_nodes{nodes = Nodes}. - --spec make_update_config(config()) -> protocol(). -make_update_config(Config) -> - #update_config{config = Config}. - -add_bytes_enqueue(Bytes, - #?STATE{msg_bytes_enqueue = Enqueue} = State) - when is_integer(Bytes) -> - State#?STATE{msg_bytes_enqueue = Enqueue + Bytes}; -add_bytes_enqueue(#{size := Bytes}, State) -> - add_bytes_enqueue(Bytes, State). - -add_bytes_drop(Bytes, - #?STATE{msg_bytes_enqueue = Enqueue} = State) - when is_integer(Bytes) -> - State#?STATE{msg_bytes_enqueue = Enqueue - Bytes}; -add_bytes_drop(#{size := Bytes}, State) -> - add_bytes_drop(Bytes, State). - -add_bytes_checkout(Bytes, - #?STATE{msg_bytes_checkout = Checkout, - msg_bytes_enqueue = Enqueue } = State) - when is_integer(Bytes) -> - State#?STATE{msg_bytes_checkout = Checkout + Bytes, - msg_bytes_enqueue = Enqueue - Bytes}; -add_bytes_checkout(#{size := Bytes}, State) -> - add_bytes_checkout(Bytes, State). - -add_bytes_settle(Bytes, - #?STATE{msg_bytes_checkout = Checkout} = State) - when is_integer(Bytes) -> - State#?STATE{msg_bytes_checkout = Checkout - Bytes}; -add_bytes_settle(#{size := Bytes}, State) -> - add_bytes_settle(Bytes, State). - -add_bytes_return(Bytes, - #?STATE{msg_bytes_checkout = Checkout, - msg_bytes_enqueue = Enqueue} = State) - when is_integer(Bytes) -> - State#?STATE{msg_bytes_checkout = Checkout - Bytes, - msg_bytes_enqueue = Enqueue + Bytes}; -add_bytes_return(#{size := Bytes}, State) -> - add_bytes_return(Bytes, State). - -add_in_memory_counts(Bytes, - #?STATE{msg_bytes_in_memory = InMemoryBytes, - msgs_ready_in_memory = InMemoryCount} = State) - when is_integer(Bytes) -> - State#?STATE{msg_bytes_in_memory = InMemoryBytes + Bytes, - msgs_ready_in_memory = InMemoryCount + 1}; -add_in_memory_counts(#{size := Bytes}, State) -> - add_in_memory_counts(Bytes, State). - -subtract_in_memory_counts(Bytes, - #?STATE{msg_bytes_in_memory = InMemoryBytes, - msgs_ready_in_memory = InMemoryCount} = State) - when is_integer(Bytes) -> - State#?STATE{msg_bytes_in_memory = InMemoryBytes - Bytes, - msgs_ready_in_memory = InMemoryCount - 1}; -subtract_in_memory_counts(#{size := Bytes}, State) -> - subtract_in_memory_counts(Bytes, State). - -message_size(#basic_message{content = Content}) -> - #content{payload_fragments_rev = PFR} = Content, - iolist_size(PFR); -message_size({'$prefix_msg', H}) -> - get_size_from_header(H); -message_size({'$empty_msg', H}) -> - get_size_from_header(H); -message_size(B) when is_binary(B) -> - byte_size(B); -message_size(Msg) -> - %% probably only hit this for testing so ok to use erts_debug - erts_debug:size(Msg). - -get_size_from_header(Size) when is_integer(Size) -> - Size; -get_size_from_header(#{size := B}) -> - B. - - -all_nodes(#?STATE{consumers = Cons0, - enqueuers = Enqs0, - waiting_consumers = WaitingConsumers0}) -> - Nodes0 = maps:fold(fun({_, P}, _, Acc) -> - Acc#{node(P) => ok} - end, #{}, Cons0), - Nodes1 = maps:fold(fun(P, _, Acc) -> - Acc#{node(P) => ok} - end, Nodes0, Enqs0), - maps:keys( - lists:foldl(fun({{_, P}, _}, Acc) -> - Acc#{node(P) => ok} - end, Nodes1, WaitingConsumers0)). - -all_pids_for(Node, #?STATE{consumers = Cons0, - enqueuers = Enqs0, - waiting_consumers = WaitingConsumers0}) -> - Cons = maps:fold(fun({_, P}, _, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, _, Acc) -> Acc - end, [], Cons0), - Enqs = maps:fold(fun(P, _, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, _, Acc) -> Acc - end, Cons, Enqs0), - lists:foldl(fun({{_, P}, _}, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, Acc) -> Acc - end, Enqs, WaitingConsumers0). - -suspected_pids_for(Node, #?STATE{consumers = Cons0, - enqueuers = Enqs0, - waiting_consumers = WaitingConsumers0}) -> - Cons = maps:fold(fun({_, P}, #consumer{status = suspected_down}, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, _, Acc) -> Acc - end, [], Cons0), - Enqs = maps:fold(fun(P, #enqueuer{status = suspected_down}, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, _, Acc) -> Acc - end, Cons, Enqs0), - lists:foldl(fun({{_, P}, - #consumer{status = suspected_down}}, Acc) - when node(P) =:= Node -> - [P | Acc]; - (_, Acc) -> Acc - end, Enqs, WaitingConsumers0). diff --git a/src/rabbit_fifo_v0.hrl b/src/rabbit_fifo_v0.hrl deleted file mode 100644 index 333ccb4d77..0000000000 --- a/src/rabbit_fifo_v0.hrl +++ /dev/null @@ -1,195 +0,0 @@ - --type option(T) :: undefined | T. - --type raw_msg() :: term(). -%% The raw message. It is opaque to rabbit_fifo. - --type msg_in_id() :: non_neg_integer(). -% a queue scoped monotonically incrementing integer used to enforce order -% in the unassigned messages map - --type msg_id() :: non_neg_integer(). -%% A consumer-scoped monotonically incrementing integer included with a -%% {@link delivery/0.}. Used to settle deliveries using -%% {@link rabbit_fifo_client:settle/3.} - --type msg_seqno() :: non_neg_integer(). -%% A sender process scoped monotonically incrementing integer included -%% in enqueue messages. Used to ensure ordering of messages send from the -%% same process - --type msg_header() :: msg_size() | - #{size := msg_size(), - delivery_count => non_neg_integer()}. -%% The message header: -%% delivery_count: the number of unsuccessful delivery attempts. -%% A non-zero value indicates a previous attempt. -%% If it only contains the size it can be condensed to an integer only - --type msg() :: {msg_header(), raw_msg()}. -%% message with a header map. - --type msg_size() :: non_neg_integer(). -%% the size in bytes of the msg payload - --type indexed_msg() :: {ra:index(), msg()}. - --type prefix_msg() :: {'$prefix_msg', msg_header()}. - --type delivery_msg() :: {msg_id(), msg()}. -%% A tuple consisting of the message id and the headered message. - --type consumer_tag() :: binary(). -%% An arbitrary binary tag used to distinguish between different consumers -%% set up by the same process. See: {@link rabbit_fifo_client:checkout/3.} - --type delivery() :: {delivery, consumer_tag(), [delivery_msg()]}. -%% Represents the delivery of one or more rabbit_fifo messages. - --type consumer_id() :: {consumer_tag(), pid()}. -%% The entity that receives messages. Uniquely identifies a consumer. - --type credit_mode() :: simple_prefetch | credited. -%% determines how credit is replenished - --type checkout_spec() :: {once | auto, Num :: non_neg_integer(), - credit_mode()} | - {dequeue, settled | unsettled} | - cancel. - --type consumer_meta() :: #{ack => boolean(), - username => binary(), - prefetch => non_neg_integer(), - args => list()}. -%% static meta data associated with a consumer - - --type applied_mfa() :: {module(), atom(), list()}. -% represents a partially applied module call - --define(RELEASE_CURSOR_EVERY, 64000). --define(RELEASE_CURSOR_EVERY_MAX, 3200000). --define(USE_AVG_HALF_LIFE, 10000.0). -%% an average QQ without any message uses about 100KB so setting this limit -%% to ~10 times that should be relatively safe. --define(GC_MEM_LIMIT_B, 2000000). - --define(MB, 1048576). --define(STATE, rabbit_fifo). - --record(consumer, - {meta = #{} :: consumer_meta(), - checked_out = #{} :: #{msg_id() => {msg_in_id(), indexed_msg()}}, - next_msg_id = 0 :: msg_id(), % part of snapshot data - %% max number of messages that can be sent - %% decremented for each delivery - credit = 0 : non_neg_integer(), - %% total number of checked out messages - ever - %% incremented for each delivery - delivery_count = 0 :: non_neg_integer(), - %% the mode of how credit is incremented - %% simple_prefetch: credit is re-filled as deliveries are settled - %% or returned. - %% credited: credit can only be changed by receiving a consumer_credit - %% command: `{consumer_credit, ReceiverDeliveryCount, Credit}' - credit_mode = simple_prefetch :: credit_mode(), % part of snapshot data - lifetime = once :: once | auto, - status = up :: up | suspected_down | cancelled - }). - --type consumer() :: #consumer{}. - --type consumer_strategy() :: competing | single_active. - --record(enqueuer, - {next_seqno = 1 :: msg_seqno(), - % out of order enqueues - sorted list - pending = [] :: [{msg_seqno(), ra:index(), raw_msg()}], - status = up :: up | suspected_down - }). - --record(cfg, - {name :: atom(), - resource :: rabbit_types:r('queue'), - release_cursor_interval :: - undefined | non_neg_integer() | - {non_neg_integer(), non_neg_integer()}, - dead_letter_handler :: option(applied_mfa()), - become_leader_handler :: option(applied_mfa()), - max_length :: option(non_neg_integer()), - max_bytes :: option(non_neg_integer()), - %% whether single active consumer is on or not for this queue - consumer_strategy = competing :: consumer_strategy(), - %% the maximum number of unsuccessful delivery attempts permitted - delivery_limit :: option(non_neg_integer()), - max_in_memory_length :: option(non_neg_integer()), - max_in_memory_bytes :: option(non_neg_integer()) - }). - --type prefix_msgs() :: {list(), list()} | - {non_neg_integer(), list(), - non_neg_integer(), list()}. - --record(?STATE, - {cfg :: #cfg{}, - % unassigned messages - messages = #{} :: #{msg_in_id() => indexed_msg()}, - % defines the lowest message in id available in the messages map - % that isn't a return - low_msg_num :: option(msg_in_id()), - % defines the next message in id to be added to the messages map - next_msg_num = 1 :: msg_in_id(), - % list of returned msg_in_ids - when checking out it picks from - % this list first before taking low_msg_num - returns = lqueue:new() :: lqueue:lqueue(prefix_msg() | - {msg_in_id(), indexed_msg()}), - % a counter of enqueues - used to trigger shadow copy points - enqueue_count = 0 :: non_neg_integer(), - % a map containing all the live processes that have ever enqueued - % a message to this queue as well as a cached value of the smallest - % ra_index of all pending enqueues - enqueuers = #{} :: #{pid() => #enqueuer{}}, - % master index of all enqueue raft indexes including pending - % enqueues - % rabbit_fifo_index can be slow when calculating the smallest - % index when there are large gaps but should be faster than gb_trees - % for normal appending operations as it's backed by a map - ra_indexes = rabbit_fifo_index:empty() :: rabbit_fifo_index:state(), - release_cursors = lqueue:new() :: lqueue:lqueue({release_cursor, - ra:index(), #?STATE{}}), - % consumers need to reflect consumer state at time of snapshot - % needs to be part of snapshot - consumers = #{} :: #{consumer_id() => #consumer{}}, - % consumers that require further service are queued here - % needs to be part of snapshot - service_queue = queue:new() :: queue:queue(consumer_id()), - %% This is a special field that is only used for snapshots - %% It represents the queued messages at the time the - %% dehydrated snapshot state was cached. - %% As release_cursors are only emitted for raft indexes where all - %% prior messages no longer contribute to the current state we can - %% replace all message payloads with their sizes (to be used for - %% overflow calculations). - %% This is done so that consumers are still served in a deterministic - %% order on recovery. - prefix_msgs = {0, [], 0, []} :: prefix_msgs(), - msg_bytes_enqueue = 0 :: non_neg_integer(), - msg_bytes_checkout = 0 :: non_neg_integer(), - %% waiting consumers, one is picked active consumer is cancelled or dies - %% used only when single active consumer is on - waiting_consumers = [] :: [{consumer_id(), consumer()}], - msg_bytes_in_memory = 0 :: non_neg_integer(), - msgs_ready_in_memory = 0 :: non_neg_integer() - }). - --type config() :: #{name := atom(), - queue_resource := rabbit_types:r('queue'), - dead_letter_handler => applied_mfa(), - become_leader_handler => applied_mfa(), - release_cursor_interval => non_neg_integer(), - max_length => non_neg_integer(), - max_bytes => non_neg_integer(), - max_in_memory_length => non_neg_integer(), - max_in_memory_bytes => non_neg_integer(), - single_active_consumer_on => boolean(), - delivery_limit => non_neg_integer()}. diff --git a/src/rabbit_file.erl b/src/rabbit_file.erl deleted file mode 100644 index f8263d9e77..0000000000 --- a/src/rabbit_file.erl +++ /dev/null @@ -1,321 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_file). - --include_lib("kernel/include/file.hrl"). - --export([is_file/1, is_dir/1, file_size/1, ensure_dir/1, wildcard/2, list_dir/1]). --export([read_term_file/1, write_term_file/2, write_file/2, write_file/3]). --export([append_file/2, ensure_parent_dirs_exist/1]). --export([rename/2, delete/1, recursive_delete/1, recursive_copy/2]). --export([lock_file/1]). --export([read_file_info/1]). --export([filename_as_a_directory/1]). - --import(file_handle_cache, [with_handle/1, with_handle/2]). - --define(TMP_EXT, ".tmp"). - -%%---------------------------------------------------------------------------- - --type ok_or_error() :: rabbit_types:ok_or_error(any()). - -%%---------------------------------------------------------------------------- - --spec is_file((file:filename())) -> boolean(). - -is_file(File) -> - case read_file_info(File) of - {ok, #file_info{type=regular}} -> true; - {ok, #file_info{type=directory}} -> true; - _ -> false - end. - --spec is_dir((file:filename())) -> boolean(). - -is_dir(Dir) -> is_dir_internal(read_file_info(Dir)). - -is_dir_no_handle(Dir) -> is_dir_internal(prim_file:read_file_info(Dir)). - -is_dir_internal({ok, #file_info{type=directory}}) -> true; -is_dir_internal(_) -> false. - --spec file_size((file:filename())) -> non_neg_integer(). - -file_size(File) -> - case read_file_info(File) of - {ok, #file_info{size=Size}} -> Size; - _ -> 0 - end. - --spec ensure_dir((file:filename())) -> ok_or_error(). - -ensure_dir(File) -> with_handle(fun () -> ensure_dir_internal(File) end). - -ensure_dir_internal("/") -> - ok; -ensure_dir_internal(File) -> - Dir = filename:dirname(File), - case is_dir_no_handle(Dir) of - true -> ok; - false -> ensure_dir_internal(Dir), - prim_file:make_dir(Dir) - end. - --spec wildcard(string(), file:filename()) -> [file:filename()]. - -wildcard(Pattern, Dir) -> - case list_dir(Dir) of - {ok, Files} -> {ok, RE} = re:compile(Pattern, [anchored]), - [File || File <- Files, - match =:= re:run(File, RE, [{capture, none}])]; - {error, _} -> [] - end. - --spec list_dir(file:filename()) -> - rabbit_types:ok_or_error2([file:filename()], any()). - -list_dir(Dir) -> with_handle(fun () -> prim_file:list_dir(Dir) end). - -read_file_info(File) -> - with_handle(fun () -> prim_file:read_file_info(File) end). - --spec read_term_file - (file:filename()) -> {'ok', [any()]} | rabbit_types:error(any()). - -read_term_file(File) -> - try - {ok, Data} = with_handle(fun () -> prim_file:read_file(File) end), - {ok, Tokens, _} = erl_scan:string(binary_to_list(Data)), - TokenGroups = group_tokens(Tokens), - {ok, [begin - {ok, Term} = erl_parse:parse_term(Tokens1), - Term - end || Tokens1 <- TokenGroups]} - catch - error:{badmatch, Error} -> Error - end. - -group_tokens(Ts) -> [lists:reverse(G) || G <- group_tokens([], Ts)]. - -group_tokens([], []) -> []; -group_tokens(Cur, []) -> [Cur]; -group_tokens(Cur, [T = {dot, _} | Ts]) -> [[T | Cur] | group_tokens([], Ts)]; -group_tokens(Cur, [T | Ts]) -> group_tokens([T | Cur], Ts). - --spec write_term_file(file:filename(), [any()]) -> ok_or_error(). - -write_term_file(File, Terms) -> - write_file(File, list_to_binary([io_lib:format("~w.~n", [Term]) || - Term <- Terms])). - --spec write_file(file:filename(), iodata()) -> ok_or_error(). - -write_file(Path, Data) -> write_file(Path, Data, []). - --spec write_file(file:filename(), iodata(), [any()]) -> ok_or_error(). - -write_file(Path, Data, Modes) -> - Modes1 = [binary, write | (Modes -- [binary, write])], - case make_binary(Data) of - Bin when is_binary(Bin) -> write_file1(Path, Bin, Modes1); - {error, _} = E -> E - end. - -%% make_binary/1 is based on the corresponding function in the -%% kernel/file.erl module of the Erlang R14B02 release, which is -%% licensed under the EPL. - -make_binary(Bin) when is_binary(Bin) -> - Bin; -make_binary(List) -> - try - iolist_to_binary(List) - catch error:Reason -> - {error, Reason} - end. - -write_file1(Path, Bin, Modes) -> - try - with_synced_copy(Path, Modes, - fun (Hdl) -> - ok = prim_file:write(Hdl, Bin) - end) - catch - error:{badmatch, Error} -> Error; - _:{error, Error} -> {error, Error} - end. - -with_synced_copy(Path, Modes, Fun) -> - case lists:member(append, Modes) of - true -> - {error, append_not_supported, Path}; - false -> - with_handle( - fun () -> - Bak = Path ++ ?TMP_EXT, - case prim_file:open(Bak, Modes) of - {ok, Hdl} -> - try - Result = Fun(Hdl), - ok = prim_file:sync(Hdl), - ok = prim_file:rename(Bak, Path), - Result - after - prim_file:close(Hdl) - end; - {error, _} = E -> E - end - end) - end. - -%% TODO the semantics of this function are rather odd. But see bug 25021. - --spec append_file(file:filename(), string()) -> ok_or_error(). - -append_file(File, Suffix) -> - case read_file_info(File) of - {ok, FInfo} -> append_file(File, FInfo#file_info.size, Suffix); - {error, enoent} -> append_file(File, 0, Suffix); - Error -> Error - end. - -append_file(_, _, "") -> - ok; -append_file(File, 0, Suffix) -> - with_handle(fun () -> - case prim_file:open([File, Suffix], [append]) of - {ok, Fd} -> prim_file:close(Fd); - Error -> Error - end - end); -append_file(File, _, Suffix) -> - case with_handle(2, fun () -> - file:copy(File, {[File, Suffix], [append]}) - end) of - {ok, _BytesCopied} -> ok; - Error -> Error - end. - --spec ensure_parent_dirs_exist(string()) -> 'ok'. - -ensure_parent_dirs_exist(Filename) -> - case ensure_dir(Filename) of - ok -> ok; - {error, Reason} -> - throw({error, {cannot_create_parent_dirs, Filename, Reason}}) - end. - --spec rename(file:filename(), file:filename()) -> ok_or_error(). - -rename(Old, New) -> with_handle(fun () -> prim_file:rename(Old, New) end). - --spec delete([file:filename()]) -> ok_or_error(). - -delete(File) -> with_handle(fun () -> prim_file:delete(File) end). - --spec recursive_delete([file:filename()]) -> - rabbit_types:ok_or_error({file:filename(), any()}). - -recursive_delete(Files) -> - with_handle( - fun () -> lists:foldl(fun (Path, ok) -> recursive_delete1(Path); - (_Path, {error, _Err} = Error) -> Error - end, ok, Files) - end). - -recursive_delete1(Path) -> - case is_dir_no_handle(Path) and not(is_symlink_no_handle(Path)) of - false -> case prim_file:delete(Path) of - ok -> ok; - {error, enoent} -> ok; %% Path doesn't exist anyway - {error, Err} -> {error, {Path, Err}} - end; - true -> case prim_file:list_dir(Path) of - {ok, FileNames} -> - case lists:foldl( - fun (FileName, ok) -> - recursive_delete1( - filename:join(Path, FileName)); - (_FileName, Error) -> - Error - end, ok, FileNames) of - ok -> - case prim_file:del_dir(Path) of - ok -> ok; - {error, Err} -> {error, {Path, Err}} - end; - {error, _Err} = Error -> - Error - end; - {error, Err} -> - {error, {Path, Err}} - end - end. - -is_symlink_no_handle(File) -> - case prim_file:read_link(File) of - {ok, _} -> true; - _ -> false - end. - --spec recursive_copy(file:filename(), file:filename()) -> - rabbit_types:ok_or_error({file:filename(), file:filename(), any()}). - -recursive_copy(Src, Dest) -> - %% Note that this uses the 'file' module and, hence, shouldn't be - %% run on many processes at once. - case is_dir(Src) of - false -> case file:copy(Src, Dest) of - {ok, _Bytes} -> ok; - {error, enoent} -> ok; %% Path doesn't exist anyway - {error, Err} -> {error, {Src, Dest, Err}} - end; - true -> case file:list_dir(Src) of - {ok, FileNames} -> - case file:make_dir(Dest) of - ok -> - lists:foldl( - fun (FileName, ok) -> - recursive_copy( - filename:join(Src, FileName), - filename:join(Dest, FileName)); - (_FileName, Error) -> - Error - end, ok, FileNames); - {error, Err} -> - {error, {Src, Dest, Err}} - end; - {error, Err} -> - {error, {Src, Dest, Err}} - end - end. - -%% TODO: When we stop supporting Erlang prior to R14, this should be -%% replaced with file:open [write, exclusive] - --spec lock_file(file:filename()) -> rabbit_types:ok_or_error('eexist'). - -lock_file(Path) -> - case is_file(Path) of - true -> {error, eexist}; - false -> with_handle( - fun () -> {ok, Lock} = prim_file:open(Path, [write]), - ok = prim_file:close(Lock) - end) - end. - --spec filename_as_a_directory(file:filename()) -> file:filename(). - -filename_as_a_directory(FileName) -> - case lists:last(FileName) of - "/" -> - FileName; - _ -> - FileName ++ "/" - end. diff --git a/src/rabbit_framing.erl b/src/rabbit_framing.erl deleted file mode 100644 index 42927b2b68..0000000000 --- a/src/rabbit_framing.erl +++ /dev/null @@ -1,36 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% TODO auto-generate - --module(rabbit_framing). - --export_type([protocol/0, - amqp_field_type/0, amqp_property_type/0, - amqp_table/0, amqp_array/0, amqp_value/0, - amqp_method_name/0, amqp_method/0, amqp_method_record/0, - amqp_method_field_name/0, amqp_property_record/0, - amqp_exception/0, amqp_exception_code/0, amqp_class_id/0]). - --type protocol() :: 'rabbit_framing_amqp_0_8' | 'rabbit_framing_amqp_0_9_1'. - --define(protocol_type(T), type(T :: rabbit_framing_amqp_0_8:T | - rabbit_framing_amqp_0_9_1:T)). - --?protocol_type(amqp_field_type()). --?protocol_type(amqp_property_type()). --?protocol_type(amqp_table()). --?protocol_type(amqp_array()). --?protocol_type(amqp_value()). --?protocol_type(amqp_method_name()). --?protocol_type(amqp_method()). --?protocol_type(amqp_method_record()). --?protocol_type(amqp_method_field_name()). --?protocol_type(amqp_property_record()). --?protocol_type(amqp_exception()). --?protocol_type(amqp_exception_code()). --?protocol_type(amqp_class_id()). diff --git a/src/rabbit_guid.erl b/src/rabbit_guid.erl deleted file mode 100644 index 01e6464332..0000000000 --- a/src/rabbit_guid.erl +++ /dev/null @@ -1,181 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_guid). - --behaviour(gen_server). - --export([start_link/0]). --export([filename/0]). --export([gen/0, gen_secure/0, string/2, binary/2, to_string/1]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --define(SERVER, ?MODULE). --define(SERIAL_FILENAME, "rabbit_serial"). - --record(state, {serial}). - -%%---------------------------------------------------------------------------- - --export_type([guid/0]). - --type guid() :: binary(). - -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, - [update_disk_serial()], []). - -%% We use this to detect a (possibly rather old) Mnesia directory, -%% since it has existed since at least 1.7.0 (as far back as I cared -%% to go). - --spec filename() -> string(). - -filename() -> - filename:join(rabbit_mnesia:dir(), ?SERIAL_FILENAME). - -update_disk_serial() -> - Filename = filename(), - Serial = case rabbit_file:read_term_file(Filename) of - {ok, [Num]} -> Num; - {ok, []} -> 0; %% [1] - {error, enoent} -> 0; - {error, Reason} -> - throw({error, {cannot_read_serial_file, Filename, Reason}}) - end, - case rabbit_file:write_term_file(Filename, [Serial + 1]) of - ok -> ok; - {error, Reason1} -> - throw({error, {cannot_write_serial_file, Filename, Reason1}}) - end, - Serial. -%% [1] a couple of users have reported startup failures due to the -%% file being empty, presumably as a result of filesystem -%% corruption. While rabbit doesn't cope with that in general, in this -%% specific case we can be more accommodating. - -%% Generate an un-hashed guid. -fresh() -> - %% We don't use erlang:now() here because a) it may return - %% duplicates when the system clock has been rewound prior to a - %% restart, or ids were generated at a high rate (which causes - %% now() to move ahead of the system time), and b) it is really - %% slow since it takes a global lock and makes a system call. - %% - %% A persisted serial number, the node, and a unique reference - %% (per node incarnation) uniquely identifies a process in space - %% and time. - Serial = gen_server:call(?SERVER, serial, infinity), - {Serial, node(), make_ref()}. - -advance_blocks({B1, B2, B3, B4}, I) -> - %% To produce a new set of blocks, we create a new 32bit block - %% hashing {B5, I}. The new hash is used as last block, and the - %% other three blocks are XORed with it. - %% - %% Doing this is convenient because it avoids cascading conflicts, - %% while being very fast. The conflicts are avoided by propagating - %% the changes through all the blocks at each round by XORing, so - %% the only occasion in which a collision will take place is when - %% all 4 blocks are the same and the counter is the same. - %% - %% The range (2^32) is provided explicitly since phash uses 2^27 - %% by default. - B5 = erlang:phash2({B1, I}, 4294967296), - {{(B2 bxor B5), (B3 bxor B5), (B4 bxor B5), B5}, I+1}. - -%% generate a GUID. This function should be used when performance is a -%% priority and predictability is not an issue. Otherwise use -%% gen_secure/0. - --spec gen() -> guid(). - -gen() -> - %% We hash a fresh GUID with md5, split it in 4 blocks, and each - %% time we need a new guid we rotate them producing a new hash - %% with the aid of the counter. Look at the comments in - %% advance_blocks/2 for details. - case get(guid) of - undefined -> <<B1:32, B2:32, B3:32, B4:32>> = Res = - erlang:md5(term_to_binary(fresh())), - put(guid, {{B1, B2, B3, B4}, 0}), - Res; - {BS, I} -> {{B1, B2, B3, B4}, _} = S = advance_blocks(BS, I), - put(guid, S), - <<B1:32, B2:32, B3:32, B4:32>> - end. - -%% generate a non-predictable GUID. -%% -%% The id is only unique within a single cluster and as long as the -%% serial store hasn't been deleted. -%% -%% If you are not concerned with predictability, gen/0 is faster. - --spec gen_secure() -> guid(). - -gen_secure() -> - %% Here instead of hashing once we hash the GUID and the counter - %% each time, so that the GUID is not predictable. - G = case get(guid_secure) of - undefined -> {fresh(), 0}; - {S, I} -> {S, I+1} - end, - put(guid_secure, G), - erlang:md5(term_to_binary(G)). - -%% generate a readable string representation of a GUID. -%% -%% employs base64url encoding, which is safer in more contexts than -%% plain base64. - --spec string(guid() | string(), any()) -> string(). - -string(G, Prefix) when is_list(Prefix) -> - Prefix ++ "-" ++ rabbit_misc:base64url(G); -string(G, Prefix) when is_binary(Prefix) -> - binary_to_list(Prefix) ++ "-" ++ rabbit_misc:base64url(G). - --spec binary(guid() | string(), any()) -> binary(). - -binary(G, Prefix) -> - list_to_binary(string(G, Prefix)). - -%% copied from https://stackoverflow.com/questions/1657204/erlang-uuid-generator -to_string(<<TL:32, TM:16, THV:16, CSR:8, CSL:8, N:48>>) -> - lists:flatten( - io_lib:format("~8.16.0b-~4.16.0b-~4.16.0b-~2.16.0b~2.16.0b-~12.16.0b", - [TL, TM, THV, CSR, CSL, N])). - -%%---------------------------------------------------------------------------- - -init([Serial]) -> - {ok, #state{serial = Serial}}. - -handle_call(serial, _From, State = #state{serial = Serial}) -> - {reply, Serial, State}; - -handle_call(_Request, _From, State) -> - {noreply, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. diff --git a/src/rabbit_health_check.erl b/src/rabbit_health_check.erl deleted file mode 100644 index 4674ca7d8e..0000000000 --- a/src/rabbit_health_check.erl +++ /dev/null @@ -1,80 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% --module(rabbit_health_check). - -%% External API --export([node/1, node/2]). - -%% Internal API --export([local/0]). - -%%---------------------------------------------------------------------------- -%% External functions -%%---------------------------------------------------------------------------- - --spec node(node(), timeout()) -> ok | {badrpc, term()} | {error_string, string()}. - -node(Node) -> - %% same default as in CLI - node(Node, 70000). -node(Node, Timeout) -> - rabbit_misc:rpc_call(Node, rabbit_health_check, local, [], Timeout). - --spec local() -> ok | {error_string, string()}. - -local() -> - rabbit_log:warning("rabbitmqctl node_health_check and its HTTP API counterpart are DEPRECATED. " - "See https://www.rabbitmq.com/monitoring.html#health-checks for replacement options."), - run_checks([list_channels, list_queues, alarms, rabbit_node_monitor]). - -%%---------------------------------------------------------------------------- -%% Internal functions -%%---------------------------------------------------------------------------- -run_checks([]) -> - ok; -run_checks([C|Cs]) -> - case node_health_check(C) of - ok -> - run_checks(Cs); - Error -> - Error - end. - -node_health_check(list_channels) -> - case rabbit_channel:info_local([pid]) of - L when is_list(L) -> - ok - end; - -node_health_check(list_queues) -> - health_check_queues(rabbit_vhost:list_names()); - -node_health_check(rabbit_node_monitor) -> - case rabbit_node_monitor:partitions() of - [] -> - ok; - L when is_list(L), length(L) > 0 -> - ErrorMsg = io_lib:format("cluster partition in effect: ~p", [L]), - {error_string, ErrorMsg} - end; - -node_health_check(alarms) -> - case proplists:get_value(alarms, rabbit:status()) of - [] -> - ok; - Alarms -> - ErrorMsg = io_lib:format("resource alarm(s) in effect:~p", [Alarms]), - {error_string, ErrorMsg} - end. - -health_check_queues([]) -> - ok; -health_check_queues([VHost|RestVHosts]) -> - case rabbit_amqqueue:info_local(VHost) of - L when is_list(L) -> - health_check_queues(RestVHosts) - end. diff --git a/src/rabbit_lager.erl b/src/rabbit_lager.erl deleted file mode 100644 index 3cbc5e431d..0000000000 --- a/src/rabbit_lager.erl +++ /dev/null @@ -1,723 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_lager). - --include_lib("rabbit_common/include/rabbit_log.hrl"). - -%% API --export([start_logger/0, log_locations/0, fold_sinks/2, - broker_is_started/0, set_log_level/1]). - -%% For test purposes --export([configure_lager/0]). - --export_type([log_location/0]). - --type log_location() :: string(). - -start_logger() -> - ok = maybe_remove_logger_handler(), - ok = app_utils:stop_applications([lager, syslog]), - ok = ensure_lager_configured(), - ok = app_utils:start_applications([lager]), - fold_sinks( - fun - (_, [], Acc) -> - Acc; - (SinkName, _, Acc) -> - lager:log(SinkName, info, self(), - "Log file opened with Lager", []), - Acc - end, ok), - ensure_log_working(). - -broker_is_started() -> - {ok, HwmCurrent} = application:get_env(lager, error_logger_hwm), - {ok, HwmOrig0} = application:get_env(lager, error_logger_hwm_original), - HwmOrig = case get_most_verbose_log_level() of - debug -> HwmOrig0 * 100; - _ -> HwmOrig0 - end, - case HwmOrig =:= HwmCurrent of - false -> - ok = application:set_env(lager, error_logger_hwm, HwmOrig), - Handlers = gen_event:which_handlers(lager_event), - lists:foreach(fun(Handler) -> - lager:set_loghwm(Handler, HwmOrig) - end, Handlers), - ok; - _ -> - ok - end. - -set_log_level(Level) -> - IsValidLevel = lists:member(Level, lager_util:levels()), - set_log_level(IsValidLevel, Level). - -set_log_level(true, Level) -> - SinksAndHandlers = [{Sink, gen_event:which_handlers(Sink)} || - Sink <- lager:list_all_sinks()], - DefaultHwm = application:get_env(lager, error_logger_hwm_original, 50), - Hwm = case Level of - debug -> DefaultHwm * 100; - _ -> DefaultHwm - end, - application:set_env(lager, error_logger_hwm, Hwm), - set_sink_log_level(SinksAndHandlers, Level, Hwm); -set_log_level(_, Level) -> - {error, {invalid_log_level, Level}}. - -set_sink_log_level([], _Level, _Hwm) -> - ok; -set_sink_log_level([{Sink, Handlers}|Rest], Level, Hwm) -> - set_sink_handler_log_level(Sink, Handlers, Level, Hwm), - set_sink_log_level(Rest, Level, Hwm). - -set_sink_handler_log_level(_Sink, [], _Level, _Hwm) -> - ok; -set_sink_handler_log_level(Sink, [Handler|Rest], Level, Hwm) - when is_atom(Handler) andalso is_integer(Hwm) -> - lager:set_loghwm(Sink, Handler, undefined, Hwm), - ok = lager:set_loglevel(Sink, Handler, undefined, Level), - set_sink_handler_log_level(Sink, Rest, Level, Hwm); -set_sink_handler_log_level(Sink, [{Handler, Id}|Rest], Level, Hwm) -> - lager:set_loghwm(Sink, Handler, Id, Hwm), - ok = lager:set_loglevel(Sink, Handler, Id, Level), - set_sink_handler_log_level(Sink, Rest, Level, Hwm); -set_sink_handler_log_level(Sink, [_|Rest], Level, Hwm) -> - set_sink_handler_log_level(Sink, Rest, Level, Hwm). - -log_locations() -> - ensure_lager_configured(), - DefaultHandlers = application:get_env(lager, handlers, []), - Sinks = application:get_env(lager, extra_sinks, []), - ExtraHandlers = [proplists:get_value(handlers, Props, []) - || {_, Props} <- Sinks], - lists:sort(log_locations1([DefaultHandlers | ExtraHandlers], [])). - -log_locations1([Handlers | Rest], Locations) -> - Locations1 = log_locations2(Handlers, Locations), - log_locations1(Rest, Locations1); -log_locations1([], Locations) -> - Locations. - -log_locations2([{lager_file_backend, Settings} | Rest], Locations) -> - FileName = lager_file_name1(Settings), - Locations1 = case lists:member(FileName, Locations) of - false -> [FileName | Locations]; - true -> Locations - end, - log_locations2(Rest, Locations1); -log_locations2([{lager_console_backend, _} | Rest], Locations) -> - Locations1 = case lists:member("<stdout>", Locations) of - false -> ["<stdout>" | Locations]; - true -> Locations - end, - log_locations2(Rest, Locations1); -log_locations2([_ | Rest], Locations) -> - log_locations2(Rest, Locations); -log_locations2([], Locations) -> - Locations. - -fold_sinks(Fun, Acc) -> - Handlers = lager_config:global_get(handlers), - Sinks = dict:to_list(lists:foldl( - fun - ({{lager_file_backend, F}, _, S}, Dict) -> - dict:append(S, F, Dict); - ({_, _, S}, Dict) -> - case dict:is_key(S, Dict) of - true -> dict:store(S, [], Dict); - false -> Dict - end - end, - dict:new(), Handlers)), - fold_sinks(Sinks, Fun, Acc). - -fold_sinks([{SinkName, FileNames} | Rest], Fun, Acc) -> - Acc1 = Fun(SinkName, FileNames, Acc), - fold_sinks(Rest, Fun, Acc1); -fold_sinks([], _, Acc) -> - Acc. - -ensure_log_working() -> - {ok, Handlers} = application:get_env(lager, handlers), - [ ensure_lager_handler_file_exist(Handler) - || Handler <- Handlers ], - Sinks = application:get_env(lager, extra_sinks, []), - ensure_extra_sinks_working(Sinks, list_expected_sinks()). - -ensure_extra_sinks_working(Sinks, [SinkName | Rest]) -> - case proplists:get_value(SinkName, Sinks) of - undefined -> throw({error, {cannot_log_to_file, unknown, - rabbit_log_lager_event_sink_undefined}}); - Sink -> - SinkHandlers = proplists:get_value(handlers, Sink, []), - [ ensure_lager_handler_file_exist(Handler) - || Handler <- SinkHandlers ] - end, - ensure_extra_sinks_working(Sinks, Rest); -ensure_extra_sinks_working(_Sinks, []) -> - ok. - -ensure_lager_handler_file_exist(Handler) -> - case lager_file_name(Handler) of - false -> ok; - FileName -> ensure_logfile_exist(FileName) - end. - -lager_file_name({lager_file_backend, Settings}) -> - lager_file_name1(Settings); -lager_file_name(_) -> - false. - -lager_file_name1(Settings) when is_list(Settings) -> - {file, FileName} = proplists:lookup(file, Settings), - lager_util:expand_path(FileName); -lager_file_name1({FileName, _}) -> lager_util:expand_path(FileName); -lager_file_name1({FileName, _, _, _, _}) -> lager_util:expand_path(FileName); -lager_file_name1(_) -> - throw({error, {cannot_log_to_file, unknown, - lager_file_backend_config_invalid}}). - - -ensure_logfile_exist(FileName) -> - LogFile = lager_util:expand_path(FileName), - case rabbit_file:read_file_info(LogFile) of - {ok,_} -> ok; - {error, Err} -> throw({error, {cannot_log_to_file, LogFile, Err}}) - end. - -ensure_lager_configured() -> - case lager_configured() of - false -> configure_lager(); - true -> ok - end. - -%% Lager should have handlers and sinks -%% Error logger forwarding to syslog should be disabled -lager_configured() -> - Sinks = lager:list_all_sinks(), - ExpectedSinks = list_expected_sinks(), - application:get_env(lager, handlers) =/= undefined - andalso - lists:all(fun(S) -> lists:member(S, Sinks) end, ExpectedSinks) - andalso - application:get_env(syslog, syslog_error_logger) =/= undefined. - -configure_lager() -> - ok = app_utils:load_applications([lager]), - %% Turn off reformatting for error_logger messages - case application:get_env(lager, error_logger_redirect) of - undefined -> application:set_env(lager, error_logger_redirect, true); - _ -> ok - end, - case application:get_env(lager, error_logger_format_raw) of - undefined -> application:set_env(lager, error_logger_format_raw, true); - _ -> ok - end, - case application:get_env(lager, log_root) of - undefined -> - %% Setting env var to 'undefined' is different from not - %% setting it at all, and lager is sensitive to this - %% difference. - case application:get_env(rabbit, lager_log_root) of - {ok, Value} -> - ok = application:set_env(lager, log_root, Value); - _ -> - ok - end; - _ -> ok - end, - case application:get_env(lager, colored) of - undefined -> - UseColor = rabbit_prelaunch_early_logging:use_colored_logging(), - application:set_env(lager, colored, UseColor); - _ -> - ok - end, - %% Set rabbit.log config variable based on environment. - prepare_rabbit_log_config(), - %% Configure syslog library. - ok = configure_syslog_error_logger(), - %% At this point we should have rabbit.log application variable - %% configured to generate RabbitMQ log handlers. - GeneratedHandlers = generate_lager_handlers(), - - %% If there are lager handlers configured, - %% both lager and generate RabbitMQ handlers are used. - %% This is because it's hard to decide clear preference rules. - %% RabbitMQ handlers can be set to [] to use only lager handlers. - Handlers = case application:get_env(lager, handlers, undefined) of - undefined -> GeneratedHandlers; - LagerHandlers -> - %% Remove handlers generated in previous starts - FormerRabbitHandlers = application:get_env(lager, rabbit_handlers, []), - GeneratedHandlers ++ remove_rabbit_handlers(LagerHandlers, - FormerRabbitHandlers) - end, - - ok = application:set_env(lager, handlers, Handlers), - ok = application:set_env(lager, rabbit_handlers, GeneratedHandlers), - - %% Setup extra sink/handlers. If they are not configured, redirect - %% messages to the default sink. To know the list of expected extra - %% sinks, we look at the 'lager_extra_sinks' compilation option. - LogConfig = application:get_env(rabbit, log, []), - LogLevels = application:get_env(rabbit, log_levels, []), - Categories = proplists:get_value(categories, LogConfig, []), - CategoriesConfig0 = case {Categories, LogLevels} of - {[], []} -> []; - {[], LogLevels} -> - io:format("Using deprecated config parameter 'log_levels'. " - "Please update your configuration file according to " - "https://rabbitmq.com/logging.html"), - lists:map(fun({Name, Level}) -> {Name, [{level, Level}]} end, - LogLevels); - {Categories, []} -> - Categories; - {Categories, _} -> - io:format("Using the deprecated config parameter 'rabbit.log_levels' together " - "with a new parameter for log categories." - " 'rabbit.log_levels' will be ignored. Please remove it from the config. More at " - "https://rabbitmq.com/logging.html"), - Categories - end, - LogLevelsFromContext = case rabbit_prelaunch:get_context() of - #{log_levels := LL} -> LL; - _ -> undefined - end, - Fun = fun - (global, _, CC) -> - CC; - (color, _, CC) -> - CC; - (CategoryS, LogLevel, CC) -> - Category = list_to_atom(CategoryS), - CCEntry = proplists:get_value( - Category, CC, []), - CCEntry1 = lists:ukeymerge( - 1, - [{level, LogLevel}], - lists:ukeysort(1, CCEntry)), - lists:keystore( - Category, 1, CC, {Category, CCEntry1}) - end, - CategoriesConfig = case LogLevelsFromContext of - undefined -> - CategoriesConfig0; - _ -> - maps:fold(Fun, - CategoriesConfig0, - LogLevelsFromContext) - end, - SinkConfigs = lists:map( - fun({Name, Config}) -> - {rabbit_log:make_internal_sink_name(Name), Config} - end, - CategoriesConfig), - LagerSinks = application:get_env(lager, extra_sinks, []), - GeneratedSinks = generate_lager_sinks( - [error_logger_lager_event | list_expected_sinks()], - SinkConfigs), - Sinks = merge_lager_sink_handlers(LagerSinks, GeneratedSinks, []), - ok = application:set_env(lager, extra_sinks, Sinks), - - case application:get_env(lager, error_logger_hwm) of - undefined -> - ok = application:set_env(lager, error_logger_hwm, 1000), - % NB: 50 is the default value in lager.app.src - ok = application:set_env(lager, error_logger_hwm_original, 50); - {ok, Val} when is_integer(Val) andalso Val < 1000 -> - ok = application:set_env(lager, error_logger_hwm, 1000), - ok = application:set_env(lager, error_logger_hwm_original, Val); - {ok, Val} when is_integer(Val) -> - ok = application:set_env(lager, error_logger_hwm_original, Val), - ok - end, - ok. - -configure_syslog_error_logger() -> - %% Disable error_logger forwarding to syslog if it's not configured - case application:get_env(syslog, syslog_error_logger) of - undefined -> - application:set_env(syslog, syslog_error_logger, false); - _ -> ok - end. - -remove_rabbit_handlers(Handlers, FormerHandlers) -> - lists:filter(fun(Handler) -> - not lists:member(Handler, FormerHandlers) - end, - Handlers). - -generate_lager_handlers() -> - LogConfig = application:get_env(rabbit, log, []), - LogHandlersConfig = lists:keydelete(categories, 1, LogConfig), - generate_lager_handlers(LogHandlersConfig). - -generate_lager_handlers(LogHandlersConfig) -> - lists:flatmap( - fun - ({file, HandlerConfig}) -> - case proplists:get_value(file, HandlerConfig, false) of - false -> []; - FileName when is_list(FileName) -> - Backend = lager_backend(file), - generate_handler(Backend, HandlerConfig) - end; - ({Other, HandlerConfig}) when - Other =:= console; Other =:= syslog; Other =:= exchange -> - case proplists:get_value(enabled, HandlerConfig, false) of - false -> []; - true -> - Backend = lager_backend(Other), - generate_handler(Backend, - lists:keydelete(enabled, 1, HandlerConfig)) - end - end, - LogHandlersConfig). - -lager_backend(file) -> lager_file_backend; -lager_backend(console) -> lager_console_backend; -lager_backend(syslog) -> syslog_lager_backend; -lager_backend(exchange) -> lager_exchange_backend. - -%% Syslog backend is using an old API for configuration and -%% does not support proplists. -generate_handler(syslog_lager_backend=Backend, HandlerConfig) -> - %% The default log level is set to `debug` because the actual - %% filtering is made at the sink level. We want to accept all - %% messages here. - DefaultConfigVal = debug, - Level = proplists:get_value(level, HandlerConfig, DefaultConfigVal), - ok = configure_handler_backend(Backend), - [{Backend, - [Level, - {}, - {lager_default_formatter, syslog_formatter_config()}]}]; -generate_handler(Backend, HandlerConfig) -> - [{Backend, - lists:ukeymerge(1, lists:ukeysort(1, HandlerConfig), - lists:ukeysort(1, default_handler_config(Backend)))}]. - -configure_handler_backend(syslog_lager_backend) -> - {ok, _} = application:ensure_all_started(syslog), - ok; -configure_handler_backend(_Backend) -> - ok. - -default_handler_config(lager_console_backend) -> - %% The default log level is set to `debug` because the actual - %% filtering is made at the sink level. We want to accept all - %% messages here. - DefaultConfigVal = debug, - [{level, DefaultConfigVal}, - {formatter_config, default_config_value({formatter_config, console})}]; -default_handler_config(lager_exchange_backend) -> - %% The default log level is set to `debug` because the actual - %% filtering is made at the sink level. We want to accept all - %% messages here. - DefaultConfigVal = debug, - [{level, DefaultConfigVal}, - {formatter_config, default_config_value({formatter_config, exchange})}]; -default_handler_config(lager_file_backend) -> - %% The default log level is set to `debug` because the actual - %% filtering is made at the sink level. We want to accept all - %% messages here. - DefaultConfigVal = debug, - [{level, DefaultConfigVal}, - {formatter_config, default_config_value({formatter_config, file})}, - {date, ""}, - {size, 0}]. - -default_config_value(level) -> - LogConfig = application:get_env(rabbit, log, []), - FoldFun = fun - ({_, Cfg}, LL) when is_list(Cfg) -> - NewLL = proplists:get_value(level, Cfg, LL), - case LL of - undefined -> - NewLL; - _ -> - MoreVerbose = lager_util:level_to_num(NewLL) > lager_util:level_to_num(LL), - case MoreVerbose of - true -> NewLL; - false -> LL - end - end; - (_, LL) -> - LL - end, - FoundLL = lists:foldl(FoldFun, undefined, LogConfig), - case FoundLL of - undefined -> info; - _ -> FoundLL - end; -default_config_value({formatter_config, console}) -> - EOL = case application:get_env(lager, colored) of - {ok, true} -> "\e[0m\r\n"; - _ -> "\r\n" - end, - [date, " ", time, " ", color, "[", severity, "] ", - {pid, ""}, - " ", message, EOL]; -default_config_value({formatter_config, _}) -> - [date, " ", time, " ", color, "[", severity, "] ", - {pid, ""}, - " ", message, "\n"]. - -syslog_formatter_config() -> - [color, "[", severity, "] ", - {pid, ""}, - " ", message, "\n"]. - -prepare_rabbit_log_config() -> - %% If RABBIT_LOGS is not set, we should ignore it. - DefaultFile = application:get_env(rabbit, lager_default_file, undefined), - %% If RABBIT_UPGRADE_LOGS is not set, we should ignore it. - UpgradeFile = application:get_env(rabbit, lager_upgrade_file, undefined), - case DefaultFile of - undefined -> ok; - false -> - set_env_default_log_disabled(); - tty -> - set_env_default_log_console(); - FileName when is_list(FileName) -> - case rabbit_prelaunch:get_context() of - %% The user explicitly sets $RABBITMQ_LOGS; - %% we should override a file location even - %% if it's set in rabbitmq.config - #{var_origins := #{main_log_file := environment}} -> - set_env_default_log_file(FileName, override); - _ -> - set_env_default_log_file(FileName, keep) - end - end, - - %% Upgrade log file never overrides the value set in rabbitmq.config - case UpgradeFile of - %% No special env for upgrade logs - redirect to the default sink - undefined -> ok; - %% Redirect logs to default output. - DefaultFile -> ok; - UpgradeFileName when is_list(UpgradeFileName) -> - set_env_upgrade_log_file(UpgradeFileName) - end. - -set_env_default_log_disabled() -> - %% Disabling all the logs. - ok = application:set_env(rabbit, log, []). - -set_env_default_log_console() -> - LogConfig = application:get_env(rabbit, log, []), - ConsoleConfig = proplists:get_value(console, LogConfig, []), - LogConfigConsole = - lists:keystore(console, 1, LogConfig, - {console, lists:keystore(enabled, 1, ConsoleConfig, - {enabled, true})}), - %% Remove the file handler - disable logging to file - LogConfigConsoleNoFile = lists:keydelete(file, 1, LogConfigConsole), - ok = application:set_env(rabbit, log, LogConfigConsoleNoFile). - -set_env_default_log_file(FileName, Override) -> - LogConfig = application:get_env(rabbit, log, []), - FileConfig = proplists:get_value(file, LogConfig, []), - NewLogConfig = case proplists:get_value(file, FileConfig, undefined) of - undefined -> - lists:keystore(file, 1, LogConfig, - {file, lists:keystore(file, 1, FileConfig, - {file, FileName})}); - _ConfiguredFileName -> - case Override of - override -> - lists:keystore( - file, 1, LogConfig, - {file, lists:keystore(file, 1, FileConfig, - {file, FileName})}); - keep -> - LogConfig - end - end, - ok = application:set_env(rabbit, log, NewLogConfig). - -set_env_upgrade_log_file(FileName) -> - LogConfig = application:get_env(rabbit, log, []), - SinksConfig = proplists:get_value(categories, LogConfig, []), - UpgradeSinkConfig = proplists:get_value(upgrade, SinksConfig, []), - FileConfig = proplists:get_value(file, SinksConfig, []), - NewLogConfig = case proplists:get_value(file, FileConfig, undefined) of - undefined -> - lists:keystore( - categories, 1, LogConfig, - {categories, - lists:keystore( - upgrade, 1, SinksConfig, - {upgrade, - lists:keystore(file, 1, UpgradeSinkConfig, - {file, FileName})})}); - %% No cahnge. We don't want to override the configured value. - _File -> LogConfig - end, - ok = application:set_env(rabbit, log, NewLogConfig). - -generate_lager_sinks(SinkNames, SinkConfigs) -> - LogLevels = case rabbit_prelaunch:get_context() of - #{log_levels := LL} -> LL; - _ -> undefined - end, - DefaultLogLevel = case LogLevels of - #{global := LogLevel} -> - LogLevel; - _ -> - default_config_value(level) - end, - lists:map(fun(SinkName) -> - SinkConfig = proplists:get_value(SinkName, SinkConfigs, []), - SinkHandlers = case proplists:get_value(file, SinkConfig, false) of - %% If no file defined - forward everything to the default backend - false -> - ForwarderLevel = proplists:get_value(level, - SinkConfig, - DefaultLogLevel), - [{lager_forwarder_backend, - [lager_util:make_internal_sink_name(lager), ForwarderLevel]}]; - %% If a file defined - add a file backend to handlers and remove all default file backends. - File -> - %% Use `debug` as a default handler to not override a handler level - Level = proplists:get_value(level, SinkConfig, DefaultLogLevel), - DefaultGeneratedHandlers = application:get_env(lager, rabbit_handlers, []), - SinkFileHandlers = case proplists:get_value(lager_file_backend, DefaultGeneratedHandlers, undefined) of - undefined -> - %% Create a new file handler. - %% `info` is a default level here. - FileLevel = proplists:get_value(level, SinkConfig, DefaultLogLevel), - generate_lager_handlers([{file, [{file, File}, {level, FileLevel}]}]); - FileHandler -> - %% Replace a filename in the handler - FileHandlerChanges = case handler_level_more_verbose(FileHandler, Level) of - true -> [{file, File}, {level, Level}]; - false -> [{file, File}] - end, - - [{lager_file_backend, - lists:ukeymerge(1, FileHandlerChanges, - lists:ukeysort(1, FileHandler))}] - end, - %% Remove all file handlers. - AllLagerHandlers = application:get_env(lager, handlers, []), - HandlersWithoutFile = lists:filter( - fun({lager_file_backend, _}) -> false; - ({_, _}) -> true - end, - AllLagerHandlers), - %% Set level for handlers which are more verbose. - %% We don't increase verbosity in sinks so it works like forwarder backend. - HandlersWithoutFileWithLevel = lists:map(fun({Name, Handler}) -> - case handler_level_more_verbose(Handler, Level) of - true -> {Name, lists:keystore(level, 1, Handler, {level, Level})}; - false -> {Name, Handler} - end - end, - HandlersWithoutFile), - - HandlersWithoutFileWithLevel ++ SinkFileHandlers - end, - {SinkName, [{handlers, SinkHandlers}, {rabbit_handlers, SinkHandlers}]} - end, - SinkNames). - -handler_level_more_verbose(Handler, Level) -> - HandlerLevel = proplists:get_value(level, Handler, default_config_value(level)), - lager_util:level_to_num(HandlerLevel) > lager_util:level_to_num(Level). - -merge_lager_sink_handlers([{Name, Sink} | RestSinks], GeneratedSinks, Agg) -> - %% rabbitmq/rabbitmq-server#2044. - %% We have to take into account that a sink's - %% handler backend may need additional configuration here. - %% {rabbit_log_federation_lager_event, [ - %% {handlers, [ - %% {lager_forwarder_backend, [lager_event,inherit]}, - %% {syslog_lager_backend, [debug]} - %% ]}, - %% {rabbit_handlers, [ - %% {lager_forwarder_backend, [lager_event,inherit]} - %% ]} - %% ]} - case lists:keytake(Name, 1, GeneratedSinks) of - {value, {Name, GenSink}, RestGeneratedSinks} -> - Handlers = proplists:get_value(handlers, Sink, []), - GenHandlers = proplists:get_value(handlers, GenSink, []), - FormerRabbitHandlers = proplists:get_value(rabbit_handlers, Sink, []), - - %% Remove handlers defined in previous starts - ConfiguredHandlers = remove_rabbit_handlers(Handlers, FormerRabbitHandlers), - NewHandlers = GenHandlers ++ ConfiguredHandlers, - ok = maybe_configure_handler_backends(NewHandlers), - MergedSink = lists:keystore(rabbit_handlers, 1, - lists:keystore(handlers, 1, Sink, - {handlers, NewHandlers}), - {rabbit_handlers, GenHandlers}), - merge_lager_sink_handlers( - RestSinks, - RestGeneratedSinks, - [{Name, MergedSink} | Agg]); - false -> - merge_lager_sink_handlers( - RestSinks, - GeneratedSinks, - [{Name, Sink} | Agg]) - end; -merge_lager_sink_handlers([], GeneratedSinks, Agg) -> GeneratedSinks ++ Agg. - -maybe_configure_handler_backends([]) -> - ok; -maybe_configure_handler_backends([{Backend, _}|Backends]) -> - ok = configure_handler_backend(Backend), - maybe_configure_handler_backends(Backends). - -list_expected_sinks() -> - rabbit_prelaunch_early_logging:list_expected_sinks(). - -maybe_remove_logger_handler() -> - M = logger, - F = remove_handler, - try - ok = erlang:apply(M, F, [default]) - catch - error:undef -> - % OK since the logger module only exists in OTP 21.1 or later - ok; - error:{badmatch, {error, {not_found, default}}} -> - % OK - this error happens when running a CLI command - ok; - Err:Reason -> - error_logger:error_msg("calling ~p:~p failed: ~p:~p~n", - [M, F, Err, Reason]) - end. - -get_most_verbose_log_level() -> - {ok, HandlersA} = application:get_env(lager, handlers), - {ok, ExtraSinks} = application:get_env(lager, extra_sinks), - HandlersB = lists:append( - [H || {_, Keys} <- ExtraSinks, - {handlers, H} <- Keys]), - get_most_verbose_log_level(HandlersA ++ HandlersB, - lager_util:level_to_num(none)). - -get_most_verbose_log_level([{_, Props} | Rest], MostVerbose) -> - LogLevel = proplists:get_value(level, Props, info), - LogLevelNum = lager_util:level_to_num(LogLevel), - case LogLevelNum > MostVerbose of - true -> - get_most_verbose_log_level(Rest, LogLevelNum); - false -> - get_most_verbose_log_level(Rest, MostVerbose) - end; -get_most_verbose_log_level([], MostVerbose) -> - lager_util:num_to_level(MostVerbose). diff --git a/src/rabbit_limiter.erl b/src/rabbit_limiter.erl deleted file mode 100644 index d3803957d3..0000000000 --- a/src/rabbit_limiter.erl +++ /dev/null @@ -1,448 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% The purpose of the limiter is to stem the flow of messages from -%% queues to channels, in order to act upon various protocol-level -%% flow control mechanisms, specifically AMQP 0-9-1's basic.qos -%% prefetch_count, our consumer prefetch extension, and AMQP 1.0's -%% link (aka consumer) credit mechanism. -%% -%% Each channel has an associated limiter process, created with -%% start_link/1, which it passes to queues on consumer creation with -%% rabbit_amqqueue:basic_consume/10, and rabbit_amqqueue:basic_get/4. -%% The latter isn't strictly necessary, since basic.get is not -%% subject to limiting, but it means that whenever a queue knows about -%% a channel, it also knows about its limiter, which is less fiddly. -%% -%% The limiter process holds state that is, in effect, shared between -%% the channel and all queues from which the channel is -%% consuming. Essentially all these queues are competing for access to -%% a single, limited resource - the ability to deliver messages via -%% the channel - and it is the job of the limiter process to mediate -%% that access. -%% -%% The limiter process is separate from the channel process for two -%% reasons: separation of concerns, and efficiency. Channels can get -%% very busy, particularly if they are also dealing with publishes. -%% With a separate limiter process all the aforementioned access -%% mediation can take place without touching the channel. -%% -%% For efficiency, both the channel and the queues keep some local -%% state, initialised from the limiter pid with new/1 and client/1, -%% respectively. In particular this allows them to avoid any -%% interaction with the limiter process when it is 'inactive', i.e. no -%% protocol-level flow control is taking place. -%% -%% This optimisation does come at the cost of some complexity though: -%% when a limiter becomes active, the channel needs to inform all its -%% consumer queues of this change in status. It does this by invoking -%% rabbit_amqqueue:activate_limit_all/2. Note that there is no inverse -%% transition, i.e. once a queue has been told about an active -%% limiter, it is not subsequently told when that limiter becomes -%% inactive. In practice it is rare for that to happen, though we -%% could optimise this case in the future. -%% -%% Consumer credit (for AMQP 1.0) and per-consumer prefetch (for AMQP -%% 0-9-1) are treated as essentially the same thing, but with the -%% exception that per-consumer prefetch gets an auto-topup when -%% acknowledgments come in. -%% -%% The bookkeeping for this is local to queues, so it is not necessary -%% to store information about it in the limiter process. But for -%% abstraction we hide it from the queue behind the limiter API, and -%% it therefore becomes part of the queue local state. -%% -%% The interactions with the limiter are as follows: -%% -%% 1. Channels tell the limiter about basic.qos prefetch counts - -%% that's what the limit_prefetch/3, unlimit_prefetch/1, -%% get_prefetch_limit/1 API functions are about. They also tell the -%% limiter queue state (via the queue) about consumer credit -%% changes and message acknowledgement - that's what credit/5 and -%% ack_from_queue/3 are for. -%% -%% 2. Queues also tell the limiter queue state about the queue -%% becoming empty (via drained/1) and consumers leaving (via -%% forget_consumer/2). -%% -%% 3. Queues register with the limiter - this happens as part of -%% activate/1. -%% -%% 4. The limiter process maintains an internal counter of 'messages -%% sent but not yet acknowledged', called the 'volume'. -%% -%% 5. Queues ask the limiter for permission (with can_send/3) whenever -%% they want to deliver a message to a channel. The limiter checks -%% whether a) the volume has not yet reached the prefetch limit, -%% and b) whether the consumer has enough credit. If so it -%% increments the volume and tells the queue to proceed. Otherwise -%% it marks the queue as requiring notification (see below) and -%% tells the queue not to proceed. -%% -%% 6. A queue that has been told to proceed (by the return value of -%% can_send/3) sends the message to the channel. Conversely, a -%% queue that has been told not to proceed, will not attempt to -%% deliver that message, or any future messages, to the -%% channel. This is accomplished by can_send/3 capturing the -%% outcome in the local state, where it can be accessed with -%% is_suspended/1. -%% -%% 7. When a channel receives an ack it tells the limiter (via ack/2) -%% how many messages were ack'ed. The limiter process decrements -%% the volume and if it falls below the prefetch_count then it -%% notifies (through rabbit_amqqueue:resume/2) all the queues -%% requiring notification, i.e. all those that had a can_send/3 -%% request denied. -%% -%% 8. Upon receipt of such a notification, queues resume delivery to -%% the channel, i.e. they will once again start asking limiter, as -%% described in (5). -%% -%% 9. When a queue has no more consumers associated with a particular -%% channel, it deactivates use of the limiter with deactivate/1, -%% which alters the local state such that no further interactions -%% with the limiter process take place until a subsequent -%% activate/1. - --module(rabbit_limiter). - --include("rabbit.hrl"). - --behaviour(gen_server2). - --export([start_link/1]). -%% channel API --export([new/1, limit_prefetch/3, unlimit_prefetch/1, is_active/1, - get_prefetch_limit/1, ack/2, pid/1]). -%% queue API --export([client/1, activate/1, can_send/3, resume/1, deactivate/1, - is_suspended/1, is_consumer_blocked/2, credit/5, ack_from_queue/3, - drained/1, forget_consumer/2]). -%% callbacks --export([init/1, terminate/2, code_change/3, handle_call/3, handle_cast/2, - handle_info/2, prioritise_call/4]). - -%%---------------------------------------------------------------------------- - --record(lstate, {pid, prefetch_limited}). --record(qstate, {pid, state, credits}). - --type lstate() :: #lstate{pid :: pid(), - prefetch_limited :: boolean()}. --type qstate() :: #qstate{pid :: pid(), - state :: 'dormant' | 'active' | 'suspended'}. - --type credit_mode() :: 'manual' | 'drain' | 'auto'. - -%%---------------------------------------------------------------------------- - --record(lim, {prefetch_count = 0, - ch_pid, - %% 'Notify' is a boolean that indicates whether a queue should be - %% notified of a change in the limit or volume that may allow it to - %% deliver more messages via the limiter's channel. - queues = maps:new(), % QPid -> {MonitorRef, Notify} - volume = 0}). - -%% mode is of type credit_mode() --record(credit, {credit = 0, mode}). - -%%---------------------------------------------------------------------------- -%% API -%%---------------------------------------------------------------------------- - --spec start_link(rabbit_types:proc_name()) -> - rabbit_types:ok_pid_or_error(). - -start_link(ProcName) -> gen_server2:start_link(?MODULE, [ProcName], []). - --spec new(pid()) -> lstate(). - -new(Pid) -> - %% this a 'call' to ensure that it is invoked at most once. - ok = gen_server:call(Pid, {new, self()}, infinity), - #lstate{pid = Pid, prefetch_limited = false}. - --spec limit_prefetch(lstate(), non_neg_integer(), non_neg_integer()) -> - lstate(). - -limit_prefetch(L, PrefetchCount, UnackedCount) when PrefetchCount > 0 -> - ok = gen_server:call( - L#lstate.pid, - {limit_prefetch, PrefetchCount, UnackedCount}, infinity), - L#lstate{prefetch_limited = true}. - --spec unlimit_prefetch(lstate()) -> lstate(). - -unlimit_prefetch(L) -> - ok = gen_server:call(L#lstate.pid, unlimit_prefetch, infinity), - L#lstate{prefetch_limited = false}. - --spec is_active(lstate()) -> boolean(). - -is_active(#lstate{prefetch_limited = Limited}) -> Limited. - --spec get_prefetch_limit(lstate()) -> non_neg_integer(). - -get_prefetch_limit(#lstate{prefetch_limited = false}) -> 0; -get_prefetch_limit(L) -> - gen_server:call(L#lstate.pid, get_prefetch_limit, infinity). - --spec ack(lstate(), non_neg_integer()) -> 'ok'. - -ack(#lstate{prefetch_limited = false}, _AckCount) -> ok; -ack(L, AckCount) -> gen_server:cast(L#lstate.pid, {ack, AckCount}). - --spec pid(lstate()) -> pid(). - -pid(#lstate{pid = Pid}) -> Pid. - --spec client(pid()) -> qstate(). - -client(Pid) -> #qstate{pid = Pid, state = dormant, credits = gb_trees:empty()}. - --spec activate(qstate()) -> qstate(). - -activate(L = #qstate{state = dormant}) -> - ok = gen_server:cast(L#qstate.pid, {register, self()}), - L#qstate{state = active}; -activate(L) -> L. - --spec can_send(qstate(), boolean(), rabbit_types:ctag()) -> - {'continue' | 'suspend', qstate()}. - -can_send(L = #qstate{pid = Pid, state = State, credits = Credits}, - AckRequired, CTag) -> - case is_consumer_blocked(L, CTag) of - false -> case (State =/= active orelse - safe_call(Pid, {can_send, self(), AckRequired}, true)) of - true -> Credits1 = decrement_credit(CTag, Credits), - {continue, L#qstate{credits = Credits1}}; - false -> {suspend, L#qstate{state = suspended}} - end; - true -> {suspend, L} - end. - -safe_call(Pid, Msg, ExitValue) -> - rabbit_misc:with_exit_handler( - fun () -> ExitValue end, - fun () -> gen_server2:call(Pid, Msg, infinity) end). - --spec resume(qstate()) -> qstate(). - -resume(L = #qstate{state = suspended}) -> - L#qstate{state = active}; -resume(L) -> L. - --spec deactivate(qstate()) -> qstate(). - -deactivate(L = #qstate{state = dormant}) -> L; -deactivate(L) -> - ok = gen_server:cast(L#qstate.pid, {unregister, self()}), - L#qstate{state = dormant}. - --spec is_suspended(qstate()) -> boolean(). - -is_suspended(#qstate{state = suspended}) -> true; -is_suspended(#qstate{}) -> false. - --spec is_consumer_blocked(qstate(), rabbit_types:ctag()) -> boolean(). - -is_consumer_blocked(#qstate{credits = Credits}, CTag) -> - case gb_trees:lookup(CTag, Credits) of - none -> false; - {value, #credit{credit = C}} when C > 0 -> false; - {value, #credit{}} -> true - end. - --spec credit - (qstate(), rabbit_types:ctag(), non_neg_integer(), credit_mode(), - boolean()) -> - {boolean(), qstate()}. - -credit(Limiter = #qstate{credits = Credits}, CTag, Crd, Mode, IsEmpty) -> - {Res, Cr} = - case IsEmpty andalso Mode =:= drain of - true -> {true, #credit{credit = 0, mode = manual}}; - false -> {false, #credit{credit = Crd, mode = Mode}} - end, - {Res, Limiter#qstate{credits = enter_credit(CTag, Cr, Credits)}}. - --spec ack_from_queue(qstate(), rabbit_types:ctag(), non_neg_integer()) -> - {boolean(), qstate()}. - -ack_from_queue(Limiter = #qstate{credits = Credits}, CTag, Credit) -> - {Credits1, Unblocked} = - case gb_trees:lookup(CTag, Credits) of - {value, C = #credit{mode = auto, credit = C0}} -> - {update_credit(CTag, C#credit{credit = C0 + Credit}, Credits), - C0 =:= 0 andalso Credit =/= 0}; - _ -> - {Credits, false} - end, - {Unblocked, Limiter#qstate{credits = Credits1}}. - --spec drained(qstate()) -> - {[{rabbit_types:ctag(), non_neg_integer()}], qstate()}. - -drained(Limiter = #qstate{credits = Credits}) -> - Drain = fun(C) -> C#credit{credit = 0, mode = manual} end, - {CTagCredits, Credits2} = - rabbit_misc:gb_trees_fold( - fun (CTag, C = #credit{credit = Crd, mode = drain}, {Acc, Creds0}) -> - {[{CTag, Crd} | Acc], update_credit(CTag, Drain(C), Creds0)}; - (_CTag, #credit{credit = _Crd, mode = _Mode}, {Acc, Creds0}) -> - {Acc, Creds0} - end, {[], Credits}, Credits), - {CTagCredits, Limiter#qstate{credits = Credits2}}. - --spec forget_consumer(qstate(), rabbit_types:ctag()) -> qstate(). - -forget_consumer(Limiter = #qstate{credits = Credits}, CTag) -> - Limiter#qstate{credits = gb_trees:delete_any(CTag, Credits)}. - -%%---------------------------------------------------------------------------- -%% Queue-local code -%%---------------------------------------------------------------------------- - -%% We want to do all the AMQP 1.0-ish link level credit calculations -%% in the queue (to do them elsewhere introduces a ton of -%% races). However, it's a big chunk of code that is conceptually very -%% linked to the limiter concept. So we get the queue to hold a bit of -%% state for us (#qstate.credits), and maintain a fiction that the -%% limiter is making the decisions... - -decrement_credit(CTag, Credits) -> - case gb_trees:lookup(CTag, Credits) of - {value, C = #credit{credit = Credit}} -> - update_credit(CTag, C#credit{credit = Credit - 1}, Credits); - none -> - Credits - end. - -enter_credit(CTag, C, Credits) -> - gb_trees:enter(CTag, ensure_credit_invariant(C), Credits). - -update_credit(CTag, C, Credits) -> - gb_trees:update(CTag, ensure_credit_invariant(C), Credits). - -ensure_credit_invariant(C = #credit{credit = 0, mode = drain}) -> - %% Using up all credit implies no need to send a 'drained' event - C#credit{mode = manual}; -ensure_credit_invariant(C) -> - C. - -%%---------------------------------------------------------------------------- -%% gen_server callbacks -%%---------------------------------------------------------------------------- - -init([ProcName]) -> ?store_proc_name(ProcName), - ?LG_PROCESS_TYPE(limiter), - {ok, #lim{}}. - -prioritise_call(get_prefetch_limit, _From, _Len, _State) -> 9; -prioritise_call(_Msg, _From, _Len, _State) -> 0. - -handle_call({new, ChPid}, _From, State = #lim{ch_pid = undefined}) -> - {reply, ok, State#lim{ch_pid = ChPid}}; - -handle_call({limit_prefetch, PrefetchCount, UnackedCount}, _From, - State = #lim{prefetch_count = 0}) -> - {reply, ok, maybe_notify(State, State#lim{prefetch_count = PrefetchCount, - volume = UnackedCount})}; -handle_call({limit_prefetch, PrefetchCount, _UnackedCount}, _From, State) -> - {reply, ok, maybe_notify(State, State#lim{prefetch_count = PrefetchCount})}; - -handle_call(unlimit_prefetch, _From, State) -> - {reply, ok, maybe_notify(State, State#lim{prefetch_count = 0, - volume = 0})}; - -handle_call(get_prefetch_limit, _From, - State = #lim{prefetch_count = PrefetchCount}) -> - {reply, PrefetchCount, State}; - -handle_call({can_send, QPid, AckRequired}, _From, - State = #lim{volume = Volume}) -> - case prefetch_limit_reached(State) of - true -> {reply, false, limit_queue(QPid, State)}; - false -> {reply, true, State#lim{volume = if AckRequired -> Volume + 1; - true -> Volume - end}} - end. - -handle_cast({ack, Count}, State = #lim{volume = Volume}) -> - NewVolume = if Volume == 0 -> 0; - true -> Volume - Count - end, - {noreply, maybe_notify(State, State#lim{volume = NewVolume})}; - -handle_cast({register, QPid}, State) -> - {noreply, remember_queue(QPid, State)}; - -handle_cast({unregister, QPid}, State) -> - {noreply, forget_queue(QPid, State)}. - -handle_info({'DOWN', _MonitorRef, _Type, QPid, _Info}, State) -> - {noreply, forget_queue(QPid, State)}. - -terminate(_, _) -> - ok. - -code_change(_, State, _) -> - {ok, State}. - -%%---------------------------------------------------------------------------- -%% Internal plumbing -%%---------------------------------------------------------------------------- - -maybe_notify(OldState, NewState) -> - case prefetch_limit_reached(OldState) andalso - not prefetch_limit_reached(NewState) of - true -> notify_queues(NewState); - false -> NewState - end. - -prefetch_limit_reached(#lim{prefetch_count = Limit, volume = Volume}) -> - Limit =/= 0 andalso Volume >= Limit. - -remember_queue(QPid, State = #lim{queues = Queues}) -> - case maps:is_key(QPid, Queues) of - false -> MRef = erlang:monitor(process, QPid), - State#lim{queues = maps:put(QPid, {MRef, false}, Queues)}; - true -> State - end. - -forget_queue(QPid, State = #lim{queues = Queues}) -> - case maps:find(QPid, Queues) of - {ok, {MRef, _}} -> true = erlang:demonitor(MRef), - State#lim{queues = maps:remove(QPid, Queues)}; - error -> State - end. - -limit_queue(QPid, State = #lim{queues = Queues}) -> - UpdateFun = fun ({MRef, _}) -> {MRef, true} end, - State#lim{queues = maps:update_with(QPid, UpdateFun, Queues)}. - -notify_queues(State = #lim{ch_pid = ChPid, queues = Queues}) -> - {QList, NewQueues} = - maps:fold(fun (_QPid, {_, false}, Acc) -> Acc; - (QPid, {MRef, true}, {L, D}) -> - {[QPid | L], maps:put(QPid, {MRef, false}, D)} - end, {[], Queues}, Queues), - case length(QList) of - 0 -> ok; - 1 -> ok = rabbit_amqqueue:resume(hd(QList), ChPid); %% common case - L -> - %% We randomly vary the position of queues in the list, - %% thus ensuring that each queue has an equal chance of - %% being notified first. - {L1, L2} = lists:split(rand:uniform(L), QList), - [[ok = rabbit_amqqueue:resume(Q, ChPid) || Q <- L3] - || L3 <- [L2, L1]], - ok - end, - State#lim{queues = NewQueues}. diff --git a/src/rabbit_log_tail.erl b/src/rabbit_log_tail.erl deleted file mode 100644 index c3faad07fc..0000000000 --- a/src/rabbit_log_tail.erl +++ /dev/null @@ -1,102 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_log_tail). - --export([tail_n_lines/2]). --export([init_tail_stream/4]). - --define(GUESS_OFFSET, 200). - -init_tail_stream(Filename, Pid, Ref, Duration) -> - RPCProc = self(), - Reader = spawn(fun() -> - link(Pid), - case file:open(Filename, [read, binary]) of - {ok, File} -> - TimeLimit = case Duration of - infinity -> infinity; - _ -> erlang:system_time(second) + Duration - end, - {ok, _} = file:position(File, eof), - RPCProc ! {Ref, opened}, - read_loop(File, Pid, Ref, TimeLimit); - {error, _} = Err -> - RPCProc ! {Ref, Err} - end - end), - receive - {Ref, opened} -> {ok, Ref}; - {Ref, {error, Err}} -> {error, Err} - after 5000 -> - exit(Reader, timeout), - {error, timeout} - end. - -read_loop(File, Pid, Ref, TimeLimit) -> - case is_integer(TimeLimit) andalso erlang:system_time(second) > TimeLimit of - true -> Pid ! {Ref, <<>>, finished}; - false -> - case file:read(File, ?GUESS_OFFSET) of - {ok, Data} -> - Pid ! {Ref, Data, confinue}, - read_loop(File, Pid, Ref, TimeLimit); - eof -> - timer:sleep(1000), - read_loop(File, Pid, Ref, TimeLimit); - {error, _} = Err -> - Pid ! {Ref, Err, finished} - end - end. - -tail_n_lines(Filename, N) -> - case file:open(Filename, [read, binary]) of - {ok, File} -> - {ok, Eof} = file:position(File, eof), - %% Eof may move. Only read up to the current one. - Result = reverse_read_n_lines(N, N, File, Eof, Eof), - file:close(File), - Result; - {error, _} = Error -> Error - end. - -reverse_read_n_lines(N, OffsetN, File, Position, Eof) -> - GuessPosition = offset(Position, OffsetN), - case read_lines_from_position(File, GuessPosition, Eof) of - {ok, Lines} -> - NLines = length(Lines), - case {NLines >= N, GuessPosition == 0} of - %% Take only N lines if there is more - {true, _} -> lists:nthtail(NLines - N, Lines); - %% Safe to assume that NLines is less then N - {_, true} -> Lines; - %% Adjust position - _ -> - reverse_read_n_lines(N, N - NLines + 1, File, GuessPosition, Eof) - end; - {error, _} = Error -> Error - end. - -read_from_position(File, GuessPosition, Eof) -> - file:pread(File, GuessPosition, max(0, Eof - GuessPosition)). - -read_lines_from_position(File, GuessPosition, Eof) -> - case read_from_position(File, GuessPosition, Eof) of - {ok, Data} -> - Lines = binary:split(Data, <<"\n">>, [global, trim]), - case {GuessPosition, Lines} of - %% If position is 0 - there are no partial lines - {0, _} -> {ok, Lines}; - %% Remove first line as it can be partial - {_, [_ | Rest]} -> {ok, Rest}; - {_, []} -> {ok, []} - end; - {error, _} = Error -> Error - end. - -offset(Base, N) -> - max(0, Base - N * ?GUESS_OFFSET). diff --git a/src/rabbit_looking_glass.erl b/src/rabbit_looking_glass.erl deleted file mode 100644 index 00b1b6d46b..0000000000 --- a/src/rabbit_looking_glass.erl +++ /dev/null @@ -1,48 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_looking_glass). - --ignore_xref([{lg, trace, 4}]). --ignore_xref([{maps, from_list, 1}]). - --export([boot/0]). --export([connections/0]). - -boot() -> - case os:getenv("RABBITMQ_TRACER") of - false -> - ok; - Value -> - Input = parse_value(Value), - rabbit_log:info( - "Enabling Looking Glass profiler, input value: ~p", - [Input] - ), - {ok, _} = application:ensure_all_started(looking_glass), - lg:trace( - Input, - lg_file_tracer, - "traces.lz4", - maps:from_list([ - {mode, profile}, - {process_dump, true}, - {running, true}, - {send, true}] - ) - ) - end. - -parse_value(Value) -> - [begin - [Mod, Fun] = string:tokens(C, ":"), - {callback, list_to_atom(Mod), list_to_atom(Fun)} - end || C <- string:tokens(Value, ",")]. - -connections() -> - Pids = [Pid || {{conns_sup, _}, Pid} <- ets:tab2list(ranch_server)], - ['_', {scope, Pids}]. diff --git a/src/rabbit_maintenance.erl b/src/rabbit_maintenance.erl deleted file mode 100644 index e5434dc888..0000000000 --- a/src/rabbit_maintenance.erl +++ /dev/null @@ -1,354 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_maintenance). - --include("rabbit.hrl"). - --export([ - is_enabled/0, - drain/0, - revive/0, - mark_as_being_drained/0, - unmark_as_being_drained/0, - is_being_drained_local_read/1, - is_being_drained_consistent_read/1, - status_local_read/1, - status_consistent_read/1, - filter_out_drained_nodes_local_read/1, - filter_out_drained_nodes_consistent_read/1, - suspend_all_client_listeners/0, - resume_all_client_listeners/0, - close_all_client_connections/0, - primary_replica_transfer_candidate_nodes/0, - random_primary_replica_transfer_candidate_node/1, - transfer_leadership_of_quorum_queues/1, - transfer_leadership_of_classic_mirrored_queues/1, - status_table_name/0, - status_table_definition/0 -]). - --define(TABLE, rabbit_node_maintenance_states). --define(FEATURE_FLAG, maintenance_mode_status). --define(DEFAULT_STATUS, regular). --define(DRAINING_STATUS, draining). - --type maintenance_status() :: ?DEFAULT_STATUS | ?DRAINING_STATUS. --type mnesia_table() :: atom(). - --export_type([ - maintenance_status/0 -]). - -%% -%% API -%% - --spec status_table_name() -> mnesia_table(). -status_table_name() -> - ?TABLE. - --spec status_table_definition() -> list(). -status_table_definition() -> - maps:to_list(#{ - record_name => node_maintenance_state, - attributes => record_info(fields, node_maintenance_state) - }). - --spec is_enabled() -> boolean(). -is_enabled() -> - rabbit_feature_flags:is_enabled(?FEATURE_FLAG). - --spec drain() -> ok. -drain() -> - case is_enabled() of - true -> do_drain(); - false -> rabbit_log:warning("Feature flag `~s` is not enabled, draining is a no-op", [?FEATURE_FLAG]) - end. - --spec do_drain() -> ok. -do_drain() -> - rabbit_log:alert("This node is being put into maintenance (drain) mode"), - mark_as_being_drained(), - rabbit_log:info("Marked this node as undergoing maintenance"), - suspend_all_client_listeners(), - rabbit_log:alert("Suspended all listeners and will no longer accept client connections"), - {ok, NConnections} = close_all_client_connections(), - %% allow plugins to react e.g. by closing their protocol connections - rabbit_event:notify(maintenance_connections_closed, #{ - reason => <<"node is being put into maintenance">> - }), - rabbit_log:alert("Closed ~b local client connections", [NConnections]), - - TransferCandidates = primary_replica_transfer_candidate_nodes(), - ReadableCandidates = readable_candidate_list(TransferCandidates), - rabbit_log:info("Node will transfer primary replicas of its queues to ~b peers: ~s", - [length(TransferCandidates), ReadableCandidates]), - transfer_leadership_of_classic_mirrored_queues(TransferCandidates), - transfer_leadership_of_quorum_queues(TransferCandidates), - stop_local_quorum_queue_followers(), - - %% allow plugins to react - rabbit_event:notify(maintenance_draining, #{ - reason => <<"node is being put into maintenance">> - }), - rabbit_log:alert("Node is ready to be shut down for maintenance or upgrade"), - - ok. - --spec revive() -> ok. -revive() -> - case is_enabled() of - true -> do_revive(); - false -> rabbit_log:warning("Feature flag `~s` is not enabled, reviving is a no-op", [?FEATURE_FLAG]) - end. - --spec do_revive() -> ok. -do_revive() -> - rabbit_log:alert("This node is being revived from maintenance (drain) mode"), - revive_local_quorum_queue_replicas(), - rabbit_log:alert("Resumed all listeners and will accept client connections again"), - resume_all_client_listeners(), - rabbit_log:alert("Resumed all listeners and will accept client connections again"), - unmark_as_being_drained(), - rabbit_log:info("Marked this node as back from maintenance and ready to serve clients"), - - %% allow plugins to react - rabbit_event:notify(maintenance_revived, #{}), - - ok. - --spec mark_as_being_drained() -> boolean(). -mark_as_being_drained() -> - rabbit_log:debug("Marking the node as undergoing maintenance"), - set_maintenance_status_status(?DRAINING_STATUS). - --spec unmark_as_being_drained() -> boolean(). -unmark_as_being_drained() -> - rabbit_log:debug("Unmarking the node as undergoing maintenance"), - set_maintenance_status_status(?DEFAULT_STATUS). - -set_maintenance_status_status(Status) -> - Res = mnesia:transaction(fun () -> - case mnesia:wread({?TABLE, node()}) of - [] -> - Row = #node_maintenance_state{ - node = node(), - status = Status - }, - mnesia:write(?TABLE, Row, write); - [Row0] -> - Row = Row0#node_maintenance_state{ - node = node(), - status = Status - }, - mnesia:write(?TABLE, Row, write) - end - end), - case Res of - {atomic, ok} -> true; - _ -> false - end. - - --spec is_being_drained_local_read(node()) -> boolean(). -is_being_drained_local_read(Node) -> - Status = status_local_read(Node), - Status =:= ?DRAINING_STATUS. - --spec is_being_drained_consistent_read(node()) -> boolean(). -is_being_drained_consistent_read(Node) -> - Status = status_consistent_read(Node), - Status =:= ?DRAINING_STATUS. - --spec status_local_read(node()) -> maintenance_status(). -status_local_read(Node) -> - case catch mnesia:dirty_read(?TABLE, Node) of - [] -> ?DEFAULT_STATUS; - [#node_maintenance_state{node = Node, status = Status}] -> - Status; - _ -> ?DEFAULT_STATUS - end. - --spec status_consistent_read(node()) -> maintenance_status(). -status_consistent_read(Node) -> - case mnesia:transaction(fun() -> mnesia:read(?TABLE, Node) end) of - {atomic, []} -> ?DEFAULT_STATUS; - {atomic, [#node_maintenance_state{node = Node, status = Status}]} -> - Status; - {atomic, _} -> ?DEFAULT_STATUS; - {aborted, _Reason} -> ?DEFAULT_STATUS - end. - - -spec filter_out_drained_nodes_local_read([node()]) -> [node()]. -filter_out_drained_nodes_local_read(Nodes) -> - lists:filter(fun(N) -> not is_being_drained_local_read(N) end, Nodes). - --spec filter_out_drained_nodes_consistent_read([node()]) -> [node()]. -filter_out_drained_nodes_consistent_read(Nodes) -> - lists:filter(fun(N) -> not is_being_drained_consistent_read(N) end, Nodes). - --spec suspend_all_client_listeners() -> rabbit_types:ok_or_error(any()). - %% Pauses all listeners on the current node except for - %% Erlang distribution (clustering and CLI tools). - %% A respausedumed listener will not accept any new client connections - %% but previously established connections won't be interrupted. -suspend_all_client_listeners() -> - Listeners = rabbit_networking:node_client_listeners(node()), - rabbit_log:info("Asked to suspend ~b client connection listeners. " - "No new client connections will be accepted until these listeners are resumed!", [length(Listeners)]), - Results = lists:foldl(local_listener_fold_fun(fun ranch:suspend_listener/1), [], Listeners), - lists:foldl(fun ok_or_first_error/2, ok, Results). - - -spec resume_all_client_listeners() -> rabbit_types:ok_or_error(any()). - %% Resumes all listeners on the current node except for - %% Erlang distribution (clustering and CLI tools). - %% A resumed listener will accept new client connections. -resume_all_client_listeners() -> - Listeners = rabbit_networking:node_client_listeners(node()), - rabbit_log:info("Asked to resume ~b client connection listeners. " - "New client connections will be accepted from now on", [length(Listeners)]), - Results = lists:foldl(local_listener_fold_fun(fun ranch:resume_listener/1), [], Listeners), - lists:foldl(fun ok_or_first_error/2, ok, Results). - - -spec close_all_client_connections() -> {'ok', non_neg_integer()}. -close_all_client_connections() -> - Pids = rabbit_networking:local_connections(), - rabbit_networking:close_connections(Pids, "Node was put into maintenance mode"), - {ok, length(Pids)}. - --spec transfer_leadership_of_quorum_queues([node()]) -> ok. -transfer_leadership_of_quorum_queues([]) -> - rabbit_log:warning("Skipping leadership transfer of quorum queues: no candidate " - "(online, not under maintenance) nodes to transfer to!"); -transfer_leadership_of_quorum_queues(_TransferCandidates) -> - %% we only transfer leadership for QQs that have local leaders - Queues = rabbit_amqqueue:list_local_leaders(), - rabbit_log:info("Will transfer leadership of ~b quorum queues with current leader on this node", - [length(Queues)]), - [begin - Name = amqqueue:get_name(Q), - rabbit_log:debug("Will trigger a leader election for local quorum queue ~s", - [rabbit_misc:rs(Name)]), - %% we trigger an election and exclude this node from the list of candidates - %% by simply shutting its local QQ replica (Ra server) - RaLeader = amqqueue:get_pid(Q), - rabbit_log:debug("Will stop Ra server ~p", [RaLeader]), - case ra:stop_server(RaLeader) of - ok -> - rabbit_log:debug("Successfully stopped Ra server ~p", [RaLeader]); - {error, nodedown} -> - rabbit_log:error("Failed to stop Ra server ~p: target node was reported as down") - end - end || Q <- Queues], - rabbit_log:info("Leadership transfer for quorum queues hosted on this node has been initiated"). - --spec transfer_leadership_of_classic_mirrored_queues([node()]) -> ok. - transfer_leadership_of_classic_mirrored_queues([]) -> - rabbit_log:warning("Skipping leadership transfer of classic mirrored queues: no candidate " - "(online, not under maintenance) nodes to transfer to!"); -transfer_leadership_of_classic_mirrored_queues(TransferCandidates) -> - Queues = rabbit_amqqueue:list_local_mirrored_classic_queues(), - ReadableCandidates = readable_candidate_list(TransferCandidates), - rabbit_log:info("Will transfer leadership of ~b classic mirrored queues hosted on this node to these peer nodes: ~s", - [length(Queues), ReadableCandidates]), - - [begin - Name = amqqueue:get_name(Q), - case random_primary_replica_transfer_candidate_node(TransferCandidates) of - {ok, Pick} -> - rabbit_log:debug("Will transfer leadership of local queue ~s to node ~s", - [rabbit_misc:rs(Name), Pick]), - case rabbit_mirror_queue_misc:transfer_leadership(Q, Pick) of - {migrated, _} -> - rabbit_log:debug("Successfully transferred leadership of queue ~s to node ~s", - [rabbit_misc:rs(Name), Pick]); - Other -> - rabbit_log:warning("Could not transfer leadership of queue ~s to node ~s: ~p", - [rabbit_misc:rs(Name), Pick, Other]) - end; - undefined -> - rabbit_log:warning("Could not transfer leadership of queue ~s: no suitable candidates?", - [Name]) - end - end || Q <- Queues], - rabbit_log:info("Leadership transfer for local classic mirrored queues is complete"). - --spec stop_local_quorum_queue_followers() -> ok. -stop_local_quorum_queue_followers() -> - Queues = rabbit_amqqueue:list_local_followers(), - rabbit_log:info("Will stop local follower replicas of ~b quorum queues on this node", - [length(Queues)]), - [begin - Name = amqqueue:get_name(Q), - rabbit_log:debug("Will stop a local follower replica of quorum queue ~s", - [rabbit_misc:rs(Name)]), - %% shut down Ra nodes so that they are not considered for leader election - {RegisteredName, _LeaderNode} = amqqueue:get_pid(Q), - RaNode = {RegisteredName, node()}, - rabbit_log:debug("Will stop Ra server ~p", [RaNode]), - case ra:stop_server(RaNode) of - ok -> - rabbit_log:debug("Successfully stopped Ra server ~p", [RaNode]); - {error, nodedown} -> - rabbit_log:error("Failed to stop Ra server ~p: target node was reported as down") - end - end || Q <- Queues], - rabbit_log:info("Stopped all local replicas of quorum queues hosted on this node"). - - -spec primary_replica_transfer_candidate_nodes() -> [node()]. -primary_replica_transfer_candidate_nodes() -> - filter_out_drained_nodes_consistent_read(rabbit_nodes:all_running() -- [node()]). - --spec random_primary_replica_transfer_candidate_node([node()]) -> {ok, node()} | undefined. -random_primary_replica_transfer_candidate_node([]) -> - undefined; -random_primary_replica_transfer_candidate_node(Candidates) -> - Nth = erlang:phash2(erlang:monotonic_time(), length(Candidates)), - Candidate = lists:nth(Nth + 1, Candidates), - {ok, Candidate}. - -revive_local_quorum_queue_replicas() -> - Queues = rabbit_amqqueue:list_local_followers(), - [begin - Name = amqqueue:get_name(Q), - rabbit_log:debug("Will trigger a leader election for local quorum queue ~s", - [rabbit_misc:rs(Name)]), - %% start local QQ replica (Ra server) of this queue - {Prefix, _Node} = amqqueue:get_pid(Q), - RaServer = {Prefix, node()}, - rabbit_log:debug("Will start Ra server ~p", [RaServer]), - case ra:restart_server(RaServer) of - ok -> - rabbit_log:debug("Successfully restarted Ra server ~p", [RaServer]); - {error, {already_started, _Pid}} -> - rabbit_log:debug("Ra server ~p is already running", [RaServer]); - {error, nodedown} -> - rabbit_log:error("Failed to restart Ra server ~p: target node was reported as down") - end - end || Q <- Queues], - rabbit_log:info("Restart of local quorum queue replicas is complete"). - -%% -%% Implementation -%% - -local_listener_fold_fun(Fun) -> - fun(#listener{node = Node, ip_address = Addr, port = Port}, Acc) when Node =:= node() -> - RanchRef = rabbit_networking:ranch_ref(Addr, Port), - [Fun(RanchRef) | Acc]; - (_, Acc) -> - Acc - end. - -ok_or_first_error(ok, Acc) -> - Acc; -ok_or_first_error({error, _} = Err, _Acc) -> - Err. - -readable_candidate_list(Nodes) -> - string:join(lists:map(fun rabbit_data_coercion:to_list/1, Nodes), ", "). diff --git a/src/rabbit_memory_monitor.erl b/src/rabbit_memory_monitor.erl deleted file mode 100644 index 5934a97cff..0000000000 --- a/src/rabbit_memory_monitor.erl +++ /dev/null @@ -1,259 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - - -%% This module handles the node-wide memory statistics. -%% It receives statistics from all queues, counts the desired -%% queue length (in seconds), and sends this information back to -%% queues. - --module(rabbit_memory_monitor). - --behaviour(gen_server2). - --export([start_link/0, register/2, deregister/1, - report_ram_duration/2, stop/0, conserve_resources/3, memory_use/1]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(process, {pid, reported, sent, callback, monitor}). - --record(state, {timer, %% 'internal_update' timer - queue_durations, %% ets #process - queue_duration_sum, %% sum of all queue_durations - queue_duration_count, %% number of elements in sum - desired_duration, %% the desired queue duration - disk_alarm %% disable paging, disk alarm has fired - }). - --define(SERVER, ?MODULE). --define(TABLE_NAME, ?MODULE). - -%% If all queues are pushed to disk (duration 0), then the sum of -%% their reported lengths will be 0. If memory then becomes available, -%% unless we manually intervene, the sum will remain 0, and the queues -%% will never get a non-zero duration. Thus when the mem use is < -%% SUM_INC_THRESHOLD, increase the sum artificially by SUM_INC_AMOUNT. --define(SUM_INC_THRESHOLD, 0.95). --define(SUM_INC_AMOUNT, 1.0). - --define(EPSILON, 0.000001). %% less than this and we clamp to 0 - -%%---------------------------------------------------------------------------- -%% Public API -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - gen_server2:start_link({local, ?SERVER}, ?MODULE, [], []). - --spec register(pid(), {atom(),atom(),[any()]}) -> 'ok'. - -register(Pid, MFA = {_M, _F, _A}) -> - gen_server2:call(?SERVER, {register, Pid, MFA}, infinity). - --spec deregister(pid()) -> 'ok'. - -deregister(Pid) -> - gen_server2:cast(?SERVER, {deregister, Pid}). - --spec report_ram_duration - (pid(), float() | 'infinity') -> number() | 'infinity'. - -report_ram_duration(Pid, QueueDuration) -> - gen_server2:call(?SERVER, - {report_ram_duration, Pid, QueueDuration}, infinity). - --spec stop() -> 'ok'. - -stop() -> - gen_server2:cast(?SERVER, stop). - -%% Paging should be enabled/disabled only in response to disk resource alarms -%% for the current node. -conserve_resources(Pid, disk, {_, Conserve, Node}) when node(Pid) =:= Node -> - gen_server2:cast(Pid, {disk_alarm, Conserve}); -conserve_resources(_Pid, _Source, _Conserve) -> - ok. - -memory_use(Type) -> - vm_memory_monitor:get_memory_use(Type). - -%%---------------------------------------------------------------------------- -%% Gen_server callbacks -%%---------------------------------------------------------------------------- - -init([]) -> - {ok, Interval} = application:get_env(rabbit, memory_monitor_interval), - {ok, TRef} = timer:send_interval(Interval, update), - - Ets = ets:new(?TABLE_NAME, [set, private, {keypos, #process.pid}]), - Alarms = rabbit_alarm:register(self(), {?MODULE, conserve_resources, []}), - {ok, internal_update( - #state { timer = TRef, - queue_durations = Ets, - queue_duration_sum = 0.0, - queue_duration_count = 0, - desired_duration = infinity, - disk_alarm = lists:member(disk, Alarms)})}. - -handle_call({report_ram_duration, Pid, QueueDuration}, From, - State = #state { queue_duration_sum = Sum, - queue_duration_count = Count, - queue_durations = Durations, - desired_duration = SendDuration }) -> - - [Proc = #process { reported = PrevQueueDuration }] = - ets:lookup(Durations, Pid), - - gen_server2:reply(From, SendDuration), - - {Sum1, Count1} = - case {PrevQueueDuration, QueueDuration} of - {infinity, infinity} -> {Sum, Count}; - {infinity, _} -> {Sum + QueueDuration, Count + 1}; - {_, infinity} -> {Sum - PrevQueueDuration, Count - 1}; - {_, _} -> {Sum - PrevQueueDuration + QueueDuration, - Count} - end, - true = ets:insert(Durations, Proc #process { reported = QueueDuration, - sent = SendDuration }), - {noreply, State #state { queue_duration_sum = zero_clamp(Sum1), - queue_duration_count = Count1 }}; - -handle_call({register, Pid, MFA}, _From, - State = #state { queue_durations = Durations }) -> - MRef = erlang:monitor(process, Pid), - true = ets:insert(Durations, #process { pid = Pid, reported = infinity, - sent = infinity, callback = MFA, - monitor = MRef }), - {reply, ok, State}; - -handle_call(_Request, _From, State) -> - {noreply, State}. - -handle_cast({disk_alarm, Alarm}, State = #state{disk_alarm = Alarm}) -> - {noreply, State}; - -handle_cast({disk_alarm, Alarm}, State) -> - {noreply, internal_update(State#state{disk_alarm = Alarm})}; - -handle_cast({deregister, Pid}, State) -> - {noreply, internal_deregister(Pid, true, State)}; - -handle_cast(stop, State) -> - {stop, normal, State}; - -handle_cast(_Request, State) -> - {noreply, State}. - -handle_info(update, State) -> - {noreply, internal_update(State)}; - -handle_info({'DOWN', _MRef, process, Pid, _Reason}, State) -> - {noreply, internal_deregister(Pid, false, State)}; - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, #state { timer = TRef }) -> - timer:cancel(TRef), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - - -%%---------------------------------------------------------------------------- -%% Internal functions -%%---------------------------------------------------------------------------- - -zero_clamp(Sum) when Sum < ?EPSILON -> 0.0; -zero_clamp(Sum) -> Sum. - -internal_deregister(Pid, Demonitor, - State = #state { queue_duration_sum = Sum, - queue_duration_count = Count, - queue_durations = Durations }) -> - case ets:lookup(Durations, Pid) of - [] -> State; - [#process { reported = PrevQueueDuration, monitor = MRef }] -> - true = case Demonitor of - true -> erlang:demonitor(MRef); - false -> true - end, - {Sum1, Count1} = - case PrevQueueDuration of - infinity -> {Sum, Count}; - _ -> {zero_clamp(Sum - PrevQueueDuration), - Count - 1} - end, - true = ets:delete(Durations, Pid), - State #state { queue_duration_sum = Sum1, - queue_duration_count = Count1 } - end. - -internal_update(State = #state{queue_durations = Durations, - desired_duration = DesiredDurationAvg, - disk_alarm = DiskAlarm}) -> - DesiredDurationAvg1 = desired_duration_average(State), - ShouldInform = should_inform_predicate(DiskAlarm), - case ShouldInform(DesiredDurationAvg, DesiredDurationAvg1) of - true -> inform_queues(ShouldInform, DesiredDurationAvg1, Durations); - false -> ok - end, - State#state{desired_duration = DesiredDurationAvg1}. - -desired_duration_average(#state{disk_alarm = true}) -> - infinity; -desired_duration_average(#state{disk_alarm = false, - queue_duration_sum = Sum, - queue_duration_count = Count}) -> - {ok, LimitThreshold} = - application:get_env(rabbit, vm_memory_high_watermark_paging_ratio), - MemoryRatio = memory_use(ratio), - if MemoryRatio =:= infinity -> - 0.0; - MemoryRatio < LimitThreshold orelse Count == 0 -> - infinity; - MemoryRatio < ?SUM_INC_THRESHOLD -> - ((Sum + ?SUM_INC_AMOUNT) / Count) / MemoryRatio; - true -> - (Sum / Count) / MemoryRatio - end. - -inform_queues(ShouldInform, DesiredDurationAvg, Durations) -> - true = - ets:foldl( - fun (Proc = #process{reported = QueueDuration, - sent = PrevSendDuration, - callback = {M, F, A}}, true) -> - case ShouldInform(PrevSendDuration, DesiredDurationAvg) - andalso ShouldInform(QueueDuration, DesiredDurationAvg) of - true -> ok = erlang:apply( - M, F, A ++ [DesiredDurationAvg]), - ets:insert( - Durations, - Proc#process{sent = DesiredDurationAvg}); - false -> true - end - end, true, Durations). - -%% In normal use, we only inform queues immediately if the desired -%% duration has decreased, we want to ensure timely paging. -should_inform_predicate(false) -> fun greater_than/2; -%% When the disk alarm has gone off though, we want to inform queues -%% immediately if the desired duration has *increased* - we want to -%% ensure timely stopping paging. -should_inform_predicate(true) -> fun (D1, D2) -> greater_than(D2, D1) end. - -greater_than(infinity, infinity) -> false; -greater_than(infinity, _D2) -> true; -greater_than(_D1, infinity) -> false; -greater_than(D1, D2) -> D1 > D2. diff --git a/src/rabbit_metrics.erl b/src/rabbit_metrics.erl deleted file mode 100644 index 10418e3884..0000000000 --- a/src/rabbit_metrics.erl +++ /dev/null @@ -1,45 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_metrics). - --behaviour(gen_server). - --export([start_link/0]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --define(SERVER, ?MODULE). - -%%---------------------------------------------------------------------------- -%% Starts the raw metrics storage and owns the ETS tables. -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - -init([]) -> - rabbit_core_metrics:init(), - {ok, none}. - -handle_call(_Request, _From, State) -> - {noreply, State}. - -handle_cast(_Request, State) -> - {noreply, State}. - -handle_info(_Msg, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. diff --git a/src/rabbit_mirror_queue_coordinator.erl b/src/rabbit_mirror_queue_coordinator.erl deleted file mode 100644 index 91a7c3ddc8..0000000000 --- a/src/rabbit_mirror_queue_coordinator.erl +++ /dev/null @@ -1,460 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mirror_queue_coordinator). - --export([start_link/4, get_gm/1, ensure_monitoring/2]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3, handle_pre_hibernate/1]). - --export([joined/2, members_changed/3, handle_msg/3, handle_terminate/2]). - --behaviour(gen_server2). --behaviour(gm). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). --include("gm_specs.hrl"). - --record(state, { q, - gm, - monitors, - death_fun, - depth_fun - }). - -%%---------------------------------------------------------------------------- -%% -%% Mirror Queues -%% -%% A queue with mirrors consists of the following: -%% -%% #amqqueue{ pid, slave_pids } -%% | | -%% +----------+ +-------+--------------+-----------...etc... -%% | | | -%% V V V -%% amqqueue_process---+ mirror-----+ mirror-----+ ...etc... -%% | BQ = master----+ | | BQ = vq | | BQ = vq | -%% | | BQ = vq | | +-+-------+ +-+-------+ -%% | +-+-------+ | | | -%% +-++-----|---------+ | | (some details elided) -%% || | | | -%% || coordinator-+ | | -%% || +-+---------+ | | -%% || | | | -%% || gm-+ -- -- -- -- gm-+- -- -- -- gm-+- -- --...etc... -%% || +--+ +--+ +--+ -%% || -%% consumers -%% -%% The master is merely an implementation of bq, and thus is invoked -%% through the normal bq interface by the amqqueue_process. The mirrors -%% meanwhile are processes in their own right (as is the -%% coordinator). The coordinator and all mirrors belong to the same gm -%% group. Every member of a gm group receives messages sent to the gm -%% group. Because the master is the bq of amqqueue_process, it doesn't -%% have sole control over its mailbox, and as a result, the master -%% itself cannot be passed messages directly (well, it could by via -%% the amqqueue:run_backing_queue callback but that would induce -%% additional unnecessary loading on the master queue process), yet it -%% needs to react to gm events, such as the death of mirrors. Thus the -%% master creates the coordinator, and it is the coordinator that is -%% the gm callback module and event handler for the master. -%% -%% Consumers are only attached to the master. Thus the master is -%% responsible for informing all mirrors when messages are fetched from -%% the bq, when they're acked, and when they're requeued. -%% -%% The basic goal is to ensure that all mirrors performs actions on -%% their bqs in the same order as the master. Thus the master -%% intercepts all events going to its bq, and suitably broadcasts -%% these events on the gm. The mirrors thus receive two streams of -%% events: one stream is via the gm, and one stream is from channels -%% directly. Whilst the stream via gm is guaranteed to be consistently -%% seen by all mirrors , the same is not true of the stream via -%% channels. For example, in the event of an unexpected death of a -%% channel during a publish, only some of the mirrors may receive that -%% publish. As a result of this problem, the messages broadcast over -%% the gm contain published content, and thus mirrors can operate -%% successfully on messages that they only receive via the gm. -%% -%% The key purpose of also sending messages directly from the channels -%% to the mirrors is that without this, in the event of the death of -%% the master, messages could be lost until a suitable mirror is -%% promoted. However, that is not the only reason. A mirror cannot send -%% confirms for a message until it has seen it from the -%% channel. Otherwise, it might send a confirm to a channel for a -%% message that it might *never* receive from that channel. This can -%% happen because new mirrors join the gm ring (and thus receive -%% messages from the master) before inserting themselves in the -%% queue's mnesia record (which is what channels look at for routing). -%% As it turns out, channels will simply ignore such bogus confirms, -%% but relying on that would introduce a dangerously tight coupling. -%% -%% Hence the mirrors have to wait until they've seen both the publish -%% via gm, and the publish via the channel before they issue the -%% confirm. Either form of publish can arrive first, and a mirror can -%% be upgraded to the master at any point during this -%% process. Confirms continue to be issued correctly, however. -%% -%% Because the mirror is a full process, it impersonates parts of the -%% amqqueue API. However, it does not need to implement all parts: for -%% example, no ack or consumer-related message can arrive directly at -%% a mirror from a channel: it is only publishes that pass both -%% directly to the mirrors and go via gm. -%% -%% Slaves can be added dynamically. When this occurs, there is no -%% attempt made to sync the current contents of the master with the -%% new mirror, thus the mirror will start empty, regardless of the state -%% of the master. Thus the mirror needs to be able to detect and ignore -%% operations which are for messages it has not received: because of -%% the strict FIFO nature of queues in general, this is -%% straightforward - all new publishes that the new mirror receives via -%% gm should be processed as normal, but fetches which are for -%% messages the mirror has never seen should be ignored. Similarly, -%% acks for messages the mirror never fetched should be -%% ignored. Similarly, we don't republish rejected messages that we -%% haven't seen. Eventually, as the master is consumed from, the -%% messages at the head of the queue which were there before the slave -%% joined will disappear, and the mirror will become fully synced with -%% the state of the master. -%% -%% The detection of the sync-status is based on the depth of the BQs, -%% where the depth is defined as the sum of the length of the BQ (as -%% per BQ:len) and the messages pending an acknowledgement. When the -%% depth of the mirror is equal to the master's, then the mirror is -%% synchronised. We only store the difference between the two for -%% simplicity. Comparing the length is not enough since we need to -%% take into account rejected messages which will make it back into -%% the master queue but can't go back in the mirror, since we don't -%% want "holes" in the mirror queue. Note that the depth, and the -%% length likewise, must always be shorter on the mirror - we assert -%% that in various places. In case mirrors are joined to an empty queue -%% which only goes on to receive publishes, they start by asking the -%% master to broadcast its depth. This is enough for mirrors to always -%% be able to work out when their head does not differ from the master -%% (and is much simpler and cheaper than getting the master to hang on -%% to the guid of the msg at the head of its queue). When a mirror is -%% promoted to a master, it unilaterally broadcasts its depth, in -%% order to solve the problem of depth requests from new mirrors being -%% unanswered by a dead master. -%% -%% Obviously, due to the async nature of communication across gm, the -%% mirrors can fall behind. This does not matter from a sync pov: if -%% they fall behind and the master dies then a) no publishes are lost -%% because all publishes go to all mirrors anyway; b) the worst that -%% happens is that acks get lost and so messages come back to -%% life. This is no worse than normal given you never get confirmation -%% that an ack has been received (not quite true with QoS-prefetch, -%% but close enough for jazz). -%% -%% Because acktags are issued by the bq independently, and because -%% there is no requirement for the master and all mirrors to use the -%% same bq, all references to msgs going over gm is by msg_id. Thus -%% upon acking, the master must convert the acktags back to msg_ids -%% (which happens to be what bq:ack returns), then sends the msg_ids -%% over gm, the mirrors must convert the msg_ids to acktags (a mapping -%% the mirrors themselves must maintain). -%% -%% When the master dies, a mirror gets promoted. This will be the -%% eldest mirror, and thus the hope is that that mirror is most likely -%% to be sync'd with the master. The design of gm is that the -%% notification of the death of the master will only appear once all -%% messages in-flight from the master have been fully delivered to all -%% members of the gm group. Thus at this point, the mirror that gets -%% promoted cannot broadcast different events in a different order -%% than the master for the same msgs: there is no possibility for the -%% same msg to be processed by the old master and the new master - if -%% it was processed by the old master then it will have been processed -%% by the mirror before the mirror was promoted, and vice versa. -%% -%% Upon promotion, all msgs pending acks are requeued as normal, the -%% mirror constructs state suitable for use in the master module, and -%% then dynamically changes into an amqqueue_process with the master -%% as the bq, and the slave's bq as the master's bq. Thus the very -%% same process that was the mirror is now a full amqqueue_process. -%% -%% It is important that we avoid memory leaks due to the death of -%% senders (i.e. channels) and partial publications. A sender -%% publishing a message may fail mid way through the publish and thus -%% only some of the mirrors will receive the message. We need the -%% mirrors to be able to detect this and tidy up as necessary to avoid -%% leaks. If we just had the master monitoring all senders then we -%% would have the possibility that a sender appears and only sends the -%% message to a few of the mirrors before dying. Those mirrors would -%% then hold on to the message, assuming they'll receive some -%% instruction eventually from the master. Thus we have both mirrors -%% and the master monitor all senders they become aware of. But there -%% is a race: if the mirror receives a DOWN of a sender, how does it -%% know whether or not the master is going to send it instructions -%% regarding those messages? -%% -%% Whilst the master monitors senders, it can't access its mailbox -%% directly, so it delegates monitoring to the coordinator. When the -%% coordinator receives a DOWN message from a sender, it informs the -%% master via a callback. This allows the master to do any tidying -%% necessary, but more importantly allows the master to broadcast a -%% sender_death message to all the mirrors , saying the sender has -%% died. Once the mirrors receive the sender_death message, they know -%% that they're not going to receive any more instructions from the gm -%% regarding that sender. However, it is possible that the coordinator -%% receives the DOWN and communicates that to the master before the -%% master has finished receiving and processing publishes from the -%% sender. This turns out not to be a problem: the sender has actually -%% died, and so will not need to receive confirms or other feedback, -%% and should further messages be "received" from the sender, the -%% master will ask the coordinator to set up a new monitor, and -%% will continue to process the messages normally. Slaves may thus -%% receive publishes via gm from previously declared "dead" senders, -%% but again, this is fine: should the mirror have just thrown out the -%% message it had received directly from the sender (due to receiving -%% a sender_death message via gm), it will be able to cope with the -%% publication purely from the master via gm. -%% -%% When a mirror receives a DOWN message for a sender, if it has not -%% received the sender_death message from the master via gm already, -%% then it will wait 20 seconds before broadcasting a request for -%% confirmation from the master that the sender really has died. -%% Should a sender have only sent a publish to mirrors , this allows -%% mirrors to inform the master of the previous existence of the -%% sender. The master will thus monitor the sender, receive the DOWN, -%% and subsequently broadcast the sender_death message, allowing the -%% mirrors to tidy up. This process can repeat for the same sender: -%% consider one mirror receives the publication, then the DOWN, then -%% asks for confirmation of death, then the master broadcasts the -%% sender_death message. Only then does another mirror receive the -%% publication and thus set up its monitoring. Eventually that slave -%% too will receive the DOWN, ask for confirmation and the master will -%% monitor the sender again, receive another DOWN, and send out -%% another sender_death message. Given the 20 second delay before -%% requesting death confirmation, this is highly unlikely, but it is a -%% possibility. -%% -%% When the 20 second timer expires, the mirror first checks to see -%% whether it still needs confirmation of the death before requesting -%% it. This prevents unnecessary traffic on gm as it allows one -%% broadcast of the sender_death message to satisfy many mirrors. -%% -%% If we consider the promotion of a mirror at this point, we have two -%% possibilities: that of the mirror that has received the DOWN and is -%% thus waiting for confirmation from the master that the sender -%% really is down; and that of the mirror that has not received the -%% DOWN. In the first case, in the act of promotion to master, the new -%% master will monitor again the dead sender, and after it has -%% finished promoting itself, it should find another DOWN waiting, -%% which it will then broadcast. This will allow mirrors to tidy up as -%% normal. In the second case, we have the possibility that -%% confirmation-of-sender-death request has been broadcast, but that -%% it was broadcast before the master failed, and that the mirror being -%% promoted does not know anything about that sender, and so will not -%% monitor it on promotion. Thus a mirror that broadcasts such a -%% request, at the point of broadcasting it, recurses, setting another -%% 20 second timer. As before, on expiry of the timer, the mirrors -%% checks to see whether it still has not received a sender_death -%% message for the dead sender, and if not, broadcasts a death -%% confirmation request. Thus this ensures that even when a master -%% dies and the new mirror has no knowledge of the dead sender, it will -%% eventually receive a death confirmation request, shall monitor the -%% dead sender, receive the DOWN and broadcast the sender_death -%% message. -%% -%% The preceding commentary deals with the possibility of mirrors -%% receiving publications from senders which the master does not, and -%% the need to prevent memory leaks in such scenarios. The inverse is -%% also possible: a partial publication may cause only the master to -%% receive a publication. It will then publish the message via gm. The -%% mirrors will receive it via gm, will publish it to their BQ and will -%% set up monitoring on the sender. They will then receive the DOWN -%% message and the master will eventually publish the corresponding -%% sender_death message. The mirror will then be able to tidy up its -%% state as normal. -%% -%% Recovery of mirrored queues is straightforward: as nodes die, the -%% remaining nodes record this, and eventually a situation is reached -%% in which only one node is alive, which is the master. This is the -%% only node which, upon recovery, will resurrect a mirrored queue: -%% nodes which die and then rejoin as a mirror will start off empty as -%% if they have no mirrored content at all. This is not surprising: to -%% achieve anything more sophisticated would require the master and -%% recovering mirror to be able to check to see whether they agree on -%% the last seen state of the queue: checking depth alone is not -%% sufficient in this case. -%% -%% For more documentation see the comments in bug 23554. -%% -%%---------------------------------------------------------------------------- - --spec start_link - (amqqueue:amqqueue(), pid() | 'undefined', - rabbit_mirror_queue_master:death_fun(), - rabbit_mirror_queue_master:depth_fun()) -> - rabbit_types:ok_pid_or_error(). - -start_link(Queue, GM, DeathFun, DepthFun) -> - gen_server2:start_link(?MODULE, [Queue, GM, DeathFun, DepthFun], []). - --spec get_gm(pid()) -> pid(). - -get_gm(CPid) -> - gen_server2:call(CPid, get_gm, infinity). - --spec ensure_monitoring(pid(), [pid()]) -> 'ok'. - -ensure_monitoring(CPid, Pids) -> - gen_server2:cast(CPid, {ensure_monitoring, Pids}). - -%% --------------------------------------------------------------------------- -%% gen_server -%% --------------------------------------------------------------------------- - -init([Q, GM, DeathFun, DepthFun]) when ?is_amqqueue(Q) -> - QueueName = amqqueue:get_name(Q), - ?store_proc_name(QueueName), - GM1 = case GM of - undefined -> - {ok, GM2} = gm:start_link( - QueueName, ?MODULE, [self()], - fun rabbit_misc:execute_mnesia_transaction/1), - receive {joined, GM2, _Members} -> - ok - end, - GM2; - _ -> - true = link(GM), - GM - end, - {ok, #state { q = Q, - gm = GM1, - monitors = pmon:new(), - death_fun = DeathFun, - depth_fun = DepthFun }, - hibernate, - {backoff, ?HIBERNATE_AFTER_MIN, ?HIBERNATE_AFTER_MIN, ?DESIRED_HIBERNATE}}. - -handle_call(get_gm, _From, State = #state { gm = GM }) -> - reply(GM, State). - -handle_cast({gm_deaths, DeadGMPids}, State = #state{q = Q}) when ?amqqueue_pid_runs_on_local_node(Q) -> - QueueName = amqqueue:get_name(Q), - MPid = amqqueue:get_pid(Q), - case rabbit_mirror_queue_misc:remove_from_queue( - QueueName, MPid, DeadGMPids) of - {ok, MPid, DeadPids, ExtraNodes} -> - rabbit_mirror_queue_misc:report_deaths(MPid, true, QueueName, - DeadPids), - rabbit_mirror_queue_misc:add_mirrors(QueueName, ExtraNodes, async), - noreply(State); - {ok, _MPid0, DeadPids, _ExtraNodes} -> - %% see rabbitmq-server#914; - %% Different mirror is now master, stop current coordinator normally. - %% Initiating queue is now mirror and the least we could do is report - %% deaths which we 'think' we saw. - %% NOTE: Reported deaths here, could be inconsistent. - rabbit_mirror_queue_misc:report_deaths(MPid, false, QueueName, - DeadPids), - {stop, shutdown, State}; - {error, not_found} -> - {stop, normal, State}; - {error, {not_synced, _}} -> - rabbit_log:error("Mirror queue ~p in unexpected state." - " Promoted to master but already a master.", - [QueueName]), - error(unexpected_mirrored_state) - end; - -handle_cast(request_depth, State = #state{depth_fun = DepthFun, q = QArg}) when ?is_amqqueue(QArg) -> - QName = amqqueue:get_name(QArg), - MPid = amqqueue:get_pid(QArg), - case rabbit_amqqueue:lookup(QName) of - {ok, QFound} when ?amqqueue_pid_equals(QFound, MPid) -> - ok = DepthFun(), - noreply(State); - _ -> - {stop, shutdown, State} - end; - -handle_cast({ensure_monitoring, Pids}, State = #state { monitors = Mons }) -> - noreply(State #state { monitors = pmon:monitor_all(Pids, Mons) }); - -handle_cast({delete_and_terminate, {shutdown, ring_shutdown}}, State) -> - {stop, normal, State}; -handle_cast({delete_and_terminate, Reason}, State) -> - {stop, Reason, State}. - -handle_info({'DOWN', _MonitorRef, process, Pid, _Reason}, - State = #state { monitors = Mons, - death_fun = DeathFun }) -> - noreply(case pmon:is_monitored(Pid, Mons) of - false -> State; - true -> ok = DeathFun(Pid), - State #state { monitors = pmon:erase(Pid, Mons) } - end); - -handle_info(Msg, State) -> - {stop, {unexpected_info, Msg}, State}. - -terminate(_Reason, #state{}) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -handle_pre_hibernate(State = #state { gm = GM }) -> - %% Since GM notifications of deaths are lazy we might not get a - %% timely notification of mirror death if policy changes when - %% everything is idle. So cause some activity just before we - %% sleep. This won't cause us to go into perpetual motion as the - %% heartbeat does not wake up coordinator or mirrors. - gm:broadcast(GM, hibernate_heartbeat), - {hibernate, State}. - -%% --------------------------------------------------------------------------- -%% GM -%% --------------------------------------------------------------------------- - -joined([CPid], Members) -> - CPid ! {joined, self(), Members}, - ok. - -members_changed([_CPid], _Births, []) -> - ok; -members_changed([CPid], _Births, Deaths) -> - ok = gen_server2:cast(CPid, {gm_deaths, Deaths}). - -handle_msg([CPid], _From, request_depth = Msg) -> - ok = gen_server2:cast(CPid, Msg); -handle_msg([CPid], _From, {ensure_monitoring, _Pids} = Msg) -> - ok = gen_server2:cast(CPid, Msg); -handle_msg([_CPid], _From, {delete_and_terminate, _Reason}) -> - %% We tell GM to stop, but we don't instruct the coordinator to - %% stop yet. The GM will first make sure all pending messages were - %% actually delivered. Then it calls handle_terminate/2 below so the - %% coordinator is stopped. - %% - %% If we stop the coordinator right now, remote mirrors could see the - %% coordinator DOWN before delete_and_terminate was delivered to all - %% GMs. One of those GM would be promoted as the master, and this GM - %% would hang forever, waiting for other GMs to stop. - {stop, {shutdown, ring_shutdown}}; -handle_msg([_CPid], _From, _Msg) -> - ok. - -handle_terminate([CPid], Reason) -> - ok = gen_server2:cast(CPid, {delete_and_terminate, Reason}), - ok. - -%% --------------------------------------------------------------------------- -%% Others -%% --------------------------------------------------------------------------- - -noreply(State) -> - {noreply, State, hibernate}. - -reply(Reply, State) -> - {reply, Reply, State, hibernate}. diff --git a/src/rabbit_mirror_queue_master.erl b/src/rabbit_mirror_queue_master.erl deleted file mode 100644 index 71146e1ce2..0000000000 --- a/src/rabbit_mirror_queue_master.erl +++ /dev/null @@ -1,578 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mirror_queue_master). - --export([init/3, terminate/2, delete_and_terminate/2, - purge/1, purge_acks/1, publish/6, publish_delivered/5, - batch_publish/4, batch_publish_delivered/4, - discard/4, fetch/2, drop/2, ack/2, requeue/2, ackfold/4, fold/3, - len/1, is_empty/1, depth/1, drain_confirmed/1, - dropwhile/2, fetchwhile/4, set_ram_duration_target/2, ram_duration/1, - needs_timeout/1, timeout/1, handle_pre_hibernate/1, resume/1, - msg_rates/1, info/2, invoke/3, is_duplicate/2, set_queue_mode/2, - zip_msgs_and_acks/4, handle_info/2]). - --export([start/2, stop/1, delete_crashed/1]). - --export([promote_backing_queue_state/8, sender_death_fun/0, depth_fun/0]). - --export([init_with_existing_bq/3, stop_mirroring/1, sync_mirrors/3]). - --behaviour(rabbit_backing_queue). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --record(state, { name, - gm, - coordinator, - backing_queue, - backing_queue_state, - seen_status, - confirmed, - known_senders, - wait_timeout - }). - --export_type([death_fun/0, depth_fun/0, stats_fun/0]). - --type death_fun() :: fun ((pid()) -> 'ok'). --type depth_fun() :: fun (() -> 'ok'). --type stats_fun() :: fun ((any()) -> 'ok'). --type master_state() :: #state { name :: rabbit_amqqueue:name(), - gm :: pid(), - coordinator :: pid(), - backing_queue :: atom(), - backing_queue_state :: any(), - seen_status :: map(), - confirmed :: [rabbit_guid:guid()], - known_senders :: sets:set() - }. - -%% For general documentation of HA design, see -%% rabbit_mirror_queue_coordinator - -%% --------------------------------------------------------------------------- -%% Backing queue -%% --------------------------------------------------------------------------- - --spec start(_, _) -> no_return(). -start(_Vhost, _DurableQueues) -> - %% This will never get called as this module will never be - %% installed as the default BQ implementation. - exit({not_valid_for_generic_backing_queue, ?MODULE}). - --spec stop(_) -> no_return(). -stop(_Vhost) -> - %% Same as start/1. - exit({not_valid_for_generic_backing_queue, ?MODULE}). - --spec delete_crashed(_) -> no_return(). -delete_crashed(_QName) -> - exit({not_valid_for_generic_backing_queue, ?MODULE}). - -init(Q, Recover, AsyncCallback) -> - {ok, BQ} = application:get_env(backing_queue_module), - BQS = BQ:init(Q, Recover, AsyncCallback), - State = #state{gm = GM} = init_with_existing_bq(Q, BQ, BQS), - ok = gm:broadcast(GM, {depth, BQ:depth(BQS)}), - State. - --spec init_with_existing_bq(amqqueue:amqqueue(), atom(), any()) -> - master_state(). - -init_with_existing_bq(Q0, BQ, BQS) when ?is_amqqueue(Q0) -> - QName = amqqueue:get_name(Q0), - case rabbit_mirror_queue_coordinator:start_link( - Q0, undefined, sender_death_fun(), depth_fun()) of - {ok, CPid} -> - GM = rabbit_mirror_queue_coordinator:get_gm(CPid), - Self = self(), - Fun = fun () -> - [Q1] = mnesia:read({rabbit_queue, QName}), - true = amqqueue:is_amqqueue(Q1), - GMPids0 = amqqueue:get_gm_pids(Q1), - GMPids1 = [{GM, Self} | GMPids0], - Q2 = amqqueue:set_gm_pids(Q1, GMPids1), - Q3 = amqqueue:set_state(Q2, live), - %% amqqueue migration: - %% The amqqueue was read from this transaction, no - %% need to handle migration. - ok = rabbit_amqqueue:store_queue(Q3) - end, - ok = rabbit_misc:execute_mnesia_transaction(Fun), - {_MNode, SNodes} = rabbit_mirror_queue_misc:suggested_queue_nodes(Q0), - %% We need synchronous add here (i.e. do not return until the - %% mirror is running) so that when queue declaration is finished - %% all mirrors are up; we don't want to end up with unsynced mirrors - %% just by declaring a new queue. But add can't be synchronous all - %% the time as it can be called by mirrors and that's - %% deadlock-prone. - rabbit_mirror_queue_misc:add_mirrors(QName, SNodes, sync), - #state{name = QName, - gm = GM, - coordinator = CPid, - backing_queue = BQ, - backing_queue_state = BQS, - seen_status = #{}, - confirmed = [], - known_senders = sets:new(), - wait_timeout = rabbit_misc:get_env(rabbit, slave_wait_timeout, 15000)}; - {error, Reason} -> - %% The GM can shutdown before the coordinator has started up - %% (lost membership or missing group), thus the start_link of - %% the coordinator returns {error, shutdown} as rabbit_amqqueue_process - % is trapping exists - throw({coordinator_not_started, Reason}) - end. - --spec stop_mirroring(master_state()) -> {atom(), any()}. - -stop_mirroring(State = #state { coordinator = CPid, - backing_queue = BQ, - backing_queue_state = BQS }) -> - unlink(CPid), - stop_all_slaves(shutdown, State), - {BQ, BQS}. - --spec sync_mirrors(stats_fun(), stats_fun(), master_state()) -> - {'ok', master_state()} | {stop, any(), master_state()}. - -sync_mirrors(HandleInfo, EmitStats, - State = #state { name = QName, - gm = GM, - backing_queue = BQ, - backing_queue_state = BQS }) -> - Log = fun (Fmt, Params) -> - rabbit_mirror_queue_misc:log_info( - QName, "Synchronising: " ++ Fmt ++ "~n", Params) - end, - Log("~p messages to synchronise", [BQ:len(BQS)]), - {ok, Q} = rabbit_amqqueue:lookup(QName), - SPids = amqqueue:get_slave_pids(Q), - SyncBatchSize = rabbit_mirror_queue_misc:sync_batch_size(Q), - Log("batch size: ~p", [SyncBatchSize]), - Ref = make_ref(), - Syncer = rabbit_mirror_queue_sync:master_prepare(Ref, QName, Log, SPids), - gm:broadcast(GM, {sync_start, Ref, Syncer, SPids}), - S = fun(BQSN) -> State#state{backing_queue_state = BQSN} end, - case rabbit_mirror_queue_sync:master_go( - Syncer, Ref, Log, HandleInfo, EmitStats, SyncBatchSize, BQ, BQS) of - {cancelled, BQS1} -> Log(" synchronisation cancelled ", []), - {ok, S(BQS1)}; - {shutdown, R, BQS1} -> {stop, R, S(BQS1)}; - {sync_died, R, BQS1} -> Log("~p", [R]), - {ok, S(BQS1)}; - {already_synced, BQS1} -> {ok, S(BQS1)}; - {ok, BQS1} -> Log("complete", []), - {ok, S(BQS1)} - end. - -terminate({shutdown, dropped} = Reason, - State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - %% Backing queue termination - this node has been explicitly - %% dropped. Normally, non-durable queues would be tidied up on - %% startup, but there's a possibility that we will be added back - %% in without this node being restarted. Thus we must do the full - %% blown delete_and_terminate now, but only locally: we do not - %% broadcast delete_and_terminate. - State#state{backing_queue_state = BQ:delete_and_terminate(Reason, BQS)}; - -terminate(Reason, - State = #state { name = QName, - backing_queue = BQ, - backing_queue_state = BQS }) -> - %% Backing queue termination. The queue is going down but - %% shouldn't be deleted. Most likely safe shutdown of this - %% node. - {ok, Q} = rabbit_amqqueue:lookup(QName), - SSPids = amqqueue:get_sync_slave_pids(Q), - case SSPids =:= [] andalso - rabbit_policy:get(<<"ha-promote-on-shutdown">>, Q) =/= <<"always">> of - true -> %% Remove the whole queue to avoid data loss - rabbit_mirror_queue_misc:log_warning( - QName, "Stopping all nodes on master shutdown since no " - "synchronised mirror (replica) is available~n", []), - stop_all_slaves(Reason, State); - false -> %% Just let some other mirror take over. - ok - end, - State #state { backing_queue_state = BQ:terminate(Reason, BQS) }. - -delete_and_terminate(Reason, State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - stop_all_slaves(Reason, State), - State#state{backing_queue_state = BQ:delete_and_terminate(Reason, BQS)}. - -stop_all_slaves(Reason, #state{name = QName, gm = GM, wait_timeout = WT}) -> - {ok, Q} = rabbit_amqqueue:lookup(QName), - SPids = amqqueue:get_slave_pids(Q), - rabbit_mirror_queue_misc:stop_all_slaves(Reason, SPids, QName, GM, WT). - -purge(State = #state { gm = GM, - backing_queue = BQ, - backing_queue_state = BQS }) -> - ok = gm:broadcast(GM, {drop, 0, BQ:len(BQS), false}), - {Count, BQS1} = BQ:purge(BQS), - {Count, State #state { backing_queue_state = BQS1 }}. - --spec purge_acks(_) -> no_return(). -purge_acks(_State) -> exit({not_implemented, {?MODULE, purge_acks}}). - -publish(Msg = #basic_message { id = MsgId }, MsgProps, IsDelivered, ChPid, Flow, - State = #state { gm = GM, - seen_status = SS, - backing_queue = BQ, - backing_queue_state = BQS }) -> - false = maps:is_key(MsgId, SS), %% ASSERTION - ok = gm:broadcast(GM, {publish, ChPid, Flow, MsgProps, Msg}, - rabbit_basic:msg_size(Msg)), - BQS1 = BQ:publish(Msg, MsgProps, IsDelivered, ChPid, Flow, BQS), - ensure_monitoring(ChPid, State #state { backing_queue_state = BQS1 }). - -batch_publish(Publishes, ChPid, Flow, - State = #state { gm = GM, - seen_status = SS, - backing_queue = BQ, - backing_queue_state = BQS }) -> - {Publishes1, false, MsgSizes} = - lists:foldl(fun ({Msg = #basic_message { id = MsgId }, - MsgProps, _IsDelivered}, {Pubs, false, Sizes}) -> - {[{Msg, MsgProps, true} | Pubs], %% [0] - false = maps:is_key(MsgId, SS), %% ASSERTION - Sizes + rabbit_basic:msg_size(Msg)} - end, {[], false, 0}, Publishes), - Publishes2 = lists:reverse(Publishes1), - ok = gm:broadcast(GM, {batch_publish, ChPid, Flow, Publishes2}, - MsgSizes), - BQS1 = BQ:batch_publish(Publishes2, ChPid, Flow, BQS), - ensure_monitoring(ChPid, State #state { backing_queue_state = BQS1 }). -%% [0] When the mirror process handles the publish command, it sets the -%% IsDelivered flag to true, so to avoid iterating over the messages -%% again at the mirror, we do it here. - -publish_delivered(Msg = #basic_message { id = MsgId }, MsgProps, - ChPid, Flow, State = #state { gm = GM, - seen_status = SS, - backing_queue = BQ, - backing_queue_state = BQS }) -> - false = maps:is_key(MsgId, SS), %% ASSERTION - ok = gm:broadcast(GM, {publish_delivered, ChPid, Flow, MsgProps, Msg}, - rabbit_basic:msg_size(Msg)), - {AckTag, BQS1} = BQ:publish_delivered(Msg, MsgProps, ChPid, Flow, BQS), - State1 = State #state { backing_queue_state = BQS1 }, - {AckTag, ensure_monitoring(ChPid, State1)}. - -batch_publish_delivered(Publishes, ChPid, Flow, - State = #state { gm = GM, - seen_status = SS, - backing_queue = BQ, - backing_queue_state = BQS }) -> - {false, MsgSizes} = - lists:foldl(fun ({Msg = #basic_message { id = MsgId }, _MsgProps}, - {false, Sizes}) -> - {false = maps:is_key(MsgId, SS), %% ASSERTION - Sizes + rabbit_basic:msg_size(Msg)} - end, {false, 0}, Publishes), - ok = gm:broadcast(GM, {batch_publish_delivered, ChPid, Flow, Publishes}, - MsgSizes), - {AckTags, BQS1} = BQ:batch_publish_delivered(Publishes, ChPid, Flow, BQS), - State1 = State #state { backing_queue_state = BQS1 }, - {AckTags, ensure_monitoring(ChPid, State1)}. - -discard(MsgId, ChPid, Flow, State = #state { gm = GM, - backing_queue = BQ, - backing_queue_state = BQS, - seen_status = SS }) -> - false = maps:is_key(MsgId, SS), %% ASSERTION - ok = gm:broadcast(GM, {discard, ChPid, Flow, MsgId}), - ensure_monitoring(ChPid, - State #state { backing_queue_state = - BQ:discard(MsgId, ChPid, Flow, BQS) }). - -dropwhile(Pred, State = #state{backing_queue = BQ, - backing_queue_state = BQS }) -> - Len = BQ:len(BQS), - {Next, BQS1} = BQ:dropwhile(Pred, BQS), - {Next, drop(Len, false, State #state { backing_queue_state = BQS1 })}. - -fetchwhile(Pred, Fun, Acc, State = #state{backing_queue = BQ, - backing_queue_state = BQS }) -> - Len = BQ:len(BQS), - {Next, Acc1, BQS1} = BQ:fetchwhile(Pred, Fun, Acc, BQS), - {Next, Acc1, drop(Len, true, State #state { backing_queue_state = BQS1 })}. - -drain_confirmed(State = #state { backing_queue = BQ, - backing_queue_state = BQS, - seen_status = SS, - confirmed = Confirmed }) -> - {MsgIds, BQS1} = BQ:drain_confirmed(BQS), - {MsgIds1, SS1} = - lists:foldl( - fun (MsgId, {MsgIdsN, SSN}) -> - %% We will never see 'discarded' here - case maps:find(MsgId, SSN) of - error -> - {[MsgId | MsgIdsN], SSN}; - {ok, published} -> - %% It was published when we were a mirror, - %% and we were promoted before we saw the - %% publish from the channel. We still - %% haven't seen the channel publish, and - %% consequently we need to filter out the - %% confirm here. We will issue the confirm - %% when we see the publish from the channel. - {MsgIdsN, maps:put(MsgId, confirmed, SSN)}; - {ok, confirmed} -> - %% Well, confirms are racy by definition. - {[MsgId | MsgIdsN], SSN} - end - end, {[], SS}, MsgIds), - {Confirmed ++ MsgIds1, State #state { backing_queue_state = BQS1, - seen_status = SS1, - confirmed = [] }}. - -fetch(AckRequired, State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - {Result, BQS1} = BQ:fetch(AckRequired, BQS), - State1 = State #state { backing_queue_state = BQS1 }, - {Result, case Result of - empty -> State1; - {_MsgId, _IsDelivered, _AckTag} -> drop_one(AckRequired, State1) - end}. - -drop(AckRequired, State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - {Result, BQS1} = BQ:drop(AckRequired, BQS), - State1 = State #state { backing_queue_state = BQS1 }, - {Result, case Result of - empty -> State1; - {_MsgId, _AckTag} -> drop_one(AckRequired, State1) - end}. - -ack(AckTags, State = #state { gm = GM, - backing_queue = BQ, - backing_queue_state = BQS }) -> - {MsgIds, BQS1} = BQ:ack(AckTags, BQS), - case MsgIds of - [] -> ok; - _ -> ok = gm:broadcast(GM, {ack, MsgIds}) - end, - {MsgIds, State #state { backing_queue_state = BQS1 }}. - -requeue(AckTags, State = #state { gm = GM, - backing_queue = BQ, - backing_queue_state = BQS }) -> - {MsgIds, BQS1} = BQ:requeue(AckTags, BQS), - ok = gm:broadcast(GM, {requeue, MsgIds}), - {MsgIds, State #state { backing_queue_state = BQS1 }}. - -ackfold(MsgFun, Acc, State = #state { backing_queue = BQ, - backing_queue_state = BQS }, AckTags) -> - {Acc1, BQS1} = BQ:ackfold(MsgFun, Acc, BQS, AckTags), - {Acc1, State #state { backing_queue_state = BQS1 }}. - -fold(Fun, Acc, State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - {Result, BQS1} = BQ:fold(Fun, Acc, BQS), - {Result, State #state { backing_queue_state = BQS1 }}. - -len(#state { backing_queue = BQ, backing_queue_state = BQS }) -> - BQ:len(BQS). - -is_empty(#state { backing_queue = BQ, backing_queue_state = BQS }) -> - BQ:is_empty(BQS). - -depth(#state { backing_queue = BQ, backing_queue_state = BQS }) -> - BQ:depth(BQS). - -set_ram_duration_target(Target, State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - State #state { backing_queue_state = - BQ:set_ram_duration_target(Target, BQS) }. - -ram_duration(State = #state { backing_queue = BQ, backing_queue_state = BQS }) -> - {Result, BQS1} = BQ:ram_duration(BQS), - {Result, State #state { backing_queue_state = BQS1 }}. - -needs_timeout(#state { backing_queue = BQ, backing_queue_state = BQS }) -> - BQ:needs_timeout(BQS). - -timeout(State = #state { backing_queue = BQ, backing_queue_state = BQS }) -> - State #state { backing_queue_state = BQ:timeout(BQS) }. - -handle_pre_hibernate(State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - State #state { backing_queue_state = BQ:handle_pre_hibernate(BQS) }. - -handle_info(Msg, State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - State #state { backing_queue_state = BQ:handle_info(Msg, BQS) }. - -resume(State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - State #state { backing_queue_state = BQ:resume(BQS) }. - -msg_rates(#state { backing_queue = BQ, backing_queue_state = BQS }) -> - BQ:msg_rates(BQS). - -info(backing_queue_status, - State = #state { backing_queue = BQ, backing_queue_state = BQS }) -> - BQ:info(backing_queue_status, BQS) ++ - [ {mirror_seen, maps:size(State #state.seen_status)}, - {mirror_senders, sets:size(State #state.known_senders)} ]; -info(Item, #state { backing_queue = BQ, backing_queue_state = BQS }) -> - BQ:info(Item, BQS). - -invoke(?MODULE, Fun, State) -> - Fun(?MODULE, State); -invoke(Mod, Fun, State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - State #state { backing_queue_state = BQ:invoke(Mod, Fun, BQS) }. - -is_duplicate(Message = #basic_message { id = MsgId }, - State = #state { seen_status = SS, - backing_queue = BQ, - backing_queue_state = BQS, - confirmed = Confirmed }) -> - %% Here, we need to deal with the possibility that we're about to - %% receive a message that we've already seen when we were a mirror - %% (we received it via gm). Thus if we do receive such message now - %% via the channel, there may be a confirm waiting to issue for - %% it. - - %% We will never see {published, ChPid, MsgSeqNo} here. - case maps:find(MsgId, SS) of - error -> - %% We permit the underlying BQ to have a peek at it, but - %% only if we ourselves are not filtering out the msg. - {Result, BQS1} = BQ:is_duplicate(Message, BQS), - {Result, State #state { backing_queue_state = BQS1 }}; - {ok, published} -> - %% It already got published when we were a mirror and no - %% confirmation is waiting. amqqueue_process will have, in - %% its msg_id_to_channel mapping, the entry for dealing - %% with the confirm when that comes back in (it's added - %% immediately after calling is_duplicate). The msg is - %% invalid. We will not see this again, nor will we be - %% further involved in confirming this message, so erase. - {{true, drop}, State #state { seen_status = maps:remove(MsgId, SS) }}; - {ok, Disposition} - when Disposition =:= confirmed - %% It got published when we were a mirror via gm, and - %% confirmed some time after that (maybe even after - %% promotion), but before we received the publish from the - %% channel, so couldn't previously know what the - %% msg_seq_no was (and thus confirm as a mirror). So we - %% need to confirm now. As above, amqqueue_process will - %% have the entry for the msg_id_to_channel mapping added - %% immediately after calling is_duplicate/2. - orelse Disposition =:= discarded -> - %% Message was discarded while we were a mirror. Confirm now. - %% As above, amqqueue_process will have the entry for the - %% msg_id_to_channel mapping. - {{true, drop}, State #state { seen_status = maps:remove(MsgId, SS), - confirmed = [MsgId | Confirmed] }} - end. - -set_queue_mode(Mode, State = #state { gm = GM, - backing_queue = BQ, - backing_queue_state = BQS }) -> - ok = gm:broadcast(GM, {set_queue_mode, Mode}), - BQS1 = BQ:set_queue_mode(Mode, BQS), - State #state { backing_queue_state = BQS1 }. - -zip_msgs_and_acks(Msgs, AckTags, Accumulator, - #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - BQ:zip_msgs_and_acks(Msgs, AckTags, Accumulator, BQS). - -%% --------------------------------------------------------------------------- -%% Other exported functions -%% --------------------------------------------------------------------------- - --spec promote_backing_queue_state - (rabbit_amqqueue:name(), pid(), atom(), any(), pid(), [any()], - map(), [pid()]) -> - master_state(). - -promote_backing_queue_state(QName, CPid, BQ, BQS, GM, AckTags, Seen, KS) -> - {_MsgIds, BQS1} = BQ:requeue(AckTags, BQS), - Len = BQ:len(BQS1), - Depth = BQ:depth(BQS1), - true = Len == Depth, %% ASSERTION: everything must have been requeued - ok = gm:broadcast(GM, {depth, Depth}), - WaitTimeout = rabbit_misc:get_env(rabbit, slave_wait_timeout, 15000), - #state { name = QName, - gm = GM, - coordinator = CPid, - backing_queue = BQ, - backing_queue_state = BQS1, - seen_status = Seen, - confirmed = [], - known_senders = sets:from_list(KS), - wait_timeout = WaitTimeout }. - --spec sender_death_fun() -> death_fun(). - -sender_death_fun() -> - Self = self(), - fun (DeadPid) -> - rabbit_amqqueue:run_backing_queue( - Self, ?MODULE, - fun (?MODULE, State = #state { gm = GM, known_senders = KS }) -> - ok = gm:broadcast(GM, {sender_death, DeadPid}), - KS1 = sets:del_element(DeadPid, KS), - State #state { known_senders = KS1 } - end) - end. - --spec depth_fun() -> depth_fun(). - -depth_fun() -> - Self = self(), - fun () -> - rabbit_amqqueue:run_backing_queue( - Self, ?MODULE, - fun (?MODULE, State = #state { gm = GM, - backing_queue = BQ, - backing_queue_state = BQS }) -> - ok = gm:broadcast(GM, {depth, BQ:depth(BQS)}), - State - end) - end. - -%% --------------------------------------------------------------------------- -%% Helpers -%% --------------------------------------------------------------------------- - -drop_one(AckRequired, State = #state { gm = GM, - backing_queue = BQ, - backing_queue_state = BQS }) -> - ok = gm:broadcast(GM, {drop, BQ:len(BQS), 1, AckRequired}), - State. - -drop(PrevLen, AckRequired, State = #state { gm = GM, - backing_queue = BQ, - backing_queue_state = BQS }) -> - Len = BQ:len(BQS), - case PrevLen - Len of - 0 -> State; - Dropped -> ok = gm:broadcast(GM, {drop, Len, Dropped, AckRequired}), - State - end. - -ensure_monitoring(ChPid, State = #state { coordinator = CPid, - known_senders = KS }) -> - case sets:is_element(ChPid, KS) of - true -> State; - false -> ok = rabbit_mirror_queue_coordinator:ensure_monitoring( - CPid, [ChPid]), - State #state { known_senders = sets:add_element(ChPid, KS) } - end. diff --git a/src/rabbit_mirror_queue_misc.erl b/src/rabbit_mirror_queue_misc.erl deleted file mode 100644 index 02f590e2fb..0000000000 --- a/src/rabbit_mirror_queue_misc.erl +++ /dev/null @@ -1,680 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mirror_queue_misc). --behaviour(rabbit_policy_validator). - --export([remove_from_queue/3, on_vhost_up/1, add_mirrors/3, - report_deaths/4, store_updated_slaves/1, - initial_queue_node/2, suggested_queue_nodes/1, actual_queue_nodes/1, - is_mirrored/1, is_mirrored_ha_nodes/1, - update_mirrors/2, update_mirrors/1, validate_policy/1, - maybe_auto_sync/1, maybe_drop_master_after_sync/1, - sync_batch_size/1, log_info/3, log_warning/3]). --export([stop_all_slaves/5]). - --export([sync_queue/1, cancel_sync_queue/1]). - --export([transfer_leadership/2, queue_length/1, get_replicas/1]). - -%% for testing only --export([module/1]). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --define(HA_NODES_MODULE, rabbit_mirror_queue_mode_nodes). - --rabbit_boot_step( - {?MODULE, - [{description, "HA policy validation"}, - {mfa, {rabbit_registry, register, - [policy_validator, <<"ha-mode">>, ?MODULE]}}, - {mfa, {rabbit_registry, register, - [policy_validator, <<"ha-params">>, ?MODULE]}}, - {mfa, {rabbit_registry, register, - [policy_validator, <<"ha-sync-mode">>, ?MODULE]}}, - {mfa, {rabbit_registry, register, - [policy_validator, <<"ha-sync-batch-size">>, ?MODULE]}}, - {mfa, {rabbit_registry, register, - [policy_validator, <<"ha-promote-on-shutdown">>, ?MODULE]}}, - {mfa, {rabbit_registry, register, - [policy_validator, <<"ha-promote-on-failure">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, recovery}]}). - - -%%---------------------------------------------------------------------------- - -%% Returns {ok, NewMPid, DeadPids, ExtraNodes} - --spec remove_from_queue - (rabbit_amqqueue:name(), pid(), [pid()]) -> - {'ok', pid(), [pid()], [node()]} | {'error', 'not_found'} | - {'error', {'not_synced', [pid()]}}. - -remove_from_queue(QueueName, Self, DeadGMPids) -> - rabbit_misc:execute_mnesia_transaction( - fun () -> - %% Someone else could have deleted the queue before we - %% get here. Or, gm group could've altered. see rabbitmq-server#914 - case mnesia:read({rabbit_queue, QueueName}) of - [] -> {error, not_found}; - [Q0] when ?is_amqqueue(Q0) -> - QPid = amqqueue:get_pid(Q0), - SPids = amqqueue:get_slave_pids(Q0), - SyncSPids = amqqueue:get_sync_slave_pids(Q0), - GMPids = amqqueue:get_gm_pids(Q0), - {DeadGM, AliveGM} = lists:partition( - fun ({GM, _}) -> - lists:member(GM, DeadGMPids) - end, GMPids), - DeadPids = [Pid || {_GM, Pid} <- DeadGM], - AlivePids = [Pid || {_GM, Pid} <- AliveGM], - Alive = [Pid || Pid <- [QPid | SPids], - lists:member(Pid, AlivePids)], - {QPid1, SPids1} = case Alive of - [] -> - %% GM altered, & if all pids are - %% perceived as dead, rather do - %% do nothing here, & trust the - %% promoted mirror to have updated - %% mnesia during the alteration. - {QPid, SPids}; - _ -> promote_slave(Alive) - end, - DoNotPromote = SyncSPids =:= [] andalso - rabbit_policy:get(<<"ha-promote-on-failure">>, Q0) =:= <<"when-synced">>, - case {{QPid, SPids}, {QPid1, SPids1}} of - {Same, Same} -> - {ok, QPid1, DeadPids, []}; - _ when QPid1 =/= QPid andalso QPid1 =:= Self andalso DoNotPromote =:= true -> - %% We have been promoted to master - %% but there are no synchronised mirrors - %% hence this node is not synchronised either - %% Bailing out. - {error, {not_synced, SPids1}}; - _ when QPid =:= QPid1 orelse QPid1 =:= Self -> - %% Either master hasn't changed, so - %% we're ok to update mnesia; or we have - %% become the master. If gm altered, - %% we have no choice but to proceed. - Q1 = amqqueue:set_pid(Q0, QPid1), - Q2 = amqqueue:set_slave_pids(Q1, SPids1), - Q3 = amqqueue:set_gm_pids(Q2, AliveGM), - store_updated_slaves(Q3), - %% If we add and remove nodes at the - %% same time we might tell the old - %% master we need to sync and then - %% shut it down. So let's check if - %% the new master needs to sync. - maybe_auto_sync(Q3), - {ok, QPid1, DeadPids, slaves_to_start_on_failure(Q3, DeadGMPids)}; - _ -> - %% Master has changed, and we're not it. - %% [1]. - Q1 = amqqueue:set_slave_pids(Q0, Alive), - Q2 = amqqueue:set_gm_pids(Q1, AliveGM), - store_updated_slaves(Q2), - {ok, QPid1, DeadPids, []} - end - end - end). -%% [1] We still update mnesia here in case the mirror that is supposed -%% to become master dies before it does do so, in which case the dead -%% old master might otherwise never get removed, which in turn might -%% prevent promotion of another mirror (e.g. us). -%% -%% Note however that we do not update the master pid. Otherwise we can -%% have the situation where a mirror updates the mnesia record for a -%% queue, promoting another mirror before that mirror realises it has -%% become the new master, which is bad because it could then mean the -%% mirror (now master) receives messages it's not ready for (for -%% example, new consumers). -%% -%% We set slave_pids to Alive rather than SPids1 since otherwise we'd -%% be removing the pid of the candidate master, which in turn would -%% prevent it from promoting itself. -%% -%% We maintain gm_pids as our source of truth, i.e. it contains the -%% most up-to-date information about which GMs and associated -%% {M,S}Pids are alive. And all pids in slave_pids always have a -%% corresponding entry in gm_pids. By contrast, due to the -%% aforementioned restriction on updating the master pid, that pid may -%% not be present in gm_pids, but only if said master has died. - -%% Sometimes a mirror dying means we need to start more on other -%% nodes - "exactly" mode can cause this to happen. -slaves_to_start_on_failure(Q, DeadGMPids) -> - %% In case Mnesia has not caught up yet, filter out nodes we know - %% to be dead.. - ClusterNodes = rabbit_nodes:all_running() -- - [node(P) || P <- DeadGMPids], - {_, OldNodes, _} = actual_queue_nodes(Q), - {_, NewNodes} = suggested_queue_nodes(Q, ClusterNodes), - NewNodes -- OldNodes. - -on_vhost_up(VHost) -> - QNames = - rabbit_misc:execute_mnesia_transaction( - fun () -> - mnesia:foldl( - fun - (Q, QNames0) when not ?amqqueue_vhost_equals(Q, VHost) -> - QNames0; - (Q, QNames0) when ?amqqueue_is_classic(Q) -> - QName = amqqueue:get_name(Q), - Pid = amqqueue:get_pid(Q), - SPids = amqqueue:get_slave_pids(Q), - %% We don't want to pass in the whole - %% cluster - we don't want a situation - %% where starting one node causes us to - %% decide to start a mirror on another - PossibleNodes0 = [node(P) || P <- [Pid | SPids]], - PossibleNodes = - case lists:member(node(), PossibleNodes0) of - true -> PossibleNodes0; - false -> [node() | PossibleNodes0] - end, - {_MNode, SNodes} = suggested_queue_nodes( - Q, PossibleNodes), - case lists:member(node(), SNodes) of - true -> [QName | QNames0]; - false -> QNames0 - end; - (_, QNames0) -> - QNames0 - end, [], rabbit_queue) - end), - [add_mirror(QName, node(), async) || QName <- QNames], - ok. - -drop_mirrors(QName, Nodes) -> - [drop_mirror(QName, Node) || Node <- Nodes], - ok. - -drop_mirror(QName, MirrorNode) -> - case rabbit_amqqueue:lookup(QName) of - {ok, Q} when ?is_amqqueue(Q) -> - Name = amqqueue:get_name(Q), - QPid = amqqueue:get_pid(Q), - SPids = amqqueue:get_slave_pids(Q), - case [Pid || Pid <- [QPid | SPids], node(Pid) =:= MirrorNode] of - [] -> - {error, {queue_not_mirrored_on_node, MirrorNode}}; - [QPid] when SPids =:= [] -> - {error, cannot_drop_only_mirror}; - [Pid] -> - log_info(Name, "Dropping queue mirror on node ~p~n", - [MirrorNode]), - exit(Pid, {shutdown, dropped}), - {ok, dropped} - end; - {error, not_found} = E -> - E - end. - --spec add_mirrors(rabbit_amqqueue:name(), [node()], 'sync' | 'async') -> - 'ok'. - -add_mirrors(QName, Nodes, SyncMode) -> - [add_mirror(QName, Node, SyncMode) || Node <- Nodes], - ok. - -add_mirror(QName, MirrorNode, SyncMode) -> - case rabbit_amqqueue:lookup(QName) of - {ok, Q} -> - rabbit_misc:with_exit_handler( - rabbit_misc:const(ok), - fun () -> - #resource{virtual_host = VHost} = amqqueue:get_name(Q), - case rabbit_vhost_sup_sup:get_vhost_sup(VHost, MirrorNode) of - {ok, _} -> - try - SPid = rabbit_amqqueue_sup_sup:start_queue_process( - MirrorNode, Q, slave), - log_info(QName, "Adding mirror on node ~p: ~p~n", - [MirrorNode, SPid]), - rabbit_mirror_queue_slave:go(SPid, SyncMode) - of - _ -> ok - catch - error:QError -> - log_warning(QName, - "Unable to start queue mirror on node '~p'. " - "Target queue supervisor is not running: ~p~n", - [MirrorNode, QError]) - end; - {error, Error} -> - log_warning(QName, - "Unable to start queue mirror on node '~p'. " - "Target virtual host is not running: ~p~n", - [MirrorNode, Error]), - ok - end - end); - {error, not_found} = E -> - E - end. - -report_deaths(_MirrorPid, _IsMaster, _QueueName, []) -> - ok; -report_deaths(MirrorPid, IsMaster, QueueName, DeadPids) -> - log_info(QueueName, "~s ~s saw deaths of mirrors~s~n", - [case IsMaster of - true -> "Master"; - false -> "Slave" - end, - rabbit_misc:pid_to_string(MirrorPid), - [[$ , rabbit_misc:pid_to_string(P)] || P <- DeadPids]]). - --spec log_info(rabbit_amqqueue:name(), string(), [any()]) -> 'ok'. - -log_info (QName, Fmt, Args) -> - rabbit_log_mirroring:info("Mirrored ~s: " ++ Fmt, - [rabbit_misc:rs(QName) | Args]). - --spec log_warning(rabbit_amqqueue:name(), string(), [any()]) -> 'ok'. - -log_warning(QName, Fmt, Args) -> - rabbit_log_mirroring:warning("Mirrored ~s: " ++ Fmt, - [rabbit_misc:rs(QName) | Args]). - --spec store_updated_slaves(amqqueue:amqqueue()) -> - amqqueue:amqqueue(). - -store_updated_slaves(Q0) when ?is_amqqueue(Q0) -> - SPids = amqqueue:get_slave_pids(Q0), - SSPids = amqqueue:get_sync_slave_pids(Q0), - RS0 = amqqueue:get_recoverable_slaves(Q0), - %% TODO now that we clear sync_slave_pids in rabbit_durable_queue, - %% do we still need this filtering? - SSPids1 = [SSPid || SSPid <- SSPids, lists:member(SSPid, SPids)], - Q1 = amqqueue:set_sync_slave_pids(Q0, SSPids1), - RS1 = update_recoverable(SPids, RS0), - Q2 = amqqueue:set_recoverable_slaves(Q1, RS1), - Q3 = amqqueue:set_state(Q2, live), - %% amqqueue migration: - %% The amqqueue was read from this transaction, no need to handle - %% migration. - ok = rabbit_amqqueue:store_queue(Q3), - %% Wake it up so that we emit a stats event - rabbit_amqqueue:notify_policy_changed(Q3), - Q3. - -%% Recoverable nodes are those which we could promote if the whole -%% cluster were to suddenly stop and we then lose the master; i.e. all -%% nodes with running mirrors , and all stopped nodes which had running -%% mirrors when they were up. -%% -%% Therefore we aim here to add new nodes with mirrors , and remove -%% running nodes without mirrors , We also try to keep the order -%% constant, and similar to the live SPids field (i.e. oldest -%% first). That's not necessarily optimal if nodes spend a long time -%% down, but we don't have a good way to predict what the optimal is -%% in that case anyway, and we assume nodes will not just be down for -%% a long time without being removed. -update_recoverable(SPids, RS) -> - SNodes = [node(SPid) || SPid <- SPids], - RunningNodes = rabbit_nodes:all_running(), - AddNodes = SNodes -- RS, - DelNodes = RunningNodes -- SNodes, %% i.e. running with no slave - (RS -- DelNodes) ++ AddNodes. - -stop_all_slaves(Reason, SPids, QName, GM, WaitTimeout) -> - PidsMRefs = [{Pid, erlang:monitor(process, Pid)} || Pid <- [GM | SPids]], - ok = gm:broadcast(GM, {delete_and_terminate, Reason}), - %% It's possible that we could be partitioned from some mirrors - %% between the lookup and the broadcast, in which case we could - %% monitor them but they would not have received the GM - %% message. So only wait for mirrors which are still - %% not-partitioned. - PendingSlavePids = lists:foldl(fun({Pid, MRef}, Acc) -> - case rabbit_mnesia:on_running_node(Pid) of - true -> - receive - {'DOWN', MRef, process, _Pid, _Info} -> - Acc - after WaitTimeout -> - rabbit_mirror_queue_misc:log_warning( - QName, "Missing 'DOWN' message from ~p in" - " node ~p~n", [Pid, node(Pid)]), - [Pid | Acc] - end; - false -> - Acc - end - end, [], PidsMRefs), - %% Normally when we remove a mirror another mirror or master will - %% notice and update Mnesia. But we just removed them all, and - %% have stopped listening ourselves. So manually clean up. - rabbit_misc:execute_mnesia_transaction(fun () -> - [Q0] = mnesia:read({rabbit_queue, QName}), - Q1 = amqqueue:set_gm_pids(Q0, []), - Q2 = amqqueue:set_slave_pids(Q1, []), - %% Restarted mirrors on running nodes can - %% ensure old incarnations are stopped using - %% the pending mirror pids. - Q3 = amqqueue:set_slave_pids_pending_shutdown(Q2, PendingSlavePids), - rabbit_mirror_queue_misc:store_updated_slaves(Q3) - end), - ok = gm:forget_group(QName). - -%%---------------------------------------------------------------------------- - -promote_slave([SPid | SPids]) -> - %% The mirror pids are maintained in descending order of age, so - %% the one to promote is the oldest. - {SPid, SPids}. - --spec initial_queue_node(amqqueue:amqqueue(), node()) -> node(). - -initial_queue_node(Q, DefNode) -> - {MNode, _SNodes} = suggested_queue_nodes(Q, DefNode, rabbit_nodes:all_running()), - MNode. - --spec suggested_queue_nodes(amqqueue:amqqueue()) -> - {node(), [node()]}. - -suggested_queue_nodes(Q) -> suggested_queue_nodes(Q, rabbit_nodes:all_running()). -suggested_queue_nodes(Q, All) -> suggested_queue_nodes(Q, node(), All). - -%% The third argument exists so we can pull a call to -%% rabbit_nodes:all_running() out of a loop or transaction -%% or both. -suggested_queue_nodes(Q, DefNode, All) when ?is_amqqueue(Q) -> - Owner = amqqueue:get_exclusive_owner(Q), - {MNode0, SNodes, SSNodes} = actual_queue_nodes(Q), - MNode = case MNode0 of - none -> DefNode; - _ -> MNode0 - end, - case Owner of - none -> Params = policy(<<"ha-params">>, Q), - case module(Q) of - {ok, M} -> M:suggested_queue_nodes( - Params, MNode, SNodes, SSNodes, All); - _ -> {MNode, []} - end; - _ -> {MNode, []} - end. - -policy(Policy, Q) -> - case rabbit_policy:get(Policy, Q) of - undefined -> none; - P -> P - end. - -module(Q) when ?is_amqqueue(Q) -> - case rabbit_policy:get(<<"ha-mode">>, Q) of - undefined -> not_mirrored; - Mode -> module(Mode) - end; - -module(Mode) when is_binary(Mode) -> - case rabbit_registry:binary_to_type(Mode) of - {error, not_found} -> not_mirrored; - T -> case rabbit_registry:lookup_module(ha_mode, T) of - {ok, Module} -> {ok, Module}; - _ -> not_mirrored - end - end. - -validate_mode(Mode) -> - case module(Mode) of - {ok, _Module} -> - ok; - not_mirrored -> - {error, "~p is not a valid ha-mode value", [Mode]} - end. - --spec is_mirrored(amqqueue:amqqueue()) -> boolean(). - -is_mirrored(Q) -> - case module(Q) of - {ok, _} -> true; - _ -> false - end. - -is_mirrored_ha_nodes(Q) -> - case module(Q) of - {ok, ?HA_NODES_MODULE} -> true; - _ -> false - end. - -actual_queue_nodes(Q) when ?is_amqqueue(Q) -> - MPid = amqqueue:get_pid(Q), - SPids = amqqueue:get_slave_pids(Q), - SSPids = amqqueue:get_sync_slave_pids(Q), - Nodes = fun (L) -> [node(Pid) || Pid <- L] end, - {case MPid of - none -> none; - _ -> node(MPid) - end, Nodes(SPids), Nodes(SSPids)}. - --spec maybe_auto_sync(amqqueue:amqqueue()) -> 'ok'. - -maybe_auto_sync(Q) when ?is_amqqueue(Q) -> - QPid = amqqueue:get_pid(Q), - case policy(<<"ha-sync-mode">>, Q) of - <<"automatic">> -> - spawn(fun() -> rabbit_amqqueue:sync_mirrors(QPid) end); - _ -> - ok - end. - -sync_queue(Q0) -> - F = fun - (Q) when ?amqqueue_is_classic(Q) -> - QPid = amqqueue:get_pid(Q), - rabbit_amqqueue:sync_mirrors(QPid); - (Q) when ?amqqueue_is_quorum(Q) -> - {error, quorum_queue_not_supported} - end, - rabbit_amqqueue:with(Q0, F). - -cancel_sync_queue(Q0) -> - F = fun - (Q) when ?amqqueue_is_classic(Q) -> - QPid = amqqueue:get_pid(Q), - rabbit_amqqueue:cancel_sync_mirrors(QPid); - (Q) when ?amqqueue_is_quorum(Q) -> - {error, quorum_queue_not_supported} - end, - rabbit_amqqueue:with(Q0, F). - -sync_batch_size(Q) when ?is_amqqueue(Q) -> - case policy(<<"ha-sync-batch-size">>, Q) of - none -> %% we need this case because none > 1 == true - default_batch_size(); - BatchSize when BatchSize > 1 -> - BatchSize; - _ -> - default_batch_size() - end. - --define(DEFAULT_BATCH_SIZE, 4096). - -default_batch_size() -> - rabbit_misc:get_env(rabbit, mirroring_sync_batch_size, - ?DEFAULT_BATCH_SIZE). - --spec update_mirrors - (amqqueue:amqqueue(), amqqueue:amqqueue()) -> 'ok'. - -update_mirrors(OldQ, NewQ) when ?amqqueue_pids_are_equal(OldQ, NewQ) -> - % Note: we do want to ensure both queues have same pid - QPid = amqqueue:get_pid(OldQ), - QPid = amqqueue:get_pid(NewQ), - case {is_mirrored(OldQ), is_mirrored(NewQ)} of - {false, false} -> ok; - _ -> rabbit_amqqueue:update_mirroring(QPid) - end. - --spec update_mirrors - (amqqueue:amqqueue()) -> 'ok'. - -update_mirrors(Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - {OldMNode, OldSNodes, _} = actual_queue_nodes(Q), - {NewMNode, NewSNodes} = suggested_queue_nodes(Q), - OldNodes = [OldMNode | OldSNodes], - NewNodes = [NewMNode | NewSNodes], - %% When a mirror dies, remove_from_queue/2 might have to add new - %% mirrors (in "exactly" mode). It will check mnesia to see which - %% mirrors there currently are. If drop_mirror/2 is invoked first - %% then when we end up in remove_from_queue/2 it will not see the - %% mirrors that add_mirror/2 will add, and also want to add them - %% (even though we are not responding to the death of a - %% mirror). Breakage ensues. - add_mirrors (QName, NewNodes -- OldNodes, async), - drop_mirrors(QName, OldNodes -- NewNodes), - %% This is for the case where no extra nodes were added but we changed to - %% a policy requiring auto-sync. - maybe_auto_sync(Q), - ok. - -queue_length(Q) -> - [{messages, M}] = rabbit_amqqueue:info(Q, [messages]), - M. - -get_replicas(Q) -> - {MNode, SNodes} = suggested_queue_nodes(Q), - [MNode] ++ SNodes. - -transfer_leadership(Q, Destination) -> - QName = amqqueue:get_name(Q), - {OldMNode, OldSNodes, _} = actual_queue_nodes(Q), - OldNodes = [OldMNode | OldSNodes], - add_mirrors(QName, [Destination] -- OldNodes, async), - drop_mirrors(QName, OldNodes -- [Destination]), - {Result, NewQ} = wait_for_new_master(QName, Destination), - update_mirrors(NewQ), - Result. - -wait_for_new_master(QName, Destination) -> - wait_for_new_master(QName, Destination, 100). - -wait_for_new_master(QName, _, 0) -> - {ok, Q} = rabbit_amqqueue:lookup(QName), - {{not_migrated, ""}, Q}; -wait_for_new_master(QName, Destination, N) -> - {ok, Q} = rabbit_amqqueue:lookup(QName), - case amqqueue:get_pid(Q) of - none -> - timer:sleep(100), - wait_for_new_master(QName, Destination, N - 1); - Pid -> - case node(Pid) of - Destination -> - {{migrated, Destination}, Q}; - _ -> - timer:sleep(100), - wait_for_new_master(QName, Destination, N - 1) - end - end. - -%% The arrival of a newly synced mirror may cause the master to die if -%% the policy does not want the master but it has been kept alive -%% because there were no synced mirrors. -%% -%% We don't just call update_mirrors/2 here since that could decide to -%% start a mirror for some other reason, and since we are the mirror ATM -%% that allows complicated deadlocks. - --spec maybe_drop_master_after_sync(amqqueue:amqqueue()) -> 'ok'. - -maybe_drop_master_after_sync(Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - MPid = amqqueue:get_pid(Q), - {DesiredMNode, DesiredSNodes} = suggested_queue_nodes(Q), - case node(MPid) of - DesiredMNode -> ok; - OldMNode -> false = lists:member(OldMNode, DesiredSNodes), %% [0] - drop_mirror(QName, OldMNode) - end, - ok. -%% [0] ASSERTION - if the policy wants the master to change, it has -%% not just shuffled it into the mirrors. All our modes ensure this -%% does not happen, but we should guard against a misbehaving plugin. - -%%---------------------------------------------------------------------------- - -validate_policy(KeyList) -> - Mode = proplists:get_value(<<"ha-mode">>, KeyList, none), - Params = proplists:get_value(<<"ha-params">>, KeyList, none), - SyncMode = proplists:get_value(<<"ha-sync-mode">>, KeyList, none), - SyncBatchSize = proplists:get_value( - <<"ha-sync-batch-size">>, KeyList, none), - PromoteOnShutdown = proplists:get_value( - <<"ha-promote-on-shutdown">>, KeyList, none), - PromoteOnFailure = proplists:get_value( - <<"ha-promote-on-failure">>, KeyList, none), - case {Mode, Params, SyncMode, SyncBatchSize, PromoteOnShutdown, PromoteOnFailure} of - {none, none, none, none, none, none} -> - ok; - {none, _, _, _, _, _} -> - {error, "ha-mode must be specified to specify ha-params, " - "ha-sync-mode or ha-promote-on-shutdown", []}; - _ -> - validate_policies( - [{Mode, fun validate_mode/1}, - {Params, ha_params_validator(Mode)}, - {SyncMode, fun validate_sync_mode/1}, - {SyncBatchSize, fun validate_sync_batch_size/1}, - {PromoteOnShutdown, fun validate_pos/1}, - {PromoteOnFailure, fun validate_pof/1}]) - end. - -ha_params_validator(Mode) -> - fun(Val) -> - {ok, M} = module(Mode), - M:validate_policy(Val) - end. - -validate_policies([]) -> - ok; -validate_policies([{Val, Validator} | Rest]) -> - case Validator(Val) of - ok -> validate_policies(Rest); - E -> E - end. - -validate_sync_mode(SyncMode) -> - case SyncMode of - <<"automatic">> -> ok; - <<"manual">> -> ok; - none -> ok; - Mode -> {error, "ha-sync-mode must be \"manual\" " - "or \"automatic\", got ~p", [Mode]} - end. - -validate_sync_batch_size(none) -> - ok; -validate_sync_batch_size(N) when is_integer(N) andalso N > 0 -> - ok; -validate_sync_batch_size(N) -> - {error, "ha-sync-batch-size takes an integer greater than 0, " - "~p given", [N]}. - -validate_pos(PromoteOnShutdown) -> - case PromoteOnShutdown of - <<"always">> -> ok; - <<"when-synced">> -> ok; - none -> ok; - Mode -> {error, "ha-promote-on-shutdown must be " - "\"always\" or \"when-synced\", got ~p", [Mode]} - end. - -validate_pof(PromoteOnShutdown) -> - case PromoteOnShutdown of - <<"always">> -> ok; - <<"when-synced">> -> ok; - none -> ok; - Mode -> {error, "ha-promote-on-failure must be " - "\"always\" or \"when-synced\", got ~p", [Mode]} - end. diff --git a/src/rabbit_mirror_queue_mode.erl b/src/rabbit_mirror_queue_mode.erl deleted file mode 100644 index 91491efc49..0000000000 --- a/src/rabbit_mirror_queue_mode.erl +++ /dev/null @@ -1,42 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mirror_queue_mode). - --behaviour(rabbit_registry_class). - --export([added_to_rabbit_registry/2, removed_from_rabbit_registry/1]). - --type master() :: node(). --type slave() :: node(). --type params() :: any(). - --callback description() -> [proplists:property()]. - -%% Called whenever we think we might need to change nodes for a -%% mirrored queue. Note that this is called from a variety of -%% contexts, both inside and outside Mnesia transactions. Ideally it -%% will be pure-functional. -%% -%% Takes: parameters set in the policy, -%% current master, -%% current mirrors, -%% current synchronised mirrors, -%% all nodes to consider -%% -%% Returns: tuple of new master, new mirrors -%% --callback suggested_queue_nodes( - params(), master(), [slave()], [slave()], [node()]) -> - {master(), [slave()]}. - -%% Are the parameters valid for this mode? --callback validate_policy(params()) -> - rabbit_policy_validator:validate_results(). - -added_to_rabbit_registry(_Type, _ModuleName) -> ok. -removed_from_rabbit_registry(_Type) -> ok. diff --git a/src/rabbit_mirror_queue_mode_all.erl b/src/rabbit_mirror_queue_mode_all.erl deleted file mode 100644 index 2da12a5972..0000000000 --- a/src/rabbit_mirror_queue_mode_all.erl +++ /dev/null @@ -1,32 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mirror_queue_mode_all). - --include("rabbit.hrl"). - --behaviour(rabbit_mirror_queue_mode). - --export([description/0, suggested_queue_nodes/5, validate_policy/1]). - --rabbit_boot_step({?MODULE, - [{description, "mirror mode all"}, - {mfa, {rabbit_registry, register, - [ha_mode, <<"all">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -description() -> - [{description, <<"Mirror queue to all nodes">>}]. - -suggested_queue_nodes(_Params, MNode, _SNodes, _SSNodes, Poss) -> - {MNode, Poss -- [MNode]}. - -validate_policy(none) -> - ok; -validate_policy(_Params) -> - {error, "ha-mode=\"all\" does not take parameters", []}. diff --git a/src/rabbit_mirror_queue_mode_exactly.erl b/src/rabbit_mirror_queue_mode_exactly.erl deleted file mode 100644 index a8aa7546ac..0000000000 --- a/src/rabbit_mirror_queue_mode_exactly.erl +++ /dev/null @@ -1,45 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mirror_queue_mode_exactly). - --include("rabbit.hrl"). - --behaviour(rabbit_mirror_queue_mode). - --export([description/0, suggested_queue_nodes/5, validate_policy/1]). - --rabbit_boot_step({?MODULE, - [{description, "mirror mode exactly"}, - {mfa, {rabbit_registry, register, - [ha_mode, <<"exactly">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -description() -> - [{description, <<"Mirror queue to a specified number of nodes">>}]. - -%% When we need to add nodes, we randomise our candidate list as a -%% crude form of load-balancing. TODO it would also be nice to -%% randomise the list of ones to remove when we have too many - we -%% would have to take account of synchronisation though. -suggested_queue_nodes(Count, MNode, SNodes, _SSNodes, Poss) -> - SCount = Count - 1, - {MNode, case SCount > length(SNodes) of - true -> Cand = shuffle((Poss -- [MNode]) -- SNodes), - SNodes ++ lists:sublist(Cand, SCount - length(SNodes)); - false -> lists:sublist(SNodes, SCount) - end}. - -shuffle(L) -> - {_, L1} = lists:unzip(lists:keysort(1, [{rand:uniform(), N} || N <- L])), - L1. - -validate_policy(N) when is_integer(N) andalso N > 0 -> - ok; -validate_policy(Params) -> - {error, "ha-mode=\"exactly\" takes an integer, ~p given", [Params]}. diff --git a/src/rabbit_mirror_queue_mode_nodes.erl b/src/rabbit_mirror_queue_mode_nodes.erl deleted file mode 100644 index f3e134ba63..0000000000 --- a/src/rabbit_mirror_queue_mode_nodes.erl +++ /dev/null @@ -1,69 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mirror_queue_mode_nodes). - --include("rabbit.hrl"). - --behaviour(rabbit_mirror_queue_mode). - --export([description/0, suggested_queue_nodes/5, validate_policy/1]). - --rabbit_boot_step({?MODULE, - [{description, "mirror mode nodes"}, - {mfa, {rabbit_registry, register, - [ha_mode, <<"nodes">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -description() -> - [{description, <<"Mirror queue to specified nodes">>}]. - -suggested_queue_nodes(PolicyNodes0, CurrentMaster, _SNodes, SSNodes, NodesRunningRabbitMQ) -> - PolicyNodes1 = [list_to_atom(binary_to_list(Node)) || Node <- PolicyNodes0], - %% If the current master is not in the nodes specified, then what we want - %% to do depends on whether there are any synchronised mirrors. If there - %% are then we can just kill the current master - the admin has asked for - %% a migration and we should give it to them. If there are not however - %% then we must keep the master around so as not to lose messages. - - PolicyNodes = case SSNodes of - [] -> lists:usort([CurrentMaster | PolicyNodes1]); - _ -> PolicyNodes1 - end, - Unavailable = PolicyNodes -- NodesRunningRabbitMQ, - AvailablePolicyNodes = PolicyNodes -- Unavailable, - case AvailablePolicyNodes of - [] -> %% We have never heard of anything? Not much we can do but - %% keep the master alive. - {CurrentMaster, []}; - _ -> case lists:member(CurrentMaster, AvailablePolicyNodes) of - true -> {CurrentMaster, - AvailablePolicyNodes -- [CurrentMaster]}; - false -> %% Make sure the new master is synced! In order to - %% get here SSNodes must not be empty. - SyncPolicyNodes = [Node || - Node <- AvailablePolicyNodes, - lists:member(Node, SSNodes)], - NewMaster = case SyncPolicyNodes of - [Node | _] -> Node; - [] -> erlang:hd(SSNodes) - end, - {NewMaster, AvailablePolicyNodes -- [NewMaster]} - end - end. - -validate_policy([]) -> - {error, "ha-mode=\"nodes\" list must be non-empty", []}; -validate_policy(Nodes) when is_list(Nodes) -> - case [I || I <- Nodes, not is_binary(I)] of - [] -> ok; - Invalid -> {error, "ha-mode=\"nodes\" takes a list of strings, " - "~p was not a string", [Invalid]} - end; -validate_policy(Params) -> - {error, "ha-mode=\"nodes\" takes a list, ~p given", [Params]}. diff --git a/src/rabbit_mirror_queue_slave.erl b/src/rabbit_mirror_queue_slave.erl deleted file mode 100644 index 0480db9cfe..0000000000 --- a/src/rabbit_mirror_queue_slave.erl +++ /dev/null @@ -1,1093 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mirror_queue_slave). - -%% For general documentation of HA design, see -%% rabbit_mirror_queue_coordinator -%% -%% We receive messages from GM and from publishers, and the gm -%% messages can arrive either before or after the 'actual' message. -%% All instructions from the GM group must be processed in the order -%% in which they're received. - --export([set_maximum_since_use/2, info/1, go/2]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3, handle_pre_hibernate/1, prioritise_call/4, - prioritise_cast/3, prioritise_info/3, format_message_queue/2]). - --export([joined/2, members_changed/3, handle_msg/3, handle_terminate/2]). - --behaviour(gen_server2). --behaviour(gm). - --include_lib("rabbit_common/include/rabbit.hrl"). - --include("amqqueue.hrl"). --include("gm_specs.hrl"). - -%%---------------------------------------------------------------------------- - --define(INFO_KEYS, - [pid, - name, - master_pid, - is_synchronised - ]). - --define(SYNC_INTERVAL, 25). %% milliseconds --define(RAM_DURATION_UPDATE_INTERVAL, 5000). --define(DEATH_TIMEOUT, 20000). %% 20 seconds - --record(state, { q, - gm, - backing_queue, - backing_queue_state, - sync_timer_ref, - rate_timer_ref, - - sender_queues, %% :: Pid -> {Q Msg, Set MsgId, ChState} - msg_id_ack, %% :: MsgId -> AckTag - - msg_id_status, - known_senders, - - %% Master depth - local depth - depth_delta - }). - -%%---------------------------------------------------------------------------- - -set_maximum_since_use(QPid, Age) -> - gen_server2:cast(QPid, {set_maximum_since_use, Age}). - -info(QPid) -> gen_server2:call(QPid, info, infinity). - -init(Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - ?store_proc_name(QName), - {ok, {not_started, Q}, hibernate, - {backoff, ?HIBERNATE_AFTER_MIN, ?HIBERNATE_AFTER_MIN, - ?DESIRED_HIBERNATE}, ?MODULE}. - -go(SPid, sync) -> gen_server2:call(SPid, go, infinity); -go(SPid, async) -> gen_server2:cast(SPid, go). - -handle_go(Q0) when ?is_amqqueue(Q0) -> - QName = amqqueue:get_name(Q0), - %% We join the GM group before we add ourselves to the amqqueue - %% record. As a result: - %% 1. We can receive msgs from GM that correspond to messages we will - %% never receive from publishers. - %% 2. When we receive a message from publishers, we must receive a - %% message from the GM group for it. - %% 3. However, that instruction from the GM group can arrive either - %% before or after the actual message. We need to be able to - %% distinguish between GM instructions arriving early, and case (1) - %% above. - %% - process_flag(trap_exit, true), %% amqqueue_process traps exits too. - {ok, GM} = gm:start_link(QName, ?MODULE, [self()], - fun rabbit_misc:execute_mnesia_transaction/1), - MRef = erlang:monitor(process, GM), - %% We ignore the DOWN message because we are also linked and - %% trapping exits, we just want to not get stuck and we will exit - %% later. - receive - {joined, GM} -> erlang:demonitor(MRef, [flush]), - ok; - {'DOWN', MRef, _, _, _} -> ok - end, - Self = self(), - Node = node(), - case rabbit_misc:execute_mnesia_transaction( - fun() -> init_it(Self, GM, Node, QName) end) of - {new, QPid, GMPids} -> - ok = file_handle_cache:register_callback( - rabbit_amqqueue, set_maximum_since_use, [Self]), - ok = rabbit_memory_monitor:register( - Self, {rabbit_amqqueue, set_ram_duration_target, [Self]}), - {ok, BQ} = application:get_env(backing_queue_module), - Q1 = amqqueue:set_pid(Q0, QPid), - _ = BQ:delete_crashed(Q1), %% For crash recovery - BQS = bq_init(BQ, Q1, new), - State = #state { q = Q1, - gm = GM, - backing_queue = BQ, - backing_queue_state = BQS, - rate_timer_ref = undefined, - sync_timer_ref = undefined, - - sender_queues = #{}, - msg_id_ack = #{}, - - msg_id_status = #{}, - known_senders = pmon:new(delegate), - - depth_delta = undefined - }, - ok = gm:broadcast(GM, request_depth), - ok = gm:validate_members(GM, [GM | [G || {G, _} <- GMPids]]), - rabbit_mirror_queue_misc:maybe_auto_sync(Q1), - {ok, State}; - {stale, StalePid} -> - rabbit_mirror_queue_misc:log_warning( - QName, "Detected stale HA master: ~p~n", [StalePid]), - gm:leave(GM), - {error, {stale_master_pid, StalePid}}; - duplicate_live_master -> - gm:leave(GM), - {error, {duplicate_live_master, Node}}; - existing -> - gm:leave(GM), - {error, normal}; - master_in_recovery -> - gm:leave(GM), - %% The queue record vanished - we must have a master starting - %% concurrently with us. In that case we can safely decide to do - %% nothing here, and the master will start us in - %% master:init_with_existing_bq/3 - {error, normal} - end. - -init_it(Self, GM, Node, QName) -> - case mnesia:read({rabbit_queue, QName}) of - [Q] when ?is_amqqueue(Q) -> - QPid = amqqueue:get_pid(Q), - SPids = amqqueue:get_slave_pids(Q), - GMPids = amqqueue:get_gm_pids(Q), - PSPids = amqqueue:get_slave_pids_pending_shutdown(Q), - case [Pid || Pid <- [QPid | SPids], node(Pid) =:= Node] of - [] -> stop_pending_slaves(QName, PSPids), - add_slave(Q, Self, GM), - {new, QPid, GMPids}; - [QPid] -> case rabbit_mnesia:is_process_alive(QPid) of - true -> duplicate_live_master; - false -> {stale, QPid} - end; - [SPid] -> case rabbit_mnesia:is_process_alive(SPid) of - true -> existing; - false -> GMPids1 = [T || T = {_, S} <- GMPids, S =/= SPid], - SPids1 = SPids -- [SPid], - Q1 = amqqueue:set_slave_pids(Q, SPids1), - Q2 = amqqueue:set_gm_pids(Q1, GMPids1), - add_slave(Q2, Self, GM), - {new, QPid, GMPids1} - end - end; - [] -> - master_in_recovery - end. - -%% Pending mirrors have been asked to stop by the master, but despite the node -%% being up these did not answer on the expected timeout. Stop local mirrors now. -stop_pending_slaves(QName, Pids) -> - [begin - rabbit_mirror_queue_misc:log_warning( - QName, "Detected a non-responsive classic queue mirror, stopping it: ~p~n", [Pid]), - case erlang:process_info(Pid, dictionary) of - undefined -> ok; - {dictionary, Dict} -> - Vhost = QName#resource.virtual_host, - {ok, AmqQSup} = rabbit_amqqueue_sup_sup:find_for_vhost(Vhost), - case proplists:get_value('$ancestors', Dict) of - [Sup, AmqQSup | _] -> - exit(Sup, kill), - exit(Pid, kill); - _ -> - ok - end - end - end || Pid <- Pids, node(Pid) =:= node(), - true =:= erlang:is_process_alive(Pid)]. - -%% Add to the end, so they are in descending order of age, see -%% rabbit_mirror_queue_misc:promote_slave/1 -add_slave(Q0, New, GM) when ?is_amqqueue(Q0) -> - SPids = amqqueue:get_slave_pids(Q0), - GMPids = amqqueue:get_gm_pids(Q0), - SPids1 = SPids ++ [New], - GMPids1 = [{GM, New} | GMPids], - Q1 = amqqueue:set_slave_pids(Q0, SPids1), - Q2 = amqqueue:set_gm_pids(Q1, GMPids1), - rabbit_mirror_queue_misc:store_updated_slaves(Q2). - -handle_call(go, _From, {not_started, Q} = NotStarted) -> - case handle_go(Q) of - {ok, State} -> {reply, ok, State}; - {error, Error} -> {stop, Error, NotStarted} - end; - -handle_call({gm_deaths, DeadGMPids}, From, - State = #state{ gm = GM, q = Q, - backing_queue = BQ, - backing_queue_state = BQS}) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - MPid = amqqueue:get_pid(Q), - Self = self(), - case rabbit_mirror_queue_misc:remove_from_queue(QName, Self, DeadGMPids) of - {error, not_found} -> - gen_server2:reply(From, ok), - {stop, normal, State}; - {error, {not_synced, _SPids}} -> - BQ:delete_and_terminate({error, not_synced}, BQS), - {stop, normal, State#state{backing_queue_state = undefined}}; - {ok, Pid, DeadPids, ExtraNodes} -> - rabbit_mirror_queue_misc:report_deaths(Self, false, QName, - DeadPids), - case Pid of - MPid -> - %% master hasn't changed - gen_server2:reply(From, ok), - rabbit_mirror_queue_misc:add_mirrors( - QName, ExtraNodes, async), - noreply(State); - Self -> - %% we've become master - QueueState = promote_me(From, State), - rabbit_mirror_queue_misc:add_mirrors( - QName, ExtraNodes, async), - {become, rabbit_amqqueue_process, QueueState, hibernate}; - _ -> - %% master has changed to not us - gen_server2:reply(From, ok), - %% see rabbitmq-server#914; - %% It's not always guaranteed that we won't have ExtraNodes. - %% If gm alters, master can change to not us with extra nodes, - %% in which case we attempt to add mirrors on those nodes. - case ExtraNodes of - [] -> void; - _ -> rabbit_mirror_queue_misc:add_mirrors( - QName, ExtraNodes, async) - end, - %% Since GM is by nature lazy we need to make sure - %% there is some traffic when a master dies, to - %% make sure all mirrors get informed of the - %% death. That is all process_death does, create - %% some traffic. - ok = gm:broadcast(GM, process_death), - Q1 = amqqueue:set_pid(Q, Pid), - State1 = State#state{q = Q1}, - noreply(State1) - end - end; - -handle_call(info, _From, State) -> - reply(infos(?INFO_KEYS, State), State). - -handle_cast(go, {not_started, Q} = NotStarted) -> - case handle_go(Q) of - {ok, State} -> {noreply, State}; - {error, Error} -> {stop, Error, NotStarted} - end; - -handle_cast({run_backing_queue, Mod, Fun}, State) -> - noreply(run_backing_queue(Mod, Fun, State)); - -handle_cast({gm, Instruction}, State = #state{q = Q0}) when ?is_amqqueue(Q0) -> - QName = amqqueue:get_name(Q0), - case rabbit_amqqueue:lookup(QName) of - {ok, Q1} when ?is_amqqueue(Q1) -> - SPids = amqqueue:get_slave_pids(Q1), - case lists:member(self(), SPids) of - true -> - handle_process_result(process_instruction(Instruction, State)); - false -> - %% Potentially a duplicated mirror caused by a partial partition, - %% will stop as a new mirror could start unaware of our presence - {stop, shutdown, State} - end; - {error, not_found} -> - %% Would not expect this to happen after fixing #953 - {stop, shutdown, State} - end; - -handle_cast({deliver, Delivery = #delivery{sender = Sender, flow = Flow}, true}, - State) -> - %% Asynchronous, non-"mandatory", deliver mode. - %% We are acking messages to the channel process that sent us - %% the message delivery. See - %% rabbit_amqqueue_process:handle_ch_down for more info. - %% If message is rejected by the master, the publish will be nacked - %% even if mirrors confirm it. No need to check for length here. - maybe_flow_ack(Sender, Flow), - noreply(maybe_enqueue_message(Delivery, State)); - -handle_cast({sync_start, Ref, Syncer}, - State = #state { depth_delta = DD, - backing_queue = BQ, - backing_queue_state = BQS }) -> - State1 = #state{rate_timer_ref = TRef} = ensure_rate_timer(State), - S = fun({MA, TRefN, BQSN}) -> - State1#state{depth_delta = undefined, - msg_id_ack = maps:from_list(MA), - rate_timer_ref = TRefN, - backing_queue_state = BQSN} - end, - case rabbit_mirror_queue_sync:slave( - DD, Ref, TRef, Syncer, BQ, BQS, - fun (BQN, BQSN) -> - BQSN1 = update_ram_duration(BQN, BQSN), - TRefN = rabbit_misc:send_after(?RAM_DURATION_UPDATE_INTERVAL, - self(), update_ram_duration), - {TRefN, BQSN1} - end) of - denied -> noreply(State1); - {ok, Res} -> noreply(set_delta(0, S(Res))); - {failed, Res} -> noreply(S(Res)); - {stop, Reason, Res} -> {stop, Reason, S(Res)} - end; - -handle_cast({set_maximum_since_use, Age}, State) -> - ok = file_handle_cache:set_maximum_since_use(Age), - noreply(State); - -handle_cast({set_ram_duration_target, Duration}, - State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - BQS1 = BQ:set_ram_duration_target(Duration, BQS), - noreply(State #state { backing_queue_state = BQS1 }); - -handle_cast(policy_changed, State) -> - %% During partial partitions, we might end up receiving messages expected by a master - %% Ignore them - noreply(State). - -handle_info(update_ram_duration, State = #state{backing_queue = BQ, - backing_queue_state = BQS}) -> - BQS1 = update_ram_duration(BQ, BQS), - %% Don't call noreply/1, we don't want to set timers - {State1, Timeout} = next_state(State #state { - rate_timer_ref = undefined, - backing_queue_state = BQS1 }), - {noreply, State1, Timeout}; - -handle_info(sync_timeout, State) -> - noreply(backing_queue_timeout( - State #state { sync_timer_ref = undefined })); - -handle_info(timeout, State) -> - noreply(backing_queue_timeout(State)); - -handle_info({'DOWN', _MonitorRef, process, ChPid, _Reason}, State) -> - local_sender_death(ChPid, State), - noreply(maybe_forget_sender(ChPid, down_from_ch, State)); - -handle_info({'EXIT', _Pid, Reason}, State) -> - {stop, Reason, State}; - -handle_info({bump_credit, Msg}, State) -> - credit_flow:handle_bump_msg(Msg), - noreply(State); - -handle_info(bump_reduce_memory_use, State) -> - noreply(State); - -%% In the event of a short partition during sync we can detect the -%% master's 'death', drop out of sync, and then receive sync messages -%% which were still in flight. Ignore them. -handle_info({sync_msg, _Ref, _Msg, _Props, _Unacked}, State) -> - noreply(State); - -handle_info({sync_complete, _Ref}, State) -> - noreply(State); - -handle_info(Msg, State) -> - {stop, {unexpected_info, Msg}, State}. - -terminate(_Reason, {not_started, _Q}) -> - ok; -terminate(_Reason, #state { backing_queue_state = undefined }) -> - %% We've received a delete_and_terminate from gm, thus nothing to - %% do here. - ok; -terminate({shutdown, dropped} = R, State = #state{backing_queue = BQ, - backing_queue_state = BQS}) -> - %% See rabbit_mirror_queue_master:terminate/2 - terminate_common(State), - BQ:delete_and_terminate(R, BQS); -terminate(shutdown, State) -> - terminate_shutdown(shutdown, State); -terminate({shutdown, _} = R, State) -> - terminate_shutdown(R, State); -terminate(Reason, State = #state{backing_queue = BQ, - backing_queue_state = BQS}) -> - terminate_common(State), - BQ:delete_and_terminate(Reason, BQS). - -%% If the Reason is shutdown, or {shutdown, _}, it is not the queue -%% being deleted: it's just the node going down. Even though we're a -%% mirror, we have no idea whether or not we'll be the only copy coming -%% back up. Thus we must assume we will be, and preserve anything we -%% have on disk. -terminate_shutdown(Reason, State = #state{backing_queue = BQ, - backing_queue_state = BQS}) -> - terminate_common(State), - BQ:terminate(Reason, BQS). - -terminate_common(State) -> - ok = rabbit_memory_monitor:deregister(self()), - stop_rate_timer(stop_sync_timer(State)). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -handle_pre_hibernate({not_started, _Q} = State) -> - {hibernate, State}; - -handle_pre_hibernate(State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - {RamDuration, BQS1} = BQ:ram_duration(BQS), - DesiredDuration = - rabbit_memory_monitor:report_ram_duration(self(), RamDuration), - BQS2 = BQ:set_ram_duration_target(DesiredDuration, BQS1), - BQS3 = BQ:handle_pre_hibernate(BQS2), - {hibernate, stop_rate_timer(State #state { backing_queue_state = BQS3 })}. - -prioritise_call(Msg, _From, _Len, _State) -> - case Msg of - info -> 9; - {gm_deaths, _Dead} -> 5; - _ -> 0 - end. - -prioritise_cast(Msg, _Len, _State) -> - case Msg of - {set_ram_duration_target, _Duration} -> 8; - {set_maximum_since_use, _Age} -> 8; - {run_backing_queue, _Mod, _Fun} -> 6; - {gm, _Msg} -> 5; - _ -> 0 - end. - -prioritise_info(Msg, _Len, _State) -> - case Msg of - update_ram_duration -> 8; - sync_timeout -> 6; - _ -> 0 - end. - -format_message_queue(Opt, MQ) -> rabbit_misc:format_message_queue(Opt, MQ). - -%% --------------------------------------------------------------------------- -%% GM -%% --------------------------------------------------------------------------- - -joined([SPid], _Members) -> SPid ! {joined, self()}, ok. - -members_changed([_SPid], _Births, []) -> - ok; -members_changed([ SPid], _Births, Deaths) -> - case rabbit_misc:with_exit_handler( - rabbit_misc:const(ok), - fun() -> - gen_server2:call(SPid, {gm_deaths, Deaths}, infinity) - end) of - ok -> ok; - {promote, CPid} -> {become, rabbit_mirror_queue_coordinator, [CPid]} - end. - -handle_msg([_SPid], _From, hibernate_heartbeat) -> - %% See rabbit_mirror_queue_coordinator:handle_pre_hibernate/1 - ok; -handle_msg([_SPid], _From, request_depth) -> - %% This is only of value to the master - ok; -handle_msg([_SPid], _From, {ensure_monitoring, _Pid}) -> - %% This is only of value to the master - ok; -handle_msg([_SPid], _From, process_death) -> - %% We must not take any notice of the master death here since it - %% comes without ordering guarantees - there could still be - %% messages from the master we have yet to receive. When we get - %% members_changed, then there will be no more messages. - ok; -handle_msg([CPid], _From, {delete_and_terminate, _Reason} = Msg) -> - ok = gen_server2:cast(CPid, {gm, Msg}), - {stop, {shutdown, ring_shutdown}}; -handle_msg([SPid], _From, {sync_start, Ref, Syncer, SPids}) -> - case lists:member(SPid, SPids) of - true -> gen_server2:cast(SPid, {sync_start, Ref, Syncer}); - false -> ok - end; -handle_msg([SPid], _From, Msg) -> - ok = gen_server2:cast(SPid, {gm, Msg}). - -handle_terminate([_SPid], _Reason) -> - ok. - -%% --------------------------------------------------------------------------- -%% Others -%% --------------------------------------------------------------------------- - -infos(Items, State) -> [{Item, i(Item, State)} || Item <- Items]. - -i(pid, _State) -> - self(); -i(name, #state{q = Q}) when ?is_amqqueue(Q) -> - amqqueue:get_name(Q); -i(master_pid, #state{q = Q}) when ?is_amqqueue(Q) -> - amqqueue:get_pid(Q); -i(is_synchronised, #state{depth_delta = DD}) -> - DD =:= 0; -i(_, _) -> - ''. - -bq_init(BQ, Q, Recover) -> - Self = self(), - BQ:init(Q, Recover, - fun (Mod, Fun) -> - rabbit_amqqueue:run_backing_queue(Self, Mod, Fun) - end). - -run_backing_queue(rabbit_mirror_queue_master, Fun, State) -> - %% Yes, this might look a little crazy, but see comments in - %% confirm_sender_death/1 - Fun(?MODULE, State); -run_backing_queue(Mod, Fun, State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - State #state { backing_queue_state = BQ:invoke(Mod, Fun, BQS) }. - -%% This feature was used by `rabbit_amqqueue_process` and -%% `rabbit_mirror_queue_slave` up-to and including RabbitMQ 3.7.x. It is -%% unused in 3.8.x and thus deprecated. We keep it to support in-place -%% upgrades to 3.8.x (i.e. mixed-version clusters), but it is a no-op -%% starting with that version. -send_mandatory(#delivery{mandatory = false}) -> - ok; -send_mandatory(#delivery{mandatory = true, - sender = SenderPid, - msg_seq_no = MsgSeqNo}) -> - gen_server2:cast(SenderPid, {mandatory_received, MsgSeqNo}). - -send_or_record_confirm(_, #delivery{ confirm = false }, MS, _State) -> - MS; -send_or_record_confirm(published, #delivery { sender = ChPid, - confirm = true, - msg_seq_no = MsgSeqNo, - message = #basic_message { - id = MsgId, - is_persistent = true } }, - MS, #state{q = Q}) when ?amqqueue_is_durable(Q) -> - maps:put(MsgId, {published, ChPid, MsgSeqNo} , MS); -send_or_record_confirm(_Status, #delivery { sender = ChPid, - confirm = true, - msg_seq_no = MsgSeqNo }, - MS, #state{q = Q} = _State) -> - ok = rabbit_classic_queue:confirm_to_sender(ChPid, - amqqueue:get_name(Q), [MsgSeqNo]), - MS. - -confirm_messages(MsgIds, State = #state{q = Q, msg_id_status = MS}) -> - QName = amqqueue:get_name(Q), - {CMs, MS1} = - lists:foldl( - fun (MsgId, {CMsN, MSN} = Acc) -> - %% We will never see 'discarded' here - case maps:find(MsgId, MSN) of - error -> - %% If it needed confirming, it'll have - %% already been done. - Acc; - {ok, published} -> - %% Still not seen it from the channel, just - %% record that it's been confirmed. - {CMsN, maps:put(MsgId, confirmed, MSN)}; - {ok, {published, ChPid, MsgSeqNo}} -> - %% Seen from both GM and Channel. Can now - %% confirm. - {rabbit_misc:gb_trees_cons(ChPid, MsgSeqNo, CMsN), - maps:remove(MsgId, MSN)}; - {ok, confirmed} -> - %% It's already been confirmed. This is - %% probably it's been both sync'd to disk - %% and then delivered and ack'd before we've - %% seen the publish from the - %% channel. Nothing to do here. - Acc - end - end, {gb_trees:empty(), MS}, MsgIds), - Fun = fun (Pid, MsgSeqNos) -> - rabbit_classic_queue:confirm_to_sender(Pid, QName, MsgSeqNos) - end, - rabbit_misc:gb_trees_foreach(Fun, CMs), - State #state { msg_id_status = MS1 }. - -handle_process_result({ok, State}) -> noreply(State); -handle_process_result({stop, State}) -> {stop, normal, State}. - --spec promote_me({pid(), term()}, #state{}) -> no_return(). - -promote_me(From, #state { q = Q0, - gm = GM, - backing_queue = BQ, - backing_queue_state = BQS, - rate_timer_ref = RateTRef, - sender_queues = SQ, - msg_id_ack = MA, - msg_id_status = MS, - known_senders = KS}) when ?is_amqqueue(Q0) -> - QName = amqqueue:get_name(Q0), - rabbit_mirror_queue_misc:log_info(QName, "Promoting mirror ~s to master~n", - [rabbit_misc:pid_to_string(self())]), - Q1 = amqqueue:set_pid(Q0, self()), - DeathFun = rabbit_mirror_queue_master:sender_death_fun(), - DepthFun = rabbit_mirror_queue_master:depth_fun(), - {ok, CPid} = rabbit_mirror_queue_coordinator:start_link(Q1, GM, DeathFun, DepthFun), - true = unlink(GM), - gen_server2:reply(From, {promote, CPid}), - - %% Everything that we're monitoring, we need to ensure our new - %% coordinator is monitoring. - MPids = pmon:monitored(KS), - ok = rabbit_mirror_queue_coordinator:ensure_monitoring(CPid, MPids), - - %% We find all the messages that we've received from channels but - %% not from gm, and pass them to the - %% queue_process:init_with_backing_queue_state to be enqueued. - %% - %% We also have to requeue messages which are pending acks: the - %% consumers from the master queue have been lost and so these - %% messages need requeuing. They might also be pending - %% confirmation, and indeed they might also be pending arrival of - %% the publication from the channel itself, if we received both - %% the publication and the fetch via gm first! Requeuing doesn't - %% affect confirmations: if the message was previously pending a - %% confirmation then it still will be, under the same msg_id. So - %% as a master, we need to be prepared to filter out the - %% publication of said messages from the channel (is_duplicate - %% (thus such requeued messages must remain in the msg_id_status - %% (MS) which becomes seen_status (SS) in the master)). - %% - %% Then there are messages we already have in the queue, which are - %% not currently pending acknowledgement: - %% 1. Messages we've only received via gm: - %% Filter out subsequent publication from channel through - %% validate_message. Might have to issue confirms then or - %% later, thus queue_process state will have to know that - %% there's a pending confirm. - %% 2. Messages received via both gm and channel: - %% Queue will have to deal with issuing confirms if necessary. - %% - %% MS contains the following three entry types: - %% - %% a) published: - %% published via gm only; pending arrival of publication from - %% channel, maybe pending confirm. - %% - %% b) {published, ChPid, MsgSeqNo}: - %% published via gm and channel; pending confirm. - %% - %% c) confirmed: - %% published via gm only, and confirmed; pending publication - %% from channel. - %% - %% d) discarded: - %% seen via gm only as discarded. Pending publication from - %% channel - %% - %% The forms a, c and d only, need to go to the master state - %% seen_status (SS). - %% - %% The form b only, needs to go through to the queue_process - %% state to form the msg_id_to_channel mapping (MTC). - %% - %% No messages that are enqueued from SQ at this point will have - %% entries in MS. - %% - %% Messages that are extracted from MA may have entries in MS, and - %% those messages are then requeued. However, as discussed above, - %% this does not affect MS, nor which bits go through to SS in - %% Master, or MTC in queue_process. - - St = [published, confirmed, discarded], - SS = maps:filter(fun (_MsgId, Status) -> lists:member(Status, St) end, MS), - AckTags = [AckTag || {_MsgId, AckTag} <- maps:to_list(MA)], - - MasterState = rabbit_mirror_queue_master:promote_backing_queue_state( - QName, CPid, BQ, BQS, GM, AckTags, SS, MPids), - - MTC = maps:fold(fun (MsgId, {published, ChPid, MsgSeqNo}, MTC0) -> - maps:put(MsgId, {ChPid, MsgSeqNo}, MTC0); - (_Msgid, _Status, MTC0) -> - MTC0 - end, #{}, MS), - Deliveries = [promote_delivery(Delivery) || - {_ChPid, {PubQ, _PendCh, _ChState}} <- maps:to_list(SQ), - Delivery <- queue:to_list(PubQ)], - AwaitGmDown = [ChPid || {ChPid, {_, _, down_from_ch}} <- maps:to_list(SQ)], - KS1 = lists:foldl(fun (ChPid0, KS0) -> - pmon:demonitor(ChPid0, KS0) - end, KS, AwaitGmDown), - rabbit_misc:store_proc_name(rabbit_amqqueue_process, QName), - rabbit_amqqueue_process:init_with_backing_queue_state( - Q1, rabbit_mirror_queue_master, MasterState, RateTRef, Deliveries, KS1, - MTC). - -%% We reset mandatory to false here because we will have sent the -%% mandatory_received already as soon as we got the message. We also -%% need to send an ack for these messages since the channel is waiting -%% for one for the via-GM case and we will not now receive one. -promote_delivery(Delivery = #delivery{sender = Sender, flow = Flow}) -> - maybe_flow_ack(Sender, Flow), - Delivery#delivery{mandatory = false}. - -noreply(State) -> - {NewState, Timeout} = next_state(State), - {noreply, ensure_rate_timer(NewState), Timeout}. - -reply(Reply, State) -> - {NewState, Timeout} = next_state(State), - {reply, Reply, ensure_rate_timer(NewState), Timeout}. - -next_state(State = #state{backing_queue = BQ, backing_queue_state = BQS}) -> - {MsgIds, BQS1} = BQ:drain_confirmed(BQS), - State1 = confirm_messages(MsgIds, - State #state { backing_queue_state = BQS1 }), - case BQ:needs_timeout(BQS1) of - false -> {stop_sync_timer(State1), hibernate }; - idle -> {stop_sync_timer(State1), ?SYNC_INTERVAL}; - timed -> {ensure_sync_timer(State1), 0 } - end. - -backing_queue_timeout(State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - State#state{backing_queue_state = BQ:timeout(BQS)}. - -ensure_sync_timer(State) -> - rabbit_misc:ensure_timer(State, #state.sync_timer_ref, - ?SYNC_INTERVAL, sync_timeout). - -stop_sync_timer(State) -> rabbit_misc:stop_timer(State, #state.sync_timer_ref). - -ensure_rate_timer(State) -> - rabbit_misc:ensure_timer(State, #state.rate_timer_ref, - ?RAM_DURATION_UPDATE_INTERVAL, - update_ram_duration). - -stop_rate_timer(State) -> rabbit_misc:stop_timer(State, #state.rate_timer_ref). - -ensure_monitoring(ChPid, State = #state { known_senders = KS }) -> - State #state { known_senders = pmon:monitor(ChPid, KS) }. - -local_sender_death(ChPid, #state { known_senders = KS }) -> - %% The channel will be monitored iff we have received a delivery - %% from it but not heard about its death from the master. So if it - %% is monitored we need to point the death out to the master (see - %% essay). - ok = case pmon:is_monitored(ChPid, KS) of - false -> ok; - true -> confirm_sender_death(ChPid) - end. - -confirm_sender_death(Pid) -> - %% We have to deal with the possibility that we'll be promoted to - %% master before this thing gets run. Consequently we set the - %% module to rabbit_mirror_queue_master so that if we do become a - %% rabbit_amqqueue_process before then, sane things will happen. - Fun = - fun (?MODULE, State = #state { known_senders = KS, - gm = GM }) -> - %% We're running still as a mirror - %% - %% See comment in local_sender_death/2; we might have - %% received a sender_death in the meanwhile so check - %% again. - ok = case pmon:is_monitored(Pid, KS) of - false -> ok; - true -> gm:broadcast(GM, {ensure_monitoring, [Pid]}), - confirm_sender_death(Pid) - end, - State; - (rabbit_mirror_queue_master, State) -> - %% We've become a master. State is now opaque to - %% us. When we became master, if Pid was still known - %% to us then we'd have set up monitoring of it then, - %% so this is now a noop. - State - end, - %% Note that we do not remove our knowledge of this ChPid until we - %% get the sender_death from GM as well as a DOWN notification. - {ok, _TRef} = timer:apply_after( - ?DEATH_TIMEOUT, rabbit_amqqueue, run_backing_queue, - [self(), rabbit_mirror_queue_master, Fun]), - ok. - -forget_sender(_, running) -> false; -forget_sender(down_from_gm, down_from_gm) -> false; %% [1] -forget_sender(down_from_ch, down_from_ch) -> false; -forget_sender(Down1, Down2) when Down1 =/= Down2 -> true. - -%% [1] If another mirror goes through confirm_sender_death/1 before we -%% do we can get two GM sender_death messages in a row for the same -%% channel - don't treat that as anything special. - -%% Record and process lifetime events from channels. Forget all about a channel -%% only when down notifications are received from both the channel and from gm. -maybe_forget_sender(ChPid, ChState, State = #state { sender_queues = SQ, - msg_id_status = MS, - known_senders = KS }) -> - case maps:find(ChPid, SQ) of - error -> - State; - {ok, {MQ, PendCh, ChStateRecord}} -> - case forget_sender(ChState, ChStateRecord) of - true -> - credit_flow:peer_down(ChPid), - State #state { sender_queues = maps:remove(ChPid, SQ), - msg_id_status = lists:foldl( - fun maps:remove/2, - MS, sets:to_list(PendCh)), - known_senders = pmon:demonitor(ChPid, KS) }; - false -> - SQ1 = maps:put(ChPid, {MQ, PendCh, ChState}, SQ), - State #state { sender_queues = SQ1 } - end - end. - -maybe_enqueue_message( - Delivery = #delivery { message = #basic_message { id = MsgId }, - sender = ChPid }, - State = #state { sender_queues = SQ, msg_id_status = MS }) -> - send_mandatory(Delivery), %% must do this before confirms - State1 = ensure_monitoring(ChPid, State), - %% We will never see {published, ChPid, MsgSeqNo} here. - case maps:find(MsgId, MS) of - error -> - {MQ, PendingCh, ChState} = get_sender_queue(ChPid, SQ), - MQ1 = queue:in(Delivery, MQ), - SQ1 = maps:put(ChPid, {MQ1, PendingCh, ChState}, SQ), - State1 #state { sender_queues = SQ1 }; - {ok, Status} -> - MS1 = send_or_record_confirm( - Status, Delivery, maps:remove(MsgId, MS), State1), - SQ1 = remove_from_pending_ch(MsgId, ChPid, SQ), - State1 #state { msg_id_status = MS1, - sender_queues = SQ1 } - end. - -get_sender_queue(ChPid, SQ) -> - case maps:find(ChPid, SQ) of - error -> {queue:new(), sets:new(), running}; - {ok, Val} -> Val - end. - -remove_from_pending_ch(MsgId, ChPid, SQ) -> - case maps:find(ChPid, SQ) of - error -> - SQ; - {ok, {MQ, PendingCh, ChState}} -> - maps:put(ChPid, {MQ, sets:del_element(MsgId, PendingCh), ChState}, - SQ) - end. - -publish_or_discard(Status, ChPid, MsgId, - State = #state { sender_queues = SQ, msg_id_status = MS }) -> - %% We really are going to do the publish/discard right now, even - %% though we may not have seen it directly from the channel. But - %% we cannot issue confirms until the latter has happened. So we - %% need to keep track of the MsgId and its confirmation status in - %% the meantime. - State1 = ensure_monitoring(ChPid, State), - {MQ, PendingCh, ChState} = get_sender_queue(ChPid, SQ), - {MQ1, PendingCh1, MS1} = - case queue:out(MQ) of - {empty, _MQ2} -> - {MQ, sets:add_element(MsgId, PendingCh), - maps:put(MsgId, Status, MS)}; - {{value, Delivery = #delivery { - message = #basic_message { id = MsgId } }}, MQ2} -> - {MQ2, PendingCh, - %% We received the msg from the channel first. Thus - %% we need to deal with confirms here. - send_or_record_confirm(Status, Delivery, MS, State1)}; - {{value, #delivery {}}, _MQ2} -> - %% The instruction was sent to us before we were - %% within the slave_pids within the #amqqueue{} - %% record. We'll never receive the message directly - %% from the channel. And the channel will not be - %% expecting any confirms from us. - {MQ, PendingCh, MS} - end, - SQ1 = maps:put(ChPid, {MQ1, PendingCh1, ChState}, SQ), - State1 #state { sender_queues = SQ1, msg_id_status = MS1 }. - - -process_instruction({publish, ChPid, Flow, MsgProps, - Msg = #basic_message { id = MsgId }}, State) -> - maybe_flow_ack(ChPid, Flow), - State1 = #state { backing_queue = BQ, backing_queue_state = BQS } = - publish_or_discard(published, ChPid, MsgId, State), - BQS1 = BQ:publish(Msg, MsgProps, true, ChPid, Flow, BQS), - {ok, State1 #state { backing_queue_state = BQS1 }}; -process_instruction({batch_publish, ChPid, Flow, Publishes}, State) -> - maybe_flow_ack(ChPid, Flow), - State1 = #state { backing_queue = BQ, backing_queue_state = BQS } = - lists:foldl(fun ({#basic_message { id = MsgId }, - _MsgProps, _IsDelivered}, St) -> - publish_or_discard(published, ChPid, MsgId, St) - end, State, Publishes), - BQS1 = BQ:batch_publish(Publishes, ChPid, Flow, BQS), - {ok, State1 #state { backing_queue_state = BQS1 }}; -process_instruction({publish_delivered, ChPid, Flow, MsgProps, - Msg = #basic_message { id = MsgId }}, State) -> - maybe_flow_ack(ChPid, Flow), - State1 = #state { backing_queue = BQ, backing_queue_state = BQS } = - publish_or_discard(published, ChPid, MsgId, State), - true = BQ:is_empty(BQS), - {AckTag, BQS1} = BQ:publish_delivered(Msg, MsgProps, ChPid, Flow, BQS), - {ok, maybe_store_ack(true, MsgId, AckTag, - State1 #state { backing_queue_state = BQS1 })}; -process_instruction({batch_publish_delivered, ChPid, Flow, Publishes}, State) -> - maybe_flow_ack(ChPid, Flow), - {MsgIds, - State1 = #state { backing_queue = BQ, backing_queue_state = BQS }} = - lists:foldl(fun ({#basic_message { id = MsgId }, _MsgProps}, - {MsgIds, St}) -> - {[MsgId | MsgIds], - publish_or_discard(published, ChPid, MsgId, St)} - end, {[], State}, Publishes), - true = BQ:is_empty(BQS), - {AckTags, BQS1} = BQ:batch_publish_delivered(Publishes, ChPid, Flow, BQS), - MsgIdsAndAcks = lists:zip(lists:reverse(MsgIds), AckTags), - State2 = lists:foldl( - fun ({MsgId, AckTag}, St) -> - maybe_store_ack(true, MsgId, AckTag, St) - end, State1 #state { backing_queue_state = BQS1 }, - MsgIdsAndAcks), - {ok, State2}; -process_instruction({discard, ChPid, Flow, MsgId}, State) -> - maybe_flow_ack(ChPid, Flow), - State1 = #state { backing_queue = BQ, backing_queue_state = BQS } = - publish_or_discard(discarded, ChPid, MsgId, State), - BQS1 = BQ:discard(MsgId, ChPid, Flow, BQS), - {ok, State1 #state { backing_queue_state = BQS1 }}; -process_instruction({drop, Length, Dropped, AckRequired}, - State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - QLen = BQ:len(BQS), - ToDrop = case QLen - Length of - N when N > 0 -> N; - _ -> 0 - end, - State1 = lists:foldl( - fun (const, StateN = #state{backing_queue_state = BQSN}) -> - {{MsgId, AckTag}, BQSN1} = BQ:drop(AckRequired, BQSN), - maybe_store_ack( - AckRequired, MsgId, AckTag, - StateN #state { backing_queue_state = BQSN1 }) - end, State, lists:duplicate(ToDrop, const)), - {ok, case AckRequired of - true -> State1; - false -> update_delta(ToDrop - Dropped, State1) - end}; -process_instruction({ack, MsgIds}, - State = #state { backing_queue = BQ, - backing_queue_state = BQS, - msg_id_ack = MA }) -> - {AckTags, MA1} = msg_ids_to_acktags(MsgIds, MA), - {MsgIds1, BQS1} = BQ:ack(AckTags, BQS), - [] = MsgIds1 -- MsgIds, %% ASSERTION - {ok, update_delta(length(MsgIds1) - length(MsgIds), - State #state { msg_id_ack = MA1, - backing_queue_state = BQS1 })}; -process_instruction({requeue, MsgIds}, - State = #state { backing_queue = BQ, - backing_queue_state = BQS, - msg_id_ack = MA }) -> - {AckTags, MA1} = msg_ids_to_acktags(MsgIds, MA), - {_MsgIds, BQS1} = BQ:requeue(AckTags, BQS), - {ok, State #state { msg_id_ack = MA1, - backing_queue_state = BQS1 }}; -process_instruction({sender_death, ChPid}, - State = #state { known_senders = KS }) -> - %% The channel will be monitored iff we have received a message - %% from it. In this case we just want to avoid doing work if we - %% never got any messages. - {ok, case pmon:is_monitored(ChPid, KS) of - false -> State; - true -> maybe_forget_sender(ChPid, down_from_gm, State) - end}; -process_instruction({depth, Depth}, - State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - {ok, set_delta(Depth - BQ:depth(BQS), State)}; - -process_instruction({delete_and_terminate, Reason}, - State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - BQ:delete_and_terminate(Reason, BQS), - {stop, State #state { backing_queue_state = undefined }}; -process_instruction({set_queue_mode, Mode}, - State = #state { backing_queue = BQ, - backing_queue_state = BQS }) -> - BQS1 = BQ:set_queue_mode(Mode, BQS), - {ok, State #state { backing_queue_state = BQS1 }}. - -maybe_flow_ack(Sender, flow) -> credit_flow:ack(Sender); -maybe_flow_ack(_Sender, noflow) -> ok. - -msg_ids_to_acktags(MsgIds, MA) -> - {AckTags, MA1} = - lists:foldl( - fun (MsgId, {Acc, MAN}) -> - case maps:find(MsgId, MA) of - error -> {Acc, MAN}; - {ok, AckTag} -> {[AckTag | Acc], maps:remove(MsgId, MAN)} - end - end, {[], MA}, MsgIds), - {lists:reverse(AckTags), MA1}. - -maybe_store_ack(false, _MsgId, _AckTag, State) -> - State; -maybe_store_ack(true, MsgId, AckTag, State = #state { msg_id_ack = MA }) -> - State #state { msg_id_ack = maps:put(MsgId, AckTag, MA) }. - -set_delta(0, State = #state { depth_delta = undefined }) -> - ok = record_synchronised(State#state.q), - State #state { depth_delta = 0 }; -set_delta(NewDelta, State = #state { depth_delta = undefined }) -> - true = NewDelta > 0, %% assertion - State #state { depth_delta = NewDelta }; -set_delta(NewDelta, State = #state { depth_delta = Delta }) -> - update_delta(NewDelta - Delta, State). - -update_delta(_DeltaChange, State = #state { depth_delta = undefined }) -> - State; -update_delta( DeltaChange, State = #state { depth_delta = 0 }) -> - 0 = DeltaChange, %% assertion: we cannot become unsync'ed - State; -update_delta( DeltaChange, State = #state { depth_delta = Delta }) -> - true = DeltaChange =< 0, %% assertion: we cannot become 'less' sync'ed - set_delta(Delta + DeltaChange, State #state { depth_delta = undefined }). - -update_ram_duration(BQ, BQS) -> - {RamDuration, BQS1} = BQ:ram_duration(BQS), - DesiredDuration = - rabbit_memory_monitor:report_ram_duration(self(), RamDuration), - BQ:set_ram_duration_target(DesiredDuration, BQS1). - -record_synchronised(Q0) when ?is_amqqueue(Q0) -> - QName = amqqueue:get_name(Q0), - Self = self(), - F = fun () -> - case mnesia:read({rabbit_queue, QName}) of - [] -> - ok; - [Q1] when ?is_amqqueue(Q1) -> - SSPids = amqqueue:get_sync_slave_pids(Q1), - SSPids1 = [Self | SSPids], - Q2 = amqqueue:set_sync_slave_pids(Q1, SSPids1), - rabbit_mirror_queue_misc:store_updated_slaves(Q2), - {ok, Q2} - end - end, - case rabbit_misc:execute_mnesia_transaction(F) of - ok -> ok; - {ok, Q2} -> rabbit_mirror_queue_misc:maybe_drop_master_after_sync(Q2) - end. diff --git a/src/rabbit_mirror_queue_sync.erl b/src/rabbit_mirror_queue_sync.erl deleted file mode 100644 index a82ee05599..0000000000 --- a/src/rabbit_mirror_queue_sync.erl +++ /dev/null @@ -1,420 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mirror_queue_sync). - --include_lib("rabbit_common/include/rabbit.hrl"). - --export([master_prepare/4, master_go/8, slave/7, conserve_resources/3]). - --define(SYNC_PROGRESS_INTERVAL, 1000000). - -%% There are three processes around, the master, the syncer and the -%% slave(s). The syncer is an intermediary, linked to the master in -%% order to make sure we do not mess with the master's credit flow or -%% set of monitors. -%% -%% Interactions -%% ------------ -%% -%% '*' indicates repeating messages. All are standard Erlang messages -%% except sync_start which is sent over GM to flush out any other -%% messages that we might have sent that way already. (credit) is the -%% usual credit_flow bump message every so often. -%% -%% Master Syncer Slave(s) -%% sync_mirrors -> || || -%% || -- (spawns) --> || || -%% || --------- sync_start (over GM) -------> || -%% || || <--- sync_ready ---- || -%% || || (or) || -%% || || <--- sync_deny ----- || -%% || <--- ready ---- || || -%% || <--- next* ---- || || } -%% || ---- msg* ----> || || } loop -%% || || ---- sync_msgs* ---> || } -%% || || <--- (credit)* ----- || } -%% || <--- next ---- || || -%% || ---- done ----> || || -%% || || -- sync_complete --> || -%% || (Dies) || - --type log_fun() :: fun ((string(), [any()]) -> 'ok'). --type bq() :: atom(). --type bqs() :: any(). --type ack() :: any(). --type slave_sync_state() :: {[{rabbit_types:msg_id(), ack()}], timer:tref(), - bqs()}. - -%% --------------------------------------------------------------------------- -%% Master - --spec master_prepare(reference(), rabbit_amqqueue:name(), - log_fun(), [pid()]) -> pid(). - -master_prepare(Ref, QName, Log, SPids) -> - MPid = self(), - spawn_link(fun () -> - ?store_proc_name(QName), - syncer(Ref, Log, MPid, SPids) - end). - --spec master_go(pid(), reference(), log_fun(), - rabbit_mirror_queue_master:stats_fun(), - rabbit_mirror_queue_master:stats_fun(), - non_neg_integer(), - bq(), bqs()) -> - {'already_synced', bqs()} | {'ok', bqs()} | - {'cancelled', bqs()} | - {'shutdown', any(), bqs()} | - {'sync_died', any(), bqs()}. - -master_go(Syncer, Ref, Log, HandleInfo, EmitStats, SyncBatchSize, BQ, BQS) -> - Args = {Syncer, Ref, Log, HandleInfo, EmitStats, rabbit_misc:get_parent()}, - receive - {'EXIT', Syncer, normal} -> {already_synced, BQS}; - {'EXIT', Syncer, Reason} -> {sync_died, Reason, BQS}; - {ready, Syncer} -> EmitStats({syncing, 0}), - master_batch_go0(Args, SyncBatchSize, - BQ, BQS) - end. - -master_batch_go0(Args, BatchSize, BQ, BQS) -> - FoldFun = - fun (Msg, MsgProps, Unacked, Acc) -> - Acc1 = append_to_acc(Msg, MsgProps, Unacked, Acc), - case maybe_master_batch_send(Acc1, BatchSize) of - true -> master_batch_send(Args, Acc1); - false -> {cont, Acc1} - end - end, - FoldAcc = {[], 0, {0, BQ:depth(BQS)}, erlang:monotonic_time()}, - bq_fold(FoldFun, FoldAcc, Args, BQ, BQS). - -master_batch_send({Syncer, Ref, Log, HandleInfo, EmitStats, Parent}, - {Batch, I, {Curr, Len}, Last}) -> - T = maybe_emit_stats(Last, I, EmitStats, Log), - HandleInfo({syncing, I}), - handle_set_maximum_since_use(), - SyncMsg = {msgs, Ref, lists:reverse(Batch)}, - NewAcc = {[], I + length(Batch), {Curr, Len}, T}, - master_send_receive(SyncMsg, NewAcc, Syncer, Ref, Parent). - -%% Either send messages when we reach the last one in the queue or -%% whenever we have accumulated BatchSize messages. -maybe_master_batch_send({_, _, {Len, Len}, _}, _BatchSize) -> - true; -maybe_master_batch_send({_, _, {Curr, _Len}, _}, BatchSize) - when Curr rem BatchSize =:= 0 -> - true; -maybe_master_batch_send(_Acc, _BatchSize) -> - false. - -bq_fold(FoldFun, FoldAcc, Args, BQ, BQS) -> - case BQ:fold(FoldFun, FoldAcc, BQS) of - {{shutdown, Reason}, BQS1} -> {shutdown, Reason, BQS1}; - {{sync_died, Reason}, BQS1} -> {sync_died, Reason, BQS1}; - {_, BQS1} -> master_done(Args, BQS1) - end. - -append_to_acc(Msg, MsgProps, Unacked, {Batch, I, {Curr, Len}, T}) -> - {[{Msg, MsgProps, Unacked} | Batch], I, {Curr + 1, Len}, T}. - -master_send_receive(SyncMsg, NewAcc, Syncer, Ref, Parent) -> - receive - {'$gen_call', From, - cancel_sync_mirrors} -> stop_syncer(Syncer, {cancel, Ref}), - gen_server2:reply(From, ok), - {stop, cancelled}; - {next, Ref} -> Syncer ! SyncMsg, - {cont, NewAcc}; - {'EXIT', Parent, Reason} -> {stop, {shutdown, Reason}}; - {'EXIT', Syncer, Reason} -> {stop, {sync_died, Reason}} - end. - -master_done({Syncer, Ref, _Log, _HandleInfo, _EmitStats, Parent}, BQS) -> - receive - {'$gen_call', From, - cancel_sync_mirrors} -> - stop_syncer(Syncer, {cancel, Ref}), - gen_server2:reply(From, ok), - {cancelled, BQS}; - {cancelled, Ref} -> - {cancelled, BQS}; - {next, Ref} -> - stop_syncer(Syncer, {done, Ref}), - {ok, BQS}; - {'EXIT', Parent, Reason} -> - {shutdown, Reason, BQS}; - {'EXIT', Syncer, Reason} -> - {sync_died, Reason, BQS} - end. - -stop_syncer(Syncer, Msg) -> - unlink(Syncer), - Syncer ! Msg, - receive {'EXIT', Syncer, _} -> ok - after 0 -> ok - end. - -maybe_emit_stats(Last, I, EmitStats, Log) -> - Interval = erlang:convert_time_unit( - erlang:monotonic_time() - Last, native, micro_seconds), - case Interval > ?SYNC_PROGRESS_INTERVAL of - true -> EmitStats({syncing, I}), - Log("~p messages", [I]), - erlang:monotonic_time(); - false -> Last - end. - -handle_set_maximum_since_use() -> - receive - {'$gen_cast', {set_maximum_since_use, Age}} -> - ok = file_handle_cache:set_maximum_since_use(Age) - after 0 -> - ok - end. - -%% Master -%% --------------------------------------------------------------------------- -%% Syncer - -syncer(Ref, Log, MPid, SPids) -> - [erlang:monitor(process, SPid) || SPid <- SPids], - %% We wait for a reply from the mirrors so that we know they are in - %% a receive block and will thus receive messages we send to them - %% *without* those messages ending up in their gen_server2 pqueue. - case await_slaves(Ref, SPids) of - [] -> Log("all mirrors already synced", []); - SPids1 -> MPid ! {ready, self()}, - Log("mirrors ~p to sync", [[node(SPid) || SPid <- SPids1]]), - syncer_check_resources(Ref, MPid, SPids1) - end. - -await_slaves(Ref, SPids) -> - [SPid || SPid <- SPids, - rabbit_mnesia:on_running_node(SPid) andalso %% [0] - receive - {sync_ready, Ref, SPid} -> true; - {sync_deny, Ref, SPid} -> false; - {'DOWN', _, process, SPid, _} -> false - end]. -%% [0] This check is in case there's been a partition which has then -%% healed in between the master retrieving the mirror pids from Mnesia -%% and sending 'sync_start' over GM. If so there might be mirrors on the -%% other side of the partition which we can monitor (since they have -%% rejoined the distributed system with us) but which did not get the -%% 'sync_start' and so will not reply. We need to act as though they are -%% down. - -syncer_check_resources(Ref, MPid, SPids) -> - rabbit_alarm:register(self(), {?MODULE, conserve_resources, []}), - %% Before we ask the master node to send the first batch of messages - %% over here, we check if one node is already short on memory. If - %% that's the case, we wait for the alarm to be cleared before - %% starting the syncer loop. - AlarmedNodes = lists:any( - fun - ({{resource_limit, memory, _}, _}) -> true; - ({_, _}) -> false - end, rabbit_alarm:get_alarms()), - if - not AlarmedNodes -> - MPid ! {next, Ref}, - syncer_loop(Ref, MPid, SPids); - true -> - case wait_for_resources(Ref, SPids) of - cancel -> MPid ! {cancelled, Ref}; - SPids1 -> MPid ! {next, Ref}, - syncer_loop(Ref, MPid, SPids1) - end - end. - -syncer_loop(Ref, MPid, SPids) -> - receive - {conserve_resources, memory, true} -> - case wait_for_resources(Ref, SPids) of - cancel -> MPid ! {cancelled, Ref}; - SPids1 -> syncer_loop(Ref, MPid, SPids1) - end; - {conserve_resources, _, _} -> - %% Ignore other alerts. - syncer_loop(Ref, MPid, SPids); - {msgs, Ref, Msgs} -> - SPids1 = wait_for_credit(SPids), - case SPids1 of - [] -> - % Die silently because there are no mirrors left. - ok; - _ -> - broadcast(SPids1, {sync_msgs, Ref, Msgs}), - MPid ! {next, Ref}, - syncer_loop(Ref, MPid, SPids1) - end; - {cancel, Ref} -> - %% We don't tell the mirrors we will die - so when we do - %% they interpret that as a failure, which is what we - %% want. - ok; - {done, Ref} -> - [SPid ! {sync_complete, Ref} || SPid <- SPids] - end. - -broadcast(SPids, Msg) -> - [begin - credit_flow:send(SPid), - SPid ! Msg - end || SPid <- SPids]. - -conserve_resources(Pid, Source, {_, Conserve, _}) -> - Pid ! {conserve_resources, Source, Conserve}, - ok. - -wait_for_credit(SPids) -> - case credit_flow:blocked() of - true -> receive - {bump_credit, Msg} -> - credit_flow:handle_bump_msg(Msg), - wait_for_credit(SPids); - {'DOWN', _, process, SPid, _} -> - credit_flow:peer_down(SPid), - wait_for_credit(lists:delete(SPid, SPids)) - end; - false -> SPids - end. - -wait_for_resources(Ref, SPids) -> - receive - {conserve_resources, memory, false} -> - SPids; - {conserve_resources, _, _} -> - %% Ignore other alerts. - wait_for_resources(Ref, SPids); - {cancel, Ref} -> - %% We don't tell the mirrors we will die - so when we do - %% they interpret that as a failure, which is what we - %% want. - cancel; - {'DOWN', _, process, SPid, _} -> - credit_flow:peer_down(SPid), - SPids1 = wait_for_credit(lists:delete(SPid, SPids)), - wait_for_resources(Ref, SPids1) - end. - -%% Syncer -%% --------------------------------------------------------------------------- -%% Slave - --spec slave(non_neg_integer(), reference(), timer:tref(), pid(), - bq(), bqs(), fun((bq(), bqs()) -> {timer:tref(), bqs()})) -> - 'denied' | - {'ok' | 'failed', slave_sync_state()} | - {'stop', any(), slave_sync_state()}. - -slave(0, Ref, _TRef, Syncer, _BQ, _BQS, _UpdateRamDuration) -> - Syncer ! {sync_deny, Ref, self()}, - denied; - -slave(_DD, Ref, TRef, Syncer, BQ, BQS, UpdateRamDuration) -> - MRef = erlang:monitor(process, Syncer), - Syncer ! {sync_ready, Ref, self()}, - {_MsgCount, BQS1} = BQ:purge(BQ:purge_acks(BQS)), - slave_sync_loop({Ref, MRef, Syncer, BQ, UpdateRamDuration, - rabbit_misc:get_parent()}, {[], TRef, BQS1}). - -slave_sync_loop(Args = {Ref, MRef, Syncer, BQ, UpdateRamDuration, Parent}, - State = {MA, TRef, BQS}) -> - receive - {'DOWN', MRef, process, Syncer, _Reason} -> - %% If the master dies half way we are not in the usual - %% half-synced state (with messages nearer the tail of the - %% queue); instead we have ones nearer the head. If we then - %% sync with a newly promoted master, or even just receive - %% messages from it, we have a hole in the middle. So the - %% only thing to do here is purge. - {_MsgCount, BQS1} = BQ:purge(BQ:purge_acks(BQS)), - credit_flow:peer_down(Syncer), - {failed, {[], TRef, BQS1}}; - {bump_credit, Msg} -> - credit_flow:handle_bump_msg(Msg), - slave_sync_loop(Args, State); - {sync_complete, Ref} -> - erlang:demonitor(MRef, [flush]), - credit_flow:peer_down(Syncer), - {ok, State}; - {'$gen_cast', {set_maximum_since_use, Age}} -> - ok = file_handle_cache:set_maximum_since_use(Age), - slave_sync_loop(Args, State); - {'$gen_cast', {set_ram_duration_target, Duration}} -> - BQS1 = BQ:set_ram_duration_target(Duration, BQS), - slave_sync_loop(Args, {MA, TRef, BQS1}); - {'$gen_cast', {run_backing_queue, Mod, Fun}} -> - BQS1 = BQ:invoke(Mod, Fun, BQS), - slave_sync_loop(Args, {MA, TRef, BQS1}); - update_ram_duration -> - {TRef1, BQS1} = UpdateRamDuration(BQ, BQS), - slave_sync_loop(Args, {MA, TRef1, BQS1}); - {sync_msgs, Ref, Batch} -> - credit_flow:ack(Syncer), - {MA1, BQS1} = process_batch(Batch, MA, BQ, BQS), - slave_sync_loop(Args, {MA1, TRef, BQS1}); - {'EXIT', Parent, Reason} -> - {stop, Reason, State}; - %% If the master throws an exception - {'$gen_cast', {gm, {delete_and_terminate, Reason}}} -> - BQ:delete_and_terminate(Reason, BQS), - {stop, Reason, {[], TRef, undefined}} - end. - -%% We are partitioning messages by the Unacked element in the tuple. -%% when unacked = true, then it's a publish_delivered message, -%% otherwise it's a publish message. -%% -%% Note that we can't first partition the batch and then publish each -%% part, since that would result in re-ordering messages, which we -%% don't want to do. -process_batch([], MA, _BQ, BQS) -> - {MA, BQS}; -process_batch(Batch, MA, BQ, BQS) -> - {_Msg, _MsgProps, Unacked} = hd(Batch), - process_batch(Batch, Unacked, [], MA, BQ, BQS). - -process_batch([{Msg, Props, true = Unacked} | Rest], true = Unacked, - Acc, MA, BQ, BQS) -> - %% publish_delivered messages don't need the IsDelivered flag, - %% therefore we just add {Msg, Props} to the accumulator. - process_batch(Rest, Unacked, [{Msg, props(Props)} | Acc], - MA, BQ, BQS); -process_batch([{Msg, Props, false = Unacked} | Rest], false = Unacked, - Acc, MA, BQ, BQS) -> - %% publish messages needs the IsDelivered flag which is set to true - %% here. - process_batch(Rest, Unacked, [{Msg, props(Props), true} | Acc], - MA, BQ, BQS); -process_batch(Batch, Unacked, Acc, MA, BQ, BQS) -> - {MA1, BQS1} = publish_batch(Unacked, lists:reverse(Acc), MA, BQ, BQS), - process_batch(Batch, MA1, BQ, BQS1). - -%% Unacked msgs are published via batch_publish. -publish_batch(false, Batch, MA, BQ, BQS) -> - batch_publish(Batch, MA, BQ, BQS); -%% Acked msgs are published via batch_publish_delivered. -publish_batch(true, Batch, MA, BQ, BQS) -> - batch_publish_delivered(Batch, MA, BQ, BQS). - - -batch_publish(Batch, MA, BQ, BQS) -> - BQS1 = BQ:batch_publish(Batch, none, noflow, BQS), - {MA, BQS1}. - -batch_publish_delivered(Batch, MA, BQ, BQS) -> - {AckTags, BQS1} = BQ:batch_publish_delivered(Batch, none, noflow, BQS), - MA1 = BQ:zip_msgs_and_acks(Batch, AckTags, MA, BQS1), - {MA1, BQS1}. - -props(Props) -> - Props#message_properties{needs_confirming = false}. diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl deleted file mode 100644 index 070c6a8205..0000000000 --- a/src/rabbit_mnesia.erl +++ /dev/null @@ -1,1117 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mnesia). - --export([%% Main interface - init/0, - join_cluster/2, - reset/0, - force_reset/0, - update_cluster_nodes/1, - change_cluster_node_type/1, - forget_cluster_node/2, - force_load_next_boot/0, - - %% Various queries to get the status of the db - status/0, - is_clustered/0, - on_running_node/1, - is_process_alive/1, - is_registered_process_alive/1, - cluster_nodes/1, - node_type/0, - dir/0, - cluster_status_from_mnesia/0, - - %% Operations on the db and utils, mainly used in `rabbit_upgrade' and `rabbit' - init_db_unchecked/2, - copy_db/1, - check_cluster_consistency/0, - ensure_mnesia_dir/0, - - %% Hooks used in `rabbit_node_monitor' - on_node_up/1, - on_node_down/1, - - %% Helpers for diagnostics commands - schema_info/1 - ]). - -%% Used internally in rpc calls --export([node_info/0, remove_node_if_mnesia_running/1]). - --ifdef(TEST). --compile(export_all). --export([init_with_lock/3]). --endif. - -%%---------------------------------------------------------------------------- - --export_type([node_type/0, cluster_status/0]). - --type node_type() :: disc | ram. --type cluster_status() :: {[node()], [node()], [node()]}. - -%%---------------------------------------------------------------------------- -%% Main interface -%%---------------------------------------------------------------------------- - --spec init() -> 'ok'. - -init() -> - ensure_mnesia_running(), - ensure_mnesia_dir(), - case is_virgin_node() of - true -> - rabbit_log:info("Node database directory at ~ts is empty. " - "Assuming we need to join an existing cluster or initialise from scratch...~n", - [dir()]), - rabbit_peer_discovery:log_configured_backend(), - rabbit_peer_discovery:maybe_init(), - init_with_lock(); - false -> - NodeType = node_type(), - init_db_and_upgrade(cluster_nodes(all), NodeType, - NodeType =:= ram, _Retry = true), - rabbit_peer_discovery:maybe_init(), - rabbit_peer_discovery:maybe_register() - end, - %% We intuitively expect the global name server to be synced when - %% Mnesia is up. In fact that's not guaranteed to be the case - - %% let's make it so. - ok = rabbit_node_monitor:global_sync(), - ok. - -init_with_lock() -> - {Retries, Timeout} = rabbit_peer_discovery:locking_retry_timeout(), - init_with_lock(Retries, Timeout, fun run_peer_discovery/0). - -init_with_lock(0, _, RunPeerDiscovery) -> - case rabbit_peer_discovery:lock_acquisition_failure_mode() of - ignore -> - rabbit_log:warning("Could not acquire a peer discovery lock, out of retries", []), - RunPeerDiscovery(), - rabbit_peer_discovery:maybe_register(); - fail -> - exit(cannot_acquire_startup_lock) - end; -init_with_lock(Retries, Timeout, RunPeerDiscovery) -> - LockResult = rabbit_peer_discovery:lock(), - rabbit_log:debug("rabbit_peer_discovery:lock returned ~p", [LockResult]), - case LockResult of - not_supported -> - rabbit_log:info("Peer discovery backend does not support locking, falling back to randomized delay"), - %% See rabbitmq/rabbitmq-server#1202 for details. - rabbit_peer_discovery:maybe_inject_randomized_delay(), - RunPeerDiscovery(), - rabbit_peer_discovery:maybe_register(); - {error, _Reason} -> - timer:sleep(Timeout), - init_with_lock(Retries - 1, Timeout, RunPeerDiscovery); - {ok, Data} -> - try - RunPeerDiscovery(), - rabbit_peer_discovery:maybe_register() - after - rabbit_peer_discovery:unlock(Data) - end - end. - --spec run_peer_discovery() -> ok | {[node()], node_type()}. -run_peer_discovery() -> - {RetriesLeft, DelayInterval} = rabbit_peer_discovery:discovery_retries(), - run_peer_discovery_with_retries(RetriesLeft, DelayInterval). - --spec run_peer_discovery_with_retries(non_neg_integer(), non_neg_integer()) -> ok | {[node()], node_type()}. -run_peer_discovery_with_retries(0, _DelayInterval) -> - ok; -run_peer_discovery_with_retries(RetriesLeft, DelayInterval) -> - FindBadNodeNames = fun - (Name, BadNames) when is_atom(Name) -> BadNames; - (Name, BadNames) -> [Name | BadNames] - end, - {DiscoveredNodes0, NodeType} = - case rabbit_peer_discovery:discover_cluster_nodes() of - {error, Reason} -> - RetriesLeft1 = RetriesLeft - 1, - rabbit_log:error("Peer discovery returned an error: ~p. Will retry after a delay of ~b ms, ~b retries left...", - [Reason, DelayInterval, RetriesLeft1]), - timer:sleep(DelayInterval), - run_peer_discovery_with_retries(RetriesLeft1, DelayInterval); - {ok, {Nodes, Type} = Config} - when is_list(Nodes) andalso (Type == disc orelse Type == disk orelse Type == ram) -> - case lists:foldr(FindBadNodeNames, [], Nodes) of - [] -> Config; - BadNames -> e({invalid_cluster_node_names, BadNames}) - end; - {ok, {_, BadType}} when BadType /= disc andalso BadType /= ram -> - e({invalid_cluster_node_type, BadType}); - {ok, _} -> - e(invalid_cluster_nodes_conf) - end, - DiscoveredNodes = lists:usort(DiscoveredNodes0), - rabbit_log:info("All discovered existing cluster peers: ~s~n", - [rabbit_peer_discovery:format_discovered_nodes(DiscoveredNodes)]), - Peers = nodes_excl_me(DiscoveredNodes), - case Peers of - [] -> - rabbit_log:info("Discovered no peer nodes to cluster with. " - "Some discovery backends can filter nodes out based on a readiness criteria. " - "Enabling debug logging might help troubleshoot."), - init_db_and_upgrade([node()], disc, false, _Retry = true); - _ -> - rabbit_log:info("Peer nodes we can cluster with: ~s~n", - [rabbit_peer_discovery:format_discovered_nodes(Peers)]), - join_discovered_peers(Peers, NodeType) - end. - -%% Attempts to join discovered, -%% reachable and compatible (in terms of Mnesia internal protocol version and such) -%% cluster peers in order. -join_discovered_peers(TryNodes, NodeType) -> - {RetriesLeft, DelayInterval} = rabbit_peer_discovery:discovery_retries(), - join_discovered_peers_with_retries(TryNodes, NodeType, RetriesLeft, DelayInterval). - -join_discovered_peers_with_retries(TryNodes, _NodeType, 0, _DelayInterval) -> - rabbit_log:warning( - "Could not successfully contact any node of: ~s (as in Erlang distribution). " - "Starting as a blank standalone node...~n", - [string:join(lists:map(fun atom_to_list/1, TryNodes), ",")]), - init_db_and_upgrade([node()], disc, false, _Retry = true); -join_discovered_peers_with_retries(TryNodes, NodeType, RetriesLeft, DelayInterval) -> - case find_reachable_peer_to_cluster_with(nodes_excl_me(TryNodes)) of - {ok, Node} -> - rabbit_log:info("Node '~s' selected for auto-clustering~n", [Node]), - {ok, {_, DiscNodes, _}} = discover_cluster0(Node), - init_db_and_upgrade(DiscNodes, NodeType, true, _Retry = true), - rabbit_connection_tracking:boot(), - rabbit_node_monitor:notify_joined_cluster(); - none -> - RetriesLeft1 = RetriesLeft - 1, - rabbit_log:error("Trying to join discovered peers failed. Will retry after a delay of ~b ms, ~b retries left...", - [DelayInterval, RetriesLeft1]), - timer:sleep(DelayInterval), - join_discovered_peers_with_retries(TryNodes, NodeType, RetriesLeft1, DelayInterval) - end. - -%% Make the node join a cluster. The node will be reset automatically -%% before we actually cluster it. The nodes provided will be used to -%% find out about the nodes in the cluster. -%% -%% This function will fail if: -%% -%% * The node is currently the only disc node of its cluster -%% * We can't connect to any of the nodes provided -%% * The node is currently already clustered with the cluster of the nodes -%% provided -%% -%% Note that we make no attempt to verify that the nodes provided are -%% all in the same cluster, we simply pick the first online node and -%% we cluster to its cluster. - --spec join_cluster(node(), node_type()) - -> ok | {ok, already_member} | {error, {inconsistent_cluster, string()}}. - -join_cluster(DiscoveryNode, NodeType) -> - ensure_mnesia_not_running(), - ensure_mnesia_dir(), - case is_only_clustered_disc_node() of - true -> e(clustering_only_disc_node); - false -> ok - end, - {ClusterNodes, _, _} = discover_cluster([DiscoveryNode]), - case me_in_nodes(ClusterNodes) of - false -> - case check_cluster_consistency(DiscoveryNode, false) of - {ok, _} -> - %% reset the node. this simplifies things and it - %% will be needed in this case - we're joining a new - %% cluster with new nodes which are not in synch - %% with the current node. It also lifts the burden - %% of resetting the node from the user. - reset_gracefully(), - - %% Join the cluster - rabbit_log:info("Clustering with ~p as ~p node~n", - [ClusterNodes, NodeType]), - ok = init_db_with_mnesia(ClusterNodes, NodeType, - true, true, _Retry = true), - rabbit_connection_tracking:boot(), - rabbit_node_monitor:notify_joined_cluster(), - ok; - {error, Reason} -> - {error, Reason} - end; - true -> - %% DiscoveryNode thinks that we are part of a cluster, but - %% do we think so ourselves? - case are_we_clustered_with(DiscoveryNode) of - true -> - rabbit_log:info("Asked to join a cluster but already a member of it: ~p~n", [ClusterNodes]), - {ok, already_member}; - false -> - Msg = format_inconsistent_cluster_message(DiscoveryNode, node()), - rabbit_log:error(Msg), - {error, {inconsistent_cluster, Msg}} - end - end. - -%% return node to its virgin state, where it is not member of any -%% cluster, has no cluster configuration, no local database, and no -%% persisted messages - --spec reset() -> 'ok'. - -reset() -> - ensure_mnesia_not_running(), - rabbit_log:info("Resetting Rabbit~n", []), - reset_gracefully(). - --spec force_reset() -> 'ok'. - -force_reset() -> - ensure_mnesia_not_running(), - rabbit_log:info("Resetting Rabbit forcefully~n", []), - wipe(). - -reset_gracefully() -> - AllNodes = cluster_nodes(all), - %% Reconnecting so that we will get an up to date nodes. We don't - %% need to check for consistency because we are resetting. - %% Force=true here so that reset still works when clustered with a - %% node which is down. - init_db_with_mnesia(AllNodes, node_type(), false, false, _Retry = false), - case is_only_clustered_disc_node() of - true -> e(resetting_only_disc_node); - false -> ok - end, - leave_cluster(), - rabbit_misc:ensure_ok(mnesia:delete_schema([node()]), cannot_delete_schema), - wipe(). - -wipe() -> - %% We need to make sure that we don't end up in a distributed - %% Erlang system with nodes while not being in an Mnesia cluster - %% with them. We don't handle that well. - [erlang:disconnect_node(N) || N <- cluster_nodes(all)], - %% remove persisted messages and any other garbage we find - ok = rabbit_file:recursive_delete(filelib:wildcard(dir() ++ "/*")), - ok = rabbit_node_monitor:reset_cluster_status(), - ok. - --spec change_cluster_node_type(node_type()) -> 'ok'. - -change_cluster_node_type(Type) -> - ensure_mnesia_not_running(), - ensure_mnesia_dir(), - case is_clustered() of - false -> e(not_clustered); - true -> ok - end, - {_, _, RunningNodes} = discover_cluster(cluster_nodes(all)), - %% We might still be marked as running by a remote node since the - %% information of us going down might not have propagated yet. - Node = case RunningNodes -- [node()] of - [] -> e(no_online_cluster_nodes); - [Node0|_] -> Node0 - end, - ok = reset(), - ok = join_cluster(Node, Type). - --spec update_cluster_nodes(node()) -> 'ok'. - -update_cluster_nodes(DiscoveryNode) -> - ensure_mnesia_not_running(), - ensure_mnesia_dir(), - Status = {AllNodes, _, _} = discover_cluster([DiscoveryNode]), - case me_in_nodes(AllNodes) of - true -> - %% As in `check_consistency/0', we can safely delete the - %% schema here, since it'll be replicated from the other - %% nodes - mnesia:delete_schema([node()]), - rabbit_node_monitor:write_cluster_status(Status), - rabbit_log:info("Updating cluster nodes from ~p~n", - [DiscoveryNode]), - init_db_with_mnesia(AllNodes, node_type(), true, true, _Retry = false); - false -> - e(inconsistent_cluster) - end, - ok. - -%% We proceed like this: try to remove the node locally. If the node -%% is offline, we remove the node if: -%% * This node is a disc node -%% * All other nodes are offline -%% * This node was, at the best of our knowledge (see comment below) -%% the last or second to last after the node we're removing to go -%% down - --spec forget_cluster_node(node(), boolean()) -> 'ok'. - -forget_cluster_node(Node, RemoveWhenOffline) -> - forget_cluster_node(Node, RemoveWhenOffline, true). - -forget_cluster_node(Node, RemoveWhenOffline, EmitNodeDeletedEvent) -> - case lists:member(Node, cluster_nodes(all)) of - true -> ok; - false -> e(not_a_cluster_node) - end, - case {RemoveWhenOffline, is_running()} of - {true, false} -> remove_node_offline_node(Node); - {true, true} -> e(online_node_offline_flag); - {false, false} -> e(offline_node_no_offline_flag); - {false, true} -> rabbit_log:info( - "Removing node ~p from cluster~n", [Node]), - case remove_node_if_mnesia_running(Node) of - ok when EmitNodeDeletedEvent -> - rabbit_event:notify(node_deleted, [{node, Node}]), - ok; - ok -> ok; - {error, _} = Err -> throw(Err) - end - end. - -remove_node_offline_node(Node) -> - %% Here `mnesia:system_info(running_db_nodes)' will RPC, but that's what we - %% want - we need to know the running nodes *now*. If the current node is a - %% RAM node it will return bogus results, but we don't care since we only do - %% this operation from disc nodes. - case {mnesia:system_info(running_db_nodes) -- [Node], node_type()} of - {[], disc} -> - start_mnesia(), - try - %% What we want to do here is replace the last node to - %% go down with the current node. The way we do this - %% is by force loading the table, and making sure that - %% they are loaded. - rabbit_table:force_load(), - rabbit_table:wait_for_replicated(_Retry = false), - %% We skip the 'node_deleted' event because the - %% application is stopped and thus, rabbit_event is not - %% enabled. - forget_cluster_node(Node, false, false), - force_load_next_boot() - after - stop_mnesia() - end; - {_, _} -> - e(removing_node_from_offline_node) - end. - -%%---------------------------------------------------------------------------- -%% Queries -%%---------------------------------------------------------------------------- - --spec status() -> [{'nodes', [{node_type(), [node()]}]} | - {'running_nodes', [node()]} | - {'partitions', [{node(), [node()]}]}]. - -status() -> - IfNonEmpty = fun (_, []) -> []; - (Type, Nodes) -> [{Type, Nodes}] - end, - [{nodes, (IfNonEmpty(disc, cluster_nodes(disc)) ++ - IfNonEmpty(ram, cluster_nodes(ram)))}] ++ - case is_running() of - true -> RunningNodes = cluster_nodes(running), - [{running_nodes, RunningNodes}, - {cluster_name, rabbit_nodes:cluster_name()}, - {partitions, mnesia_partitions(RunningNodes)}]; - false -> [] - end. - -mnesia_partitions(Nodes) -> - Replies = rabbit_node_monitor:partitions(Nodes), - [Reply || Reply = {_, R} <- Replies, R =/= []]. - -is_running() -> mnesia:system_info(is_running) =:= yes. - --spec is_clustered() -> boolean(). - -is_clustered() -> AllNodes = cluster_nodes(all), - AllNodes =/= [] andalso AllNodes =/= [node()]. - --spec on_running_node(pid()) -> boolean(). - -on_running_node(Pid) -> lists:member(node(Pid), cluster_nodes(running)). - -%% This requires the process be in the same running cluster as us -%% (i.e. not partitioned or some random node). -%% -%% See also rabbit_misc:is_process_alive/1 which does not. - --spec is_process_alive(pid() | {atom(), node()}) -> boolean(). - -is_process_alive(Pid) when is_pid(Pid) -> - on_running_node(Pid) andalso - rpc:call(node(Pid), erlang, is_process_alive, [Pid]) =:= true; -is_process_alive({Name, Node}) -> - lists:member(Node, cluster_nodes(running)) andalso - rpc:call(Node, rabbit_mnesia, is_registered_process_alive, [Name]) =:= true. - --spec is_registered_process_alive(atom()) -> boolean(). - -is_registered_process_alive(Name) -> - is_pid(whereis(Name)). - --spec cluster_nodes('all' | 'disc' | 'ram' | 'running') -> [node()]. - -cluster_nodes(WhichNodes) -> cluster_status(WhichNodes). - -%% This function is the actual source of information, since it gets -%% the data from mnesia. Obviously it'll work only when mnesia is -%% running. - --spec cluster_status_from_mnesia() -> rabbit_types:ok_or_error2( - cluster_status(), any()). - -cluster_status_from_mnesia() -> - case is_running() of - false -> - {error, mnesia_not_running}; - true -> - %% If the tables are not present, it means that - %% `init_db/3' hasn't been run yet. In other words, either - %% we are a virgin node or a restarted RAM node. In both - %% cases we're not interested in what mnesia has to say. - NodeType = case mnesia:system_info(use_dir) of - true -> disc; - false -> ram - end, - case rabbit_table:is_present() of - true -> AllNodes = mnesia:system_info(db_nodes), - DiscCopies = mnesia:table_info(schema, disc_copies), - DiscNodes = case NodeType of - disc -> nodes_incl_me(DiscCopies); - ram -> DiscCopies - end, - %% `mnesia:system_info(running_db_nodes)' is safe since - %% we know that mnesia is running - RunningNodes = mnesia:system_info(running_db_nodes), - {ok, {AllNodes, DiscNodes, RunningNodes}}; - false -> {error, tables_not_present} - end - end. - -cluster_status(WhichNodes) -> - {AllNodes, DiscNodes, RunningNodes} = Nodes = - case cluster_status_from_mnesia() of - {ok, Nodes0} -> - Nodes0; - {error, _Reason} -> - {AllNodes0, DiscNodes0, RunningNodes0} = - rabbit_node_monitor:read_cluster_status(), - %% The cluster status file records the status when the node is - %% online, but we know for sure that the node is offline now, so - %% we can remove it from the list of running nodes. - {AllNodes0, DiscNodes0, nodes_excl_me(RunningNodes0)} - end, - case WhichNodes of - status -> Nodes; - all -> AllNodes; - disc -> DiscNodes; - ram -> AllNodes -- DiscNodes; - running -> RunningNodes - end. - -node_info() -> - {rabbit_misc:otp_release(), rabbit_misc:version(), - mnesia:system_info(protocol_version), - cluster_status_from_mnesia()}. - --spec node_type() -> node_type(). - -node_type() -> - {_AllNodes, DiscNodes, _RunningNodes} = - rabbit_node_monitor:read_cluster_status(), - case DiscNodes =:= [] orelse me_in_nodes(DiscNodes) of - true -> disc; - false -> ram - end. - --spec dir() -> file:filename(). - -dir() -> mnesia:system_info(directory). - -%%---------------------------------------------------------------------------- -%% Operations on the db -%%---------------------------------------------------------------------------- - -%% Adds the provided nodes to the mnesia cluster, creating a new -%% schema if there is the need to and catching up if there are other -%% nodes in the cluster already. It also updates the cluster status -%% file. -init_db(ClusterNodes, NodeType, CheckOtherNodes) -> - NodeIsVirgin = is_virgin_node(), - rabbit_log:debug("Does data directory looks like that of a blank (uninitialised) node? ~p", [NodeIsVirgin]), - Nodes = change_extra_db_nodes(ClusterNodes, CheckOtherNodes), - %% Note that we use `system_info' here and not the cluster status - %% since when we start rabbit for the first time the cluster - %% status will say we are a disc node but the tables won't be - %% present yet. - WasDiscNode = mnesia:system_info(use_dir), - case {Nodes, WasDiscNode, NodeType} of - {[], _, ram} -> - %% Standalone ram node, we don't want that - throw({error, cannot_create_standalone_ram_node}); - {[], false, disc} -> - %% RAM -> disc, starting from scratch - ok = create_schema(); - {[], true, disc} -> - %% First disc node up - maybe_force_load(), - ok; - {[_ | _], _, _} -> - %% Subsequent node in cluster, catch up - maybe_force_load(), - ok = rabbit_table:wait_for_replicated(_Retry = true), - ok = rabbit_table:ensure_local_copies(NodeType) - end, - ensure_feature_flags_are_in_sync(Nodes, NodeIsVirgin), - ensure_schema_integrity(), - rabbit_node_monitor:update_cluster_status(), - ok. - --spec init_db_unchecked([node()], node_type()) -> 'ok'. - -init_db_unchecked(ClusterNodes, NodeType) -> - init_db(ClusterNodes, NodeType, false). - -init_db_and_upgrade(ClusterNodes, NodeType, CheckOtherNodes, Retry) -> - ok = init_db(ClusterNodes, NodeType, CheckOtherNodes), - ok = case rabbit_upgrade:maybe_upgrade_local() of - ok -> ok; - starting_from_scratch -> rabbit_version:record_desired(); - version_not_available -> schema_ok_or_move() - end, - %% `maybe_upgrade_local' restarts mnesia, so ram nodes will forget - %% about the cluster - case NodeType of - ram -> start_mnesia(), - change_extra_db_nodes(ClusterNodes, false); - disc -> ok - end, - %% ...and all nodes will need to wait for tables - rabbit_table:wait_for_replicated(Retry), - ok. - -init_db_with_mnesia(ClusterNodes, NodeType, - CheckOtherNodes, CheckConsistency, Retry) -> - start_mnesia(CheckConsistency), - try - init_db_and_upgrade(ClusterNodes, NodeType, CheckOtherNodes, Retry) - after - stop_mnesia() - end. - --spec ensure_mnesia_dir() -> 'ok'. - -ensure_mnesia_dir() -> - MnesiaDir = dir() ++ "/", - case filelib:ensure_dir(MnesiaDir) of - {error, Reason} -> - throw({error, {cannot_create_mnesia_dir, MnesiaDir, Reason}}); - ok -> - ok - end. - -ensure_mnesia_running() -> - case mnesia:system_info(is_running) of - yes -> - ok; - starting -> - wait_for(mnesia_running), - ensure_mnesia_running(); - Reason when Reason =:= no; Reason =:= stopping -> - throw({error, mnesia_not_running}) - end. - -ensure_mnesia_not_running() -> - case mnesia:system_info(is_running) of - no -> - ok; - stopping -> - wait_for(mnesia_not_running), - ensure_mnesia_not_running(); - Reason when Reason =:= yes; Reason =:= starting -> - throw({error, mnesia_unexpectedly_running}) - end. - -ensure_feature_flags_are_in_sync(Nodes, NodeIsVirgin) -> - Ret = rabbit_feature_flags:sync_feature_flags_with_cluster( - Nodes, NodeIsVirgin), - case Ret of - ok -> ok; - {error, Reason} -> throw({error, {incompatible_feature_flags, Reason}}) - end. - -ensure_schema_integrity() -> - case rabbit_table:check_schema_integrity(_Retry = true) of - ok -> - ok; - {error, Reason} -> - throw({error, {schema_integrity_check_failed, Reason}}) - end. - --spec copy_db(file:filename()) -> rabbit_types:ok_or_error(any()). - -copy_db(Destination) -> - ok = ensure_mnesia_not_running(), - rabbit_file:recursive_copy(dir(), Destination). - -force_load_filename() -> - filename:join(dir(), "force_load"). - --spec force_load_next_boot() -> 'ok'. - -force_load_next_boot() -> - rabbit_file:write_file(force_load_filename(), <<"">>). - -maybe_force_load() -> - case rabbit_file:is_file(force_load_filename()) of - true -> rabbit_table:force_load(), - rabbit_file:delete(force_load_filename()); - false -> ok - end. - -%% This does not guarantee us much, but it avoids some situations that -%% will definitely end up badly - --spec check_cluster_consistency() -> 'ok'. - -check_cluster_consistency() -> - %% We want to find 0 or 1 consistent nodes. - case lists:foldl( - fun (Node, {error, _}) -> check_cluster_consistency(Node, true); - (_Node, {ok, Status}) -> {ok, Status} - end, {error, not_found}, nodes_excl_me(cluster_nodes(all))) - of - {ok, Status = {RemoteAllNodes, _, _}} -> - case ordsets:is_subset(ordsets:from_list(cluster_nodes(all)), - ordsets:from_list(RemoteAllNodes)) of - true -> - ok; - false -> - %% We delete the schema here since we think we are - %% clustered with nodes that are no longer in the - %% cluster and there is no other way to remove - %% them from our schema. On the other hand, we are - %% sure that there is another online node that we - %% can use to sync the tables with. There is a - %% race here: if between this check and the - %% `init_db' invocation the cluster gets - %% disbanded, we're left with a node with no - %% mnesia data that will try to connect to offline - %% nodes. - mnesia:delete_schema([node()]) - end, - rabbit_node_monitor:write_cluster_status(Status); - {error, not_found} -> - ok; - {error, _} = E -> - throw(E) - end. - -check_cluster_consistency(Node, CheckNodesConsistency) -> - case remote_node_info(Node) of - {badrpc, _Reason} -> - {error, not_found}; - {_OTP, Rabbit, DelegateModuleHash, _Status} when is_binary(DelegateModuleHash) -> - %% when a delegate module .beam file hash is present - %% in the tuple, we are dealing with an old version - rabbit_version:version_error("Rabbit", rabbit_misc:version(), Rabbit); - {_OTP, _Rabbit, _Protocol, {error, _}} -> - {error, not_found}; - {OTP, Rabbit, Protocol, {ok, Status}} when CheckNodesConsistency -> - case check_consistency(Node, OTP, Rabbit, Protocol, Status) of - {error, _} = E -> E; - {ok, Res} -> {ok, Res} - end; - {OTP, Rabbit, Protocol, {ok, Status}} -> - case check_consistency(Node, OTP, Rabbit, Protocol) of - {error, _} = E -> E; - ok -> {ok, Status} - end - end. - -remote_node_info(Node) -> - case rpc:call(Node, rabbit_mnesia, node_info, []) of - {badrpc, _} = Error -> Error; - %% RabbitMQ prior to 3.6.2 - {OTP, Rabbit, Status} -> {OTP, Rabbit, unsupported, Status}; - %% RabbitMQ 3.6.2 or later - {OTP, Rabbit, Protocol, Status} -> {OTP, Rabbit, Protocol, Status} - end. - - -%%-------------------------------------------------------------------- -%% Hooks for `rabbit_node_monitor' -%%-------------------------------------------------------------------- - --spec on_node_up(node()) -> 'ok'. - -on_node_up(Node) -> - case running_disc_nodes() of - [Node] -> rabbit_log:info("cluster contains disc nodes again~n"); - _ -> ok - end. - --spec on_node_down(node()) -> 'ok'. - -on_node_down(_Node) -> - case running_disc_nodes() of - [] -> rabbit_log:info("only running disc node went down~n"); - _ -> ok - end. - -running_disc_nodes() -> - {_AllNodes, DiscNodes, RunningNodes} = cluster_status(status), - ordsets:to_list(ordsets:intersection(ordsets:from_list(DiscNodes), - ordsets:from_list(RunningNodes))). - -%%-------------------------------------------------------------------- -%% Helpers for diagnostics commands -%%-------------------------------------------------------------------- - -schema_info(Items) -> - Tables = mnesia:system_info(tables), - [info(Table, Items) || Table <- Tables]. - -info(Table, Items) -> - All = [{name, Table} | mnesia:table_info(Table, all)], - [{Item, proplists:get_value(Item, All)} || Item <- Items]. - -%%-------------------------------------------------------------------- -%% Internal helpers -%%-------------------------------------------------------------------- - -discover_cluster(Nodes) -> - case lists:foldl(fun (_, {ok, Res}) -> {ok, Res}; - (Node, _) -> discover_cluster0(Node) - end, {error, no_nodes_provided}, Nodes) of - {ok, Res} -> Res; - {error, E} -> throw({error, E}); - {badrpc, Reason} -> throw({badrpc_multi, Reason, Nodes}) - end. - -discover_cluster0(Node) when Node == node() -> - {error, cannot_cluster_node_with_itself}; -discover_cluster0(Node) -> - rpc:call(Node, rabbit_mnesia, cluster_status_from_mnesia, []). - -schema_ok_or_move() -> - case rabbit_table:check_schema_integrity(_Retry = false) of - ok -> - ok; - {error, Reason} -> - %% NB: we cannot use rabbit_log here since it may not have been - %% started yet - rabbit_log:warning("schema integrity check failed: ~p~n" - "moving database to backup location " - "and recreating schema from scratch~n", - [Reason]), - ok = move_db(), - ok = create_schema() - end. - -%% We only care about disc nodes since ram nodes are supposed to catch -%% up only -create_schema() -> - stop_mnesia(), - rabbit_log:debug("Will bootstrap a schema database..."), - rabbit_misc:ensure_ok(mnesia:create_schema([node()]), cannot_create_schema), - rabbit_log:debug("Bootstraped a schema database successfully"), - start_mnesia(), - - rabbit_log:debug("Will create schema database tables"), - ok = rabbit_table:create(), - rabbit_log:debug("Created schema database tables successfully"), - rabbit_log:debug("Will check schema database integrity..."), - ensure_schema_integrity(), - rabbit_log:debug("Schema database schema integrity check passed"), - ok = rabbit_version:record_desired(). - -move_db() -> - stop_mnesia(), - MnesiaDir = filename:dirname(dir() ++ "/"), - {{Year, Month, Day}, {Hour, Minute, Second}} = erlang:universaltime(), - BackupDir = rabbit_misc:format( - "~s_~w~2..0w~2..0w~2..0w~2..0w~2..0w", - [MnesiaDir, Year, Month, Day, Hour, Minute, Second]), - case file:rename(MnesiaDir, BackupDir) of - ok -> - %% NB: we cannot use rabbit_log here since it may not have - %% been started yet - rabbit_log:warning("moved database from ~s to ~s~n", - [MnesiaDir, BackupDir]), - ok; - {error, Reason} -> throw({error, {cannot_backup_mnesia, - MnesiaDir, BackupDir, Reason}}) - end, - ensure_mnesia_dir(), - start_mnesia(), - ok. - -remove_node_if_mnesia_running(Node) -> - case is_running() of - false -> - {error, mnesia_not_running}; - true -> - %% Deleting the the schema copy of the node will result in - %% the node being removed from the cluster, with that - %% change being propagated to all nodes - case mnesia:del_table_copy(schema, Node) of - {atomic, ok} -> - rabbit_amqqueue:forget_all_durable(Node), - rabbit_node_monitor:notify_left_cluster(Node), - ok; - {aborted, Reason} -> - {error, {failed_to_remove_node, Node, Reason}} - end - end. - -leave_cluster() -> - case nodes_excl_me(cluster_nodes(all)) of - [] -> ok; - AllNodes -> case lists:any(fun leave_cluster/1, AllNodes) of - true -> ok; - false -> e(no_running_cluster_nodes) - end - end. - -leave_cluster(Node) -> - case rpc:call(Node, - rabbit_mnesia, remove_node_if_mnesia_running, [node()]) of - ok -> true; - {error, mnesia_not_running} -> false; - {error, Reason} -> throw({error, Reason}); - {badrpc, nodedown} -> false - end. - -wait_for(Condition) -> - rabbit_log:info("Waiting for ~p...~n", [Condition]), - timer:sleep(1000). - -start_mnesia(CheckConsistency) -> - case CheckConsistency of - true -> check_cluster_consistency(); - false -> ok - end, - rabbit_misc:ensure_ok(mnesia:start(), cannot_start_mnesia), - ensure_mnesia_running(). - -start_mnesia() -> - start_mnesia(true). - -stop_mnesia() -> - stopped = mnesia:stop(), - ensure_mnesia_not_running(). - -change_extra_db_nodes(ClusterNodes0, CheckOtherNodes) -> - ClusterNodes = nodes_excl_me(ClusterNodes0), - case {mnesia:change_config(extra_db_nodes, ClusterNodes), ClusterNodes} of - {{ok, []}, [_|_]} when CheckOtherNodes -> - throw({error, {failed_to_cluster_with, ClusterNodes, - "Mnesia could not connect to any nodes."}}); - {{ok, Nodes}, _} -> - Nodes - end. - -check_consistency(Node, OTP, Rabbit, ProtocolVersion) -> - rabbit_misc:sequence_error( - [check_mnesia_or_otp_consistency(Node, ProtocolVersion, OTP), - check_rabbit_consistency(Node, Rabbit)]). - -check_consistency(Node, OTP, Rabbit, ProtocolVersion, Status) -> - rabbit_misc:sequence_error( - [check_mnesia_or_otp_consistency(Node, ProtocolVersion, OTP), - check_rabbit_consistency(Node, Rabbit), - check_nodes_consistency(Node, Status)]). - -check_nodes_consistency(Node, RemoteStatus = {RemoteAllNodes, _, _}) -> - case me_in_nodes(RemoteAllNodes) of - true -> - {ok, RemoteStatus}; - false -> - {error, {inconsistent_cluster, - format_inconsistent_cluster_message(node(), Node)}} - end. - -check_mnesia_or_otp_consistency(_Node, unsupported, OTP) -> - rabbit_version:check_otp_consistency(OTP); -check_mnesia_or_otp_consistency(Node, ProtocolVersion, _) -> - check_mnesia_consistency(Node, ProtocolVersion). - -check_mnesia_consistency(Node, ProtocolVersion) -> - % If mnesia is running we will just check protocol version - % If it's not running, we don't want it to join cluster until all checks pass - % so we start it without `dir` env variable to prevent - % joining cluster and/or corrupting data - with_running_or_clean_mnesia(fun() -> - case negotiate_protocol([Node]) of - [Node] -> ok; - [] -> - LocalVersion = mnesia:system_info(protocol_version), - {error, {inconsistent_cluster, - rabbit_misc:format("Mnesia protocol negotiation failed." - " Local version: ~p." - " Remote version ~p", - [LocalVersion, ProtocolVersion])}} - end - end). - -negotiate_protocol([Node]) -> - mnesia_monitor:negotiate_protocol([Node]). - -with_running_or_clean_mnesia(Fun) -> - IsMnesiaRunning = case mnesia:system_info(is_running) of - yes -> true; - no -> false; - stopping -> - ensure_mnesia_not_running(), - false; - starting -> - ensure_mnesia_running(), - true - end, - case IsMnesiaRunning of - true -> Fun(); - false -> - SavedMnesiaDir = dir(), - application:unset_env(mnesia, dir), - SchemaLoc = application:get_env(mnesia, schema_location, opt_disc), - application:set_env(mnesia, schema_location, ram), - mnesia:start(), - Result = Fun(), - application:stop(mnesia), - application:set_env(mnesia, dir, SavedMnesiaDir), - application:set_env(mnesia, schema_location, SchemaLoc), - Result - end. - -check_rabbit_consistency(RemoteNode, RemoteVersion) -> - rabbit_misc:sequence_error( - [rabbit_version:check_version_consistency( - rabbit_misc:version(), RemoteVersion, "Rabbit", - fun rabbit_misc:version_minor_equivalent/2), - rabbit_feature_flags:check_node_compatibility(RemoteNode)]). - -%% This is fairly tricky. We want to know if the node is in the state -%% that a `reset' would leave it in. We cannot simply check if the -%% mnesia tables aren't there because restarted RAM nodes won't have -%% tables while still being non-virgin. What we do instead is to -%% check if the mnesia directory is non existent or empty, with the -%% exception of certain files and directories, which can be there very early -%% on node boot. -is_virgin_node() -> - case rabbit_file:list_dir(dir()) of - {error, enoent} -> - true; - {ok, []} -> - true; - {ok, List0} -> - IgnoredFiles0 = - [rabbit_node_monitor:cluster_status_filename(), - rabbit_node_monitor:running_nodes_filename(), - rabbit_node_monitor:default_quorum_filename(), - rabbit_node_monitor:quorum_filename(), - rabbit_feature_flags:enabled_feature_flags_list_file()], - IgnoredFiles = [filename:basename(File) || File <- IgnoredFiles0], - rabbit_log:debug("Files and directories found in node's data directory: ~s, of them to be ignored: ~s", - [string:join(lists:usort(List0), ", "), string:join(lists:usort(IgnoredFiles), ", ")]), - List = List0 -- IgnoredFiles, - rabbit_log:debug("Files and directories found in node's data directory sans ignored ones: ~s", [string:join(lists:usort(List), ", ")]), - List =:= [] - end. - -find_reachable_peer_to_cluster_with([]) -> - none; -find_reachable_peer_to_cluster_with([Node | Nodes]) -> - Fail = fun (Fmt, Args) -> - rabbit_log:warning( - "Could not auto-cluster with node ~s: " ++ Fmt, [Node | Args]), - find_reachable_peer_to_cluster_with(Nodes) - end, - case remote_node_info(Node) of - {badrpc, _} = Reason -> - Fail("~p~n", [Reason]); - %% old delegate hash check - {_OTP, RMQ, Hash, _} when is_binary(Hash) -> - Fail("version ~s~n", [RMQ]); - {_OTP, _RMQ, _Protocol, {error, _} = E} -> - Fail("~p~n", [E]); - {OTP, RMQ, Protocol, _} -> - case check_consistency(Node, OTP, RMQ, Protocol) of - {error, _} -> Fail("versions ~p~n", - [{OTP, RMQ}]); - ok -> {ok, Node} - end - end. - -is_only_clustered_disc_node() -> - node_type() =:= disc andalso is_clustered() andalso - cluster_nodes(disc) =:= [node()]. - -are_we_clustered_with(Node) -> - lists:member(Node, mnesia_lib:all_nodes()). - -me_in_nodes(Nodes) -> lists:member(node(), Nodes). - -nodes_incl_me(Nodes) -> lists:usort([node()|Nodes]). - -nodes_excl_me(Nodes) -> Nodes -- [node()]. - --spec e(any()) -> no_return(). - -e(Tag) -> throw({error, {Tag, error_description(Tag)}}). - -error_description({invalid_cluster_node_names, BadNames}) -> - "In the 'cluster_nodes' configuration key, the following node names " - "are invalid: " ++ lists:flatten(io_lib:format("~p", [BadNames])); -error_description({invalid_cluster_node_type, BadType}) -> - "In the 'cluster_nodes' configuration key, the node type is invalid " - "(expected 'disc' or 'ram'): " ++ - lists:flatten(io_lib:format("~p", [BadType])); -error_description(invalid_cluster_nodes_conf) -> - "The 'cluster_nodes' configuration key is invalid, it must be of the " - "form {[Nodes], Type}, where Nodes is a list of node names and " - "Type is either 'disc' or 'ram'"; -error_description(clustering_only_disc_node) -> - "You cannot cluster a node if it is the only disc node in its existing " - " cluster. If new nodes joined while this node was offline, use " - "'update_cluster_nodes' to add them manually."; -error_description(resetting_only_disc_node) -> - "You cannot reset a node when it is the only disc node in a cluster. " - "Please convert another node of the cluster to a disc node first."; -error_description(not_clustered) -> - "Non-clustered nodes can only be disc nodes."; -error_description(no_online_cluster_nodes) -> - "Could not find any online cluster nodes. If the cluster has changed, " - "you can use the 'update_cluster_nodes' command."; -error_description(inconsistent_cluster) -> - "The nodes provided do not have this node as part of the cluster."; -error_description(not_a_cluster_node) -> - "The node selected is not in the cluster."; -error_description(online_node_offline_flag) -> - "You set the --offline flag, which is used to remove nodes remotely from " - "offline nodes, but this node is online."; -error_description(offline_node_no_offline_flag) -> - "You are trying to remove a node from an offline node. That is dangerous, " - "but can be done with the --offline flag. Please consult the manual " - "for rabbitmqctl for more information."; -error_description(removing_node_from_offline_node) -> - "To remove a node remotely from an offline node, the node you are removing " - "from must be a disc node and all the other nodes must be offline."; -error_description(no_running_cluster_nodes) -> - "You cannot leave a cluster if no online nodes are present.". - -format_inconsistent_cluster_message(Thinker, Dissident) -> - rabbit_misc:format("Node ~p thinks it's clustered " - "with node ~p, but ~p disagrees", - [Thinker, Dissident, Dissident]). diff --git a/src/rabbit_mnesia_rename.erl b/src/rabbit_mnesia_rename.erl deleted file mode 100644 index e0d88c0f5e..0000000000 --- a/src/rabbit_mnesia_rename.erl +++ /dev/null @@ -1,276 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_mnesia_rename). --include("rabbit.hrl"). - --export([rename/2]). --export([maybe_finish/1]). - --define(CONVERT_TABLES, [schema, rabbit_durable_queue]). - -%% Supports renaming the nodes in the Mnesia database. In order to do -%% this, we take a backup of the database, traverse the backup -%% changing node names and pids as we go, then restore it. -%% -%% That's enough for a standalone node, for clusters the story is more -%% complex. We can take pairs of nodes From and To, but backing up and -%% restoring the database changes schema cookies, so if we just do -%% this on all nodes the cluster will refuse to re-form with -%% "Incompatible schema cookies.". Therefore we do something similar -%% to what we do for upgrades - the first node in the cluster to -%% restart becomes the authority, and other nodes wipe their own -%% Mnesia state and rejoin. They also need to tell Mnesia the old node -%% is not coming back. -%% -%% If we are renaming nodes one at a time then the running cluster -%% might not be aware that a rename has taken place, so after we wipe -%% and rejoin we then update any tables (in practice just -%% rabbit_durable_queue) which should be aware that we have changed. - -%%---------------------------------------------------------------------------- - --spec rename(node(), [{node(), node()}]) -> 'ok'. - -rename(Node, NodeMapList) -> - try - %% Check everything is correct and figure out what we are - %% changing from and to. - {FromNode, ToNode, NodeMap} = prepare(Node, NodeMapList), - - %% We backup and restore Mnesia even if other nodes are - %% running at the time, and defer the final decision about - %% whether to use our mutated copy or rejoin the cluster until - %% we restart. That means we might be mutating our copy of the - %% database while the cluster is running. *Do not* contact the - %% cluster while this is happening, we are likely to get - %% confused. - application:set_env(kernel, dist_auto_connect, never), - - %% Take a copy we can restore from if we abandon the - %% rename. We don't restore from the "backup" since restoring - %% that changes schema cookies and might stop us rejoining the - %% cluster. - ok = rabbit_mnesia:copy_db(mnesia_copy_dir()), - - %% And make the actual changes - become(FromNode), - take_backup(before_backup_name()), - convert_backup(NodeMap, before_backup_name(), after_backup_name()), - ok = rabbit_file:write_term_file(rename_config_name(), - [{FromNode, ToNode}]), - convert_config_files(NodeMap), - become(ToNode), - restore_backup(after_backup_name()), - ok - after - stop_mnesia() - end. - -prepare(Node, NodeMapList) -> - %% If we have a previous rename and haven't started since, give up. - case rabbit_file:is_dir(dir()) of - true -> exit({rename_in_progress, - "Restart node under old name to roll back"}); - false -> ok = rabbit_file:ensure_dir(mnesia_copy_dir()) - end, - - %% Check we don't have two nodes mapped to the same node - {FromNodes, ToNodes} = lists:unzip(NodeMapList), - case length(FromNodes) - length(lists:usort(ToNodes)) of - 0 -> ok; - _ -> exit({duplicate_node, ToNodes}) - end, - - %% Figure out which node we are before and after the change - FromNode = case [From || {From, To} <- NodeMapList, - To =:= Node] of - [N] -> N; - [] -> Node - end, - NodeMap = dict:from_list(NodeMapList), - ToNode = case dict:find(FromNode, NodeMap) of - {ok, N2} -> N2; - error -> FromNode - end, - - %% Check that we are in the cluster, all old nodes are in the - %% cluster, and no new nodes are. - Nodes = rabbit_mnesia:cluster_nodes(all), - case {FromNodes -- Nodes, ToNodes -- (ToNodes -- Nodes), - lists:member(Node, Nodes ++ ToNodes)} of - {[], [], true} -> ok; - {[], [], false} -> exit({i_am_not_involved, Node}); - {F, [], _} -> exit({nodes_not_in_cluster, F}); - {_, T, _} -> exit({nodes_already_in_cluster, T}) - end, - {FromNode, ToNode, NodeMap}. - -take_backup(Backup) -> - start_mnesia(), - %% We backup only local tables: in particular, this excludes the - %% connection tracking tables which have no local replica. - LocalTables = mnesia:system_info(local_tables), - {ok, Name, _Nodes} = mnesia:activate_checkpoint([ - {max, LocalTables} - ]), - ok = mnesia:backup_checkpoint(Name, Backup), - stop_mnesia(). - -restore_backup(Backup) -> - ok = mnesia:install_fallback(Backup, [{scope, local}]), - start_mnesia(), - stop_mnesia(), - rabbit_mnesia:force_load_next_boot(). - --spec maybe_finish([node()]) -> 'ok'. - -maybe_finish(AllNodes) -> - case rabbit_file:read_term_file(rename_config_name()) of - {ok, [{FromNode, ToNode}]} -> finish(FromNode, ToNode, AllNodes); - _ -> ok - end. - -finish(FromNode, ToNode, AllNodes) -> - case node() of - ToNode -> - case rabbit_upgrade:nodes_running(AllNodes) of - [] -> finish_primary(FromNode, ToNode); - _ -> finish_secondary(FromNode, ToNode, AllNodes) - end; - FromNode -> - rabbit_log:info( - "Abandoning rename from ~s to ~s since we are still ~s~n", - [FromNode, ToNode, FromNode]), - [{ok, _} = file:copy(backup_of_conf(F), F) || F <- config_files()], - ok = rabbit_file:recursive_delete([rabbit_mnesia:dir()]), - ok = rabbit_file:recursive_copy( - mnesia_copy_dir(), rabbit_mnesia:dir()), - delete_rename_files(); - _ -> - %% Boot will almost certainly fail but we might as - %% well just log this - rabbit_log:info( - "Rename attempted from ~s to ~s but we are ~s - ignoring.~n", - [FromNode, ToNode, node()]) - end. - -finish_primary(FromNode, ToNode) -> - rabbit_log:info("Restarting as primary after rename from ~s to ~s~n", - [FromNode, ToNode]), - delete_rename_files(), - ok. - -finish_secondary(FromNode, ToNode, AllNodes) -> - rabbit_log:info("Restarting as secondary after rename from ~s to ~s~n", - [FromNode, ToNode]), - rabbit_upgrade:secondary_upgrade(AllNodes), - rename_in_running_mnesia(FromNode, ToNode), - delete_rename_files(), - ok. - -dir() -> rabbit_mnesia:dir() ++ "-rename". -before_backup_name() -> dir() ++ "/backup-before". -after_backup_name() -> dir() ++ "/backup-after". -rename_config_name() -> dir() ++ "/pending.config". -mnesia_copy_dir() -> dir() ++ "/mnesia-copy". - -delete_rename_files() -> ok = rabbit_file:recursive_delete([dir()]). - -start_mnesia() -> rabbit_misc:ensure_ok(mnesia:start(), cannot_start_mnesia), - rabbit_table:force_load(), - rabbit_table:wait_for_replicated(_Retry = false). -stop_mnesia() -> stopped = mnesia:stop(). - -convert_backup(NodeMap, FromBackup, ToBackup) -> - mnesia:traverse_backup( - FromBackup, ToBackup, - fun - (Row, Acc) -> - case lists:member(element(1, Row), ?CONVERT_TABLES) of - true -> {[update_term(NodeMap, Row)], Acc}; - false -> {[Row], Acc} - end - end, switched). - -config_files() -> - [rabbit_node_monitor:running_nodes_filename(), - rabbit_node_monitor:cluster_status_filename()]. - -backup_of_conf(Path) -> - filename:join([dir(), filename:basename(Path)]). - -convert_config_files(NodeMap) -> - [convert_config_file(NodeMap, Path) || Path <- config_files()]. - -convert_config_file(NodeMap, Path) -> - {ok, Term} = rabbit_file:read_term_file(Path), - {ok, _} = file:copy(Path, backup_of_conf(Path)), - ok = rabbit_file:write_term_file(Path, update_term(NodeMap, Term)). - -lookup_node(OldNode, NodeMap) -> - case dict:find(OldNode, NodeMap) of - {ok, NewNode} -> NewNode; - error -> OldNode - end. - -mini_map(FromNode, ToNode) -> dict:from_list([{FromNode, ToNode}]). - -update_term(NodeMap, L) when is_list(L) -> - [update_term(NodeMap, I) || I <- L]; -update_term(NodeMap, T) when is_tuple(T) -> - list_to_tuple(update_term(NodeMap, tuple_to_list(T))); -update_term(NodeMap, Node) when is_atom(Node) -> - lookup_node(Node, NodeMap); -update_term(NodeMap, Pid) when is_pid(Pid) -> - rabbit_misc:pid_change_node(Pid, lookup_node(node(Pid), NodeMap)); -update_term(_NodeMap, Term) -> - Term. - -rename_in_running_mnesia(FromNode, ToNode) -> - All = rabbit_mnesia:cluster_nodes(all), - Running = rabbit_nodes:all_running(), - case {lists:member(FromNode, Running), lists:member(ToNode, All)} of - {false, true} -> ok; - {true, _} -> exit({old_node_running, FromNode}); - {_, false} -> exit({new_node_not_in_cluster, ToNode}) - end, - {atomic, ok} = mnesia:del_table_copy(schema, FromNode), - Map = mini_map(FromNode, ToNode), - {atomic, _} = transform_table(rabbit_durable_queue, Map), - ok. - -transform_table(Table, Map) -> - mnesia:sync_transaction( - fun () -> - mnesia:lock({table, Table}, write), - transform_table(Table, Map, mnesia:first(Table)) - end). - -transform_table(_Table, _Map, '$end_of_table') -> - ok; -transform_table(Table, Map, Key) -> - [Term] = mnesia:read(Table, Key, write), - ok = mnesia:write(Table, update_term(Map, Term), write), - transform_table(Table, Map, mnesia:next(Table, Key)). - -become(BecomeNode) -> - error_logger:tty(false), - case net_adm:ping(BecomeNode) of - pong -> exit({node_running, BecomeNode}); - pang -> ok = net_kernel:stop(), - io:format(" * Impersonating node: ~s...", [BecomeNode]), - {ok, _} = start_distribution(BecomeNode), - io:format(" done~n", []), - Dir = mnesia:system_info(directory), - io:format(" * Mnesia directory : ~s~n", [Dir]) - end. - -start_distribution(Name) -> - rabbit_nodes:ensure_epmd(), - NameType = rabbit_nodes_common:name_type(Name), - net_kernel:start([Name, NameType]). diff --git a/src/rabbit_msg_file.erl b/src/rabbit_msg_file.erl deleted file mode 100644 index 1a24f690a0..0000000000 --- a/src/rabbit_msg_file.erl +++ /dev/null @@ -1,114 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_msg_file). - --export([append/3, read/2, scan/4]). - -%%---------------------------------------------------------------------------- - --include("rabbit_msg_store.hrl"). - --define(INTEGER_SIZE_BYTES, 8). --define(INTEGER_SIZE_BITS, (8 * ?INTEGER_SIZE_BYTES)). --define(WRITE_OK_SIZE_BITS, 8). --define(WRITE_OK_MARKER, 255). --define(FILE_PACKING_ADJUSTMENT, (1 + ?INTEGER_SIZE_BYTES)). --define(MSG_ID_SIZE_BYTES, 16). --define(MSG_ID_SIZE_BITS, (8 * ?MSG_ID_SIZE_BYTES)). --define(SCAN_BLOCK_SIZE, 4194304). %% 4MB - -%%---------------------------------------------------------------------------- - --type io_device() :: any(). --type position() :: non_neg_integer(). --type msg_size() :: non_neg_integer(). --type file_size() :: non_neg_integer(). --type message_accumulator(A) :: - fun (({rabbit_types:msg_id(), msg_size(), position(), binary()}, A) -> - A). - -%%---------------------------------------------------------------------------- - --spec append(io_device(), rabbit_types:msg_id(), msg()) -> - rabbit_types:ok_or_error2(msg_size(), any()). - -append(FileHdl, MsgId, MsgBody) - when is_binary(MsgId) andalso size(MsgId) =:= ?MSG_ID_SIZE_BYTES -> - MsgBodyBin = term_to_binary(MsgBody), - MsgBodyBinSize = size(MsgBodyBin), - Size = MsgBodyBinSize + ?MSG_ID_SIZE_BYTES, - case file_handle_cache:append(FileHdl, - <<Size:?INTEGER_SIZE_BITS, - MsgId:?MSG_ID_SIZE_BYTES/binary, - MsgBodyBin:MsgBodyBinSize/binary, - ?WRITE_OK_MARKER:?WRITE_OK_SIZE_BITS>>) of - ok -> {ok, Size + ?FILE_PACKING_ADJUSTMENT}; - KO -> KO - end. - --spec read(io_device(), msg_size()) -> - rabbit_types:ok_or_error2({rabbit_types:msg_id(), msg()}, - any()). - -read(FileHdl, TotalSize) -> - Size = TotalSize - ?FILE_PACKING_ADJUSTMENT, - BodyBinSize = Size - ?MSG_ID_SIZE_BYTES, - case file_handle_cache:read(FileHdl, TotalSize) of - {ok, <<Size:?INTEGER_SIZE_BITS, - MsgId:?MSG_ID_SIZE_BYTES/binary, - MsgBodyBin:BodyBinSize/binary, - ?WRITE_OK_MARKER:?WRITE_OK_SIZE_BITS>>} -> - {ok, {MsgId, binary_to_term(MsgBodyBin)}}; - KO -> KO - end. - --spec scan(io_device(), file_size(), message_accumulator(A), A) -> - {'ok', A, position()}. - -scan(FileHdl, FileSize, Fun, Acc) when FileSize >= 0 -> - scan(FileHdl, FileSize, <<>>, 0, 0, Fun, Acc). - -scan(_FileHdl, FileSize, _Data, FileSize, ScanOffset, _Fun, Acc) -> - {ok, Acc, ScanOffset}; -scan(FileHdl, FileSize, Data, ReadOffset, ScanOffset, Fun, Acc) -> - Read = lists:min([?SCAN_BLOCK_SIZE, (FileSize - ReadOffset)]), - case file_handle_cache:read(FileHdl, Read) of - {ok, Data1} -> - {Data2, Acc1, ScanOffset1} = - scanner(<<Data/binary, Data1/binary>>, ScanOffset, Fun, Acc), - ReadOffset1 = ReadOffset + size(Data1), - scan(FileHdl, FileSize, Data2, ReadOffset1, ScanOffset1, Fun, Acc1); - _KO -> - {ok, Acc, ScanOffset} - end. - -scanner(<<>>, Offset, _Fun, Acc) -> - {<<>>, Acc, Offset}; -scanner(<<0:?INTEGER_SIZE_BITS, _Rest/binary>>, Offset, _Fun, Acc) -> - {<<>>, Acc, Offset}; %% Nothing to do other than stop. -scanner(<<Size:?INTEGER_SIZE_BITS, MsgIdAndMsg:Size/binary, - WriteMarker:?WRITE_OK_SIZE_BITS, Rest/binary>>, Offset, Fun, Acc) -> - TotalSize = Size + ?FILE_PACKING_ADJUSTMENT, - case WriteMarker of - ?WRITE_OK_MARKER -> - %% Here we take option 5 from - %% https://www.erlang.org/cgi-bin/ezmlm-cgi?2:mss:1569 in - %% which we read the MsgId as a number, and then convert it - %% back to a binary in order to work around bugs in - %% Erlang's GC. - <<MsgIdNum:?MSG_ID_SIZE_BITS, Msg/binary>> = - <<MsgIdAndMsg:Size/binary>>, - <<MsgId:?MSG_ID_SIZE_BYTES/binary>> = - <<MsgIdNum:?MSG_ID_SIZE_BITS>>, - scanner(Rest, Offset + TotalSize, Fun, - Fun({MsgId, TotalSize, Offset, Msg}, Acc)); - _ -> - scanner(Rest, Offset + TotalSize, Fun, Acc) - end; -scanner(Data, Offset, _Fun, Acc) -> - {Data, Acc, Offset}. diff --git a/src/rabbit_msg_record.erl b/src/rabbit_msg_record.erl deleted file mode 100644 index 3ebe14cb9f..0000000000 --- a/src/rabbit_msg_record.erl +++ /dev/null @@ -1,400 +0,0 @@ --module(rabbit_msg_record). - --export([ - init/1, - to_iodata/1, - from_amqp091/2, - to_amqp091/1, - add_message_annotations/2, - message_annotation/2, - message_annotation/3 - ]). - --include("rabbit.hrl"). --include("rabbit_framing.hrl"). --include_lib("amqp10_common/include/amqp10_framing.hrl"). - --type maybe(T) :: T | undefined. --type amqp10_data() :: #'v1_0.data'{} | - [#'v1_0.amqp_sequence'{} | #'v1_0.data'{}] | - #'v1_0.amqp_value'{}. --record(msg, - { - % header :: maybe(#'v1_0.header'{}), - % delivery_annotations :: maybe(#'v1_0.delivery_annotations'{}), - message_annotations :: maybe(#'v1_0.message_annotations'{}), - properties :: maybe(#'v1_0.properties'{}), - application_properties :: maybe(#'v1_0.application_properties'{}), - data :: maybe(amqp10_data()) - % footer :: maybe(#'v1_0.footer'{}) - }). - -%% holds static or rarely changing fields --record(cfg, {}). --record(?MODULE, {cfg :: #cfg{}, - msg :: #msg{}, - %% holds a list of modifications to various sections - changes = [] :: list()}). - --opaque state() :: #?MODULE{}. - --export_type([ - state/0 - ]). - -%% this module acts as a wrapper / converter for the internal binar storage format -%% (AMQP 1.0) and any format it needs to be converted to / from. -%% Efficiency is key. No unnecessary allocations or work should be done until it -%% is absolutely needed - -%% init from an AMQP 1.0 encoded binary --spec init(binary()) -> state(). -init(Bin) when is_binary(Bin) -> - %% TODO: delay parsing until needed - {MA, P, AP, D} = decode(amqp10_framing:decode_bin(Bin), - {undefined, undefined, undefined, undefined}), - #?MODULE{cfg = #cfg{}, - msg = #msg{properties = P, - application_properties = AP, - message_annotations = MA, - data = D}}. - -decode([], Acc) -> - Acc; -decode([#'v1_0.message_annotations'{} = MA | Rem], {_, P, AP, D}) -> - decode(Rem, {MA, P, AP, D}); -decode([#'v1_0.properties'{} = P | Rem], {MA, _, AP, D}) -> - decode(Rem, {MA, P, AP, D}); -decode([#'v1_0.application_properties'{} = AP | Rem], {MA, P, _, D}) -> - decode(Rem, {MA, P, AP, D}); -decode([#'v1_0.data'{} = D | Rem], {MA, P, AP, _}) -> - decode(Rem, {MA, P, AP, D}). - -amqp10_properties_empty(#'v1_0.properties'{message_id = undefined, - user_id = undefined, - to = undefined, - % subject = wrap(utf8, RKey), - reply_to = undefined, - correlation_id = undefined, - content_type = undefined, - content_encoding = undefined, - creation_time = undefined}) -> - true; -amqp10_properties_empty(_) -> - false. - -%% to realise the final binary data representation --spec to_iodata(state()) -> iodata(). -to_iodata(#?MODULE{msg = #msg{properties = P, - application_properties = AP, - message_annotations = MA, - data = Data}}) -> - [ - case MA of - #'v1_0.message_annotations'{content = []} -> - <<>>; - _ -> - amqp10_framing:encode_bin(MA) - end, - case amqp10_properties_empty(P) of - true -> <<>>; - false -> - amqp10_framing:encode_bin(P) - end, - case AP of - #'v1_0.application_properties'{content = []} -> - <<>>; - _ -> - amqp10_framing:encode_bin(AP) - end, - amqp10_framing:encode_bin(Data) - ]. - -%% TODO: refine type spec here --spec add_message_annotations(#{binary() => {atom(), term()}}, state()) -> - state(). -add_message_annotations(Anns, - #?MODULE{msg = - #msg{message_annotations = MA0} = Msg} = State) -> - Content = maps:fold( - fun (K, {T, V}, Acc) -> - map_add(symbol, K, T, V, Acc) - end, - case MA0 of - undefined -> []; - #'v1_0.message_annotations'{content = C} -> C - end, - Anns), - - State#?MODULE{msg = - Msg#msg{message_annotations = - #'v1_0.message_annotations'{content = Content}}}. - -%% TODO: refine --type amqp10_term() :: {atom(), term()}. - --spec message_annotation(binary(), state()) -> undefined | amqp10_term(). -message_annotation(Key, State) -> - message_annotation(Key, State, undefined). - --spec message_annotation(binary(), state(), undefined | amqp10_term()) -> - undefined | amqp10_term(). -message_annotation(_Key, #?MODULE{msg = #msg{message_annotations = undefined}}, - Default) -> - Default; -message_annotation(Key, - #?MODULE{msg = - #msg{message_annotations = - #'v1_0.message_annotations'{content = Content}}}, - Default) - when is_binary(Key) -> - case lists:search(fun ({{symbol, K}, _}) -> K == Key end, Content) of - {value, {_K, V}} -> - V; - false -> - Default - end. - - -%% take a binary AMQP 1.0 input function, -%% parses it and returns the current parse state -%% this is the input function from storage and from, e.g. socket input --spec from_amqp091(#'P_basic'{}, iodata()) -> state(). -from_amqp091(#'P_basic'{message_id = MsgId, - expiration = Expiration, - delivery_mode = DelMode, - headers = Headers, - user_id = UserId, - reply_to = ReplyTo, - type = Type, - priority = Priority, - app_id = AppId, - correlation_id = CorrId, - content_type = ContentType, - content_encoding = ContentEncoding, - timestamp = Timestamp - }, Data) -> - %% TODO: support parsing properties bin directly? - ConvertedTs = case Timestamp of - undefined -> - undefined; - _ -> - Timestamp * 1000 - end, - P = #'v1_0.properties'{message_id = wrap(utf8, MsgId), - user_id = wrap(binary, UserId), - to = undefined, - % subject = wrap(utf8, RKey), - reply_to = wrap(utf8, ReplyTo), - correlation_id = wrap(utf8, CorrId), - content_type = wrap(symbol, ContentType), - content_encoding = wrap(symbol, ContentEncoding), - creation_time = wrap(timestamp, ConvertedTs)}, - - APC0 = [{wrap(utf8, K), from_091(T, V)} || {K, T, V} - <- case Headers of - undefined -> []; - _ -> Headers - end], - %% properties that do not map directly to AMQP 1.0 properties are stored - %% in application properties - APC = map_add(utf8, <<"x-basic-type">>, utf8, Type, - map_add(utf8, <<"x-basic-app-id">>, utf8, AppId, APC0)), - - MAC = map_add(symbol, <<"x-basic-priority">>, ubyte, Priority, - map_add(symbol, <<"x-basic-delivery-mode">>, ubyte, DelMode, - map_add(symbol, <<"x-basic-expiration">>, utf8, Expiration, []))), - - AP = #'v1_0.application_properties'{content = APC}, - MA = #'v1_0.message_annotations'{content = MAC}, - #?MODULE{cfg = #cfg{}, - msg = #msg{properties = P, - application_properties = AP, - message_annotations = MA, - data = #'v1_0.data'{content = Data}}}. - -map_add(_T, _Key, _Type, undefined, Acc) -> - Acc; -map_add(KeyType, Key, Type, Value, Acc) -> - [{wrap(KeyType, Key), wrap(Type, Value)} | Acc]. - --spec to_amqp091(state()) -> {#'P_basic'{}, iodata()}. -to_amqp091(#?MODULE{msg = #msg{properties = P, - application_properties = APR, - message_annotations = MAR, - data = #'v1_0.data'{content = Payload}}}) -> - #'v1_0.properties'{message_id = MsgId, - user_id = UserId, - reply_to = ReplyTo0, - correlation_id = CorrId, - content_type = ContentType, - content_encoding = ContentEncoding, - creation_time = Timestamp} = case P of - undefined -> - #'v1_0.properties'{}; - _ -> - P - end, - - AP0 = case APR of - #'v1_0.application_properties'{content = AC} -> AC; - _ -> [] - end, - MA0 = case MAR of - #'v1_0.message_annotations'{content = MC} -> MC; - _ -> [] - end, - - {Type, AP1} = amqp10_map_get(utf8(<<"x-basic-type">>), AP0), - {AppId, AP} = amqp10_map_get(utf8(<<"x-basic-app-id">>), AP1), - - {Priority, MA1} = amqp10_map_get(symbol(<<"x-basic-priority">>), MA0), - {DelMode, MA2} = amqp10_map_get(symbol(<<"x-basic-delivery-mode">>), MA1), - {Expiration, _MA} = amqp10_map_get(symbol(<<"x-basic-expiration">>), MA2), - - Headers0 = [to_091(unwrap(K), V) || {K, V} <- AP], - {Headers1, MsgId091} = message_id(MsgId, <<"x-message-id-type">>, Headers0), - {Headers, CorrId091} = message_id(CorrId, <<"x-correlation-id-type">>, Headers1), - - BP = #'P_basic'{message_id = MsgId091, - delivery_mode = DelMode, - expiration = Expiration, - user_id = unwrap(UserId), - headers = case Headers of - [] -> undefined; - _ -> Headers - end, - reply_to = unwrap(ReplyTo0), - type = Type, - app_id = AppId, - priority = Priority, - correlation_id = CorrId091, - content_type = unwrap(ContentType), - content_encoding = unwrap(ContentEncoding), - timestamp = case unwrap(Timestamp) of - undefined -> - undefined; - Ts -> - Ts div 1000 - end - }, - {BP, Payload}. - -%%% Internal - -amqp10_map_get(K, AP0) -> - case lists:keytake(K, 1, AP0) of - false -> - {undefined, AP0}; - {value, {_, V}, AP} -> - {unwrap(V), AP} - end. - -wrap(_Type, undefined) -> - undefined; -wrap(Type, Val) -> - {Type, Val}. - -unwrap(undefined) -> - undefined; -unwrap({_Type, V}) -> - V. - -% symbol_for(#'v1_0.properties'{}) -> -% {symbol, <<"amqp:properties:list">>}; - -% number_for(#'v1_0.properties'{}) -> -% {ulong, 115}; -% encode(Frame = #'v1_0.properties'{}) -> -% amqp10_framing:encode_described(list, 115, Frame); - -% encode_described(list, CodeNumber, Frame) -> -% {described, {ulong, CodeNumber}, -% {list, lists:map(fun encode/1, tl(tuple_to_list(Frame)))}}; - -% -spec generate(amqp10_type()) -> iolist(). -% generate({described, Descriptor, Value}) -> -% DescBin = generate(Descriptor), -% ValueBin = generate(Value), -% [ ?DESCRIBED_BIN, DescBin, ValueBin ]. - -to_091(Key, {utf8, V}) when is_binary(V) -> {Key, longstr, V}; -to_091(Key, {long, V}) -> {Key, long, V}; -to_091(Key, {byte, V}) -> {Key, byte, V}; -to_091(Key, {ubyte, V}) -> {Key, unsignedbyte, V}; -to_091(Key, {short, V}) -> {Key, short, V}; -to_091(Key, {ushort, V}) -> {Key, unsignedshort, V}; -to_091(Key, {uint, V}) -> {Key, unsignedint, V}; -to_091(Key, {int, V}) -> {Key, signedint, V}; -to_091(Key, {double, V}) -> {Key, double, V}; -to_091(Key, {float, V}) -> {Key, float, V}; -%% NB: header values can never be shortstr! -to_091(Key, {timestamp, V}) -> {Key, timestamp, V div 1000}; -to_091(Key, {binary, V}) -> {Key, binary, V}; -to_091(Key, {boolean, V}) -> {Key, bool, V}; -to_091(Key, true) -> {Key, bool, true}; -to_091(Key, false) -> {Key, bool, false}. - -from_091(longstr, V) when is_binary(V) -> {utf8, V}; -from_091(long, V) -> {long, V}; -from_091(unsignedbyte, V) -> {ubyte, V}; -from_091(short, V) -> {short, V}; -from_091(unsignedshort, V) -> {ushort, V}; -from_091(unsignedint, V) -> {uint, V}; -from_091(signedint, V) -> {int, V}; -from_091(double, V) -> {double, V}; -from_091(float, V) -> {float, V}; -from_091(bool, V) -> {boolean, V}; -from_091(binary, V) -> {binary, V}; -from_091(timestamp, V) -> {timestamp, V * 1000}; -from_091(byte, V) -> {byte, V}. - -% convert_header(signedint, V) -> [$I, <<V:32/signed>>]; -% convert_header(decimal, V) -> {Before, After} = V, -% [$D, Before, <<After:32>>]; -% convert_header(timestamp, V) -> [$T, <<V:64>>]; -% % convert_header(table, V) -> [$F | table_to_binary(V)]; -% % convert_header(array, V) -> [$A | array_to_binary(V)]; -% convert_header(byte, V) -> [$b, <<V:8/signed>>]; -% convert_header(double, V) -> [$d, <<V:64/float>>]; -% convert_header(float, V) -> [$f, <<V:32/float>>]; -% convert_header(short, V) -> [$s, <<V:16/signed>>]; -% convert_header(binary, V) -> [$x | long_string_to_binary(V)]; -% convert_header(unsignedbyte, V) -> [$B, <<V:8/unsigned>>]; -% convert_header(unsignedshort, V) -> [$u, <<V:16/unsigned>>]; -% convert_header(unsignedint, V) -> [$i, <<V:32/unsigned>>]; -% convert_header(void, _V) -> [$V]. - -utf8(T) -> {utf8, T}. -symbol(T) -> {symbol, T}. - -message_id({uuid, UUID}, HKey, H0) -> - H = [{HKey, longstr, <<"uuid">>} | H0], - {H, rabbit_data_coercion:to_binary(rabbit_guid:to_string(UUID))}; -message_id({ulong, N}, HKey, H0) -> - H = [{HKey, longstr, <<"ulong">>} | H0], - {H, erlang:integer_to_binary(N)}; -message_id({binary, B}, HKey, H0) -> - E = base64:encode(B), - case byte_size(E) > 256 of - true -> - K = binary:replace(HKey, <<"-type">>, <<>>), - {[{K, longstr, B} | H0], undefined}; - false -> - H = [{HKey, longstr, <<"binary">>} | H0], - {H, E} - end; -message_id({utf8, S}, HKey, H0) -> - case byte_size(S) > 256 of - true -> - K = binary:replace(HKey, <<"-type">>, <<>>), - {[{K, longstr, S} | H0], undefined}; - false -> - {H0, S} - end; -message_id(MsgId, _, H) -> - {H, unwrap(MsgId)}. - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). --endif. diff --git a/src/rabbit_msg_store.erl b/src/rabbit_msg_store.erl deleted file mode 100644 index 4851e56248..0000000000 --- a/src/rabbit_msg_store.erl +++ /dev/null @@ -1,2245 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_msg_store). - --behaviour(gen_server2). - --export([start_link/4, start_global_store_link/4, successfully_recovered_state/1, - client_init/4, client_terminate/1, client_delete_and_terminate/1, - client_ref/1, close_all_indicated/1, - write/3, write_flow/3, read/2, contains/2, remove/2]). - --export([set_maximum_since_use/2, combine_files/3, - delete_file/2]). %% internal - --export([scan_file_for_valid_messages/1]). %% salvage tool - --export([transform_dir/3, force_recovery/2]). %% upgrade - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3, prioritise_call/4, prioritise_cast/3, - prioritise_info/3, format_message_queue/2]). - -%%---------------------------------------------------------------------------- - --include("rabbit_msg_store.hrl"). - --define(SYNC_INTERVAL, 25). %% milliseconds --define(CLEAN_FILENAME, "clean.dot"). --define(FILE_SUMMARY_FILENAME, "file_summary.ets"). --define(TRANSFORM_TMP, "transform_tmp"). - --define(BINARY_MODE, [raw, binary]). --define(READ_MODE, [read]). --define(READ_AHEAD_MODE, [read_ahead | ?READ_MODE]). --define(WRITE_MODE, [write]). - --define(FILE_EXTENSION, ".rdq"). --define(FILE_EXTENSION_TMP, ".rdt"). - --define(HANDLE_CACHE_BUFFER_SIZE, 1048576). %% 1MB - - %% i.e. two pairs, so GC does not go idle when busy --define(MAXIMUM_SIMULTANEOUS_GC_FILES, 4). - -%%---------------------------------------------------------------------------- - --record(msstate, - { - %% store directory - dir, - %% the module for index ops, - %% rabbit_msg_store_ets_index by default - index_module, - %% where are messages? - index_state, - %% current file name as number - current_file, - %% current file handle since the last fsync? - current_file_handle, - %% file handle cache - file_handle_cache, - %% TRef for our interval timer - sync_timer_ref, - %% sum of valid data in all files - sum_valid_data, - %% sum of file sizes - sum_file_size, - %% things to do once GC completes - pending_gc_completion, - %% pid of our GC - gc_pid, - %% tid of the shared file handles table - file_handles_ets, - %% tid of the file summary table - file_summary_ets, - %% tid of current file cache table - cur_file_cache_ets, - %% tid of writes/removes in flight - flying_ets, - %% set of dying clients - dying_clients, - %% map of references of all registered clients - %% to callbacks - clients, - %% boolean: did we recover state? - successfully_recovered, - %% how big are our files allowed to get? - file_size_limit, - %% client ref to synced messages mapping - cref_to_msg_ids, - %% See CREDIT_DISC_BOUND in rabbit.hrl - credit_disc_bound - }). - --record(client_msstate, - { server, - client_ref, - file_handle_cache, - index_state, - index_module, - dir, - gc_pid, - file_handles_ets, - file_summary_ets, - cur_file_cache_ets, - flying_ets, - credit_disc_bound - }). - --record(file_summary, - {file, valid_total_size, left, right, file_size, locked, readers}). - --record(gc_state, - { dir, - index_module, - index_state, - file_summary_ets, - file_handles_ets, - msg_store - }). - --record(dying_client, - { client_ref, - file, - offset - }). - -%%---------------------------------------------------------------------------- - --export_type([gc_state/0, file_num/0]). - --type gc_state() :: #gc_state { dir :: file:filename(), - index_module :: atom(), - index_state :: any(), - file_summary_ets :: ets:tid(), - file_handles_ets :: ets:tid(), - msg_store :: server() - }. - --type server() :: pid() | atom(). --type client_ref() :: binary(). --type file_num() :: non_neg_integer(). --type client_msstate() :: #client_msstate { - server :: server(), - client_ref :: client_ref(), - file_handle_cache :: map(), - index_state :: any(), - index_module :: atom(), - dir :: file:filename(), - gc_pid :: pid(), - file_handles_ets :: ets:tid(), - file_summary_ets :: ets:tid(), - cur_file_cache_ets :: ets:tid(), - flying_ets :: ets:tid(), - credit_disc_bound :: {pos_integer(), pos_integer()}}. --type msg_ref_delta_gen(A) :: - fun ((A) -> 'finished' | - {rabbit_types:msg_id(), non_neg_integer(), A}). --type maybe_msg_id_fun() :: - 'undefined' | fun ((gb_sets:set(), 'written' | 'ignored') -> any()). --type maybe_close_fds_fun() :: 'undefined' | fun (() -> 'ok'). --type deletion_thunk() :: fun (() -> boolean()). - -%%---------------------------------------------------------------------------- - -%% We run GC whenever (garbage / sum_file_size) > ?GARBAGE_FRACTION -%% It is not recommended to set this to < 0.5 --define(GARBAGE_FRACTION, 0.5). - -%% Message store is responsible for storing messages -%% on disk and loading them back. The store handles both -%% persistent messages and transient ones (when a node -%% is under RAM pressure and needs to page messages out -%% to disk). The store is responsible for locating messages -%% on disk and maintaining an index. -%% -%% There are two message stores per node: one for transient -%% and one for persistent messages. -%% -%% Queue processes interact with the stores via clients. -%% -%% The components: -%% -%% Index: this is a mapping from MsgId to #msg_location{}. -%% By default, it's in ETS, but other implementations can -%% be used. -%% FileSummary: this maps File to #file_summary{} and is stored -%% in ETS. -%% -%% The basic idea is that messages are appended to the current file up -%% until that file becomes too big (> file_size_limit). At that point, -%% the file is closed and a new file is created on the _right_ of the -%% old file which is used for new messages. Files are named -%% numerically ascending, thus the file with the lowest name is the -%% eldest file. -%% -%% We need to keep track of which messages are in which files (this is -%% the index); how much useful data is in each file and which files -%% are on the left and right of each other. This is the purpose of the -%% file summary ETS table. -%% -%% As messages are removed from files, holes appear in these -%% files. The field ValidTotalSize contains the total amount of useful -%% data left in the file. This is needed for garbage collection. -%% -%% When we discover that a file is now empty, we delete it. When we -%% discover that it can be combined with the useful data in either its -%% left or right neighbour, and overall, across all the files, we have -%% ((the amount of garbage) / (the sum of all file sizes)) > -%% ?GARBAGE_FRACTION, we start a garbage collection run concurrently, -%% which will compact the two files together. This keeps disk -%% utilisation high and aids performance. We deliberately do this -%% lazily in order to prevent doing GC on files which are soon to be -%% emptied (and hence deleted). -%% -%% Given the compaction between two files, the left file (i.e. elder -%% file) is considered the ultimate destination for the good data in -%% the right file. If necessary, the good data in the left file which -%% is fragmented throughout the file is written out to a temporary -%% file, then read back in to form a contiguous chunk of good data at -%% the start of the left file. Thus the left file is garbage collected -%% and compacted. Then the good data from the right file is copied -%% onto the end of the left file. Index and file summary tables are -%% updated. -%% -%% On non-clean startup, we scan the files we discover, dealing with -%% the possibilities of a crash having occurred during a compaction -%% (this consists of tidyup - the compaction is deliberately designed -%% such that data is duplicated on disk rather than risking it being -%% lost), and rebuild the file summary and index ETS table. -%% -%% So, with this design, messages move to the left. Eventually, they -%% should end up in a contiguous block on the left and are then never -%% rewritten. But this isn't quite the case. If in a file there is one -%% message that is being ignored, for some reason, and messages in the -%% file to the right and in the current block are being read all the -%% time then it will repeatedly be the case that the good data from -%% both files can be combined and will be written out to a new -%% file. Whenever this happens, our shunned message will be rewritten. -%% -%% So, provided that we combine messages in the right order, -%% (i.e. left file, bottom to top, right file, bottom to top), -%% eventually our shunned message will end up at the bottom of the -%% left file. The compaction/combining algorithm is smart enough to -%% read in good data from the left file that is scattered throughout -%% (i.e. C and D in the below diagram), then truncate the file to just -%% above B (i.e. truncate to the limit of the good contiguous region -%% at the start of the file), then write C and D on top and then write -%% E, F and G from the right file on top. Thus contiguous blocks of -%% good data at the bottom of files are not rewritten. -%% -%% +-------+ +-------+ +-------+ -%% | X | | G | | G | -%% +-------+ +-------+ +-------+ -%% | D | | X | | F | -%% +-------+ +-------+ +-------+ -%% | X | | X | | E | -%% +-------+ +-------+ +-------+ -%% | C | | F | ===> | D | -%% +-------+ +-------+ +-------+ -%% | X | | X | | C | -%% +-------+ +-------+ +-------+ -%% | B | | X | | B | -%% +-------+ +-------+ +-------+ -%% | A | | E | | A | -%% +-------+ +-------+ +-------+ -%% left right left -%% -%% From this reasoning, we do have a bound on the number of times the -%% message is rewritten. From when it is inserted, there can be no -%% files inserted between it and the head of the queue, and the worst -%% case is that every time it is rewritten, it moves one position lower -%% in the file (for it to stay at the same position requires that -%% there are no holes beneath it, which means truncate would be used -%% and so it would not be rewritten at all). Thus this seems to -%% suggest the limit is the number of messages ahead of it in the -%% queue, though it's likely that that's pessimistic, given the -%% requirements for compaction/combination of files. -%% -%% The other property that we have is the bound on the lowest -%% utilisation, which should be 50% - worst case is that all files are -%% fractionally over half full and can't be combined (equivalent is -%% alternating full files and files with only one tiny message in -%% them). -%% -%% Messages are reference-counted. When a message with the same msg id -%% is written several times we only store it once, and only remove it -%% from the store when it has been removed the same number of times. -%% -%% The reference counts do not persist. Therefore the initialisation -%% function must be provided with a generator that produces ref count -%% deltas for all recovered messages. This is only used on startup -%% when the shutdown was non-clean. -%% -%% Read messages with a reference count greater than one are entered -%% into a message cache. The purpose of the cache is not especially -%% performance, though it can help there too, but prevention of memory -%% explosion. It ensures that as messages with a high reference count -%% are read from several processes they are read back as the same -%% binary object rather than multiples of identical binary -%% objects. -%% -%% Reads can be performed directly by clients without calling to the -%% server. This is safe because multiple file handles can be used to -%% read files. However, locking is used by the concurrent GC to make -%% sure that reads are not attempted from files which are in the -%% process of being garbage collected. -%% -%% When a message is removed, its reference count is decremented. Even -%% if the reference count becomes 0, its entry is not removed. This is -%% because in the event of the same message being sent to several -%% different queues, there is the possibility of one queue writing and -%% removing the message before other queues write it at all. Thus -%% accommodating 0-reference counts allows us to avoid unnecessary -%% writes here. Of course, there are complications: the file to which -%% the message has already been written could be locked pending -%% deletion or GC, which means we have to rewrite the message as the -%% original copy will now be lost. -%% -%% The server automatically defers reads, removes and contains calls -%% that occur which refer to files which are currently being -%% GC'd. Contains calls are only deferred in order to ensure they do -%% not overtake removes. -%% -%% The current file to which messages are being written has a -%% write-back cache. This is written to immediately by clients and can -%% be read from by clients too. This means that there are only ever -%% writes made to the current file, thus eliminating delays due to -%% flushing write buffers in order to be able to safely read from the -%% current file. The one exception to this is that on start up, the -%% cache is not populated with msgs found in the current file, and -%% thus in this case only, reads may have to come from the file -%% itself. The effect of this is that even if the msg_store process is -%% heavily overloaded, clients can still write and read messages with -%% very low latency and not block at all. -%% -%% Clients of the msg_store are required to register before using the -%% msg_store. This provides them with the necessary client-side state -%% to allow them to directly access the various caches and files. When -%% they terminate, they should deregister. They can do this by calling -%% either client_terminate/1 or client_delete_and_terminate/1. The -%% differences are: (a) client_terminate is synchronous. As a result, -%% if the msg_store is badly overloaded and has lots of in-flight -%% writes and removes to process, this will take some time to -%% return. However, once it does return, you can be sure that all the -%% actions you've issued to the msg_store have been processed. (b) Not -%% only is client_delete_and_terminate/1 asynchronous, but it also -%% permits writes and subsequent removes from the current -%% (terminating) client which are still in flight to be safely -%% ignored. Thus from the point of view of the msg_store itself, and -%% all from the same client: -%% -%% (T) = termination; (WN) = write of msg N; (RN) = remove of msg N -%% --> W1, W2, W1, R1, T, W3, R2, W2, R1, R2, R3, W4 --> -%% -%% The client obviously sent T after all the other messages (up to -%% W4), but because the msg_store prioritises messages, the T can be -%% promoted and thus received early. -%% -%% Thus at the point of the msg_store receiving T, we have messages 1 -%% and 2 with a refcount of 1. After T, W3 will be ignored because -%% it's an unknown message, as will R3, and W4. W2, R1 and R2 won't be -%% ignored because the messages that they refer to were already known -%% to the msg_store prior to T. However, it can be a little more -%% complex: after the first R2, the refcount of msg 2 is 0. At that -%% point, if a GC occurs or file deletion, msg 2 could vanish, which -%% would then mean that the subsequent W2 and R2 are then ignored. -%% -%% The use case then for client_delete_and_terminate/1 is if the -%% client wishes to remove everything it's written to the msg_store: -%% it issues removes for all messages it's written and not removed, -%% and then calls client_delete_and_terminate/1. At that point, any -%% in-flight writes (and subsequent removes) can be ignored, but -%% removes and writes for messages the msg_store already knows about -%% will continue to be processed normally (which will normally just -%% involve modifying the reference count, which is fast). Thus we save -%% disk bandwidth for writes which are going to be immediately removed -%% again by the the terminating client. -%% -%% We use a separate set to keep track of the dying clients in order -%% to keep that set, which is inspected on every write and remove, as -%% small as possible. Inspecting the set of all clients would degrade -%% performance with many healthy clients and few, if any, dying -%% clients, which is the typical case. -%% -%% Client termination messages are stored in a separate ets index to -%% avoid filling primary message store index and message files with -%% client termination messages. -%% -%% When the msg_store has a backlog (i.e. it has unprocessed messages -%% in its mailbox / gen_server priority queue), a further optimisation -%% opportunity arises: we can eliminate pairs of 'write' and 'remove' -%% from the same client for the same message. A typical occurrence of -%% these is when an empty durable queue delivers persistent messages -%% to ack'ing consumers. The queue will asynchronously ask the -%% msg_store to 'write' such messages, and when they are acknowledged -%% it will issue a 'remove'. That 'remove' may be issued before the -%% msg_store has processed the 'write'. There is then no point going -%% ahead with the processing of that 'write'. -%% -%% To detect this situation a 'flying_ets' table is shared between the -%% clients and the server. The table is keyed on the combination of -%% client (reference) and msg id, and the value represents an -%% integration of all the writes and removes currently "in flight" for -%% that message between the client and server - '+1' means all the -%% writes/removes add up to a single 'write', '-1' to a 'remove', and -%% '0' to nothing. (NB: the integration can never add up to more than -%% one 'write' or 'read' since clients must not write/remove a message -%% more than once without first removing/writing it). -%% -%% Maintaining this table poses two challenges: 1) both the clients -%% and the server access and update the table, which causes -%% concurrency issues, 2) we must ensure that entries do not stay in -%% the table forever, since that would constitute a memory leak. We -%% address the former by carefully modelling all operations as -%% sequences of atomic actions that produce valid results in all -%% possible interleavings. We address the latter by deleting table -%% entries whenever the server finds a 0-valued entry during the -%% processing of a write/remove. 0 is essentially equivalent to "no -%% entry". If, OTOH, the value is non-zero we know there is at least -%% one other 'write' or 'remove' in flight, so we get an opportunity -%% later to delete the table entry when processing these. -%% -%% There are two further complications. We need to ensure that 1) -%% eliminated writes still get confirmed, and 2) the write-back cache -%% doesn't grow unbounded. These are quite straightforward to -%% address. See the comments in the code. -%% -%% For notes on Clean Shutdown and startup, see documentation in -%% rabbit_variable_queue. - -%%---------------------------------------------------------------------------- -%% public API -%%---------------------------------------------------------------------------- - --spec start_link - (atom(), file:filename(), [binary()] | 'undefined', - {msg_ref_delta_gen(A), A}) -> rabbit_types:ok_pid_or_error(). - -start_link(Type, Dir, ClientRefs, StartupFunState) when is_atom(Type) -> - gen_server2:start_link(?MODULE, - [Type, Dir, ClientRefs, StartupFunState], - [{timeout, infinity}]). - -start_global_store_link(Type, Dir, ClientRefs, StartupFunState) when is_atom(Type) -> - gen_server2:start_link({local, Type}, ?MODULE, - [Type, Dir, ClientRefs, StartupFunState], - [{timeout, infinity}]). - --spec successfully_recovered_state(server()) -> boolean(). - -successfully_recovered_state(Server) -> - gen_server2:call(Server, successfully_recovered_state, infinity). - --spec client_init(server(), client_ref(), maybe_msg_id_fun(), - maybe_close_fds_fun()) -> client_msstate(). - -client_init(Server, Ref, MsgOnDiskFun, CloseFDsFun) when is_pid(Server); is_atom(Server) -> - {IState, IModule, Dir, GCPid, - FileHandlesEts, FileSummaryEts, CurFileCacheEts, FlyingEts} = - gen_server2:call( - Server, {new_client_state, Ref, self(), MsgOnDiskFun, CloseFDsFun}, - infinity), - CreditDiscBound = rabbit_misc:get_env(rabbit, msg_store_credit_disc_bound, - ?CREDIT_DISC_BOUND), - #client_msstate { server = Server, - client_ref = Ref, - file_handle_cache = #{}, - index_state = IState, - index_module = IModule, - dir = Dir, - gc_pid = GCPid, - file_handles_ets = FileHandlesEts, - file_summary_ets = FileSummaryEts, - cur_file_cache_ets = CurFileCacheEts, - flying_ets = FlyingEts, - credit_disc_bound = CreditDiscBound }. - --spec client_terminate(client_msstate()) -> 'ok'. - -client_terminate(CState = #client_msstate { client_ref = Ref }) -> - close_all_handles(CState), - ok = server_call(CState, {client_terminate, Ref}). - --spec client_delete_and_terminate(client_msstate()) -> 'ok'. - -client_delete_and_terminate(CState = #client_msstate { client_ref = Ref }) -> - close_all_handles(CState), - ok = server_cast(CState, {client_dying, Ref}), - ok = server_cast(CState, {client_delete, Ref}). - --spec client_ref(client_msstate()) -> client_ref(). - -client_ref(#client_msstate { client_ref = Ref }) -> Ref. - --spec write_flow(rabbit_types:msg_id(), msg(), client_msstate()) -> 'ok'. - -write_flow(MsgId, Msg, - CState = #client_msstate { - server = Server, - credit_disc_bound = CreditDiscBound }) -> - %% Here we are tracking messages sent by the - %% rabbit_amqqueue_process process via the - %% rabbit_variable_queue. We are accessing the - %% rabbit_amqqueue_process process dictionary. - credit_flow:send(Server, CreditDiscBound), - client_write(MsgId, Msg, flow, CState). - --spec write(rabbit_types:msg_id(), msg(), client_msstate()) -> 'ok'. - -write(MsgId, Msg, CState) -> client_write(MsgId, Msg, noflow, CState). - --spec read(rabbit_types:msg_id(), client_msstate()) -> - {rabbit_types:ok(msg()) | 'not_found', client_msstate()}. - -read(MsgId, - CState = #client_msstate { cur_file_cache_ets = CurFileCacheEts }) -> - file_handle_cache_stats:update(msg_store_read), - %% Check the cur file cache - case ets:lookup(CurFileCacheEts, MsgId) of - [] -> - Defer = fun() -> {server_call(CState, {read, MsgId}), CState} end, - case index_lookup_positive_ref_count(MsgId, CState) of - not_found -> Defer(); - MsgLocation -> client_read1(MsgLocation, Defer, CState) - end; - [{MsgId, Msg, _CacheRefCount}] -> - {{ok, Msg}, CState} - end. - --spec contains(rabbit_types:msg_id(), client_msstate()) -> boolean(). - -contains(MsgId, CState) -> server_call(CState, {contains, MsgId}). - --spec remove([rabbit_types:msg_id()], client_msstate()) -> 'ok'. - -remove([], _CState) -> ok; -remove(MsgIds, CState = #client_msstate { client_ref = CRef }) -> - [client_update_flying(-1, MsgId, CState) || MsgId <- MsgIds], - server_cast(CState, {remove, CRef, MsgIds}). - --spec set_maximum_since_use(server(), non_neg_integer()) -> 'ok'. - -set_maximum_since_use(Server, Age) when is_pid(Server); is_atom(Server) -> - gen_server2:cast(Server, {set_maximum_since_use, Age}). - -%%---------------------------------------------------------------------------- -%% Client-side-only helpers -%%---------------------------------------------------------------------------- - -server_call(#client_msstate { server = Server }, Msg) -> - gen_server2:call(Server, Msg, infinity). - -server_cast(#client_msstate { server = Server }, Msg) -> - gen_server2:cast(Server, Msg). - -client_write(MsgId, Msg, Flow, - CState = #client_msstate { cur_file_cache_ets = CurFileCacheEts, - client_ref = CRef }) -> - file_handle_cache_stats:update(msg_store_write), - ok = client_update_flying(+1, MsgId, CState), - ok = update_msg_cache(CurFileCacheEts, MsgId, Msg), - ok = server_cast(CState, {write, CRef, MsgId, Flow}). - -client_read1(#msg_location { msg_id = MsgId, file = File } = MsgLocation, Defer, - CState = #client_msstate { file_summary_ets = FileSummaryEts }) -> - case ets:lookup(FileSummaryEts, File) of - [] -> %% File has been GC'd and no longer exists. Go around again. - read(MsgId, CState); - [#file_summary { locked = Locked, right = Right }] -> - client_read2(Locked, Right, MsgLocation, Defer, CState) - end. - -client_read2(false, undefined, _MsgLocation, Defer, _CState) -> - %% Although we've already checked both caches and not found the - %% message there, the message is apparently in the - %% current_file. We can only arrive here if we are trying to read - %% a message which we have not written, which is very odd, so just - %% defer. - %% - %% OR, on startup, the cur_file_cache is not populated with the - %% contents of the current file, thus reads from the current file - %% will end up here and will need to be deferred. - Defer(); -client_read2(true, _Right, _MsgLocation, Defer, _CState) -> - %% Of course, in the mean time, the GC could have run and our msg - %% is actually in a different file, unlocked. However, deferring is - %% the safest and simplest thing to do. - Defer(); -client_read2(false, _Right, - MsgLocation = #msg_location { msg_id = MsgId, file = File }, - Defer, - CState = #client_msstate { file_summary_ets = FileSummaryEts }) -> - %% It's entirely possible that everything we're doing from here on - %% is for the wrong file, or a non-existent file, as a GC may have - %% finished. - safe_ets_update_counter( - FileSummaryEts, File, {#file_summary.readers, +1}, - fun (_) -> client_read3(MsgLocation, Defer, CState) end, - fun () -> read(MsgId, CState) end). - -client_read3(#msg_location { msg_id = MsgId, file = File }, Defer, - CState = #client_msstate { file_handles_ets = FileHandlesEts, - file_summary_ets = FileSummaryEts, - gc_pid = GCPid, - client_ref = Ref }) -> - Release = - fun() -> ok = case ets:update_counter(FileSummaryEts, File, - {#file_summary.readers, -1}) of - 0 -> case ets:lookup(FileSummaryEts, File) of - [#file_summary { locked = true }] -> - rabbit_msg_store_gc:no_readers( - GCPid, File); - _ -> ok - end; - _ -> ok - end - end, - %% If a GC involving the file hasn't already started, it won't - %% start now. Need to check again to see if we've been locked in - %% the meantime, between lookup and update_counter (thus GC - %% started before our +1. In fact, it could have finished by now - %% too). - case ets:lookup(FileSummaryEts, File) of - [] -> %% GC has deleted our file, just go round again. - read(MsgId, CState); - [#file_summary { locked = true }] -> - %% If we get a badarg here, then the GC has finished and - %% deleted our file. Try going around again. Otherwise, - %% just defer. - %% - %% badarg scenario: we lookup, msg_store locks, GC starts, - %% GC ends, we +1 readers, msg_store ets:deletes (and - %% unlocks the dest) - try Release(), - Defer() - catch error:badarg -> read(MsgId, CState) - end; - [#file_summary { locked = false }] -> - %% Ok, we're definitely safe to continue - a GC involving - %% the file cannot start up now, and isn't running, so - %% nothing will tell us from now on to close the handle if - %% it's already open. - %% - %% Finally, we need to recheck that the msg is still at - %% the same place - it's possible an entire GC ran between - %% us doing the lookup and the +1 on the readers. (Same as - %% badarg scenario above, but we don't have a missing file - %% - we just have the /wrong/ file). - case index_lookup(MsgId, CState) of - #msg_location { file = File } = MsgLocation -> - %% Still the same file. - {ok, CState1} = close_all_indicated(CState), - %% We are now guaranteed that the mark_handle_open - %% call will either insert_new correctly, or will - %% fail, but find the value is open, not close. - mark_handle_open(FileHandlesEts, File, Ref), - %% Could the msg_store now mark the file to be - %% closed? No: marks for closing are issued only - %% when the msg_store has locked the file. - %% This will never be the current file - {Msg, CState2} = read_from_disk(MsgLocation, CState1), - Release(), %% this MUST NOT fail with badarg - {{ok, Msg}, CState2}; - #msg_location {} = MsgLocation -> %% different file! - Release(), %% this MUST NOT fail with badarg - client_read1(MsgLocation, Defer, CState); - not_found -> %% it seems not to exist. Defer, just to be sure. - try Release() %% this can badarg, same as locked case, above - catch error:badarg -> ok - end, - Defer() - end - end. - -client_update_flying(Diff, MsgId, #client_msstate { flying_ets = FlyingEts, - client_ref = CRef }) -> - Key = {MsgId, CRef}, - case ets:insert_new(FlyingEts, {Key, Diff}) of - true -> ok; - false -> try ets:update_counter(FlyingEts, Key, {2, Diff}) of - 0 -> ok; - Diff -> ok; - Err -> throw({bad_flying_ets_update, Diff, Err, Key}) - catch error:badarg -> - %% this is guaranteed to succeed since the - %% server only removes and updates flying_ets - %% entries; it never inserts them - true = ets:insert_new(FlyingEts, {Key, Diff}) - end, - ok - end. - -clear_client(CRef, State = #msstate { cref_to_msg_ids = CTM, - dying_clients = DyingClients }) -> - State #msstate { cref_to_msg_ids = maps:remove(CRef, CTM), - dying_clients = maps:remove(CRef, DyingClients) }. - - -%%---------------------------------------------------------------------------- -%% gen_server callbacks -%%---------------------------------------------------------------------------- - - -init([Type, BaseDir, ClientRefs, StartupFunState]) -> - process_flag(trap_exit, true), - - ok = file_handle_cache:register_callback(?MODULE, set_maximum_since_use, - [self()]), - - Dir = filename:join(BaseDir, atom_to_list(Type)), - Name = filename:join(filename:basename(BaseDir), atom_to_list(Type)), - - {ok, IndexModule} = application:get_env(rabbit, msg_store_index_module), - rabbit_log:info("Message store ~tp: using ~p to provide index~n", [Name, IndexModule]), - - AttemptFileSummaryRecovery = - case ClientRefs of - undefined -> ok = rabbit_file:recursive_delete([Dir]), - ok = filelib:ensure_dir(filename:join(Dir, "nothing")), - false; - _ -> ok = filelib:ensure_dir(filename:join(Dir, "nothing")), - recover_crashed_compactions(Dir) - end, - %% if we found crashed compactions we trust neither the - %% file_summary nor the location index. Note the file_summary is - %% left empty here if it can't be recovered. - {FileSummaryRecovered, FileSummaryEts} = - recover_file_summary(AttemptFileSummaryRecovery, Dir), - {CleanShutdown, IndexState, ClientRefs1} = - recover_index_and_client_refs(IndexModule, FileSummaryRecovered, - ClientRefs, Dir, Name), - Clients = maps:from_list( - [{CRef, {undefined, undefined, undefined}} || - CRef <- ClientRefs1]), - %% CleanShutdown => msg location index and file_summary both - %% recovered correctly. - true = case {FileSummaryRecovered, CleanShutdown} of - {true, false} -> ets:delete_all_objects(FileSummaryEts); - _ -> true - end, - %% CleanShutdown <=> msg location index and file_summary both - %% recovered correctly. - - FileHandlesEts = ets:new(rabbit_msg_store_shared_file_handles, - [ordered_set, public]), - CurFileCacheEts = ets:new(rabbit_msg_store_cur_file, [set, public]), - FlyingEts = ets:new(rabbit_msg_store_flying, [set, public]), - - {ok, FileSizeLimit} = application:get_env(rabbit, msg_store_file_size_limit), - - {ok, GCPid} = rabbit_msg_store_gc:start_link( - #gc_state { dir = Dir, - index_module = IndexModule, - index_state = IndexState, - file_summary_ets = FileSummaryEts, - file_handles_ets = FileHandlesEts, - msg_store = self() - }), - - CreditDiscBound = rabbit_misc:get_env(rabbit, msg_store_credit_disc_bound, - ?CREDIT_DISC_BOUND), - - State = #msstate { dir = Dir, - index_module = IndexModule, - index_state = IndexState, - current_file = 0, - current_file_handle = undefined, - file_handle_cache = #{}, - sync_timer_ref = undefined, - sum_valid_data = 0, - sum_file_size = 0, - pending_gc_completion = maps:new(), - gc_pid = GCPid, - file_handles_ets = FileHandlesEts, - file_summary_ets = FileSummaryEts, - cur_file_cache_ets = CurFileCacheEts, - flying_ets = FlyingEts, - dying_clients = #{}, - clients = Clients, - successfully_recovered = CleanShutdown, - file_size_limit = FileSizeLimit, - cref_to_msg_ids = #{}, - credit_disc_bound = CreditDiscBound - }, - %% If we didn't recover the msg location index then we need to - %% rebuild it now. - Cleanliness = case CleanShutdown of - true -> "clean"; - false -> "unclean" - end, - rabbit_log:debug("Rebuilding message location index after ~s shutdown...~n", - [Cleanliness]), - {Offset, State1 = #msstate { current_file = CurFile }} = - build_index(CleanShutdown, StartupFunState, State), - rabbit_log:debug("Finished rebuilding index~n", []), - %% read is only needed so that we can seek - {ok, CurHdl} = open_file(Dir, filenum_to_name(CurFile), - [read | ?WRITE_MODE]), - {ok, Offset} = file_handle_cache:position(CurHdl, Offset), - ok = file_handle_cache:truncate(CurHdl), - - {ok, maybe_compact(State1 #msstate { current_file_handle = CurHdl }), - hibernate, - {backoff, ?HIBERNATE_AFTER_MIN, ?HIBERNATE_AFTER_MIN, ?DESIRED_HIBERNATE}}. - -prioritise_call(Msg, _From, _Len, _State) -> - case Msg of - successfully_recovered_state -> 7; - {new_client_state, _Ref, _Pid, _MODC, _CloseFDsFun} -> 7; - {read, _MsgId} -> 2; - _ -> 0 - end. - -prioritise_cast(Msg, _Len, _State) -> - case Msg of - {combine_files, _Source, _Destination, _Reclaimed} -> 8; - {delete_file, _File, _Reclaimed} -> 8; - {set_maximum_since_use, _Age} -> 8; - {client_dying, _Pid} -> 7; - _ -> 0 - end. - -prioritise_info(Msg, _Len, _State) -> - case Msg of - sync -> 8; - _ -> 0 - end. - -handle_call(successfully_recovered_state, _From, State) -> - reply(State #msstate.successfully_recovered, State); - -handle_call({new_client_state, CRef, CPid, MsgOnDiskFun, CloseFDsFun}, _From, - State = #msstate { dir = Dir, - index_state = IndexState, - index_module = IndexModule, - file_handles_ets = FileHandlesEts, - file_summary_ets = FileSummaryEts, - cur_file_cache_ets = CurFileCacheEts, - flying_ets = FlyingEts, - clients = Clients, - gc_pid = GCPid }) -> - Clients1 = maps:put(CRef, {CPid, MsgOnDiskFun, CloseFDsFun}, Clients), - erlang:monitor(process, CPid), - reply({IndexState, IndexModule, Dir, GCPid, FileHandlesEts, FileSummaryEts, - CurFileCacheEts, FlyingEts}, - State #msstate { clients = Clients1 }); - -handle_call({client_terminate, CRef}, _From, State) -> - reply(ok, clear_client(CRef, State)); - -handle_call({read, MsgId}, From, State) -> - State1 = read_message(MsgId, From, State), - noreply(State1); - -handle_call({contains, MsgId}, From, State) -> - State1 = contains_message(MsgId, From, State), - noreply(State1). - -handle_cast({client_dying, CRef}, - State = #msstate { dying_clients = DyingClients, - current_file_handle = CurHdl, - current_file = CurFile }) -> - {ok, CurOffset} = file_handle_cache:current_virtual_offset(CurHdl), - DyingClients1 = maps:put(CRef, - #dying_client{client_ref = CRef, - file = CurFile, - offset = CurOffset}, - DyingClients), - noreply(State #msstate { dying_clients = DyingClients1 }); - -handle_cast({client_delete, CRef}, - State = #msstate { clients = Clients }) -> - State1 = State #msstate { clients = maps:remove(CRef, Clients) }, - noreply(clear_client(CRef, State1)); - -handle_cast({write, CRef, MsgId, Flow}, - State = #msstate { cur_file_cache_ets = CurFileCacheEts, - clients = Clients, - credit_disc_bound = CreditDiscBound }) -> - case Flow of - flow -> {CPid, _, _} = maps:get(CRef, Clients), - %% We are going to process a message sent by the - %% rabbit_amqqueue_process. Now we are accessing the - %% msg_store process dictionary. - credit_flow:ack(CPid, CreditDiscBound); - noflow -> ok - end, - true = 0 =< ets:update_counter(CurFileCacheEts, MsgId, {3, -1}), - case update_flying(-1, MsgId, CRef, State) of - process -> - [{MsgId, Msg, _PWC}] = ets:lookup(CurFileCacheEts, MsgId), - noreply(write_message(MsgId, Msg, CRef, State)); - ignore -> - %% A 'remove' has already been issued and eliminated the - %% 'write'. - State1 = blind_confirm(CRef, gb_sets:singleton(MsgId), - ignored, State), - %% If all writes get eliminated, cur_file_cache_ets could - %% grow unbounded. To prevent that we delete the cache - %% entry here, but only if the message isn't in the - %% current file. That way reads of the message can - %% continue to be done client side, from either the cache - %% or the non-current files. If the message *is* in the - %% current file then the cache entry will be removed by - %% the normal logic for that in write_message/4 and - %% maybe_roll_to_new_file/2. - case index_lookup(MsgId, State1) of - [#msg_location { file = File }] - when File == State1 #msstate.current_file -> - ok; - _ -> - true = ets:match_delete(CurFileCacheEts, {MsgId, '_', 0}) - end, - noreply(State1) - end; - -handle_cast({remove, CRef, MsgIds}, State) -> - {RemovedMsgIds, State1} = - lists:foldl( - fun (MsgId, {Removed, State2}) -> - case update_flying(+1, MsgId, CRef, State2) of - process -> {[MsgId | Removed], - remove_message(MsgId, CRef, State2)}; - ignore -> {Removed, State2} - end - end, {[], State}, MsgIds), - noreply(maybe_compact(client_confirm(CRef, gb_sets:from_list(RemovedMsgIds), - ignored, State1))); - -handle_cast({combine_files, Source, Destination, Reclaimed}, - State = #msstate { sum_file_size = SumFileSize, - file_handles_ets = FileHandlesEts, - file_summary_ets = FileSummaryEts, - clients = Clients }) -> - ok = cleanup_after_file_deletion(Source, State), - %% see comment in cleanup_after_file_deletion, and client_read3 - true = mark_handle_to_close(Clients, FileHandlesEts, Destination, false), - true = ets:update_element(FileSummaryEts, Destination, - {#file_summary.locked, false}), - State1 = State #msstate { sum_file_size = SumFileSize - Reclaimed }, - noreply(maybe_compact(run_pending([Source, Destination], State1))); - -handle_cast({delete_file, File, Reclaimed}, - State = #msstate { sum_file_size = SumFileSize }) -> - ok = cleanup_after_file_deletion(File, State), - State1 = State #msstate { sum_file_size = SumFileSize - Reclaimed }, - noreply(maybe_compact(run_pending([File], State1))); - -handle_cast({set_maximum_since_use, Age}, State) -> - ok = file_handle_cache:set_maximum_since_use(Age), - noreply(State). - -handle_info(sync, State) -> - noreply(internal_sync(State)); - -handle_info(timeout, State) -> - noreply(internal_sync(State)); - -handle_info({'DOWN', _MRef, process, Pid, _Reason}, State) -> - %% similar to what happens in - %% rabbit_amqqueue_process:handle_ch_down but with a relation of - %% msg_store -> rabbit_amqqueue_process instead of - %% rabbit_amqqueue_process -> rabbit_channel. - credit_flow:peer_down(Pid), - noreply(State); - -handle_info({'EXIT', _Pid, Reason}, State) -> - {stop, Reason, State}. - -terminate(_Reason, State = #msstate { index_state = IndexState, - index_module = IndexModule, - current_file_handle = CurHdl, - gc_pid = GCPid, - file_handles_ets = FileHandlesEts, - file_summary_ets = FileSummaryEts, - cur_file_cache_ets = CurFileCacheEts, - flying_ets = FlyingEts, - clients = Clients, - dir = Dir }) -> - rabbit_log:info("Stopping message store for directory '~s'", [Dir]), - %% stop the gc first, otherwise it could be working and we pull - %% out the ets tables from under it. - ok = rabbit_msg_store_gc:stop(GCPid), - State1 = case CurHdl of - undefined -> State; - _ -> State2 = internal_sync(State), - ok = file_handle_cache:close(CurHdl), - State2 - end, - State3 = close_all_handles(State1), - case store_file_summary(FileSummaryEts, Dir) of - ok -> ok; - {error, FSErr} -> - rabbit_log:error("Unable to store file summary" - " for vhost message store for directory ~p~n" - "Error: ~p~n", - [Dir, FSErr]) - end, - [true = ets:delete(T) || T <- [FileSummaryEts, FileHandlesEts, - CurFileCacheEts, FlyingEts]], - IndexModule:terminate(IndexState), - case store_recovery_terms([{client_refs, maps:keys(Clients)}, - {index_module, IndexModule}], Dir) of - ok -> - rabbit_log:info("Message store for directory '~s' is stopped", [Dir]), - ok; - {error, RTErr} -> - rabbit_log:error("Unable to save message store recovery terms" - " for directory ~p~nError: ~p~n", - [Dir, RTErr]) - end, - State3 #msstate { index_state = undefined, - current_file_handle = undefined }. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -format_message_queue(Opt, MQ) -> rabbit_misc:format_message_queue(Opt, MQ). - -%%---------------------------------------------------------------------------- -%% general helper functions -%%---------------------------------------------------------------------------- - -noreply(State) -> - {State1, Timeout} = next_state(State), - {noreply, State1, Timeout}. - -reply(Reply, State) -> - {State1, Timeout} = next_state(State), - {reply, Reply, State1, Timeout}. - -next_state(State = #msstate { sync_timer_ref = undefined, - cref_to_msg_ids = CTM }) -> - case maps:size(CTM) of - 0 -> {State, hibernate}; - _ -> {start_sync_timer(State), 0} - end; -next_state(State = #msstate { cref_to_msg_ids = CTM }) -> - case maps:size(CTM) of - 0 -> {stop_sync_timer(State), hibernate}; - _ -> {State, 0} - end. - -start_sync_timer(State) -> - rabbit_misc:ensure_timer(State, #msstate.sync_timer_ref, - ?SYNC_INTERVAL, sync). - -stop_sync_timer(State) -> - rabbit_misc:stop_timer(State, #msstate.sync_timer_ref). - -internal_sync(State = #msstate { current_file_handle = CurHdl, - cref_to_msg_ids = CTM }) -> - State1 = stop_sync_timer(State), - CGs = maps:fold(fun (CRef, MsgIds, NS) -> - case gb_sets:is_empty(MsgIds) of - true -> NS; - false -> [{CRef, MsgIds} | NS] - end - end, [], CTM), - ok = case CGs of - [] -> ok; - _ -> file_handle_cache:sync(CurHdl) - end, - lists:foldl(fun ({CRef, MsgIds}, StateN) -> - client_confirm(CRef, MsgIds, written, StateN) - end, State1, CGs). - -update_flying(Diff, MsgId, CRef, #msstate { flying_ets = FlyingEts }) -> - Key = {MsgId, CRef}, - NDiff = -Diff, - case ets:lookup(FlyingEts, Key) of - [] -> ignore; - [{_, Diff}] -> ignore; %% [1] - [{_, NDiff}] -> ets:update_counter(FlyingEts, Key, {2, Diff}), - true = ets:delete_object(FlyingEts, {Key, 0}), - process; - [{_, 0}] -> true = ets:delete_object(FlyingEts, {Key, 0}), - ignore; - [{_, Err}] -> throw({bad_flying_ets_record, Diff, Err, Key}) - end. -%% [1] We can get here, for example, in the following scenario: There -%% is a write followed by a remove in flight. The counter will be 0, -%% so on processing the write the server attempts to delete the -%% entry. If at that point the client injects another write it will -%% either insert a new entry, containing +1, or increment the existing -%% entry to +1, thus preventing its removal. Either way therefore when -%% the server processes the read, the counter will be +1. - -write_action({true, not_found}, _MsgId, State) -> - {ignore, undefined, State}; -write_action({true, #msg_location { file = File }}, _MsgId, State) -> - {ignore, File, State}; -write_action({false, not_found}, _MsgId, State) -> - {write, State}; -write_action({Mask, #msg_location { ref_count = 0, file = File, - total_size = TotalSize }}, - MsgId, State = #msstate { file_summary_ets = FileSummaryEts }) -> - case {Mask, ets:lookup(FileSummaryEts, File)} of - {false, [#file_summary { locked = true }]} -> - ok = index_delete(MsgId, State), - {write, State}; - {false_if_increment, [#file_summary { locked = true }]} -> - %% The msg for MsgId is older than the client death - %% message, but as it is being GC'd currently we'll have - %% to write a new copy, which will then be younger, so - %% ignore this write. - {ignore, File, State}; - {_Mask, [#file_summary {}]} -> - ok = index_update_ref_count(MsgId, 1, State), - State1 = adjust_valid_total_size(File, TotalSize, State), - {confirm, File, State1} - end; -write_action({_Mask, #msg_location { ref_count = RefCount, file = File }}, - MsgId, State) -> - ok = index_update_ref_count(MsgId, RefCount + 1, State), - %% We already know about it, just update counter. Only update - %% field otherwise bad interaction with concurrent GC - {confirm, File, State}. - -write_message(MsgId, Msg, CRef, - State = #msstate { cur_file_cache_ets = CurFileCacheEts }) -> - case write_action(should_mask_action(CRef, MsgId, State), MsgId, State) of - {write, State1} -> - write_message(MsgId, Msg, - record_pending_confirm(CRef, MsgId, State1)); - {ignore, CurFile, State1 = #msstate { current_file = CurFile }} -> - State1; - {ignore, _File, State1} -> - true = ets:delete_object(CurFileCacheEts, {MsgId, Msg, 0}), - State1; - {confirm, CurFile, State1 = #msstate { current_file = CurFile }}-> - record_pending_confirm(CRef, MsgId, State1); - {confirm, _File, State1} -> - true = ets:delete_object(CurFileCacheEts, {MsgId, Msg, 0}), - update_pending_confirms( - fun (MsgOnDiskFun, CTM) -> - MsgOnDiskFun(gb_sets:singleton(MsgId), written), - CTM - end, CRef, State1) - end. - -remove_message(MsgId, CRef, - State = #msstate { file_summary_ets = FileSummaryEts }) -> - case should_mask_action(CRef, MsgId, State) of - {true, _Location} -> - State; - {false_if_increment, #msg_location { ref_count = 0 }} -> - %% CRef has tried to both write and remove this msg whilst - %% it's being GC'd. - %% - %% ASSERTION: [#file_summary { locked = true }] = - %% ets:lookup(FileSummaryEts, File), - State; - {_Mask, #msg_location { ref_count = RefCount, file = File, - total_size = TotalSize }} - when RefCount > 0 -> - %% only update field, otherwise bad interaction with - %% concurrent GC - Dec = fun () -> index_update_ref_count( - MsgId, RefCount - 1, State) end, - case RefCount of - %% don't remove from cur_file_cache_ets here because - %% there may be further writes in the mailbox for the - %% same msg. - 1 -> case ets:lookup(FileSummaryEts, File) of - [#file_summary { locked = true }] -> - add_to_pending_gc_completion( - {remove, MsgId, CRef}, File, State); - [#file_summary {}] -> - ok = Dec(), - delete_file_if_empty( - File, adjust_valid_total_size( - File, -TotalSize, State)) - end; - _ -> ok = Dec(), - State - end - end. - -write_message(MsgId, Msg, - State = #msstate { current_file_handle = CurHdl, - current_file = CurFile, - sum_valid_data = SumValid, - sum_file_size = SumFileSize, - file_summary_ets = FileSummaryEts }) -> - {ok, CurOffset} = file_handle_cache:current_virtual_offset(CurHdl), - {ok, TotalSize} = rabbit_msg_file:append(CurHdl, MsgId, Msg), - ok = index_insert( - #msg_location { msg_id = MsgId, ref_count = 1, file = CurFile, - offset = CurOffset, total_size = TotalSize }, State), - [#file_summary { right = undefined, locked = false }] = - ets:lookup(FileSummaryEts, CurFile), - [_,_] = ets:update_counter(FileSummaryEts, CurFile, - [{#file_summary.valid_total_size, TotalSize}, - {#file_summary.file_size, TotalSize}]), - maybe_roll_to_new_file(CurOffset + TotalSize, - State #msstate { - sum_valid_data = SumValid + TotalSize, - sum_file_size = SumFileSize + TotalSize }). - -read_message(MsgId, From, State) -> - case index_lookup_positive_ref_count(MsgId, State) of - not_found -> gen_server2:reply(From, not_found), - State; - MsgLocation -> read_message1(From, MsgLocation, State) - end. - -read_message1(From, #msg_location { msg_id = MsgId, file = File, - offset = Offset } = MsgLoc, - State = #msstate { current_file = CurFile, - current_file_handle = CurHdl, - file_summary_ets = FileSummaryEts, - cur_file_cache_ets = CurFileCacheEts }) -> - case File =:= CurFile of - true -> {Msg, State1} = - %% can return [] if msg in file existed on startup - case ets:lookup(CurFileCacheEts, MsgId) of - [] -> - {ok, RawOffSet} = - file_handle_cache:current_raw_offset(CurHdl), - ok = case Offset >= RawOffSet of - true -> file_handle_cache:flush(CurHdl); - false -> ok - end, - read_from_disk(MsgLoc, State); - [{MsgId, Msg1, _CacheRefCount}] -> - {Msg1, State} - end, - gen_server2:reply(From, {ok, Msg}), - State1; - false -> [#file_summary { locked = Locked }] = - ets:lookup(FileSummaryEts, File), - case Locked of - true -> add_to_pending_gc_completion({read, MsgId, From}, - File, State); - false -> {Msg, State1} = read_from_disk(MsgLoc, State), - gen_server2:reply(From, {ok, Msg}), - State1 - end - end. - -read_from_disk(#msg_location { msg_id = MsgId, file = File, offset = Offset, - total_size = TotalSize }, State) -> - {Hdl, State1} = get_read_handle(File, State), - {ok, Offset} = file_handle_cache:position(Hdl, Offset), - {ok, {MsgId, Msg}} = - case rabbit_msg_file:read(Hdl, TotalSize) of - {ok, {MsgId, _}} = Obj -> - Obj; - Rest -> - {error, {misread, [{old_state, State}, - {file_num, File}, - {offset, Offset}, - {msg_id, MsgId}, - {read, Rest}, - {proc_dict, get()} - ]}} - end, - {Msg, State1}. - -contains_message(MsgId, From, - State = #msstate { pending_gc_completion = Pending }) -> - case index_lookup_positive_ref_count(MsgId, State) of - not_found -> - gen_server2:reply(From, false), - State; - #msg_location { file = File } -> - case maps:is_key(File, Pending) of - true -> add_to_pending_gc_completion( - {contains, MsgId, From}, File, State); - false -> gen_server2:reply(From, true), - State - end - end. - -add_to_pending_gc_completion( - Op, File, State = #msstate { pending_gc_completion = Pending }) -> - State #msstate { pending_gc_completion = - rabbit_misc:maps_cons(File, Op, Pending) }. - -run_pending(Files, State) -> - lists:foldl( - fun (File, State1 = #msstate { pending_gc_completion = Pending }) -> - Pending1 = maps:remove(File, Pending), - lists:foldl( - fun run_pending_action/2, - State1 #msstate { pending_gc_completion = Pending1 }, - lists:reverse(maps:get(File, Pending))) - end, State, Files). - -run_pending_action({read, MsgId, From}, State) -> - read_message(MsgId, From, State); -run_pending_action({contains, MsgId, From}, State) -> - contains_message(MsgId, From, State); -run_pending_action({remove, MsgId, CRef}, State) -> - remove_message(MsgId, CRef, State). - -safe_ets_update_counter(Tab, Key, UpdateOp, SuccessFun, FailThunk) -> - try - SuccessFun(ets:update_counter(Tab, Key, UpdateOp)) - catch error:badarg -> FailThunk() - end. - -update_msg_cache(CacheEts, MsgId, Msg) -> - case ets:insert_new(CacheEts, {MsgId, Msg, 1}) of - true -> ok; - false -> safe_ets_update_counter( - CacheEts, MsgId, {3, +1}, fun (_) -> ok end, - fun () -> update_msg_cache(CacheEts, MsgId, Msg) end) - end. - -adjust_valid_total_size(File, Delta, State = #msstate { - sum_valid_data = SumValid, - file_summary_ets = FileSummaryEts }) -> - [_] = ets:update_counter(FileSummaryEts, File, - [{#file_summary.valid_total_size, Delta}]), - State #msstate { sum_valid_data = SumValid + Delta }. - -maps_store(Key, Val, Dict) -> - false = maps:is_key(Key, Dict), - maps:put(Key, Val, Dict). - -update_pending_confirms(Fun, CRef, - State = #msstate { clients = Clients, - cref_to_msg_ids = CTM }) -> - case maps:get(CRef, Clients) of - {_CPid, undefined, _CloseFDsFun} -> State; - {_CPid, MsgOnDiskFun, _CloseFDsFun} -> CTM1 = Fun(MsgOnDiskFun, CTM), - State #msstate { - cref_to_msg_ids = CTM1 } - end. - -record_pending_confirm(CRef, MsgId, State) -> - update_pending_confirms( - fun (_MsgOnDiskFun, CTM) -> - NewMsgIds = case maps:find(CRef, CTM) of - error -> gb_sets:singleton(MsgId); - {ok, MsgIds} -> gb_sets:add(MsgId, MsgIds) - end, - maps:put(CRef, NewMsgIds, CTM) - end, CRef, State). - -client_confirm(CRef, MsgIds, ActionTaken, State) -> - update_pending_confirms( - fun (MsgOnDiskFun, CTM) -> - case maps:find(CRef, CTM) of - {ok, Gs} -> MsgOnDiskFun(gb_sets:intersection(Gs, MsgIds), - ActionTaken), - MsgIds1 = rabbit_misc:gb_sets_difference( - Gs, MsgIds), - case gb_sets:is_empty(MsgIds1) of - true -> maps:remove(CRef, CTM); - false -> maps:put(CRef, MsgIds1, CTM) - end; - error -> CTM - end - end, CRef, State). - -blind_confirm(CRef, MsgIds, ActionTaken, State) -> - update_pending_confirms( - fun (MsgOnDiskFun, CTM) -> MsgOnDiskFun(MsgIds, ActionTaken), CTM end, - CRef, State). - -%% Detect whether the MsgId is older or younger than the client's death -%% msg (if there is one). If the msg is older than the client death -%% msg, and it has a 0 ref_count we must only alter the ref_count, not -%% rewrite the msg - rewriting it would make it younger than the death -%% msg and thus should be ignored. Note that this (correctly) returns -%% false when testing to remove the death msg itself. -should_mask_action(CRef, MsgId, - State = #msstate{dying_clients = DyingClients}) -> - case {maps:find(CRef, DyingClients), index_lookup(MsgId, State)} of - {error, Location} -> - {false, Location}; - {{ok, _}, not_found} -> - {true, not_found}; - {{ok, Client}, #msg_location { file = File, offset = Offset, - ref_count = RefCount } = Location} -> - #dying_client{file = DeathFile, offset = DeathOffset} = Client, - {case {{DeathFile, DeathOffset} < {File, Offset}, RefCount} of - {true, _} -> true; - {false, 0} -> false_if_increment; - {false, _} -> false - end, Location} - end. - -%%---------------------------------------------------------------------------- -%% file helper functions -%%---------------------------------------------------------------------------- - -open_file(File, Mode) -> - file_handle_cache:open_with_absolute_path( - File, ?BINARY_MODE ++ Mode, - [{write_buffer, ?HANDLE_CACHE_BUFFER_SIZE}, - {read_buffer, ?HANDLE_CACHE_BUFFER_SIZE}]). - -open_file(Dir, FileName, Mode) -> - open_file(form_filename(Dir, FileName), Mode). - -close_handle(Key, CState = #client_msstate { file_handle_cache = FHC }) -> - CState #client_msstate { file_handle_cache = close_handle(Key, FHC) }; - -close_handle(Key, State = #msstate { file_handle_cache = FHC }) -> - State #msstate { file_handle_cache = close_handle(Key, FHC) }; - -close_handle(Key, FHC) -> - case maps:find(Key, FHC) of - {ok, Hdl} -> ok = file_handle_cache:close(Hdl), - maps:remove(Key, FHC); - error -> FHC - end. - -mark_handle_open(FileHandlesEts, File, Ref) -> - %% This is fine to fail (already exists). Note it could fail with - %% the value being close, and not have it updated to open. - ets:insert_new(FileHandlesEts, {{Ref, File}, open}), - true. - -%% See comment in client_read3 - only call this when the file is locked -mark_handle_to_close(ClientRefs, FileHandlesEts, File, Invoke) -> - [ begin - case (ets:update_element(FileHandlesEts, Key, {2, close}) - andalso Invoke) of - true -> case maps:get(Ref, ClientRefs) of - {_CPid, _MsgOnDiskFun, undefined} -> - ok; - {_CPid, _MsgOnDiskFun, CloseFDsFun} -> - ok = CloseFDsFun() - end; - false -> ok - end - end || {{Ref, _File} = Key, open} <- - ets:match_object(FileHandlesEts, {{'_', File}, open}) ], - true. - -safe_file_delete_fun(File, Dir, FileHandlesEts) -> - fun () -> safe_file_delete(File, Dir, FileHandlesEts) end. - -safe_file_delete(File, Dir, FileHandlesEts) -> - %% do not match on any value - it's the absence of the row that - %% indicates the client has really closed the file. - case ets:match_object(FileHandlesEts, {{'_', File}, '_'}, 1) of - {[_|_], _Cont} -> false; - _ -> ok = file:delete( - form_filename(Dir, filenum_to_name(File))), - true - end. - --spec close_all_indicated - (client_msstate()) -> rabbit_types:ok(client_msstate()). - -close_all_indicated(#client_msstate { file_handles_ets = FileHandlesEts, - client_ref = Ref } = - CState) -> - Objs = ets:match_object(FileHandlesEts, {{Ref, '_'}, close}), - {ok, lists:foldl(fun ({Key = {_Ref, File}, close}, CStateM) -> - true = ets:delete(FileHandlesEts, Key), - close_handle(File, CStateM) - end, CState, Objs)}. - -close_all_handles(CState = #client_msstate { file_handles_ets = FileHandlesEts, - file_handle_cache = FHC, - client_ref = Ref }) -> - ok = maps:fold(fun (File, Hdl, ok) -> - true = ets:delete(FileHandlesEts, {Ref, File}), - file_handle_cache:close(Hdl) - end, ok, FHC), - CState #client_msstate { file_handle_cache = #{} }; - -close_all_handles(State = #msstate { file_handle_cache = FHC }) -> - ok = maps:fold(fun (_Key, Hdl, ok) -> file_handle_cache:close(Hdl) end, - ok, FHC), - State #msstate { file_handle_cache = #{} }. - -get_read_handle(FileNum, CState = #client_msstate { file_handle_cache = FHC, - dir = Dir }) -> - {Hdl, FHC2} = get_read_handle(FileNum, FHC, Dir), - {Hdl, CState #client_msstate { file_handle_cache = FHC2 }}; - -get_read_handle(FileNum, State = #msstate { file_handle_cache = FHC, - dir = Dir }) -> - {Hdl, FHC2} = get_read_handle(FileNum, FHC, Dir), - {Hdl, State #msstate { file_handle_cache = FHC2 }}. - -get_read_handle(FileNum, FHC, Dir) -> - case maps:find(FileNum, FHC) of - {ok, Hdl} -> {Hdl, FHC}; - error -> {ok, Hdl} = open_file(Dir, filenum_to_name(FileNum), - ?READ_MODE), - {Hdl, maps:put(FileNum, Hdl, FHC)} - end. - -preallocate(Hdl, FileSizeLimit, FinalPos) -> - {ok, FileSizeLimit} = file_handle_cache:position(Hdl, FileSizeLimit), - ok = file_handle_cache:truncate(Hdl), - {ok, FinalPos} = file_handle_cache:position(Hdl, FinalPos), - ok. - -truncate_and_extend_file(Hdl, Lowpoint, Highpoint) -> - {ok, Lowpoint} = file_handle_cache:position(Hdl, Lowpoint), - ok = file_handle_cache:truncate(Hdl), - ok = preallocate(Hdl, Highpoint, Lowpoint). - -form_filename(Dir, Name) -> filename:join(Dir, Name). - -filenum_to_name(File) -> integer_to_list(File) ++ ?FILE_EXTENSION. - -filename_to_num(FileName) -> list_to_integer(filename:rootname(FileName)). - -list_sorted_filenames(Dir, Ext) -> - lists:sort(fun (A, B) -> filename_to_num(A) < filename_to_num(B) end, - filelib:wildcard("*" ++ Ext, Dir)). - -%%---------------------------------------------------------------------------- -%% index -%%---------------------------------------------------------------------------- - -index_lookup_positive_ref_count(Key, State) -> - case index_lookup(Key, State) of - not_found -> not_found; - #msg_location { ref_count = 0 } -> not_found; - #msg_location {} = MsgLocation -> MsgLocation - end. - -index_update_ref_count(Key, RefCount, State) -> - index_update_fields(Key, {#msg_location.ref_count, RefCount}, State). - -index_lookup(Key, #gc_state { index_module = Index, - index_state = State }) -> - Index:lookup(Key, State); - -index_lookup(Key, #client_msstate { index_module = Index, - index_state = State }) -> - Index:lookup(Key, State); - -index_lookup(Key, #msstate { index_module = Index, index_state = State }) -> - Index:lookup(Key, State). - -index_insert(Obj, #msstate { index_module = Index, index_state = State }) -> - Index:insert(Obj, State). - -index_update(Obj, #msstate { index_module = Index, index_state = State }) -> - Index:update(Obj, State). - -index_update_fields(Key, Updates, #msstate{ index_module = Index, - index_state = State }) -> - Index:update_fields(Key, Updates, State); -index_update_fields(Key, Updates, #gc_state{ index_module = Index, - index_state = State }) -> - Index:update_fields(Key, Updates, State). - -index_delete(Key, #msstate { index_module = Index, index_state = State }) -> - Index:delete(Key, State). - -index_delete_object(Obj, #gc_state{ index_module = Index, - index_state = State }) -> - Index:delete_object(Obj, State). - -index_clean_up_temporary_reference_count_entries( - #msstate { index_module = Index, - index_state = State }) -> - Index:clean_up_temporary_reference_count_entries_without_file(State). - -%%---------------------------------------------------------------------------- -%% shutdown and recovery -%%---------------------------------------------------------------------------- - -recover_index_and_client_refs(IndexModule, _Recover, undefined, Dir, _Name) -> - {false, IndexModule:new(Dir), []}; -recover_index_and_client_refs(IndexModule, false, _ClientRefs, Dir, Name) -> - rabbit_log:warning("Message store ~tp: rebuilding indices from scratch~n", [Name]), - {false, IndexModule:new(Dir), []}; -recover_index_and_client_refs(IndexModule, true, ClientRefs, Dir, Name) -> - Fresh = fun (ErrorMsg, ErrorArgs) -> - rabbit_log:warning("Message store ~tp : " ++ ErrorMsg ++ "~n" - "rebuilding indices from scratch~n", - [Name | ErrorArgs]), - {false, IndexModule:new(Dir), []} - end, - case read_recovery_terms(Dir) of - {false, Error} -> - Fresh("failed to read recovery terms: ~p", [Error]); - {true, Terms} -> - RecClientRefs = proplists:get_value(client_refs, Terms, []), - RecIndexModule = proplists:get_value(index_module, Terms), - case (lists:sort(ClientRefs) =:= lists:sort(RecClientRefs) - andalso IndexModule =:= RecIndexModule) of - true -> case IndexModule:recover(Dir) of - {ok, IndexState1} -> - {true, IndexState1, ClientRefs}; - {error, Error} -> - Fresh("failed to recover index: ~p", [Error]) - end; - false -> Fresh("recovery terms differ from present", []) - end - end. - -store_recovery_terms(Terms, Dir) -> - rabbit_file:write_term_file(filename:join(Dir, ?CLEAN_FILENAME), Terms). - -read_recovery_terms(Dir) -> - Path = filename:join(Dir, ?CLEAN_FILENAME), - case rabbit_file:read_term_file(Path) of - {ok, Terms} -> case file:delete(Path) of - ok -> {true, Terms}; - {error, Error} -> {false, Error} - end; - {error, Error} -> {false, Error} - end. - -store_file_summary(Tid, Dir) -> - ets:tab2file(Tid, filename:join(Dir, ?FILE_SUMMARY_FILENAME), - [{extended_info, [object_count]}]). - -recover_file_summary(false, _Dir) -> - %% TODO: the only reason for this to be an *ordered*_set is so - %% that a) maybe_compact can start a traversal from the eldest - %% file, and b) build_index in fast recovery mode can easily - %% identify the current file. It's awkward to have both that - %% odering and the left/right pointers in the entries - replacing - %% the former with some additional bit of state would be easy, but - %% ditching the latter would be neater. - {false, ets:new(rabbit_msg_store_file_summary, - [ordered_set, public, {keypos, #file_summary.file}])}; -recover_file_summary(true, Dir) -> - Path = filename:join(Dir, ?FILE_SUMMARY_FILENAME), - case ets:file2tab(Path) of - {ok, Tid} -> ok = file:delete(Path), - {true, Tid}; - {error, _Error} -> recover_file_summary(false, Dir) - end. - -count_msg_refs(Gen, Seed, State) -> - case Gen(Seed) of - finished -> - ok; - {_MsgId, 0, Next} -> - count_msg_refs(Gen, Next, State); - {MsgId, Delta, Next} -> - ok = case index_lookup(MsgId, State) of - not_found -> - index_insert(#msg_location { msg_id = MsgId, - file = undefined, - ref_count = Delta }, - State); - #msg_location { ref_count = RefCount } = StoreEntry -> - NewRefCount = RefCount + Delta, - case NewRefCount of - 0 -> index_delete(MsgId, State); - _ -> index_update(StoreEntry #msg_location { - ref_count = NewRefCount }, - State) - end - end, - count_msg_refs(Gen, Next, State) - end. - -recover_crashed_compactions(Dir) -> - FileNames = list_sorted_filenames(Dir, ?FILE_EXTENSION), - TmpFileNames = list_sorted_filenames(Dir, ?FILE_EXTENSION_TMP), - lists:foreach( - fun (TmpFileName) -> - NonTmpRelatedFileName = - filename:rootname(TmpFileName) ++ ?FILE_EXTENSION, - true = lists:member(NonTmpRelatedFileName, FileNames), - ok = recover_crashed_compaction( - Dir, TmpFileName, NonTmpRelatedFileName) - end, TmpFileNames), - TmpFileNames == []. - -recover_crashed_compaction(Dir, TmpFileName, NonTmpRelatedFileName) -> - %% Because a msg can legitimately appear multiple times in the - %% same file, identifying the contents of the tmp file and where - %% they came from is non-trivial. If we are recovering a crashed - %% compaction then we will be rebuilding the index, which can cope - %% with duplicates appearing. Thus the simplest and safest thing - %% to do is to append the contents of the tmp file to its main - %% file. - {ok, TmpHdl} = open_file(Dir, TmpFileName, ?READ_MODE), - {ok, MainHdl} = open_file(Dir, NonTmpRelatedFileName, - ?READ_MODE ++ ?WRITE_MODE), - {ok, _End} = file_handle_cache:position(MainHdl, eof), - Size = filelib:file_size(form_filename(Dir, TmpFileName)), - {ok, Size} = file_handle_cache:copy(TmpHdl, MainHdl, Size), - ok = file_handle_cache:close(MainHdl), - ok = file_handle_cache:delete(TmpHdl), - ok. - -scan_file_for_valid_messages(File) -> - case open_file(File, ?READ_MODE) of - {ok, Hdl} -> Valid = rabbit_msg_file:scan( - Hdl, filelib:file_size(File), - fun scan_fun/2, []), - ok = file_handle_cache:close(Hdl), - Valid; - {error, enoent} -> {ok, [], 0}; - {error, Reason} -> {error, {unable_to_scan_file, - filename:basename(File), - Reason}} - end. - -scan_file_for_valid_messages(Dir, FileName) -> - scan_file_for_valid_messages(form_filename(Dir, FileName)). - -scan_fun({MsgId, TotalSize, Offset, _Msg}, Acc) -> - [{MsgId, TotalSize, Offset} | Acc]. - -%% Takes the list in *ascending* order (i.e. eldest message -%% first). This is the opposite of what scan_file_for_valid_messages -%% produces. The list of msgs that is produced is youngest first. -drop_contiguous_block_prefix(L) -> drop_contiguous_block_prefix(L, 0). - -drop_contiguous_block_prefix([], ExpectedOffset) -> - {ExpectedOffset, []}; -drop_contiguous_block_prefix([#msg_location { offset = ExpectedOffset, - total_size = TotalSize } | Tail], - ExpectedOffset) -> - ExpectedOffset1 = ExpectedOffset + TotalSize, - drop_contiguous_block_prefix(Tail, ExpectedOffset1); -drop_contiguous_block_prefix(MsgsAfterGap, ExpectedOffset) -> - {ExpectedOffset, MsgsAfterGap}. - -build_index(true, _StartupFunState, - State = #msstate { file_summary_ets = FileSummaryEts }) -> - ets:foldl( - fun (#file_summary { valid_total_size = ValidTotalSize, - file_size = FileSize, - file = File }, - {_Offset, State1 = #msstate { sum_valid_data = SumValid, - sum_file_size = SumFileSize }}) -> - {FileSize, State1 #msstate { - sum_valid_data = SumValid + ValidTotalSize, - sum_file_size = SumFileSize + FileSize, - current_file = File }} - end, {0, State}, FileSummaryEts); -build_index(false, {MsgRefDeltaGen, MsgRefDeltaGenInit}, - State = #msstate { dir = Dir }) -> - rabbit_log:debug("Rebuilding message refcount...~n", []), - ok = count_msg_refs(MsgRefDeltaGen, MsgRefDeltaGenInit, State), - rabbit_log:debug("Done rebuilding message refcount~n", []), - {ok, Pid} = gatherer:start_link(), - case [filename_to_num(FileName) || - FileName <- list_sorted_filenames(Dir, ?FILE_EXTENSION)] of - [] -> rebuild_index(Pid, [State #msstate.current_file], - State); - Files -> {Offset, State1} = rebuild_index(Pid, Files, State), - {Offset, lists:foldl(fun delete_file_if_empty/2, - State1, Files)} - end. - -build_index_worker(Gatherer, State = #msstate { dir = Dir }, - Left, File, Files) -> - FileName = filenum_to_name(File), - rabbit_log:debug("Rebuilding message location index from ~p (~B file(s) remaining)~n", - [form_filename(Dir, FileName), length(Files)]), - {ok, Messages, FileSize} = - scan_file_for_valid_messages(Dir, FileName), - {ValidMessages, ValidTotalSize} = - lists:foldl( - fun (Obj = {MsgId, TotalSize, Offset}, {VMAcc, VTSAcc}) -> - case index_lookup(MsgId, State) of - #msg_location { file = undefined } = StoreEntry -> - ok = index_update(StoreEntry #msg_location { - file = File, offset = Offset, - total_size = TotalSize }, - State), - {[Obj | VMAcc], VTSAcc + TotalSize}; - _ -> - {VMAcc, VTSAcc} - end - end, {[], 0}, Messages), - {Right, FileSize1} = - case Files of - %% if it's the last file, we'll truncate to remove any - %% rubbish above the last valid message. This affects the - %% file size. - [] -> {undefined, case ValidMessages of - [] -> 0; - _ -> {_MsgId, TotalSize, Offset} = - lists:last(ValidMessages), - Offset + TotalSize - end}; - [F|_] -> {F, FileSize} - end, - ok = gatherer:in(Gatherer, #file_summary { - file = File, - valid_total_size = ValidTotalSize, - left = Left, - right = Right, - file_size = FileSize1, - locked = false, - readers = 0 }), - ok = gatherer:finish(Gatherer). - -enqueue_build_index_workers(_Gatherer, _Left, [], _State) -> - exit(normal); -enqueue_build_index_workers(Gatherer, Left, [File|Files], State) -> - ok = worker_pool:dispatch_sync( - fun () -> - link(Gatherer), - ok = build_index_worker(Gatherer, State, - Left, File, Files), - unlink(Gatherer), - ok - end), - enqueue_build_index_workers(Gatherer, File, Files, State). - -reduce_index(Gatherer, LastFile, - State = #msstate { file_summary_ets = FileSummaryEts, - sum_valid_data = SumValid, - sum_file_size = SumFileSize }) -> - case gatherer:out(Gatherer) of - empty -> - ok = gatherer:stop(Gatherer), - ok = index_clean_up_temporary_reference_count_entries(State), - Offset = case ets:lookup(FileSummaryEts, LastFile) of - [] -> 0; - [#file_summary { file_size = FileSize }] -> FileSize - end, - {Offset, State #msstate { current_file = LastFile }}; - {value, #file_summary { valid_total_size = ValidTotalSize, - file_size = FileSize } = FileSummary} -> - true = ets:insert_new(FileSummaryEts, FileSummary), - reduce_index(Gatherer, LastFile, - State #msstate { - sum_valid_data = SumValid + ValidTotalSize, - sum_file_size = SumFileSize + FileSize }) - end. - -rebuild_index(Gatherer, Files, State) -> - lists:foreach(fun (_File) -> - ok = gatherer:fork(Gatherer) - end, Files), - Pid = spawn( - fun () -> - enqueue_build_index_workers(Gatherer, undefined, - Files, State) - end), - erlang:monitor(process, Pid), - reduce_index(Gatherer, lists:last(Files), State). - -%%---------------------------------------------------------------------------- -%% garbage collection / compaction / aggregation -- internal -%%---------------------------------------------------------------------------- - -maybe_roll_to_new_file( - Offset, - State = #msstate { dir = Dir, - current_file_handle = CurHdl, - current_file = CurFile, - file_summary_ets = FileSummaryEts, - cur_file_cache_ets = CurFileCacheEts, - file_size_limit = FileSizeLimit }) - when Offset >= FileSizeLimit -> - State1 = internal_sync(State), - ok = file_handle_cache:close(CurHdl), - NextFile = CurFile + 1, - {ok, NextHdl} = open_file(Dir, filenum_to_name(NextFile), ?WRITE_MODE), - true = ets:insert_new(FileSummaryEts, #file_summary { - file = NextFile, - valid_total_size = 0, - left = CurFile, - right = undefined, - file_size = 0, - locked = false, - readers = 0 }), - true = ets:update_element(FileSummaryEts, CurFile, - {#file_summary.right, NextFile}), - true = ets:match_delete(CurFileCacheEts, {'_', '_', 0}), - maybe_compact(State1 #msstate { current_file_handle = NextHdl, - current_file = NextFile }); -maybe_roll_to_new_file(_, State) -> - State. - -maybe_compact(State = #msstate { sum_valid_data = SumValid, - sum_file_size = SumFileSize, - gc_pid = GCPid, - pending_gc_completion = Pending, - file_summary_ets = FileSummaryEts, - file_size_limit = FileSizeLimit }) - when SumFileSize > 2 * FileSizeLimit andalso - (SumFileSize - SumValid) / SumFileSize > ?GARBAGE_FRACTION -> - %% TODO: the algorithm here is sub-optimal - it may result in a - %% complete traversal of FileSummaryEts. - First = ets:first(FileSummaryEts), - case First =:= '$end_of_table' orelse - maps:size(Pending) >= ?MAXIMUM_SIMULTANEOUS_GC_FILES of - true -> - State; - false -> - case find_files_to_combine(FileSummaryEts, FileSizeLimit, - ets:lookup(FileSummaryEts, First)) of - not_found -> - State; - {Src, Dst} -> - Pending1 = maps_store(Dst, [], - maps_store(Src, [], Pending)), - State1 = close_handle(Src, close_handle(Dst, State)), - true = ets:update_element(FileSummaryEts, Src, - {#file_summary.locked, true}), - true = ets:update_element(FileSummaryEts, Dst, - {#file_summary.locked, true}), - ok = rabbit_msg_store_gc:combine(GCPid, Src, Dst), - State1 #msstate { pending_gc_completion = Pending1 } - end - end; -maybe_compact(State) -> - State. - -find_files_to_combine(FileSummaryEts, FileSizeLimit, - [#file_summary { file = Dst, - valid_total_size = DstValid, - right = Src, - locked = DstLocked }]) -> - case Src of - undefined -> - not_found; - _ -> - [#file_summary { file = Src, - valid_total_size = SrcValid, - left = Dst, - right = SrcRight, - locked = SrcLocked }] = Next = - ets:lookup(FileSummaryEts, Src), - case SrcRight of - undefined -> not_found; - _ -> case (DstValid + SrcValid =< FileSizeLimit) andalso - (DstValid > 0) andalso (SrcValid > 0) andalso - not (DstLocked orelse SrcLocked) of - true -> {Src, Dst}; - false -> find_files_to_combine( - FileSummaryEts, FileSizeLimit, Next) - end - end - end. - -delete_file_if_empty(File, State = #msstate { current_file = File }) -> - State; -delete_file_if_empty(File, State = #msstate { - gc_pid = GCPid, - file_summary_ets = FileSummaryEts, - pending_gc_completion = Pending }) -> - [#file_summary { valid_total_size = ValidData, - locked = false }] = - ets:lookup(FileSummaryEts, File), - case ValidData of - %% don't delete the file_summary_ets entry for File here - %% because we could have readers which need to be able to - %% decrement the readers count. - 0 -> true = ets:update_element(FileSummaryEts, File, - {#file_summary.locked, true}), - ok = rabbit_msg_store_gc:delete(GCPid, File), - Pending1 = maps_store(File, [], Pending), - close_handle(File, - State #msstate { pending_gc_completion = Pending1 }); - _ -> State - end. - -cleanup_after_file_deletion(File, - #msstate { file_handles_ets = FileHandlesEts, - file_summary_ets = FileSummaryEts, - clients = Clients }) -> - %% Ensure that any clients that have open fhs to the file close - %% them before using them again. This has to be done here (given - %% it's done in the msg_store, and not the gc), and not when - %% starting up the GC, because if done when starting up the GC, - %% the client could find the close, and close and reopen the fh, - %% whilst the GC is waiting for readers to disappear, before it's - %% actually done the GC. - true = mark_handle_to_close(Clients, FileHandlesEts, File, true), - [#file_summary { left = Left, - right = Right, - locked = true, - readers = 0 }] = ets:lookup(FileSummaryEts, File), - %% We'll never delete the current file, so right is never undefined - true = Right =/= undefined, %% ASSERTION - true = ets:update_element(FileSummaryEts, Right, - {#file_summary.left, Left}), - %% ensure the double linked list is maintained - true = case Left of - undefined -> true; %% File is the eldest file (left-most) - _ -> ets:update_element(FileSummaryEts, Left, - {#file_summary.right, Right}) - end, - true = ets:delete(FileSummaryEts, File), - ok. - -%%---------------------------------------------------------------------------- -%% garbage collection / compaction / aggregation -- external -%%---------------------------------------------------------------------------- - --spec combine_files(non_neg_integer(), non_neg_integer(), gc_state()) -> - {ok, deletion_thunk()} | {defer, [non_neg_integer()]}. - -combine_files(Source, Destination, - State = #gc_state { file_summary_ets = FileSummaryEts }) -> - [#file_summary{locked = true} = SourceSummary] = - ets:lookup(FileSummaryEts, Source), - - [#file_summary{locked = true} = DestinationSummary] = - ets:lookup(FileSummaryEts, Destination), - - case {SourceSummary, DestinationSummary} of - {#file_summary{readers = 0}, #file_summary{readers = 0}} -> - {ok, do_combine_files(SourceSummary, DestinationSummary, - Source, Destination, State)}; - _ -> - rabbit_log:debug("Asked to combine files ~p and ~p but they have active readers. Deferring.", - [Source, Destination]), - DeferredFiles = [FileSummary#file_summary.file - || FileSummary <- [SourceSummary, DestinationSummary], - FileSummary#file_summary.readers /= 0], - {defer, DeferredFiles} - end. - -do_combine_files(SourceSummary, DestinationSummary, - Source, Destination, - State = #gc_state { file_summary_ets = FileSummaryEts, - file_handles_ets = FileHandlesEts, - dir = Dir, - msg_store = Server }) -> - #file_summary { - readers = 0, - left = Destination, - valid_total_size = SourceValid, - file_size = SourceFileSize, - locked = true } = SourceSummary, - #file_summary { - readers = 0, - right = Source, - valid_total_size = DestinationValid, - file_size = DestinationFileSize, - locked = true } = DestinationSummary, - - SourceName = filenum_to_name(Source), - DestinationName = filenum_to_name(Destination), - {ok, SourceHdl} = open_file(Dir, SourceName, - ?READ_AHEAD_MODE), - {ok, DestinationHdl} = open_file(Dir, DestinationName, - ?READ_AHEAD_MODE ++ ?WRITE_MODE), - TotalValidData = SourceValid + DestinationValid, - %% if DestinationValid =:= DestinationContiguousTop then we don't - %% need a tmp file - %% if they're not equal, then we need to write out everything past - %% the DestinationContiguousTop to a tmp file then truncate, - %% copy back in, and then copy over from Source - %% otherwise we just truncate straight away and copy over from Source - {DestinationWorkList, DestinationValid} = - load_and_vacuum_message_file(Destination, State), - {DestinationContiguousTop, DestinationWorkListTail} = - drop_contiguous_block_prefix(DestinationWorkList), - case DestinationWorkListTail of - [] -> ok = truncate_and_extend_file( - DestinationHdl, DestinationContiguousTop, TotalValidData); - _ -> Tmp = filename:rootname(DestinationName) ++ ?FILE_EXTENSION_TMP, - {ok, TmpHdl} = open_file(Dir, Tmp, ?READ_AHEAD_MODE++?WRITE_MODE), - ok = copy_messages( - DestinationWorkListTail, DestinationContiguousTop, - DestinationValid, DestinationHdl, TmpHdl, Destination, - State), - TmpSize = DestinationValid - DestinationContiguousTop, - %% so now Tmp contains everything we need to salvage - %% from Destination, and index_state has been updated to - %% reflect the compaction of Destination so truncate - %% Destination and copy from Tmp back to the end - {ok, 0} = file_handle_cache:position(TmpHdl, 0), - ok = truncate_and_extend_file( - DestinationHdl, DestinationContiguousTop, TotalValidData), - {ok, TmpSize} = - file_handle_cache:copy(TmpHdl, DestinationHdl, TmpSize), - %% position in DestinationHdl should now be DestinationValid - ok = file_handle_cache:sync(DestinationHdl), - ok = file_handle_cache:delete(TmpHdl) - end, - {SourceWorkList, SourceValid} = load_and_vacuum_message_file(Source, State), - ok = copy_messages(SourceWorkList, DestinationValid, TotalValidData, - SourceHdl, DestinationHdl, Destination, State), - %% tidy up - ok = file_handle_cache:close(DestinationHdl), - ok = file_handle_cache:close(SourceHdl), - - %% don't update dest.right, because it could be changing at the - %% same time - true = ets:update_element( - FileSummaryEts, Destination, - [{#file_summary.valid_total_size, TotalValidData}, - {#file_summary.file_size, TotalValidData}]), - - Reclaimed = SourceFileSize + DestinationFileSize - TotalValidData, - rabbit_log:debug("Combined segment files number ~p (source) and ~p (destination), reclaimed ~p bytes", - [Source, Destination, Reclaimed]), - gen_server2:cast(Server, {combine_files, Source, Destination, Reclaimed}), - safe_file_delete_fun(Source, Dir, FileHandlesEts). - --spec delete_file(non_neg_integer(), gc_state()) -> {ok, deletion_thunk()} | {defer, [non_neg_integer()]}. - -delete_file(File, State = #gc_state { file_summary_ets = FileSummaryEts, - file_handles_ets = FileHandlesEts, - dir = Dir, - msg_store = Server }) -> - case ets:lookup(FileSummaryEts, File) of - [#file_summary { valid_total_size = 0, - locked = true, - file_size = FileSize, - readers = 0 }] -> - {[], 0} = load_and_vacuum_message_file(File, State), - gen_server2:cast(Server, {delete_file, File, FileSize}), - {ok, safe_file_delete_fun(File, Dir, FileHandlesEts)}; - [#file_summary{readers = Readers}] when Readers > 0 -> - rabbit_log:debug("Asked to delete file ~p but it has active readers. Deferring.", - [File]), - {defer, [File]} - end. - -load_and_vacuum_message_file(File, State = #gc_state { dir = Dir }) -> - %% Messages here will be end-of-file at start-of-list - {ok, Messages, _FileSize} = - scan_file_for_valid_messages(Dir, filenum_to_name(File)), - %% foldl will reverse so will end up with msgs in ascending offset order - lists:foldl( - fun ({MsgId, TotalSize, Offset}, Acc = {List, Size}) -> - case index_lookup(MsgId, State) of - #msg_location { file = File, total_size = TotalSize, - offset = Offset, ref_count = 0 } = Entry -> - ok = index_delete_object(Entry, State), - Acc; - #msg_location { file = File, total_size = TotalSize, - offset = Offset } = Entry -> - {[ Entry | List ], TotalSize + Size}; - _ -> - Acc - end - end, {[], 0}, Messages). - -copy_messages(WorkList, InitOffset, FinalOffset, SourceHdl, DestinationHdl, - Destination, State) -> - Copy = fun ({BlockStart, BlockEnd}) -> - BSize = BlockEnd - BlockStart, - {ok, BlockStart} = - file_handle_cache:position(SourceHdl, BlockStart), - {ok, BSize} = - file_handle_cache:copy(SourceHdl, DestinationHdl, BSize) - end, - case - lists:foldl( - fun (#msg_location { msg_id = MsgId, offset = Offset, - total_size = TotalSize }, - {CurOffset, Block = {BlockStart, BlockEnd}}) -> - %% CurOffset is in the DestinationFile. - %% Offset, BlockStart and BlockEnd are in the SourceFile - %% update MsgLocation to reflect change of file and offset - ok = index_update_fields(MsgId, - [{#msg_location.file, Destination}, - {#msg_location.offset, CurOffset}], - State), - {CurOffset + TotalSize, - case BlockEnd of - undefined -> - %% base case, called only for the first list elem - {Offset, Offset + TotalSize}; - Offset -> - %% extend the current block because the - %% next msg follows straight on - {BlockStart, BlockEnd + TotalSize}; - _ -> - %% found a gap, so actually do the work for - %% the previous block - Copy(Block), - {Offset, Offset + TotalSize} - end} - end, {InitOffset, {undefined, undefined}}, WorkList) of - {FinalOffset, Block} -> - case WorkList of - [] -> ok; - _ -> Copy(Block), %% do the last remaining block - ok = file_handle_cache:sync(DestinationHdl) - end; - {FinalOffsetZ, _Block} -> - {gc_error, [{expected, FinalOffset}, - {got, FinalOffsetZ}, - {destination, Destination}]} - end. - --spec force_recovery(file:filename(), server()) -> 'ok'. - -force_recovery(BaseDir, Store) -> - Dir = filename:join(BaseDir, atom_to_list(Store)), - case file:delete(filename:join(Dir, ?CLEAN_FILENAME)) of - ok -> ok; - {error, enoent} -> ok - end, - recover_crashed_compactions(BaseDir), - ok. - -foreach_file(D, Fun, Files) -> - [ok = Fun(filename:join(D, File)) || File <- Files]. - -foreach_file(D1, D2, Fun, Files) -> - [ok = Fun(filename:join(D1, File), filename:join(D2, File)) || File <- Files]. - --spec transform_dir(file:filename(), server(), - fun ((any()) -> (rabbit_types:ok_or_error2(msg(), any())))) -> 'ok'. - -transform_dir(BaseDir, Store, TransformFun) -> - Dir = filename:join(BaseDir, atom_to_list(Store)), - TmpDir = filename:join(Dir, ?TRANSFORM_TMP), - TransformFile = fun (A, B) -> transform_msg_file(A, B, TransformFun) end, - CopyFile = fun (Src, Dst) -> {ok, _Bytes} = file:copy(Src, Dst), ok end, - case filelib:is_dir(TmpDir) of - true -> throw({error, transform_failed_previously}); - false -> FileList = list_sorted_filenames(Dir, ?FILE_EXTENSION), - foreach_file(Dir, TmpDir, TransformFile, FileList), - foreach_file(Dir, fun file:delete/1, FileList), - foreach_file(TmpDir, Dir, CopyFile, FileList), - foreach_file(TmpDir, fun file:delete/1, FileList), - ok = file:del_dir(TmpDir) - end. - -transform_msg_file(FileOld, FileNew, TransformFun) -> - ok = rabbit_file:ensure_parent_dirs_exist(FileNew), - {ok, RefOld} = file_handle_cache:open_with_absolute_path( - FileOld, [raw, binary, read], []), - {ok, RefNew} = file_handle_cache:open_with_absolute_path( - FileNew, [raw, binary, write], - [{write_buffer, ?HANDLE_CACHE_BUFFER_SIZE}]), - {ok, _Acc, _IgnoreSize} = - rabbit_msg_file:scan( - RefOld, filelib:file_size(FileOld), - fun({MsgId, _Size, _Offset, BinMsg}, ok) -> - {ok, MsgNew} = case binary_to_term(BinMsg) of - <<>> -> {ok, <<>>}; %% dying client marker - Msg -> TransformFun(Msg) - end, - {ok, _} = rabbit_msg_file:append(RefNew, MsgId, MsgNew), - ok - end, ok), - ok = file_handle_cache:close(RefOld), - ok = file_handle_cache:close(RefNew), - ok. diff --git a/src/rabbit_msg_store_ets_index.erl b/src/rabbit_msg_store_ets_index.erl deleted file mode 100644 index 294417b5ba..0000000000 --- a/src/rabbit_msg_store_ets_index.erl +++ /dev/null @@ -1,76 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_msg_store_ets_index). - --include("rabbit_msg_store.hrl"). - --behaviour(rabbit_msg_store_index). - --export([new/1, recover/1, - lookup/2, insert/2, update/2, update_fields/3, delete/2, - delete_object/2, clean_up_temporary_reference_count_entries_without_file/1, terminate/1]). - --define(MSG_LOC_NAME, rabbit_msg_store_ets_index). --define(FILENAME, "msg_store_index.ets"). - --record(state, { table, dir }). - -new(Dir) -> - file:delete(filename:join(Dir, ?FILENAME)), - Tid = ets:new(?MSG_LOC_NAME, [set, public, {keypos, #msg_location.msg_id}]), - #state { table = Tid, dir = Dir }. - -recover(Dir) -> - Path = filename:join(Dir, ?FILENAME), - case ets:file2tab(Path) of - {ok, Tid} -> file:delete(Path), - {ok, #state { table = Tid, dir = Dir }}; - Error -> Error - end. - -lookup(Key, State) -> - case ets:lookup(State #state.table, Key) of - [] -> not_found; - [Entry] -> Entry - end. - -insert(Obj, State) -> - true = ets:insert_new(State #state.table, Obj), - ok. - -update(Obj, State) -> - true = ets:insert(State #state.table, Obj), - ok. - -update_fields(Key, Updates, State) -> - true = ets:update_element(State #state.table, Key, Updates), - ok. - -delete(Key, State) -> - true = ets:delete(State #state.table, Key), - ok. - -delete_object(Obj, State) -> - true = ets:delete_object(State #state.table, Obj), - ok. - -clean_up_temporary_reference_count_entries_without_file(State) -> - MatchHead = #msg_location { file = undefined, _ = '_' }, - ets:select_delete(State #state.table, [{MatchHead, [], [true]}]), - ok. - -terminate(#state { table = MsgLocations, dir = Dir }) -> - case ets:tab2file(MsgLocations, filename:join(Dir, ?FILENAME), - [{extended_info, [object_count]}]) of - ok -> ok; - {error, Err} -> - rabbit_log:error("Unable to save message store index" - " for directory ~p.~nError: ~p~n", - [Dir, Err]) - end, - ets:delete(MsgLocations). diff --git a/src/rabbit_msg_store_gc.erl b/src/rabbit_msg_store_gc.erl deleted file mode 100644 index 41addc5fa6..0000000000 --- a/src/rabbit_msg_store_gc.erl +++ /dev/null @@ -1,125 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_msg_store_gc). - --behaviour(gen_server2). - --export([start_link/1, combine/3, delete/2, no_readers/2, stop/1]). - --export([set_maximum_since_use/2]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3, prioritise_cast/3]). - --record(state, - { pending_no_readers, - on_action, - msg_store_state - }). - --include("rabbit.hrl"). - -%%---------------------------------------------------------------------------- - --spec start_link(rabbit_msg_store:gc_state()) -> - rabbit_types:ok_pid_or_error(). - -start_link(MsgStoreState) -> - gen_server2:start_link(?MODULE, [MsgStoreState], - [{timeout, infinity}]). - --spec combine(pid(), rabbit_msg_store:file_num(), - rabbit_msg_store:file_num()) -> 'ok'. - -combine(Server, Source, Destination) -> - gen_server2:cast(Server, {combine, Source, Destination}). - --spec delete(pid(), rabbit_msg_store:file_num()) -> 'ok'. - -delete(Server, File) -> - gen_server2:cast(Server, {delete, File}). - --spec no_readers(pid(), rabbit_msg_store:file_num()) -> 'ok'. - -no_readers(Server, File) -> - gen_server2:cast(Server, {no_readers, File}). - --spec stop(pid()) -> 'ok'. - -stop(Server) -> - gen_server2:call(Server, stop, infinity). - --spec set_maximum_since_use(pid(), non_neg_integer()) -> 'ok'. - -set_maximum_since_use(Pid, Age) -> - gen_server2:cast(Pid, {set_maximum_since_use, Age}). - -%%---------------------------------------------------------------------------- - -init([MsgStoreState]) -> - ok = file_handle_cache:register_callback(?MODULE, set_maximum_since_use, - [self()]), - {ok, #state { pending_no_readers = #{}, - on_action = [], - msg_store_state = MsgStoreState }, hibernate, - {backoff, ?HIBERNATE_AFTER_MIN, ?HIBERNATE_AFTER_MIN, ?DESIRED_HIBERNATE}}. - -prioritise_cast({set_maximum_since_use, _Age}, _Len, _State) -> 8; -prioritise_cast(_Msg, _Len, _State) -> 0. - -handle_call(stop, _From, State) -> - {stop, normal, ok, State}. - -handle_cast({combine, Source, Destination}, State) -> - {noreply, attempt_action(combine, [Source, Destination], State), hibernate}; - -handle_cast({delete, File}, State) -> - {noreply, attempt_action(delete, [File], State), hibernate}; - -handle_cast({no_readers, File}, - State = #state { pending_no_readers = Pending }) -> - {noreply, case maps:find(File, Pending) of - error -> - State; - {ok, {Action, Files}} -> - Pending1 = maps:remove(File, Pending), - attempt_action( - Action, Files, - State #state { pending_no_readers = Pending1 }) - end, hibernate}; - -handle_cast({set_maximum_since_use, Age}, State) -> - ok = file_handle_cache:set_maximum_since_use(Age), - {noreply, State, hibernate}. - -handle_info(Info, State) -> - {stop, {unhandled_info, Info}, State}. - -terminate(_Reason, State) -> - State. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -attempt_action(Action, Files, - State = #state { pending_no_readers = Pending, - on_action = Thunks, - msg_store_state = MsgStoreState }) -> - case do_action(Action, Files, MsgStoreState) of - {ok, OkThunk} -> - State#state{on_action = lists:filter(fun (Thunk) -> not Thunk() end, - [OkThunk | Thunks])}; - {defer, [File | _]} -> - Pending1 = maps:put(File, {Action, Files}, Pending), - State #state { pending_no_readers = Pending1 } - end. - -do_action(combine, [Source, Destination], MsgStoreState) -> - rabbit_msg_store:combine_files(Source, Destination, MsgStoreState); -do_action(delete, [File], MsgStoreState) -> - rabbit_msg_store:delete_file(File, MsgStoreState). diff --git a/src/rabbit_networking.erl b/src/rabbit_networking.erl deleted file mode 100644 index 433b1d7540..0000000000 --- a/src/rabbit_networking.erl +++ /dev/null @@ -1,663 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_networking). - -%% This module contains various functions that deal with networking, -%% TCP and TLS listeners, and connection information. -%% -%% It also contains a boot step — boot/0 — that starts networking machinery. -%% This module primarily covers AMQP 0-9-1 but some bits are reused in -%% plugins that provide protocol support, e.g. STOMP or MQTT. -%% -%% Functions in this module take care of normalising TCP listener options, -%% including dual IP stack cases, and starting the AMQP 0-9-1 listener(s). -%% -%% See also tcp_listener_sup and tcp_listener. - --export([boot/0, start_tcp_listener/2, start_ssl_listener/3, - stop_tcp_listener/1, on_node_down/1, active_listeners/0, - node_listeners/1, node_client_listeners/1, - register_connection/1, unregister_connection/1, - register_non_amqp_connection/1, unregister_non_amqp_connection/1, - connections/0, non_amqp_connections/0, connection_info_keys/0, - connection_info/1, connection_info/2, - connection_info_all/0, connection_info_all/1, - emit_connection_info_all/4, emit_connection_info_local/3, - close_connection/2, close_connections/2, close_all_connections/1, - force_connection_event_refresh/1, force_non_amqp_connection_event_refresh/1, - handshake/2, tcp_host/1, - ranch_ref/1, ranch_ref/2, ranch_ref_of_protocol/1, - listener_of_protocol/1, stop_ranch_listener_of_protocol/1]). - -%% Used by TCP-based transports, e.g. STOMP adapter --export([tcp_listener_addresses/1, tcp_listener_spec/9, - ensure_ssl/0, fix_ssl_options/1, poodle_check/1]). - --export([tcp_listener_started/4, tcp_listener_stopped/4]). - --deprecated([{force_connection_event_refresh, 1, eventually}]). - --export([ - local_connections/0, - local_non_amqp_connections/0, - %% prefer local_connections/0 - connections_local/0 -]). - --include("rabbit.hrl"). --include("rabbit_misc.hrl"). - -%% IANA-suggested ephemeral port range is 49152 to 65535 --define(FIRST_TEST_BIND_PORT, 49152). - -%%---------------------------------------------------------------------------- - --export_type([ip_port/0, hostname/0]). - --type hostname() :: rabbit_net:hostname(). --type ip_port() :: rabbit_net:ip_port(). - --type family() :: atom(). --type listener_config() :: ip_port() | - {hostname(), ip_port()} | - {hostname(), ip_port(), family()}. --type address() :: {inet:ip_address(), ip_port(), family()}. --type name_prefix() :: atom(). --type protocol() :: atom(). --type label() :: string(). - --spec boot() -> 'ok' | no_return(). - -boot() -> - ok = record_distribution_listener(), - _ = application:start(ranch), - rabbit_log:debug("Started Ranch"), - %% Failures will throw exceptions - _ = boot_listeners(fun boot_tcp/1, application:get_env(rabbit, num_tcp_acceptors, 10), "TCP"), - _ = boot_listeners(fun boot_tls/1, application:get_env(rabbit, num_ssl_acceptors, 10), "TLS"), - ok. - -boot_listeners(Fun, NumAcceptors, Type) -> - case Fun(NumAcceptors) of - ok -> - ok; - {error, {could_not_start_listener, Address, Port, Details}} = Error -> - rabbit_log:error("Failed to start ~s listener [~s]:~p, error: ~p", - [Type, Address, Port, Details]), - throw(Error) - end. - -boot_tcp(NumAcceptors) -> - {ok, TcpListeners} = application:get_env(tcp_listeners), - case lists:foldl(fun(Listener, ok) -> - start_tcp_listener(Listener, NumAcceptors); - (_Listener, Error) -> - Error - end, - ok, TcpListeners) of - ok -> ok; - {error, _} = Error -> Error - end. - -boot_tls(NumAcceptors) -> - case application:get_env(ssl_listeners) of - {ok, []} -> - ok; - {ok, SslListeners} -> - SslOpts = ensure_ssl(), - case poodle_check('AMQP') of - ok -> [start_ssl_listener(L, SslOpts, NumAcceptors) || L <- SslListeners]; - danger -> ok - end, - ok - end. - --spec ensure_ssl() -> rabbit_types:infos(). - -ensure_ssl() -> - {ok, SslAppsConfig} = application:get_env(rabbit, ssl_apps), - ok = app_utils:start_applications(SslAppsConfig), - {ok, SslOptsConfig0} = application:get_env(rabbit, ssl_options), - rabbit_ssl_options:fix(SslOptsConfig0). - --spec poodle_check(atom()) -> 'ok' | 'danger'. - -poodle_check(Context) -> - {ok, Vsn} = application:get_key(ssl, vsn), - case rabbit_misc:version_compare(Vsn, "5.3", gte) of %% R16B01 - true -> ok; - false -> case application:get_env(rabbit, ssl_allow_poodle_attack) of - {ok, true} -> ok; - _ -> log_poodle_fail(Context), - danger - end - end. - -log_poodle_fail(Context) -> - rabbit_log:error( - "The installed version of Erlang (~s) contains the bug OTP-10905,~n" - "which makes it impossible to disable SSLv3. This makes the system~n" - "vulnerable to the POODLE attack. SSL listeners for ~s have therefore~n" - "been disabled.~n~n" - "You are advised to upgrade to a recent Erlang version; R16B01 is the~n" - "first version in which this bug is fixed, but later is usually~n" - "better.~n~n" - "If you cannot upgrade now and want to re-enable SSL listeners, you can~n" - "set the config item 'ssl_allow_poodle_attack' to 'true' in the~n" - "'rabbit' section of your configuration file.~n", - [rabbit_misc:otp_release(), Context]). - -fix_ssl_options(Config) -> - rabbit_ssl_options:fix(Config). - --spec tcp_listener_addresses(listener_config()) -> [address()]. - -tcp_listener_addresses(Port) when is_integer(Port) -> - tcp_listener_addresses_auto(Port); -tcp_listener_addresses({"auto", Port}) -> - %% Variant to prevent lots of hacking around in bash and batch files - tcp_listener_addresses_auto(Port); -tcp_listener_addresses({Host, Port}) -> - %% auto: determine family IPv4 / IPv6 after converting to IP address - tcp_listener_addresses({Host, Port, auto}); -tcp_listener_addresses({Host, Port, Family0}) - when is_integer(Port) andalso (Port >= 0) andalso (Port =< 65535) -> - [{IPAddress, Port, Family} || - {IPAddress, Family} <- getaddr(Host, Family0)]; -tcp_listener_addresses({_Host, Port, _Family0}) -> - rabbit_log:error("invalid port ~p - not 0..65535~n", [Port]), - throw({error, {invalid_port, Port}}). - -tcp_listener_addresses_auto(Port) -> - lists:append([tcp_listener_addresses(Listener) || - Listener <- port_to_listeners(Port)]). - --spec tcp_listener_spec - (name_prefix(), address(), [gen_tcp:listen_option()], module(), module(), - any(), protocol(), non_neg_integer(), label()) -> - supervisor:child_spec(). - -tcp_listener_spec(NamePrefix, {IPAddress, Port, Family}, SocketOpts, - Transport, ProtoSup, ProtoOpts, Protocol, NumAcceptors, Label) -> - Args = [IPAddress, Port, Transport, [Family | SocketOpts], ProtoSup, ProtoOpts, - {?MODULE, tcp_listener_started, [Protocol, SocketOpts]}, - {?MODULE, tcp_listener_stopped, [Protocol, SocketOpts]}, - NumAcceptors, Label], - {rabbit_misc:tcp_name(NamePrefix, IPAddress, Port), - {tcp_listener_sup, start_link, Args}, - transient, infinity, supervisor, [tcp_listener_sup]}. - --spec ranch_ref(#listener{} | [{atom(), any()}] | 'undefined') -> ranch:ref() | undefined. -ranch_ref(#listener{port = Port}) -> - [{IPAddress, Port, _Family} | _] = tcp_listener_addresses(Port), - {acceptor, IPAddress, Port}; -ranch_ref(Listener) when is_list(Listener) -> - Port = rabbit_misc:pget(port, Listener), - [{IPAddress, Port, _Family} | _] = tcp_listener_addresses(Port), - {acceptor, IPAddress, Port}; -ranch_ref(undefined) -> - undefined. - --spec ranch_ref(inet:ip_address(), ip_port()) -> ranch:ref(). - -%% Returns a reference that identifies a TCP listener in Ranch. -ranch_ref(IPAddress, Port) -> - {acceptor, IPAddress, Port}. - --spec ranch_ref_of_protocol(atom()) -> ranch:ref() | undefined. -ranch_ref_of_protocol(Protocol) -> - ranch_ref(listener_of_protocol(Protocol)). - --spec listener_of_protocol(atom()) -> #listener{}. -listener_of_protocol(Protocol) -> - rabbit_misc:execute_mnesia_transaction( - fun() -> - MatchSpec = #listener{ - node = node(), - protocol = Protocol, - _ = '_' - }, - case mnesia:match_object(rabbit_listener, MatchSpec, read) of - [] -> undefined; - [Row] -> Row - end - end). - --spec stop_ranch_listener_of_protocol(atom()) -> ok | {error, not_found}. -stop_ranch_listener_of_protocol(Protocol) -> - case rabbit_networking:ranch_ref_of_protocol(Protocol) of - undefined -> ok; - Ref -> - rabbit_log:debug("Stopping Ranch listener for protocol ~s", [Protocol]), - ranch:stop_listener(Ref) - end. - --spec start_tcp_listener( - listener_config(), integer()) -> 'ok' | {'error', term()}. - -start_tcp_listener(Listener, NumAcceptors) -> - start_listener(Listener, NumAcceptors, amqp, "TCP listener", tcp_opts()). - --spec start_ssl_listener( - listener_config(), rabbit_types:infos(), integer()) -> 'ok' | {'error', term()}. - -start_ssl_listener(Listener, SslOpts, NumAcceptors) -> - start_listener(Listener, NumAcceptors, 'amqp/ssl', "TLS (SSL) listener", tcp_opts() ++ SslOpts). - - --spec start_listener( - listener_config(), integer(), protocol(), label(), list()) -> 'ok' | {'error', term()}. -start_listener(Listener, NumAcceptors, Protocol, Label, Opts) -> - lists:foldl(fun (Address, ok) -> - start_listener0(Address, NumAcceptors, Protocol, Label, Opts); - (_Address, {error, _} = Error) -> - Error - end, ok, tcp_listener_addresses(Listener)). - -start_listener0(Address, NumAcceptors, Protocol, Label, Opts) -> - Transport = transport(Protocol), - Spec = tcp_listener_spec(rabbit_tcp_listener_sup, Address, Opts, - Transport, rabbit_connection_sup, [], Protocol, - NumAcceptors, Label), - case supervisor:start_child(rabbit_sup, Spec) of - {ok, _} -> ok; - {error, {{shutdown, {failed_to_start_child, _, - {shutdown, {failed_to_start_child, _, - {listen_error, _, PosixError}}}}}, _}} -> - {IPAddress, Port, _Family} = Address, - {error, {could_not_start_listener, rabbit_misc:ntoa(IPAddress), Port, PosixError}}; - {error, Other} -> - {IPAddress, Port, _Family} = Address, - {error, {could_not_start_listener, rabbit_misc:ntoa(IPAddress), Port, Other}} - end. - -transport(Protocol) -> - case Protocol of - amqp -> ranch_tcp; - 'amqp/ssl' -> ranch_ssl - end. - --spec stop_tcp_listener(listener_config()) -> 'ok'. - -stop_tcp_listener(Listener) -> - [stop_tcp_listener0(Address) || - Address <- tcp_listener_addresses(Listener)], - ok. - -stop_tcp_listener0({IPAddress, Port, _Family}) -> - Name = rabbit_misc:tcp_name(rabbit_tcp_listener_sup, IPAddress, Port), - ok = supervisor:terminate_child(rabbit_sup, Name), - ok = supervisor:delete_child(rabbit_sup, Name). - --spec tcp_listener_started - (_, _, - string() | - {byte(),byte(),byte(),byte()} | - {char(),char(),char(),char(),char(),char(),char(),char()}, _) -> - 'ok'. - -tcp_listener_started(Protocol, Opts, IPAddress, Port) -> - %% We need the ip to distinguish e.g. 0.0.0.0 and 127.0.0.1 - %% We need the host so we can distinguish multiple instances of the above - %% in a cluster. - ok = mnesia:dirty_write( - rabbit_listener, - #listener{node = node(), - protocol = Protocol, - host = tcp_host(IPAddress), - ip_address = IPAddress, - port = Port, - opts = Opts}). - --spec tcp_listener_stopped - (_, _, - string() | - {byte(),byte(),byte(),byte()} | - {char(),char(),char(),char(),char(),char(),char(),char()}, - _) -> - 'ok'. - -tcp_listener_stopped(Protocol, Opts, IPAddress, Port) -> - ok = mnesia:dirty_delete_object( - rabbit_listener, - #listener{node = node(), - protocol = Protocol, - host = tcp_host(IPAddress), - ip_address = IPAddress, - port = Port, - opts = Opts}). - --spec record_distribution_listener() -> ok | no_return(). - -record_distribution_listener() -> - {Name, Host} = rabbit_nodes:parts(node()), - case erl_epmd:port_please(list_to_atom(Name), Host, infinity) of - {port, Port, _Version} -> - tcp_listener_started(clustering, [], {0,0,0,0,0,0,0,0}, Port); - noport -> - throw({error, no_epmd_port}) - end. - --spec active_listeners() -> [rabbit_types:listener()]. - -active_listeners() -> - rabbit_misc:dirty_read_all(rabbit_listener). - --spec node_listeners(node()) -> [rabbit_types:listener()]. - -node_listeners(Node) -> - mnesia:dirty_read(rabbit_listener, Node). - --spec node_client_listeners(node()) -> [rabbit_types:listener()]. - -node_client_listeners(Node) -> - case node_listeners(Node) of - [] -> []; - Xs -> - lists:filter(fun (#listener{protocol = clustering}) -> false; - (_) -> true - end, Xs) - end. - --spec on_node_down(node()) -> 'ok'. - -on_node_down(Node) -> - case lists:member(Node, nodes()) of - false -> - rabbit_log:info( - "Node ~s is down, deleting its listeners~n", [Node]), - ok = mnesia:dirty_delete(rabbit_listener, Node); - true -> - rabbit_log:info( - "Keeping ~s listeners: the node is already back~n", [Node]) - end. - --spec register_connection(pid()) -> ok. - -register_connection(Pid) -> pg_local:join(rabbit_connections, Pid). - --spec unregister_connection(pid()) -> ok. - -unregister_connection(Pid) -> pg_local:leave(rabbit_connections, Pid). - --spec connections() -> [rabbit_types:connection()]. - -connections() -> - Nodes = rabbit_nodes:all_running(), - rabbit_misc:append_rpc_all_nodes(Nodes, rabbit_networking, connections_local, [], ?RPC_TIMEOUT). - --spec local_connections() -> [rabbit_types:connection()]. -%% @doc Returns pids of AMQP 0-9-1 and AMQP 1.0 connections local to this node. -local_connections() -> - connections_local(). - --spec connections_local() -> [rabbit_types:connection()]. -%% @deprecated Prefer {@link local_connections} -connections_local() -> pg_local:get_members(rabbit_connections). - --spec register_non_amqp_connection(pid()) -> ok. - -register_non_amqp_connection(Pid) -> pg_local:join(rabbit_non_amqp_connections, Pid). - --spec unregister_non_amqp_connection(pid()) -> ok. - -unregister_non_amqp_connection(Pid) -> pg_local:leave(rabbit_non_amqp_connections, Pid). - --spec non_amqp_connections() -> [rabbit_types:connection()]. - -non_amqp_connections() -> - Nodes = rabbit_nodes:all_running(), - rabbit_misc:append_rpc_all_nodes(Nodes, rabbit_networking, local_non_amqp_connections, [], ?RPC_TIMEOUT). - --spec local_non_amqp_connections() -> [rabbit_types:connection()]. -local_non_amqp_connections() -> - pg_local:get_members(rabbit_non_amqp_connections). - --spec connection_info_keys() -> rabbit_types:info_keys(). - -connection_info_keys() -> rabbit_reader:info_keys(). - --spec connection_info(rabbit_types:connection()) -> rabbit_types:infos(). - -connection_info(Pid) -> rabbit_reader:info(Pid). - --spec connection_info(rabbit_types:connection(), rabbit_types:info_keys()) -> - rabbit_types:infos(). - -connection_info(Pid, Items) -> rabbit_reader:info(Pid, Items). - --spec connection_info_all() -> [rabbit_types:infos()]. - -connection_info_all() -> cmap(fun (Q) -> connection_info(Q) end). - --spec connection_info_all(rabbit_types:info_keys()) -> - [rabbit_types:infos()]. - -connection_info_all(Items) -> cmap(fun (Q) -> connection_info(Q, Items) end). - -emit_connection_info_all(Nodes, Items, Ref, AggregatorPid) -> - Pids = [ spawn_link(Node, rabbit_networking, emit_connection_info_local, [Items, Ref, AggregatorPid]) || Node <- Nodes ], - rabbit_control_misc:await_emitters_termination(Pids), - ok. - -emit_connection_info_local(Items, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map_with_exit_handler( - AggregatorPid, Ref, fun(Q) -> connection_info(Q, Items) end, - connections_local()). - --spec close_connection(pid(), string()) -> 'ok'. - -close_connection(Pid, Explanation) -> - case lists:member(Pid, connections()) of - true -> - Res = rabbit_reader:shutdown(Pid, Explanation), - rabbit_log:info("Closing connection ~p because ~p~n", [Pid, Explanation]), - Res; - false -> - rabbit_log:warning("Asked to close connection ~p (reason: ~p) " - "but no running cluster node reported it as an active connection. Was it already closed? ~n", - [Pid, Explanation]), - ok - end. - --spec close_connections([pid()], string()) -> 'ok'. -close_connections(Pids, Explanation) -> - [close_connection(Pid, Explanation) || Pid <- Pids], - ok. - -%% Meant to be used by tests only --spec close_all_connections(string()) -> 'ok'. -close_all_connections(Explanation) -> - Pids = connections(), - [close_connection(Pid, Explanation) || Pid <- Pids], - ok. - --spec force_connection_event_refresh(reference()) -> 'ok'. -force_connection_event_refresh(Ref) -> - [rabbit_reader:force_event_refresh(C, Ref) || C <- connections()], - ok. - --spec force_non_amqp_connection_event_refresh(reference()) -> 'ok'. -force_non_amqp_connection_event_refresh(Ref) -> - [gen_server:cast(Pid, {force_event_refresh, Ref}) || Pid <- non_amqp_connections()], - ok. - --spec failed_to_recv_proxy_header(_, _) -> no_return(). -failed_to_recv_proxy_header(Ref, Error) -> - Msg = case Error of - closed -> "error when receiving proxy header: TCP socket was ~p prematurely"; - _Other -> "error when receiving proxy header: ~p" - end, - rabbit_log:debug(Msg, [Error]), - % The following call will clean up resources then exit - _ = ranch:handshake(Ref), - exit({shutdown, failed_to_recv_proxy_header}). - -handshake(Ref, ProxyProtocolEnabled) -> - case ProxyProtocolEnabled of - true -> - case ranch:recv_proxy_header(Ref, 3000) of - {error, Error} -> - failed_to_recv_proxy_header(Ref, Error); - {error, protocol_error, Error} -> - failed_to_recv_proxy_header(Ref, Error); - {ok, ProxyInfo} -> - {ok, Sock} = ranch:handshake(Ref), - setup_socket(Sock), - {ok, {rabbit_proxy_socket, Sock, ProxyInfo}} - end; - false -> - {ok, Sock} = ranch:handshake(Ref), - setup_socket(Sock), - {ok, Sock} - end. - -setup_socket(Sock) -> - ok = tune_buffer_size(Sock), - ok = file_handle_cache:obtain(). - -tune_buffer_size(Sock) -> - case tune_buffer_size1(Sock) of - ok -> ok; - {error, _} -> rabbit_net:fast_close(Sock), - exit(normal) - end. - -tune_buffer_size1(Sock) -> - case rabbit_net:getopts(Sock, [sndbuf, recbuf, buffer]) of - {ok, BufSizes} -> BufSz = lists:max([Sz || {_Opt, Sz} <- BufSizes]), - rabbit_net:setopts(Sock, [{buffer, BufSz}]); - Error -> Error - end. - -%%-------------------------------------------------------------------- - -tcp_host(IPAddress) -> - rabbit_net:tcp_host(IPAddress). - -cmap(F) -> rabbit_misc:filter_exit_map(F, connections()). - -tcp_opts() -> - {ok, ConfigOpts} = application:get_env(rabbit, tcp_listen_options), - ConfigOpts. - -%% inet_parse:address takes care of ip string, like "0.0.0.0" -%% inet:getaddr returns immediately for ip tuple {0,0,0,0}, -%% and runs 'inet_gethost' port process for dns lookups. -%% On Windows inet:getaddr runs dns resolver for ip string, which may fail. -getaddr(Host, Family) -> - case inet_parse:address(Host) of - {ok, IPAddress} -> [{IPAddress, resolve_family(IPAddress, Family)}]; - {error, _} -> gethostaddr(Host, Family) - end. - -gethostaddr(Host, auto) -> - Lookups = [{Family, inet:getaddr(Host, Family)} || Family <- [inet, inet6]], - case [{IP, Family} || {Family, {ok, IP}} <- Lookups] of - [] -> host_lookup_error(Host, Lookups); - IPs -> IPs - end; - -gethostaddr(Host, Family) -> - case inet:getaddr(Host, Family) of - {ok, IPAddress} -> [{IPAddress, Family}]; - {error, Reason} -> host_lookup_error(Host, Reason) - end. - --spec host_lookup_error(_, _) -> no_return(). -host_lookup_error(Host, Reason) -> - rabbit_log:error("invalid host ~p - ~p~n", [Host, Reason]), - throw({error, {invalid_host, Host, Reason}}). - -resolve_family({_,_,_,_}, auto) -> inet; -resolve_family({_,_,_,_,_,_,_,_}, auto) -> inet6; -resolve_family(IP, auto) -> throw({error, {strange_family, IP}}); -resolve_family(_, F) -> F. - -%%-------------------------------------------------------------------- - -%% There are three kinds of machine (for our purposes). -%% -%% * Those which treat IPv4 addresses as a special kind of IPv6 address -%% ("Single stack") -%% - Linux by default, Windows Vista and later -%% - We also treat any (hypothetical?) IPv6-only machine the same way -%% * Those which consider IPv6 and IPv4 to be completely separate things -%% ("Dual stack") -%% - OpenBSD, Windows XP / 2003, Linux if so configured -%% * Those which do not support IPv6. -%% - Ancient/weird OSes, Linux if so configured -%% -%% How to reconfigure Linux to test this: -%% Single stack (default): -%% echo 0 > /proc/sys/net/ipv6/bindv6only -%% Dual stack: -%% echo 1 > /proc/sys/net/ipv6/bindv6only -%% IPv4 only: -%% add ipv6.disable=1 to GRUB_CMDLINE_LINUX_DEFAULT in /etc/default/grub then -%% sudo update-grub && sudo reboot -%% -%% This matters in (and only in) the case where the sysadmin (or the -%% app descriptor) has only supplied a port and we wish to bind to -%% "all addresses". This means different things depending on whether -%% we're single or dual stack. On single stack binding to "::" -%% implicitly includes all IPv4 addresses, and subsequently attempting -%% to bind to "0.0.0.0" will fail. On dual stack, binding to "::" will -%% only bind to IPv6 addresses, and we need another listener bound to -%% "0.0.0.0" for IPv4. Finally, on IPv4-only systems we of course only -%% want to bind to "0.0.0.0". -%% -%% Unfortunately it seems there is no way to detect single vs dual stack -%% apart from attempting to bind to the port. -port_to_listeners(Port) -> - IPv4 = {"0.0.0.0", Port, inet}, - IPv6 = {"::", Port, inet6}, - case ipv6_status(?FIRST_TEST_BIND_PORT) of - single_stack -> [IPv6]; - ipv6_only -> [IPv6]; - dual_stack -> [IPv6, IPv4]; - ipv4_only -> [IPv4] - end. - -ipv6_status(TestPort) -> - IPv4 = [inet, {ip, {0,0,0,0}}], - IPv6 = [inet6, {ip, {0,0,0,0,0,0,0,0}}], - case gen_tcp:listen(TestPort, IPv6) of - {ok, LSock6} -> - case gen_tcp:listen(TestPort, IPv4) of - {ok, LSock4} -> - %% Dual stack - gen_tcp:close(LSock6), - gen_tcp:close(LSock4), - dual_stack; - %% Checking the error here would only let us - %% distinguish single stack IPv6 / IPv4 vs IPv6 only, - %% which we figure out below anyway. - {error, _} -> - gen_tcp:close(LSock6), - case gen_tcp:listen(TestPort, IPv4) of - %% Single stack - {ok, LSock4} -> gen_tcp:close(LSock4), - single_stack; - %% IPv6-only machine. Welcome to the future. - {error, eafnosupport} -> ipv6_only; %% Linux - {error, eprotonosupport}-> ipv6_only; %% FreeBSD - %% Dual stack machine with something already - %% on IPv4. - {error, _} -> ipv6_status(TestPort + 1) - end - end; - %% IPv4-only machine. Welcome to the 90s. - {error, eafnosupport} -> %% Linux - ipv4_only; - {error, eprotonosupport} -> %% FreeBSD - ipv4_only; - %% Port in use - {error, _} -> - ipv6_status(TestPort + 1) - end. diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl deleted file mode 100644 index b56180c54c..0000000000 --- a/src/rabbit_node_monitor.erl +++ /dev/null @@ -1,926 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_node_monitor). - -%% Transitional step until we can require Erlang/OTP 21 and -%% use the now recommended try/catch syntax for obtaining the stack trace. --compile(nowarn_deprecated_function). - --behaviour(gen_server). - --export([start_link/0]). --export([running_nodes_filename/0, - cluster_status_filename/0, quorum_filename/0, default_quorum_filename/0, - prepare_cluster_status_files/0, - write_cluster_status/1, read_cluster_status/0, - update_cluster_status/0, reset_cluster_status/0]). --export([notify_node_up/0, notify_joined_cluster/0, notify_left_cluster/1]). --export([partitions/0, partitions/1, status/1, subscribe/1]). --export([pause_partition_guard/0]). --export([global_sync/0]). - -%% gen_server callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - - %% Utils --export([all_rabbit_nodes_up/0, run_outside_applications/2, ping_all/0, - alive_nodes/1, alive_rabbit_nodes/1]). - --define(SERVER, ?MODULE). --define(NODE_REPLY_TIMEOUT, 5000). --define(RABBIT_UP_RPC_TIMEOUT, 2000). --define(RABBIT_DOWN_PING_INTERVAL, 1000). - --record(state, {monitors, partitions, subscribers, down_ping_timer, - keepalive_timer, autoheal, guid, node_guids}). - -%%---------------------------------------------------------------------------- -%% Start -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - -%%---------------------------------------------------------------------------- -%% Cluster file operations -%%---------------------------------------------------------------------------- - -%% The cluster file information is kept in two files. The "cluster -%% status file" contains all the clustered nodes and the disc nodes. -%% The "running nodes file" contains the currently running nodes or -%% the running nodes at shutdown when the node is down. -%% -%% We strive to keep the files up to date and we rely on this -%% assumption in various situations. Obviously when mnesia is offline -%% the information we have will be outdated, but it cannot be -%% otherwise. - --spec running_nodes_filename() -> string(). - -running_nodes_filename() -> - filename:join(rabbit_mnesia:dir(), "nodes_running_at_shutdown"). - --spec cluster_status_filename() -> string(). - -cluster_status_filename() -> - filename:join(rabbit_mnesia:dir(), "cluster_nodes.config"). - -quorum_filename() -> - ra_env:data_dir(). - -default_quorum_filename() -> - filename:join(rabbit_mnesia:dir(), "quorum"). - --spec prepare_cluster_status_files() -> 'ok' | no_return(). - -prepare_cluster_status_files() -> - rabbit_mnesia:ensure_mnesia_dir(), - RunningNodes1 = case try_read_file(running_nodes_filename()) of - {ok, [Nodes]} when is_list(Nodes) -> Nodes; - {ok, Other} -> corrupt_cluster_status_files(Other); - {error, enoent} -> [] - end, - ThisNode = [node()], - %% The running nodes file might contain a set or a list, in case - %% of the legacy file - RunningNodes2 = lists:usort(ThisNode ++ RunningNodes1), - {AllNodes1, DiscNodes} = - case try_read_file(cluster_status_filename()) of - {ok, [{AllNodes, DiscNodes0}]} -> - {AllNodes, DiscNodes0}; - {ok, [AllNodes0]} when is_list(AllNodes0) -> - {legacy_cluster_nodes(AllNodes0), legacy_disc_nodes(AllNodes0)}; - {ok, Files} -> - corrupt_cluster_status_files(Files); - {error, enoent} -> - LegacyNodes = legacy_cluster_nodes([]), - {LegacyNodes, LegacyNodes} - end, - AllNodes2 = lists:usort(AllNodes1 ++ RunningNodes2), - ok = write_cluster_status({AllNodes2, DiscNodes, RunningNodes2}). - --spec corrupt_cluster_status_files(any()) -> no_return(). - -corrupt_cluster_status_files(F) -> - throw({error, corrupt_cluster_status_files, F}). - --spec write_cluster_status(rabbit_mnesia:cluster_status()) -> 'ok'. - -write_cluster_status({All, Disc, Running}) -> - ClusterStatusFN = cluster_status_filename(), - Res = case rabbit_file:write_term_file(ClusterStatusFN, [{All, Disc}]) of - ok -> - RunningNodesFN = running_nodes_filename(), - {RunningNodesFN, - rabbit_file:write_term_file(RunningNodesFN, [Running])}; - E1 = {error, _} -> - {ClusterStatusFN, E1} - end, - case Res of - {_, ok} -> ok; - {FN, {error, E2}} -> throw({error, {could_not_write_file, FN, E2}}) - end. - --spec read_cluster_status() -> rabbit_mnesia:cluster_status(). - -read_cluster_status() -> - case {try_read_file(cluster_status_filename()), - try_read_file(running_nodes_filename())} of - {{ok, [{All, Disc}]}, {ok, [Running]}} when is_list(Running) -> - {All, Disc, Running}; - {Stat, Run} -> - throw({error, {corrupt_or_missing_cluster_files, Stat, Run}}) - end. - --spec update_cluster_status() -> 'ok'. - -update_cluster_status() -> - {ok, Status} = rabbit_mnesia:cluster_status_from_mnesia(), - write_cluster_status(Status). - --spec reset_cluster_status() -> 'ok'. - -reset_cluster_status() -> - write_cluster_status({[node()], [node()], [node()]}). - -%%---------------------------------------------------------------------------- -%% Cluster notifications -%%---------------------------------------------------------------------------- - --spec notify_node_up() -> 'ok'. - -notify_node_up() -> - gen_server:cast(?SERVER, notify_node_up). - --spec notify_joined_cluster() -> 'ok'. - -notify_joined_cluster() -> - Nodes = rabbit_nodes:all_running() -- [node()], - gen_server:abcast(Nodes, ?SERVER, - {joined_cluster, node(), rabbit_mnesia:node_type()}), - ok. - --spec notify_left_cluster(node()) -> 'ok'. - -notify_left_cluster(Node) -> - Nodes = rabbit_nodes:all_running(), - gen_server:abcast(Nodes, ?SERVER, {left_cluster, Node}), - ok. - -%%---------------------------------------------------------------------------- -%% Server calls -%%---------------------------------------------------------------------------- - --spec partitions() -> [node()]. - -partitions() -> - gen_server:call(?SERVER, partitions, infinity). - --spec partitions([node()]) -> [{node(), [node()]}]. - -partitions(Nodes) -> - {Replies, _} = gen_server:multi_call(Nodes, ?SERVER, partitions, ?NODE_REPLY_TIMEOUT), - Replies. - --spec status([node()]) -> {[{node(), [node()]}], [node()]}. - -status(Nodes) -> - gen_server:multi_call(Nodes, ?SERVER, status, infinity). - --spec subscribe(pid()) -> 'ok'. - -subscribe(Pid) -> - gen_server:cast(?SERVER, {subscribe, Pid}). - -%%---------------------------------------------------------------------------- -%% pause_minority/pause_if_all_down safety -%%---------------------------------------------------------------------------- - -%% If we are in a minority and pause_minority mode then a) we are -%% going to shut down imminently and b) we should not confirm anything -%% until then, since anything we confirm is likely to be lost. -%% -%% The same principles apply to a node which isn't part of the preferred -%% partition when we are in pause_if_all_down mode. -%% -%% We could confirm something by having an HA queue see the pausing -%% state (and fail over into it) before the node monitor stops us, or -%% by using unmirrored queues and just having them vanish (and -%% confirming messages as thrown away). -%% -%% So we have channels call in here before issuing confirms, to do a -%% lightweight check that we have not entered a pausing state. - --spec pause_partition_guard() -> 'ok' | 'pausing'. - -pause_partition_guard() -> - case get(pause_partition_guard) of - not_pause_mode -> - ok; - undefined -> - {ok, M} = application:get_env(rabbit, cluster_partition_handling), - case M of - pause_minority -> - pause_minority_guard([], ok); - {pause_if_all_down, PreferredNodes, _} -> - pause_if_all_down_guard(PreferredNodes, [], ok); - _ -> - put(pause_partition_guard, not_pause_mode), - ok - end; - {minority_mode, Nodes, LastState} -> - pause_minority_guard(Nodes, LastState); - {pause_if_all_down_mode, PreferredNodes, Nodes, LastState} -> - pause_if_all_down_guard(PreferredNodes, Nodes, LastState) - end. - -pause_minority_guard(LastNodes, LastState) -> - case nodes() of - LastNodes -> LastState; - _ -> NewState = case majority() of - false -> pausing; - true -> ok - end, - put(pause_partition_guard, - {minority_mode, nodes(), NewState}), - NewState - end. - -pause_if_all_down_guard(PreferredNodes, LastNodes, LastState) -> - case nodes() of - LastNodes -> LastState; - _ -> NewState = case in_preferred_partition(PreferredNodes) of - false -> pausing; - true -> ok - end, - put(pause_partition_guard, - {pause_if_all_down_mode, PreferredNodes, nodes(), - NewState}), - NewState - end. - -%%---------------------------------------------------------------------------- -%% "global" hang workaround. -%%---------------------------------------------------------------------------- - -%% This code works around a possible inconsistency in the "global" -%% state, causing global:sync/0 to never return. -%% -%% 1. A process is spawned. -%% 2. If after 15", global:sync() didn't return, the "global" -%% state is parsed. -%% 3. If it detects that a sync is blocked for more than 10", -%% the process sends fake nodedown/nodeup events to the two -%% nodes involved (one local, one remote). -%% 4. Both "global" instances restart their synchronisation. -%% 5. globao:sync() finally returns. -%% -%% FIXME: Remove this workaround, once we got rid of the change to -%% "dist_auto_connect" and fixed the bugs uncovered. - -global_sync() -> - Pid = spawn(fun workaround_global_hang/0), - ok = global:sync(), - Pid ! global_sync_done, - ok. - -workaround_global_hang() -> - receive - global_sync_done -> - ok - after 10000 -> - find_blocked_global_peers() - end. - -find_blocked_global_peers() -> - Snapshot1 = snapshot_global_dict(), - timer:sleep(10000), - Snapshot2 = snapshot_global_dict(), - find_blocked_global_peers1(Snapshot2, Snapshot1). - -snapshot_global_dict() -> - {status, _, _, [Dict | _]} = sys:get_status(global_name_server), - [E || {{sync_tag_his, _}, _} = E <- Dict]. - -find_blocked_global_peers1([{{sync_tag_his, Peer}, _} = Item | Rest], - OlderSnapshot) -> - case lists:member(Item, OlderSnapshot) of - true -> unblock_global_peer(Peer); - false -> ok - end, - find_blocked_global_peers1(Rest, OlderSnapshot); -find_blocked_global_peers1([], _) -> - ok. - -unblock_global_peer(PeerNode) -> - ThisNode = node(), - PeerState = rpc:call(PeerNode, sys, get_status, [global_name_server]), - error_logger:info_msg( - "Global hang workaround: global state on ~s seems broken~n" - " * Peer global state: ~p~n" - " * Local global state: ~p~n" - "Faking nodedown/nodeup between ~s and ~s~n", - [PeerNode, PeerState, sys:get_status(global_name_server), - PeerNode, ThisNode]), - {global_name_server, ThisNode} ! {nodedown, PeerNode}, - {global_name_server, PeerNode} ! {nodedown, ThisNode}, - {global_name_server, ThisNode} ! {nodeup, PeerNode}, - {global_name_server, PeerNode} ! {nodeup, ThisNode}, - ok. - -%%---------------------------------------------------------------------------- -%% gen_server callbacks -%%---------------------------------------------------------------------------- - -init([]) -> - %% We trap exits so that the supervisor will not just kill us. We - %% want to be sure that we are not going to be killed while - %% writing out the cluster status files - bad things can then - %% happen. - process_flag(trap_exit, true), - net_kernel:monitor_nodes(true, [nodedown_reason]), - {ok, _} = mnesia:subscribe(system), - %% If the node has been restarted, Mnesia can trigger a system notification - %% before the monitor subscribes to receive them. To avoid autoheal blocking due to - %% the inconsistent database event never arriving, we being monitoring all running - %% nodes as early as possible. The rest of the monitoring ops will only be triggered - %% when notifications arrive. - Nodes = possibly_partitioned_nodes(), - startup_log(Nodes), - Monitors = lists:foldl(fun(Node, Monitors0) -> - pmon:monitor({rabbit, Node}, Monitors0) - end, pmon:new(), Nodes), - {ok, ensure_keepalive_timer(#state{monitors = Monitors, - subscribers = pmon:new(), - partitions = [], - guid = rabbit_guid:gen(), - node_guids = maps:new(), - autoheal = rabbit_autoheal:init()})}. - -handle_call(partitions, _From, State = #state{partitions = Partitions}) -> - {reply, Partitions, State}; - -handle_call(status, _From, State = #state{partitions = Partitions}) -> - {reply, [{partitions, Partitions}, - {nodes, [node() | nodes()]}], State}; - -handle_call(_Request, _From, State) -> - {noreply, State}. - -handle_cast(notify_node_up, State = #state{guid = GUID}) -> - Nodes = rabbit_nodes:all_running() -- [node()], - gen_server:abcast(Nodes, ?SERVER, - {node_up, node(), rabbit_mnesia:node_type(), GUID}), - %% register other active rabbits with this rabbit - DiskNodes = rabbit_mnesia:cluster_nodes(disc), - [gen_server:cast(?SERVER, {node_up, N, case lists:member(N, DiskNodes) of - true -> disc; - false -> ram - end}) || N <- Nodes], - {noreply, State}; - -%%---------------------------------------------------------------------------- -%% Partial partition detection -%% -%% Every node generates a GUID each time it starts, and announces that -%% GUID in 'node_up', with 'announce_guid' sent by return so the new -%% node knows the GUIDs of the others. These GUIDs are sent in all the -%% partial partition related messages to ensure that we ignore partial -%% partition messages from before we restarted (to avoid getting stuck -%% in a loop). -%% -%% When one node gets nodedown from another, it then sends -%% 'check_partial_partition' to all the nodes it still thinks are -%% alive. If any of those (intermediate) nodes still see the "down" -%% node as up, they inform it that this has happened. The original -%% node (in 'ignore', 'pause_if_all_down' or 'autoheal' mode) will then -%% disconnect from the intermediate node to "upgrade" to a full -%% partition. -%% -%% In pause_minority mode it will instead immediately pause until all -%% nodes come back. This is because the contract for pause_minority is -%% that nodes should never sit in a partitioned state - if it just -%% disconnected, it would become a minority, pause, realise it's not -%% in a minority any more, and come back, still partitioned (albeit no -%% longer partially). -%% ---------------------------------------------------------------------------- - -handle_cast({node_up, Node, NodeType, GUID}, - State = #state{guid = MyGUID, - node_guids = GUIDs}) -> - cast(Node, {announce_guid, node(), MyGUID}), - GUIDs1 = maps:put(Node, GUID, GUIDs), - handle_cast({node_up, Node, NodeType}, State#state{node_guids = GUIDs1}); - -handle_cast({announce_guid, Node, GUID}, State = #state{node_guids = GUIDs}) -> - {noreply, State#state{node_guids = maps:put(Node, GUID, GUIDs)}}; - -handle_cast({check_partial_partition, Node, Rep, NodeGUID, MyGUID, RepGUID}, - State = #state{guid = MyGUID, - node_guids = GUIDs}) -> - case lists:member(Node, rabbit_nodes:all_running()) andalso - maps:find(Node, GUIDs) =:= {ok, NodeGUID} of - true -> spawn_link( %%[1] - fun () -> - case rpc:call(Node, rabbit, is_running, []) of - {badrpc, _} -> ok; - _ -> - rabbit_log:warning("Received a 'DOWN' message" - " from ~p but still can" - " communicate with it ~n", - [Node]), - cast(Rep, {partial_partition, - Node, node(), RepGUID}) - end - end); - false -> ok - end, - {noreply, State}; -%% [1] We checked that we haven't heard the node go down - but we -%% really should make sure we can actually communicate with -%% it. Otherwise there's a race where we falsely detect a partial -%% partition. -%% -%% Now of course the rpc:call/4 may take a long time to return if -%% connectivity with the node is actually interrupted - but that's OK, -%% we only really want to do something in a timely manner if -%% connectivity is OK. However, of course as always we must not block -%% the node monitor, so we do the check in a separate process. - -handle_cast({check_partial_partition, _Node, _Reporter, - _NodeGUID, _GUID, _ReporterGUID}, State) -> - {noreply, State}; - -handle_cast({partial_partition, NotReallyDown, Proxy, MyGUID}, - State = #state{guid = MyGUID}) -> - FmtBase = "Partial partition detected:~n" - " * We saw DOWN from ~s~n" - " * We can still see ~s which can see ~s~n", - ArgsBase = [NotReallyDown, Proxy, NotReallyDown], - case application:get_env(rabbit, cluster_partition_handling) of - {ok, pause_minority} -> - rabbit_log:error( - FmtBase ++ " * pause_minority mode enabled~n" - "We will therefore pause until the *entire* cluster recovers~n", - ArgsBase), - await_cluster_recovery(fun all_nodes_up/0), - {noreply, State}; - {ok, {pause_if_all_down, PreferredNodes, _}} -> - case in_preferred_partition(PreferredNodes) of - true -> rabbit_log:error( - FmtBase ++ "We will therefore intentionally " - "disconnect from ~s~n", ArgsBase ++ [Proxy]), - upgrade_to_full_partition(Proxy); - false -> rabbit_log:info( - FmtBase ++ "We are about to pause, no need " - "for further actions~n", ArgsBase) - end, - {noreply, State}; - {ok, _} -> - rabbit_log:error( - FmtBase ++ "We will therefore intentionally disconnect from ~s~n", - ArgsBase ++ [Proxy]), - upgrade_to_full_partition(Proxy), - {noreply, State} - end; - -handle_cast({partial_partition, _GUID, _Reporter, _Proxy}, State) -> - {noreply, State}; - -%% Sometimes it appears the Erlang VM does not give us nodedown -%% messages reliably when another node disconnects from us. Therefore -%% we are told just before the disconnection so we can reciprocate. -handle_cast({partial_partition_disconnect, Other}, State) -> - rabbit_log:error("Partial partition disconnect from ~s~n", [Other]), - disconnect(Other), - {noreply, State}; - -%% Note: when updating the status file, we can't simply write the -%% mnesia information since the message can (and will) overtake the -%% mnesia propagation. -handle_cast({node_up, Node, NodeType}, - State = #state{monitors = Monitors}) -> - rabbit_log:info("rabbit on node ~p up~n", [Node]), - {AllNodes, DiscNodes, RunningNodes} = read_cluster_status(), - write_cluster_status({add_node(Node, AllNodes), - case NodeType of - disc -> add_node(Node, DiscNodes); - ram -> DiscNodes - end, - add_node(Node, RunningNodes)}), - ok = handle_live_rabbit(Node), - Monitors1 = case pmon:is_monitored({rabbit, Node}, Monitors) of - true -> - Monitors; - false -> - pmon:monitor({rabbit, Node}, Monitors) - end, - {noreply, maybe_autoheal(State#state{monitors = Monitors1})}; - -handle_cast({joined_cluster, Node, NodeType}, State) -> - {AllNodes, DiscNodes, RunningNodes} = read_cluster_status(), - write_cluster_status({add_node(Node, AllNodes), - case NodeType of - disc -> add_node(Node, DiscNodes); - ram -> DiscNodes - end, - RunningNodes}), - {noreply, State}; - -handle_cast({left_cluster, Node}, State) -> - {AllNodes, DiscNodes, RunningNodes} = read_cluster_status(), - write_cluster_status({del_node(Node, AllNodes), del_node(Node, DiscNodes), - del_node(Node, RunningNodes)}), - {noreply, State}; - -handle_cast({subscribe, Pid}, State = #state{subscribers = Subscribers}) -> - {noreply, State#state{subscribers = pmon:monitor(Pid, Subscribers)}}; - -handle_cast(keepalive, State) -> - {noreply, State}; - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info({'DOWN', _MRef, process, {rabbit, Node}, _Reason}, - State = #state{monitors = Monitors, subscribers = Subscribers}) -> - rabbit_log:info("rabbit on node ~p down~n", [Node]), - {AllNodes, DiscNodes, RunningNodes} = read_cluster_status(), - write_cluster_status({AllNodes, DiscNodes, del_node(Node, RunningNodes)}), - [P ! {node_down, Node} || P <- pmon:monitored(Subscribers)], - {noreply, handle_dead_rabbit( - Node, - State#state{monitors = pmon:erase({rabbit, Node}, Monitors)})}; - -handle_info({'DOWN', _MRef, process, Pid, _Reason}, - State = #state{subscribers = Subscribers}) -> - {noreply, State#state{subscribers = pmon:erase(Pid, Subscribers)}}; - -handle_info({nodedown, Node, Info}, State = #state{guid = MyGUID, - node_guids = GUIDs}) -> - rabbit_log:info("node ~p down: ~p~n", - [Node, proplists:get_value(nodedown_reason, Info)]), - Check = fun (N, CheckGUID, DownGUID) -> - cast(N, {check_partial_partition, - Node, node(), DownGUID, CheckGUID, MyGUID}) - end, - case maps:find(Node, GUIDs) of - {ok, DownGUID} -> Alive = rabbit_nodes:all_running() - -- [node(), Node], - [case maps:find(N, GUIDs) of - {ok, CheckGUID} -> Check(N, CheckGUID, DownGUID); - error -> ok - end || N <- Alive]; - error -> ok - end, - {noreply, handle_dead_node(Node, State)}; - -handle_info({nodeup, Node, _Info}, State) -> - rabbit_log:info("node ~p up~n", [Node]), - {noreply, State}; - -handle_info({mnesia_system_event, - {inconsistent_database, running_partitioned_network, Node}}, - State = #state{partitions = Partitions, - monitors = Monitors}) -> - %% We will not get a node_up from this node - yet we should treat it as - %% up (mostly). - State1 = case pmon:is_monitored({rabbit, Node}, Monitors) of - true -> State; - false -> State#state{ - monitors = pmon:monitor({rabbit, Node}, Monitors)} - end, - ok = handle_live_rabbit(Node), - Partitions1 = lists:usort([Node | Partitions]), - {noreply, maybe_autoheal(State1#state{partitions = Partitions1})}; - -handle_info({autoheal_msg, Msg}, State = #state{autoheal = AState, - partitions = Partitions}) -> - AState1 = rabbit_autoheal:handle_msg(Msg, AState, Partitions), - {noreply, State#state{autoheal = AState1}}; - -handle_info(ping_down_nodes, State) -> - %% We ping nodes when some are down to ensure that we find out - %% about healed partitions quickly. We ping all nodes rather than - %% just the ones we know are down for simplicity; it's not expensive - %% to ping the nodes that are up, after all. - State1 = State#state{down_ping_timer = undefined}, - Self = self(), - %% We ping in a separate process since in a partition it might - %% take some noticeable length of time and we don't want to block - %% the node monitor for that long. - spawn_link(fun () -> - ping_all(), - case all_nodes_up() of - true -> ok; - false -> Self ! ping_down_nodes_again - end - end), - {noreply, State1}; - -handle_info(ping_down_nodes_again, State) -> - {noreply, ensure_ping_timer(State)}; - -handle_info(ping_up_nodes, State) -> - %% In this case we need to ensure that we ping "quickly" - - %% i.e. only nodes that we know to be up. - [cast(N, keepalive) || N <- alive_nodes() -- [node()]], - {noreply, ensure_keepalive_timer(State#state{keepalive_timer = undefined})}; - -handle_info({'EXIT', _, _} = Info, State = #state{autoheal = AState0}) -> - AState = rabbit_autoheal:process_down(Info, AState0), - {noreply, State#state{autoheal = AState}}; - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, State) -> - rabbit_misc:stop_timer(State, #state.down_ping_timer), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%---------------------------------------------------------------------------- -%% Functions that call the module specific hooks when nodes go up/down -%%---------------------------------------------------------------------------- - -handle_dead_node(Node, State = #state{autoheal = Autoheal}) -> - %% In general in rabbit_node_monitor we care about whether the - %% rabbit application is up rather than the node; we do this so - %% that we can respond in the same way to "rabbitmqctl stop_app" - %% and "rabbitmqctl stop" as much as possible. - %% - %% However, for pause_minority and pause_if_all_down modes we can't do - %% this, since we depend on looking at whether other nodes are up - %% to decide whether to come back up ourselves - if we decide that - %% based on the rabbit application we would go down and never come - %% back. - case application:get_env(rabbit, cluster_partition_handling) of - {ok, pause_minority} -> - case majority([Node]) of - true -> ok; - false -> await_cluster_recovery(fun majority/0) - end, - State; - {ok, {pause_if_all_down, PreferredNodes, HowToRecover}} -> - case in_preferred_partition(PreferredNodes, [Node]) of - true -> ok; - false -> await_cluster_recovery( - fun in_preferred_partition/0) - end, - case HowToRecover of - autoheal -> State#state{autoheal = - rabbit_autoheal:node_down(Node, Autoheal)}; - _ -> State - end; - {ok, ignore} -> - State; - {ok, autoheal} -> - State#state{autoheal = rabbit_autoheal:node_down(Node, Autoheal)}; - {ok, Term} -> - rabbit_log:warning("cluster_partition_handling ~p unrecognised, " - "assuming 'ignore'~n", [Term]), - State - end. - -await_cluster_recovery(Condition) -> - rabbit_log:warning("Cluster minority/secondary status detected - " - "awaiting recovery~n", []), - run_outside_applications(fun () -> - rabbit:stop(), - wait_for_cluster_recovery(Condition) - end, false), - ok. - -run_outside_applications(Fun, WaitForExistingProcess) -> - spawn_link(fun () -> - %% Ignore exit messages from the monitor - the link is needed - %% to ensure the monitor detects abnormal exits from this process - %% and can reset the 'restarting' status on the autoheal, avoiding - %% a deadlock. The monitor is restarted when rabbit does, so messages - %% in the other direction should be ignored. - process_flag(trap_exit, true), - %% If our group leader is inside an application we are about - %% to stop, application:stop/1 does not return. - group_leader(whereis(init), self()), - register_outside_app_process(Fun, WaitForExistingProcess) - end). - -register_outside_app_process(Fun, WaitForExistingProcess) -> - %% Ensure only one such process at a time, the exit(badarg) is - %% harmless if one is already running. - %% - %% If WaitForExistingProcess is false, the given fun is simply not - %% executed at all and the process exits. - %% - %% If WaitForExistingProcess is true, we wait for the end of the - %% currently running process before executing the given function. - try register(rabbit_outside_app_process, self()) of - true -> - do_run_outside_app_fun(Fun) - catch - error:badarg when WaitForExistingProcess -> - MRef = erlang:monitor(process, rabbit_outside_app_process), - receive - {'DOWN', MRef, _, _, _} -> - %% The existing process exited, let's try to - %% register again. - register_outside_app_process(Fun, WaitForExistingProcess) - end; - error:badarg -> - ok - end. - -do_run_outside_app_fun(Fun) -> - try - Fun() - catch _:E:Stacktrace -> - rabbit_log:error( - "rabbit_outside_app_process:~n~p~n~p~n", - [E, Stacktrace]) - end. - -wait_for_cluster_recovery(Condition) -> - ping_all(), - case Condition() of - true -> rabbit:start(); - false -> timer:sleep(?RABBIT_DOWN_PING_INTERVAL), - wait_for_cluster_recovery(Condition) - end. - -handle_dead_rabbit(Node, State = #state{partitions = Partitions, - autoheal = Autoheal}) -> - %% TODO: This may turn out to be a performance hog when there are - %% lots of nodes. We really only need to execute some of these - %% statements on *one* node, rather than all of them. - ok = rabbit_networking:on_node_down(Node), - ok = rabbit_amqqueue:on_node_down(Node), - ok = rabbit_alarm:on_node_down(Node), - ok = rabbit_mnesia:on_node_down(Node), - %% If we have been partitioned, and we are now in the only remaining - %% partition, we no longer care about partitions - forget them. Note - %% that we do not attempt to deal with individual (other) partitions - %% going away. It's only safe to forget anything about partitions when - %% there are no partitions. - Down = Partitions -- alive_rabbit_nodes(), - NoLongerPartitioned = rabbit_nodes:all_running(), - Partitions1 = case Partitions -- Down -- NoLongerPartitioned of - [] -> []; - _ -> Partitions - end, - ensure_ping_timer( - State#state{partitions = Partitions1, - autoheal = rabbit_autoheal:rabbit_down(Node, Autoheal)}). - -ensure_ping_timer(State) -> - rabbit_misc:ensure_timer( - State, #state.down_ping_timer, ?RABBIT_DOWN_PING_INTERVAL, - ping_down_nodes). - -ensure_keepalive_timer(State) -> - {ok, Interval} = application:get_env(rabbit, cluster_keepalive_interval), - rabbit_misc:ensure_timer( - State, #state.keepalive_timer, Interval, ping_up_nodes). - -handle_live_rabbit(Node) -> - ok = rabbit_amqqueue:on_node_up(Node), - ok = rabbit_alarm:on_node_up(Node), - ok = rabbit_mnesia:on_node_up(Node). - -maybe_autoheal(State = #state{partitions = []}) -> - State; - -maybe_autoheal(State = #state{autoheal = AState}) -> - case all_nodes_up() of - true -> State#state{autoheal = rabbit_autoheal:maybe_start(AState)}; - false -> State - end. - -%%-------------------------------------------------------------------- -%% Internal utils -%%-------------------------------------------------------------------- - -try_read_file(FileName) -> - case rabbit_file:read_term_file(FileName) of - {ok, Term} -> {ok, Term}; - {error, enoent} -> {error, enoent}; - {error, E} -> throw({error, {cannot_read_file, FileName, E}}) - end. - -legacy_cluster_nodes(Nodes) -> - %% We get all the info that we can, including the nodes from - %% mnesia, which will be there if the node is a disc node (empty - %% list otherwise) - lists:usort(Nodes ++ mnesia:system_info(db_nodes)). - -legacy_disc_nodes(AllNodes) -> - case AllNodes == [] orelse lists:member(node(), AllNodes) of - true -> [node()]; - false -> [] - end. - -add_node(Node, Nodes) -> lists:usort([Node | Nodes]). - -del_node(Node, Nodes) -> Nodes -- [Node]. - -cast(Node, Msg) -> gen_server:cast({?SERVER, Node}, Msg). - -upgrade_to_full_partition(Proxy) -> - cast(Proxy, {partial_partition_disconnect, node()}), - disconnect(Proxy). - -%% When we call this, it's because we want to force Mnesia to detect a -%% partition. But if we just disconnect_node/1 then Mnesia won't -%% detect a very short partition. So we want to force a slightly -%% longer disconnect. Unfortunately we don't have a way to blacklist -%% individual nodes; the best we can do is turn off auto-connect -%% altogether. -disconnect(Node) -> - application:set_env(kernel, dist_auto_connect, never), - erlang:disconnect_node(Node), - timer:sleep(1000), - application:unset_env(kernel, dist_auto_connect), - ok. - -%%-------------------------------------------------------------------- - -%% mnesia:system_info(db_nodes) (and hence -%% rabbit_nodes:all_running()) does not return all nodes -%% when partitioned, just those that we are sharing Mnesia state -%% with. So we have a small set of replacement functions -%% here. "rabbit" in a function's name implies we test if the rabbit -%% application is up, not just the node. - -%% As we use these functions to decide what to do in pause_minority or -%% pause_if_all_down states, they *must* be fast, even in the case where -%% TCP connections are timing out. So that means we should be careful -%% about whether we connect to nodes which are currently disconnected. - -majority() -> - majority([]). - -majority(NodesDown) -> - Nodes = rabbit_mnesia:cluster_nodes(all), - AliveNodes = alive_nodes(Nodes) -- NodesDown, - length(AliveNodes) / length(Nodes) > 0.5. - -in_preferred_partition() -> - {ok, {pause_if_all_down, PreferredNodes, _}} = - application:get_env(rabbit, cluster_partition_handling), - in_preferred_partition(PreferredNodes). - -in_preferred_partition(PreferredNodes) -> - in_preferred_partition(PreferredNodes, []). - -in_preferred_partition(PreferredNodes, NodesDown) -> - Nodes = rabbit_mnesia:cluster_nodes(all), - RealPreferredNodes = [N || N <- PreferredNodes, lists:member(N, Nodes)], - AliveNodes = alive_nodes(RealPreferredNodes) -- NodesDown, - RealPreferredNodes =:= [] orelse AliveNodes =/= []. - -all_nodes_up() -> - Nodes = rabbit_mnesia:cluster_nodes(all), - length(alive_nodes(Nodes)) =:= length(Nodes). - --spec all_rabbit_nodes_up() -> boolean(). - -all_rabbit_nodes_up() -> - Nodes = rabbit_mnesia:cluster_nodes(all), - length(alive_rabbit_nodes(Nodes)) =:= length(Nodes). - --spec alive_nodes([node()]) -> [node()]. - -alive_nodes() -> alive_nodes(rabbit_mnesia:cluster_nodes(all)). -alive_nodes(Nodes) -> [N || N <- Nodes, lists:member(N, [node()|nodes()])]. - --spec alive_rabbit_nodes([node()]) -> [node()]. - -alive_rabbit_nodes() -> alive_rabbit_nodes(rabbit_mnesia:cluster_nodes(all)). - -alive_rabbit_nodes(Nodes) -> - [N || N <- alive_nodes(Nodes), rabbit:is_running(N)]. - -%% This one is allowed to connect! - --spec ping_all() -> 'ok'. - -ping_all() -> - [net_adm:ping(N) || N <- rabbit_mnesia:cluster_nodes(all)], - ok. - -possibly_partitioned_nodes() -> - alive_rabbit_nodes() -- rabbit_nodes:all_running(). - -startup_log([]) -> - rabbit_log:info("Starting rabbit_node_monitor~n", []); -startup_log(Nodes) -> - rabbit_log:info("Starting rabbit_node_monitor, might be partitioned from ~p~n", - [Nodes]). diff --git a/src/rabbit_nodes.erl b/src/rabbit_nodes.erl deleted file mode 100644 index 3034a4d513..0000000000 --- a/src/rabbit_nodes.erl +++ /dev/null @@ -1,157 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_nodes). - --export([names/1, diagnostics/1, make/1, make/2, parts/1, cookie_hash/0, - is_running/2, is_process_running/2, - cluster_name/0, set_cluster_name/1, set_cluster_name/2, ensure_epmd/0, - all_running/0, name_type/0, running_count/0, total_count/0, - await_running_count/2, is_single_node_cluster/0, - boot/0]). --export([persistent_cluster_id/0, seed_internal_cluster_id/0, seed_user_provided_cluster_name/0]). - --include_lib("kernel/include/inet.hrl"). --include_lib("rabbit_common/include/rabbit.hrl"). - --define(SAMPLING_INTERVAL, 1000). - --define(INTERNAL_CLUSTER_ID_PARAM_NAME, internal_cluster_id). - -%%---------------------------------------------------------------------------- -%% API -%%---------------------------------------------------------------------------- - -boot() -> - seed_internal_cluster_id(), - seed_user_provided_cluster_name(). - -name_type() -> - #{nodename_type := NodeType} = rabbit_prelaunch:get_context(), - NodeType. - --spec names(string()) -> - rabbit_types:ok_or_error2([{string(), integer()}], term()). - -names(Hostname) -> - rabbit_nodes_common:names(Hostname). - --spec diagnostics([node()]) -> string(). - -diagnostics(Nodes) -> - rabbit_nodes_common:diagnostics(Nodes). - -make(NameOrParts) -> - rabbit_nodes_common:make(NameOrParts). - -make(ShortName, Hostname) -> - make({ShortName, Hostname}). - -parts(NodeStr) -> - rabbit_nodes_common:parts(NodeStr). - --spec cookie_hash() -> string(). - -cookie_hash() -> - rabbit_nodes_common:cookie_hash(). - --spec is_running(node(), atom()) -> boolean(). - -is_running(Node, Application) -> - rabbit_nodes_common:is_running(Node, Application). - --spec is_process_running(node(), atom()) -> boolean(). - -is_process_running(Node, Process) -> - rabbit_nodes_common:is_process_running(Node, Process). - --spec cluster_name() -> binary(). - -cluster_name() -> - rabbit_runtime_parameters:value_global( - cluster_name, cluster_name_default()). - -cluster_name_default() -> - {ID, _} = parts(node()), - FQDN = rabbit_net:hostname(), - list_to_binary(atom_to_list(make({ID, FQDN}))). - --spec persistent_cluster_id() -> binary(). -persistent_cluster_id() -> - case rabbit_runtime_parameters:lookup_global(?INTERNAL_CLUSTER_ID_PARAM_NAME) of - not_found -> - seed_internal_cluster_id(), - persistent_cluster_id(); - Param -> - #{value := Val, name := ?INTERNAL_CLUSTER_ID_PARAM_NAME} = maps:from_list(Param), - Val - end. - --spec seed_internal_cluster_id() -> binary(). -seed_internal_cluster_id() -> - case rabbit_runtime_parameters:lookup_global(?INTERNAL_CLUSTER_ID_PARAM_NAME) of - not_found -> - Id = rabbit_guid:binary(rabbit_guid:gen(), "rabbitmq-cluster-id"), - rabbit_log:info("Initialising internal cluster ID to '~s'", [Id]), - rabbit_runtime_parameters:set_global(?INTERNAL_CLUSTER_ID_PARAM_NAME, Id, ?INTERNAL_USER), - Id; - Param -> - #{value := Val, name := ?INTERNAL_CLUSTER_ID_PARAM_NAME} = maps:from_list(Param), - Val - end. - -seed_user_provided_cluster_name() -> - case application:get_env(rabbit, cluster_name) of - undefined -> ok; - {ok, Name} -> - rabbit_log:info("Setting cluster name to '~s' as configured", [Name]), - set_cluster_name(rabbit_data_coercion:to_binary(Name)) - end. - --spec set_cluster_name(binary()) -> 'ok'. - -set_cluster_name(Name) -> - set_cluster_name(Name, ?INTERNAL_USER). - --spec set_cluster_name(binary(), rabbit_types:username()) -> 'ok'. - -set_cluster_name(Name, Username) -> - %% Cluster name should be binary - BinaryName = rabbit_data_coercion:to_binary(Name), - rabbit_runtime_parameters:set_global(cluster_name, BinaryName, Username). - -ensure_epmd() -> - rabbit_nodes_common:ensure_epmd(). - --spec all_running() -> [node()]. -all_running() -> rabbit_mnesia:cluster_nodes(running). - --spec running_count() -> integer(). -running_count() -> length(all_running()). - --spec total_count() -> integer(). -total_count() -> length(rabbit_mnesia:cluster_nodes(all)). - --spec is_single_node_cluster() -> boolean(). -is_single_node_cluster() -> - total_count() =:= 1. - --spec await_running_count(integer(), integer()) -> 'ok' | {'error', atom()}. -await_running_count(TargetCount, Timeout) -> - Retries = round(Timeout/?SAMPLING_INTERVAL), - await_running_count_with_retries(TargetCount, Retries). - -await_running_count_with_retries(1, _Retries) -> ok; -await_running_count_with_retries(_TargetCount, Retries) when Retries =:= 0 -> - {error, timeout}; -await_running_count_with_retries(TargetCount, Retries) -> - case running_count() >= TargetCount of - true -> ok; - false -> - timer:sleep(?SAMPLING_INTERVAL), - await_running_count_with_retries(TargetCount, Retries - 1) - end. diff --git a/src/rabbit_osiris_metrics.erl b/src/rabbit_osiris_metrics.erl deleted file mode 100644 index 7b2574c7e1..0000000000 --- a/src/rabbit_osiris_metrics.erl +++ /dev/null @@ -1,103 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License -%% at https://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and -%% limitations under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% Copyright (c) 2012-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_osiris_metrics). - --behaviour(gen_server). - --export([start_link/0]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --define(TICK_TIMEOUT, 5000). --define(SERVER, ?MODULE). - --define(STATISTICS_KEYS, - [policy, - operator_policy, - effective_policy_definition, - state, - leader, - online, - members - ]). - --record(state, {timeout :: non_neg_integer()}). - -%%---------------------------------------------------------------------------- -%% Starts the raw metrics storage and owns the ETS tables. -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). - -init([]) -> - Timeout = application:get_env(rabbit, stream_tick_interval, - ?TICK_TIMEOUT), - erlang:send_after(Timeout, self(), tick), - {ok, #state{timeout = Timeout}}. - -handle_call(_Request, _From, State) -> - {noreply, State}. - -handle_cast(_Request, State) -> - {noreply, State}. - -handle_info(tick, #state{timeout = Timeout} = State) -> - Data = osiris_counters:overview(), - maps:map( - fun ({osiris_writer, QName}, #{offset := Offs, - first_offset := FstOffs}) -> - COffs = Offs + 1 - FstOffs, - rabbit_core_metrics:queue_stats(QName, COffs, 0, COffs, 0), - Infos = try - %% TODO complete stats! - case rabbit_amqqueue:lookup(QName) of - {ok, Q} -> - rabbit_stream_queue:info(Q, ?STATISTICS_KEYS); - _ -> - [] - end - catch - _:_ -> - %% It's possible that the writer has died but - %% it's still on the amqqueue record, so the - %% `erlang:process_info/2` calls will return - %% `undefined` and crash with a badmatch. - %% At least for now, skipping the metrics might - %% be the best option. Otherwise this brings - %% down `rabbit_sup` and the whole `rabbit` app. - [] - end, - rabbit_core_metrics:queue_stats(QName, Infos), - rabbit_event:notify(queue_stats, Infos ++ [{name, QName}, - {messages, COffs}, - {messages_ready, COffs}, - {messages_unacknowledged, 0}]), - ok; - (_, _V) -> - ok - end, Data), - erlang:send_after(Timeout, self(), tick), - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. diff --git a/src/rabbit_parameter_validation.erl b/src/rabbit_parameter_validation.erl deleted file mode 100644 index 66287ec799..0000000000 --- a/src/rabbit_parameter_validation.erl +++ /dev/null @@ -1,88 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_parameter_validation). - --export([number/2, integer/2, binary/2, boolean/2, list/2, regex/2, proplist/3, enum/1]). - -number(_Name, Term) when is_number(Term) -> - ok; - -number(Name, Term) -> - {error, "~s should be a number, actually was ~p", [Name, Term]}. - -integer(_Name, Term) when is_integer(Term) -> - ok; - -integer(Name, Term) -> - {error, "~s should be a number, actually was ~p", [Name, Term]}. - -binary(_Name, Term) when is_binary(Term) -> - ok; - -binary(Name, Term) -> - {error, "~s should be binary, actually was ~p", [Name, Term]}. - -boolean(_Name, Term) when is_boolean(Term) -> - ok; -boolean(Name, Term) -> - {error, "~s should be boolean, actually was ~p", [Name, Term]}. - -list(_Name, Term) when is_list(Term) -> - ok; - -list(Name, Term) -> - {error, "~s should be list, actually was ~p", [Name, Term]}. - -regex(Name, Term) when is_binary(Term) -> - case re:compile(Term) of - {ok, _} -> ok; - {error, Reason} -> {error, "~s should be regular expression " - "but is invalid: ~p", [Name, Reason]} - end; -regex(Name, Term) -> - {error, "~s should be a binary but was ~p", [Name, Term]}. - -proplist(Name, Constraints, Term) when is_list(Term) -> - {Results, Remainder} - = lists:foldl( - fun ({Key, Fun, Needed}, {Results0, Term0}) -> - case {lists:keytake(Key, 1, Term0), Needed} of - {{value, {Key, Value}, Term1}, _} -> - {[Fun(Key, Value) | Results0], - Term1}; - {false, mandatory} -> - {[{error, "Key \"~s\" not found in ~s", - [Key, Name]} | Results0], Term0}; - {false, optional} -> - {Results0, Term0} - end - end, {[], Term}, Constraints), - case Remainder of - [] -> Results; - _ -> [{error, "Unrecognised terms ~p in ~s", [Remainder, Name]} - | Results] - end; - -proplist(Name, Constraints, Term0) when is_map(Term0) -> - Term = maps:to_list(Term0), - proplist(Name, Constraints, Term); - -proplist(Name, _Constraints, Term) -> - {error, "~s not a list ~p", [Name, Term]}. - -enum(OptionsA) -> - Options = [list_to_binary(atom_to_list(O)) || O <- OptionsA], - fun (Name, Term) when is_binary(Term) -> - case lists:member(Term, Options) of - true -> ok; - false -> {error, "~s should be one of ~p, actually was ~p", - [Name, Options, Term]} - end; - (Name, Term) -> - {error, "~s should be binary, actually was ~p", [Name, Term]} - end. diff --git a/src/rabbit_password.erl b/src/rabbit_password.erl deleted file mode 100644 index 6a5254b707..0000000000 --- a/src/rabbit_password.erl +++ /dev/null @@ -1,52 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_password). --include("rabbit.hrl"). - --define(DEFAULT_HASHING_MODULE, rabbit_password_hashing_sha256). - -%% -%% API -%% - --export([hash/1, hash/2, generate_salt/0, salted_hash/2, salted_hash/3, - hashing_mod/0, hashing_mod/1]). - -hash(Cleartext) -> - hash(hashing_mod(), Cleartext). - -hash(HashingMod, Cleartext) -> - SaltBin = generate_salt(), - Hash = salted_hash(HashingMod, SaltBin, Cleartext), - <<SaltBin/binary, Hash/binary>>. - -generate_salt() -> - Salt = rand:uniform(16#ffffffff), - <<Salt:32>>. - -salted_hash(Salt, Cleartext) -> - salted_hash(hashing_mod(), Salt, Cleartext). - -salted_hash(Mod, Salt, Cleartext) -> - Fun = fun Mod:hash/1, - Fun(<<Salt/binary, Cleartext/binary>>). - -hashing_mod() -> - rabbit_misc:get_env(rabbit, password_hashing_module, - ?DEFAULT_HASHING_MODULE). - -hashing_mod(rabbit_password_hashing_sha256) -> - rabbit_password_hashing_sha256; -hashing_mod(rabbit_password_hashing_md5) -> - rabbit_password_hashing_md5; -%% fall back to the hashing function that's been used prior to 3.6.0 -hashing_mod(undefined) -> - rabbit_password_hashing_md5; -%% if a custom module is configured, simply use it -hashing_mod(CustomMod) when is_atom(CustomMod) -> - CustomMod. diff --git a/src/rabbit_password_hashing_md5.erl b/src/rabbit_password_hashing_md5.erl deleted file mode 100644 index 1e306673ca..0000000000 --- a/src/rabbit_password_hashing_md5.erl +++ /dev/null @@ -1,19 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% Legacy hashing implementation, only used as a last resort when -%% #internal_user.hashing_algorithm is md5 or undefined (the case in -%% pre-3.6.0 user records). - --module(rabbit_password_hashing_md5). - --behaviour(rabbit_password_hashing). - --export([hash/1]). - -hash(Binary) -> - erlang:md5(Binary). diff --git a/src/rabbit_password_hashing_sha256.erl b/src/rabbit_password_hashing_sha256.erl deleted file mode 100644 index 3ccc298efd..0000000000 --- a/src/rabbit_password_hashing_sha256.erl +++ /dev/null @@ -1,15 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_password_hashing_sha256). - --behaviour(rabbit_password_hashing). - --export([hash/1]). - -hash(Binary) -> - crypto:hash(sha256, Binary). diff --git a/src/rabbit_password_hashing_sha512.erl b/src/rabbit_password_hashing_sha512.erl deleted file mode 100644 index c5edf8888a..0000000000 --- a/src/rabbit_password_hashing_sha512.erl +++ /dev/null @@ -1,15 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_password_hashing_sha512). - --behaviour(rabbit_password_hashing). - --export([hash/1]). - -hash(Binary) -> - crypto:hash(sha512, Binary). diff --git a/src/rabbit_peer_discovery.erl b/src/rabbit_peer_discovery.erl deleted file mode 100644 index 1688579450..0000000000 --- a/src/rabbit_peer_discovery.erl +++ /dev/null @@ -1,326 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_peer_discovery). - -%% -%% API -%% - --export([maybe_init/0, discover_cluster_nodes/0, backend/0, node_type/0, - normalize/1, format_discovered_nodes/1, log_configured_backend/0, - register/0, unregister/0, maybe_register/0, maybe_unregister/0, - maybe_inject_randomized_delay/0, lock/0, unlock/1, - discovery_retries/0]). --export([append_node_prefix/1, node_prefix/0, locking_retry_timeout/0, - lock_acquisition_failure_mode/0]). - --define(DEFAULT_BACKEND, rabbit_peer_discovery_classic_config). - -%% what node type is used by default for this node when joining -%% a new cluster as a virgin node --define(DEFAULT_NODE_TYPE, disc). - -%% default node prefix to attach to discovered hostnames --define(DEFAULT_PREFIX, "rabbit"). - -%% default randomized delay range, in seconds --define(DEFAULT_STARTUP_RANDOMIZED_DELAY, {5, 60}). - -%% default discovery retries and interval. --define(DEFAULT_DISCOVERY_RETRY_COUNT, 10). --define(DEFAULT_DISCOVERY_RETRY_INTERVAL_MS, 500). - --define(NODENAME_PART_SEPARATOR, "@"). - --spec backend() -> atom(). - -backend() -> - case application:get_env(rabbit, cluster_formation) of - {ok, Proplist} -> - proplists:get_value(peer_discovery_backend, Proplist, ?DEFAULT_BACKEND); - undefined -> - ?DEFAULT_BACKEND - end. - - - --spec node_type() -> rabbit_types:node_type(). - -node_type() -> - case application:get_env(rabbit, cluster_formation) of - {ok, Proplist} -> - proplists:get_value(node_type, Proplist, ?DEFAULT_NODE_TYPE); - undefined -> - ?DEFAULT_NODE_TYPE - end. - --spec locking_retry_timeout() -> {Retries :: integer(), Timeout :: integer()}. - -locking_retry_timeout() -> - case application:get_env(rabbit, cluster_formation) of - {ok, Proplist} -> - Retries = proplists:get_value(lock_retry_limit, Proplist, 10), - Timeout = proplists:get_value(lock_retry_timeout, Proplist, 30000), - {Retries, Timeout}; - undefined -> - {10, 30000} - end. - --spec lock_acquisition_failure_mode() -> ignore | fail. - -lock_acquisition_failure_mode() -> - case application:get_env(rabbit, cluster_formation) of - {ok, Proplist} -> - proplists:get_value(lock_acquisition_failure_mode, Proplist, fail); - undefined -> - fail - end. - --spec log_configured_backend() -> ok. - -log_configured_backend() -> - rabbit_log:info("Configured peer discovery backend: ~s~n", [backend()]). - -maybe_init() -> - Backend = backend(), - code:ensure_loaded(Backend), - case erlang:function_exported(Backend, init, 0) of - true -> - rabbit_log:debug("Peer discovery backend supports initialisation"), - case Backend:init() of - ok -> - rabbit_log:debug("Peer discovery backend initialisation succeeded"), - ok; - {error, Error} -> - rabbit_log:warning("Peer discovery backend initialisation failed: ~p.", [Error]), - ok - end; - false -> - rabbit_log:debug("Peer discovery backend does not support initialisation"), - ok - end. - - -%% This module doesn't currently sanity-check the return value of -%% `Backend:list_nodes()`. Therefore, it could return something invalid: -%% thus the `{œk, any()} in the spec. -%% -%% `rabbit_mnesia:init_from_config()` does some verifications. - --spec discover_cluster_nodes() -> - {ok, {Nodes :: [node()], NodeType :: rabbit_types:node_type()} | any()} | - {error, Reason :: string()}. - -discover_cluster_nodes() -> - Backend = backend(), - normalize(Backend:list_nodes()). - - --spec maybe_register() -> ok. - -maybe_register() -> - Backend = backend(), - case Backend:supports_registration() of - true -> - register(), - Backend:post_registration(); - false -> - rabbit_log:info("Peer discovery backend ~s does not support registration, skipping registration.", [Backend]), - ok - end. - - --spec maybe_unregister() -> ok. - -maybe_unregister() -> - Backend = backend(), - case Backend:supports_registration() of - true -> - unregister(); - false -> - rabbit_log:info("Peer discovery backend ~s does not support registration, skipping unregistration.", [Backend]), - ok - end. - --spec discovery_retries() -> {Retries :: integer(), Interval :: integer()}. - -discovery_retries() -> - case application:get_env(rabbit, cluster_formation) of - {ok, Proplist} -> - Retries = proplists:get_value(discovery_retry_limit, Proplist, ?DEFAULT_DISCOVERY_RETRY_COUNT), - Interval = proplists:get_value(discovery_retry_interval, Proplist, ?DEFAULT_DISCOVERY_RETRY_INTERVAL_MS), - {Retries, Interval}; - undefined -> - {?DEFAULT_DISCOVERY_RETRY_COUNT, ?DEFAULT_DISCOVERY_RETRY_INTERVAL_MS} - end. - - --spec maybe_inject_randomized_delay() -> ok. -maybe_inject_randomized_delay() -> - Backend = backend(), - case Backend:supports_registration() of - true -> - rabbit_log:info("Peer discovery backend ~s supports registration.", [Backend]), - inject_randomized_delay(); - false -> - rabbit_log:info("Peer discovery backend ~s does not support registration, skipping randomized startup delay.", [Backend]), - ok - end. - --spec inject_randomized_delay() -> ok. - -inject_randomized_delay() -> - {Min, Max} = randomized_delay_range_in_ms(), - case {Min, Max} of - %% When the max value is set to 0, consider the delay to be disabled. - %% In addition, `rand:uniform/1` will fail with a "no function clause" - %% when the argument is 0. - {_, 0} -> - rabbit_log:info("Randomized delay range's upper bound is set to 0. Considering it disabled."), - ok; - {_, N} when is_number(N) -> - rand:seed(exsplus), - RandomVal = rand:uniform(round(N)), - rabbit_log:debug("Randomized startup delay: configured range is from ~p to ~p milliseconds, PRNG pick: ~p...", - [Min, Max, RandomVal]), - Effective = case RandomVal < Min of - true -> Min; - false -> RandomVal - end, - rabbit_log:info("Will wait for ~p milliseconds before proceeding with registration...", [Effective]), - timer:sleep(Effective), - ok - end. - --spec randomized_delay_range_in_ms() -> {integer(), integer()}. - -randomized_delay_range_in_ms() -> - Backend = backend(), - Default = case erlang:function_exported(Backend, randomized_startup_delay_range, 0) of - true -> Backend:randomized_startup_delay_range(); - false -> ?DEFAULT_STARTUP_RANDOMIZED_DELAY - end, - {Min, Max} = case application:get_env(rabbit, cluster_formation) of - {ok, Proplist} -> - proplists:get_value(randomized_startup_delay_range, Proplist, Default); - undefined -> - Default - end, - {Min * 1000, Max * 1000}. - - --spec register() -> ok. - -register() -> - Backend = backend(), - rabbit_log:info("Will register with peer discovery backend ~s", [Backend]), - case Backend:register() of - ok -> ok; - {error, Error} -> - rabbit_log:error("Failed to register with peer discovery backend ~s: ~p", - [Backend, Error]), - ok - end. - - --spec unregister() -> ok. - -unregister() -> - Backend = backend(), - rabbit_log:info("Will unregister with peer discovery backend ~s", [Backend]), - case Backend:unregister() of - ok -> ok; - {error, Error} -> - rabbit_log:error("Failed to unregister with peer discovery backend ~s: ~p", - [Backend, Error]), - ok - end. - --spec lock() -> {ok, Data :: term()} | not_supported | {error, Reason :: string()}. - -lock() -> - Backend = backend(), - rabbit_log:info("Will try to lock with peer discovery backend ~s", [Backend]), - case Backend:lock(node()) of - {error, Reason} = Error -> - rabbit_log:error("Failed to lock with peer discovery backend ~s: ~p", - [Backend, Reason]), - Error; - Any -> - Any - end. - --spec unlock(Data :: term()) -> ok | {error, Reason :: string()}. - -unlock(Data) -> - Backend = backend(), - rabbit_log:info("Will try to unlock with peer discovery backend ~s", [Backend]), - case Backend:unlock(Data) of - {error, Reason} = Error -> - rabbit_log:error("Failed to unlock with peer discovery backend ~s: ~p, " - "lock data: ~p", - [Backend, Reason, Data]), - Error; - Any -> - Any - end. - -%% -%% Implementation -%% - --spec normalize(Nodes :: [node()] | - {Nodes :: [node()], - NodeType :: rabbit_types:node_type()} | - {ok, Nodes :: [node()]} | - {ok, {Nodes :: [node()], - NodeType :: rabbit_types:node_type()}} | - {error, Reason :: string()}) -> - {ok, {Nodes :: [node()], NodeType :: rabbit_types:node_type()}} | - {error, Reason :: string()}. - -normalize(Nodes) when is_list(Nodes) -> - {ok, {Nodes, disc}}; -normalize({Nodes, NodeType}) when is_list(Nodes) andalso is_atom(NodeType) -> - {ok, {Nodes, NodeType}}; -normalize({ok, Nodes}) when is_list(Nodes) -> - {ok, {Nodes, disc}}; -normalize({ok, {Nodes, NodeType}}) when is_list(Nodes) andalso is_atom(NodeType) -> - {ok, {Nodes, NodeType}}; -normalize({error, Reason}) -> - {error, Reason}. - --spec format_discovered_nodes(Nodes :: list()) -> string(). - -format_discovered_nodes(Nodes) -> - %% NOTE: in OTP 21 string:join/2 is deprecated but still available. - %% Its recommended replacement is not a drop-in one, though, so - %% we will not be switching just yet. - string:join(lists:map(fun rabbit_data_coercion:to_list/1, Nodes), ", "). - - - --spec node_prefix() -> string(). - -node_prefix() -> - case string:tokens(atom_to_list(node()), ?NODENAME_PART_SEPARATOR) of - [Prefix, _] -> Prefix; - [_] -> ?DEFAULT_PREFIX - end. - - - --spec append_node_prefix(Value :: binary() | string()) -> string(). - -append_node_prefix(Value) when is_binary(Value) orelse is_list(Value) -> - Val = rabbit_data_coercion:to_list(Value), - Hostname = case string:tokens(Val, ?NODENAME_PART_SEPARATOR) of - [_ExistingPrefix, HN] -> HN; - [HN] -> HN - end, - string:join([node_prefix(), Hostname], ?NODENAME_PART_SEPARATOR). diff --git a/src/rabbit_peer_discovery_classic_config.erl b/src/rabbit_peer_discovery_classic_config.erl deleted file mode 100644 index 8bc7382a75..0000000000 --- a/src/rabbit_peer_discovery_classic_config.erl +++ /dev/null @@ -1,75 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_peer_discovery_classic_config). --behaviour(rabbit_peer_discovery_backend). - --include("rabbit.hrl"). - --export([list_nodes/0, supports_registration/0, register/0, unregister/0, - post_registration/0, lock/1, unlock/1]). - -%% -%% API -%% - --spec list_nodes() -> {ok, {Nodes :: [node()], rabbit_types:node_type()}} | - {error, Reason :: string()}. - -list_nodes() -> - case application:get_env(rabbit, cluster_nodes, {[], disc}) of - {_Nodes, _NodeType} = Pair -> {ok, Pair}; - Nodes when is_list(Nodes) -> {ok, {Nodes, disc}} - end. - --spec supports_registration() -> boolean(). - -supports_registration() -> - %% If we don't have any nodes configured, skip randomized delay and similar operations - %% as we don't want to delay startup for no reason. MK. - has_any_peer_nodes_configured(). - --spec register() -> ok. - -register() -> - ok. - --spec unregister() -> ok. - -unregister() -> - ok. - --spec post_registration() -> ok. - -post_registration() -> - ok. - --spec lock(Node :: atom()) -> not_supported. - -lock(_Node) -> - not_supported. - --spec unlock(Data :: term()) -> ok. - -unlock(_Data) -> - ok. - -%% -%% Helpers -%% - -has_any_peer_nodes_configured() -> - case application:get_env(rabbit, cluster_nodes, []) of - {[], _NodeType} -> - false; - {Nodes, _NodeType} when is_list(Nodes) -> - true; - [] -> - false; - Nodes when is_list(Nodes) -> - true - end. diff --git a/src/rabbit_peer_discovery_dns.erl b/src/rabbit_peer_discovery_dns.erl deleted file mode 100644 index 6e343a6e2d..0000000000 --- a/src/rabbit_peer_discovery_dns.erl +++ /dev/null @@ -1,113 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_peer_discovery_dns). --behaviour(rabbit_peer_discovery_backend). - --include("rabbit.hrl"). - --export([list_nodes/0, supports_registration/0, register/0, unregister/0, - post_registration/0, lock/1, unlock/1]). -%% for tests --export([discover_nodes/2, discover_hostnames/2]). - -%% -%% API -%% - --spec list_nodes() -> - {ok, {Nodes :: [node()], rabbit_types:node_type()}}. - -list_nodes() -> - case application:get_env(rabbit, cluster_formation) of - undefined -> - {ok, {[], disc}}; - {ok, ClusterFormation} -> - case proplists:get_value(peer_discovery_dns, ClusterFormation) of - undefined -> - rabbit_log:warning("Peer discovery backend is set to ~s " - "but final config does not contain rabbit.cluster_formation.peer_discovery_dns. " - "Cannot discover any nodes because seed hostname is not configured!", - [?MODULE]), - {ok, {[], disc}}; - Proplist -> - Hostname = rabbit_data_coercion:to_list(proplists:get_value(hostname, Proplist)), - - {ok, {discover_nodes(Hostname, net_kernel:longnames()), rabbit_peer_discovery:node_type()}} - end - end. - - --spec supports_registration() -> boolean(). - -supports_registration() -> - false. - - --spec register() -> ok. - -register() -> - ok. - --spec unregister() -> ok. - -unregister() -> - ok. - --spec post_registration() -> ok. - -post_registration() -> - ok. - --spec lock(Node :: atom()) -> not_supported. - -lock(_Node) -> - not_supported. - --spec unlock(Data :: term()) -> ok. - -unlock(_Data) -> - ok. - -%% -%% Implementation -%% - -discover_nodes(SeedHostname, LongNamesUsed) -> - [list_to_atom(rabbit_peer_discovery:append_node_prefix(H)) || - H <- discover_hostnames(SeedHostname, LongNamesUsed)]. - -discover_hostnames(SeedHostname, LongNamesUsed) -> - lookup(SeedHostname, LongNamesUsed, ipv4) ++ - lookup(SeedHostname, LongNamesUsed, ipv6). - -decode_record(ipv4) -> - a; -decode_record(ipv6) -> - aaaa. - -lookup(SeedHostname, LongNamesUsed, IPv) -> - IPs = inet_res:lookup(SeedHostname, in, decode_record(IPv)), - rabbit_log:info("Addresses discovered via ~s records of ~s: ~s", - [string:to_upper(atom_to_list(decode_record(IPv))), - SeedHostname, - string:join([inet_parse:ntoa(IP) || IP <- IPs], ", ")]), - Hosts = [extract_host(inet:gethostbyaddr(A), LongNamesUsed, A) || - A <- IPs], - lists:filter(fun(E) -> E =/= error end, Hosts). - - -%% long node names are used -extract_host({ok, {hostent, FQDN, _, _, _, _}}, true, _Address) -> - FQDN; -%% short node names are used -extract_host({ok, {hostent, FQDN, _, _, _, _}}, false, _Address) -> - lists:nth(1, string:tokens(FQDN, ".")); -extract_host({error, Error}, _, Address) -> - rabbit_log:error("Reverse DNS lookup for address ~s failed: ~p", - [inet_parse:ntoa(Address), Error]), - error. diff --git a/src/rabbit_plugins.erl b/src/rabbit_plugins.erl deleted file mode 100644 index 5697ffc29a..0000000000 --- a/src/rabbit_plugins.erl +++ /dev/null @@ -1,699 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2011-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_plugins). --include_lib("rabbit_common/include/rabbit.hrl"). --include_lib("stdlib/include/zip.hrl"). - --export([setup/0, active/0, read_enabled/1, list/1, list/2, dependencies/3, running_plugins/0]). --export([ensure/1]). --export([validate_plugins/1, format_invalid_plugins/1]). --export([is_strictly_plugin/1, strictly_plugins/2, strictly_plugins/1]). --export([plugins_dir/0, plugin_names/1, plugins_expand_dir/0, enabled_plugins_file/0]). - -% Export for testing purpose. --export([is_version_supported/2, validate_plugins/2]). -%%---------------------------------------------------------------------------- - --type plugin_name() :: atom(). - -%%---------------------------------------------------------------------------- - --spec ensure(string()) -> {'ok', [atom()], [atom()]} | {error, any()}. - -ensure(FileJustChanged) -> - case rabbit:is_running() of - true -> ensure1(FileJustChanged); - false -> {error, rabbit_not_running} - end. - -ensure1(FileJustChanged0) -> - {ok, OurFile0} = application:get_env(rabbit, enabled_plugins_file), - FileJustChanged = filename:nativename(FileJustChanged0), - OurFile = filename:nativename(OurFile0), - case OurFile of - FileJustChanged -> - Enabled = read_enabled(OurFile), - Wanted = prepare_plugins(Enabled), - Current = active(), - Start = Wanted -- Current, - Stop = Current -- Wanted, - rabbit:start_apps(Start), - %% We need sync_notify here since mgmt will attempt to look at all - %% the modules for the disabled plugins - if they are unloaded - %% that won't work. - ok = rabbit_event:sync_notify(plugins_changed, [{enabled, Start}, - {disabled, Stop}]), - %% The app_utils module stops the apps in reverse order, so we should - %% pass them here in dependency order. - rabbit:stop_apps(lists:reverse(Stop)), - clean_plugins(Stop), - case {Start, Stop} of - {[], []} -> - ok; - {[], _} -> - rabbit_log:info("Plugins changed; disabled ~p~n", - [Stop]); - {_, []} -> - rabbit_log:info("Plugins changed; enabled ~p~n", - [Start]); - {_, _} -> - rabbit_log:info("Plugins changed; enabled ~p, disabled ~p~n", - [Start, Stop]) - end, - {ok, Start, Stop}; - _ -> - {error, {enabled_plugins_mismatch, FileJustChanged, OurFile}} - end. - --spec plugins_expand_dir() -> file:filename(). -plugins_expand_dir() -> - case application:get_env(rabbit, plugins_expand_dir) of - {ok, ExpandDir} -> - ExpandDir; - _ -> - filename:join([rabbit_mnesia:dir(), "plugins_expand_dir"]) - end. - --spec plugins_dir() -> file:filename(). -plugins_dir() -> - case application:get_env(rabbit, plugins_dir) of - {ok, PluginsDistDir} -> - PluginsDistDir; - _ -> - filename:join([rabbit_mnesia:dir(), "plugins_dir_stub"]) - end. - --spec enabled_plugins_file() -> file:filename(). -enabled_plugins_file() -> - case application:get_env(rabbit, enabled_plugins_file) of - {ok, Val} -> - Val; - _ -> - filename:join([rabbit_mnesia:dir(), "enabled_plugins"]) - end. - --spec enabled_plugins() -> [atom()]. -enabled_plugins() -> - case application:get_env(rabbit, enabled_plugins_file) of - {ok, EnabledFile} -> - read_enabled(EnabledFile); - _ -> - [] - end. - -%% @doc Prepares the file system and installs all enabled plugins. - --spec setup() -> [plugin_name()]. - -setup() -> - ExpandDir = plugins_expand_dir(), - %% Eliminate the contents of the destination directory - case delete_recursively(ExpandDir) of - ok -> ok; - {error, E1} -> throw({error, {cannot_delete_plugins_expand_dir, - [ExpandDir, E1]}}) - end, - Enabled = enabled_plugins(), - prepare_plugins(Enabled). - -%% @doc Lists the plugins which are currently running. - --spec active() -> [plugin_name()]. - -active() -> - InstalledPlugins = plugin_names(list(plugins_dir())), - [App || {App, _, _} <- rabbit_misc:which_applications(), - lists:member(App, InstalledPlugins)]. - -%% @doc Get the list of plugins which are ready to be enabled. - --spec list(string()) -> [#plugin{}]. - -list(PluginsPath) -> - list(PluginsPath, false). - --spec list(string(), boolean()) -> [#plugin{}]. - -list(PluginsPath, IncludeRequiredDeps) -> - {AllPlugins, LoadingProblems} = discover_plugins(split_path(PluginsPath)), - {UniquePlugins, DuplicateProblems} = remove_duplicate_plugins(AllPlugins), - Plugins1 = maybe_keep_required_deps(IncludeRequiredDeps, UniquePlugins), - Plugins2 = remove_plugins(Plugins1), - maybe_report_plugin_loading_problems(LoadingProblems ++ DuplicateProblems), - ensure_dependencies(Plugins2). - -%% @doc Read the list of enabled plugins from the supplied term file. - --spec read_enabled(file:filename()) -> [plugin_name()]. - -read_enabled(PluginsFile) -> - case rabbit_file:read_term_file(PluginsFile) of - {ok, [Plugins]} -> Plugins; - {ok, []} -> []; - {ok, [_|_]} -> throw({error, {malformed_enabled_plugins_file, - PluginsFile}}); - {error, enoent} -> []; - {error, Reason} -> throw({error, {cannot_read_enabled_plugins_file, - PluginsFile, Reason}}) - end. - -%% @doc Calculate the dependency graph from <i>Sources</i>. -%% When Reverse =:= true the bottom/leaf level applications are returned in -%% the resulting list, otherwise they're skipped. - --spec dependencies(boolean(), [plugin_name()], [#plugin{}]) -> - [plugin_name()]. - -dependencies(Reverse, Sources, AllPlugins) -> - {ok, G} = rabbit_misc:build_acyclic_graph( - fun ({App, _Deps}) -> [{App, App}] end, - fun ({App, Deps}) -> [{App, Dep} || Dep <- Deps] end, - [{Name, Deps} || #plugin{name = Name, - dependencies = Deps} <- AllPlugins]), - Dests = case Reverse of - false -> digraph_utils:reachable(Sources, G); - true -> digraph_utils:reaching(Sources, G) - end, - OrderedDests = digraph_utils:postorder(digraph_utils:subgraph(G, Dests)), - true = digraph:delete(G), - OrderedDests. - -%% Filter real plugins from application dependencies - --spec is_strictly_plugin(#plugin{}) -> boolean(). - -is_strictly_plugin(#plugin{extra_dependencies = ExtraDeps}) -> - lists:member(rabbit, ExtraDeps). - --spec strictly_plugins([plugin_name()], [#plugin{}]) -> [plugin_name()]. - -strictly_plugins(Plugins, AllPlugins) -> - lists:filter( - fun(Name) -> - is_strictly_plugin(lists:keyfind(Name, #plugin.name, AllPlugins)) - end, Plugins). - --spec strictly_plugins([plugin_name()]) -> [plugin_name()]. - -strictly_plugins(Plugins) -> - AllPlugins = list(plugins_dir()), - lists:filter( - fun(Name) -> - is_strictly_plugin(lists:keyfind(Name, #plugin.name, AllPlugins)) - end, Plugins). - -%% For a few known cases, an externally provided plugin can be trusted. -%% In this special case, it overrides the plugin. -is_plugin_provided_by_otp(#plugin{name = eldap}) -> - %% eldap was added to Erlang/OTP R15B01 (ERTS 5.9.1). In this case, - %% we prefer this version to the plugin. - rabbit_misc:version_compare(erlang:system_info(version), "5.9.1", gte); -is_plugin_provided_by_otp(_) -> - false. - -%% Make sure we don't list OTP apps in here, and also that we detect -%% missing dependencies. -ensure_dependencies(Plugins) -> - Names = plugin_names(Plugins), - NotThere = [Dep || #plugin{dependencies = Deps} <- Plugins, - Dep <- Deps, - not lists:member(Dep, Names)], - {OTP, Missing} = lists:partition(fun is_loadable/1, lists:usort(NotThere)), - case Missing of - [] -> ok; - _ -> Blame = [Name || #plugin{name = Name, - dependencies = Deps} <- Plugins, - lists:any(fun (Dep) -> - lists:member(Dep, Missing) - end, Deps)], - throw({error, {missing_dependencies, Missing, Blame}}) - end, - [P#plugin{dependencies = Deps -- OTP, - extra_dependencies = Deps -- (Deps -- OTP)} - || P = #plugin{dependencies = Deps} <- Plugins]. - -is_loadable(App) -> - case application:load(App) of - {error, {already_loaded, _}} -> true; - ok -> application:unload(App), - true; - _ -> false - end. - - -%% List running plugins along with their version. --spec running_plugins() -> {ok, [{atom(), Vsn :: string()}]}. -running_plugins() -> - ActivePlugins = active(), - {ok, [{App, Vsn} || {App, _ , Vsn} <- rabbit_misc:which_applications(), lists:member(App, ActivePlugins)]}. - -%%---------------------------------------------------------------------------- - -prepare_plugins(Enabled) -> - ExpandDir = plugins_expand_dir(), - AllPlugins = list(plugins_dir()), - Wanted = dependencies(false, Enabled, AllPlugins), - WantedPlugins = lookup_plugins(Wanted, AllPlugins), - {ValidPlugins, Problems} = validate_plugins(WantedPlugins), - maybe_warn_about_invalid_plugins(Problems), - case filelib:ensure_dir(ExpandDir ++ "/") of - ok -> ok; - {error, E2} -> throw({error, {cannot_create_plugins_expand_dir, - [ExpandDir, E2]}}) - end, - [prepare_plugin(Plugin, ExpandDir) || Plugin <- ValidPlugins], - Wanted. - -maybe_warn_about_invalid_plugins([]) -> - ok; -maybe_warn_about_invalid_plugins(InvalidPlugins) -> - %% TODO: error message formatting - rabbit_log:warning(format_invalid_plugins(InvalidPlugins)). - - -format_invalid_plugins(InvalidPlugins) -> - lists:flatten(["Failed to enable some plugins: \r\n" - | [format_invalid_plugin(Plugin) - || Plugin <- InvalidPlugins]]). - -format_invalid_plugin({Name, Errors}) -> - [io_lib:format(" ~p:~n", [Name]) - | [format_invalid_plugin_error(Err) || Err <- Errors]]. - -format_invalid_plugin_error({missing_dependency, Dep}) -> - io_lib:format(" Dependency is missing or invalid: ~p~n", [Dep]); -%% a plugin doesn't support the effective broker version -format_invalid_plugin_error({broker_version_mismatch, Version, Required}) -> - io_lib:format(" Plugin doesn't support current server version." - " Actual broker version: ~p, supported by the plugin: ~p~n", - [Version, format_required_versions(Required)]); -%% one of dependencies of a plugin doesn't match its version requirements -format_invalid_plugin_error({{dependency_version_mismatch, Version, Required}, Name}) -> - io_lib:format(" Version '~p' of dependency '~p' is unsupported." - " Version ranges supported by the plugin: ~p~n", - [Version, Name, Required]); -format_invalid_plugin_error(Err) -> - io_lib:format(" Unknown error ~p~n", [Err]). - -format_required_versions(Versions) -> - lists:map(fun(V) -> - case re:run(V, "^[0-9]*\.[0-9]*\.", [{capture, all, list}]) of - {match, [Sub]} -> - lists:flatten(io_lib:format("~s-~sx", [V, Sub])); - _ -> - V - end - end, Versions). - -validate_plugins(Plugins) -> - application:load(rabbit), - RabbitVersion = RabbitVersion = case application:get_key(rabbit, vsn) of - undefined -> "0.0.0"; - {ok, Val} -> Val - end, - validate_plugins(Plugins, RabbitVersion). - -validate_plugins(Plugins, BrokerVersion) -> - lists:foldl( - fun(#plugin{name = Name, - broker_version_requirements = BrokerVersionReqs, - dependency_version_requirements = DepsVersions} = Plugin, - {Plugins0, Errors}) -> - case is_version_supported(BrokerVersion, BrokerVersionReqs) of - true -> - case BrokerVersion of - "0.0.0" -> - rabbit_log:warning( - "Running development version of the broker." - " Requirement ~p for plugin ~p is ignored.", - [BrokerVersionReqs, Name]); - _ -> ok - end, - case check_plugins_versions(Name, Plugins0, DepsVersions) of - ok -> {[Plugin | Plugins0], Errors}; - {error, Err} -> {Plugins0, [{Name, Err} | Errors]} - end; - false -> - Error = [{broker_version_mismatch, BrokerVersion, BrokerVersionReqs}], - {Plugins0, [{Name, Error} | Errors]} - end - end, - {[],[]}, - Plugins). - -check_plugins_versions(PluginName, AllPlugins, RequiredVersions) -> - ExistingVersions = [{Name, Vsn} - || #plugin{name = Name, version = Vsn} <- AllPlugins], - Problems = lists:foldl( - fun({Name, Versions}, Acc) -> - case proplists:get_value(Name, ExistingVersions) of - undefined -> [{missing_dependency, Name} | Acc]; - Version -> - case is_version_supported(Version, Versions) of - true -> - case Version of - "" -> - rabbit_log:warning( - "~p plugin version is not defined." - " Requirement ~p for plugin ~p is ignored", - [Versions, PluginName]); - _ -> ok - end, - Acc; - false -> - [{{dependency_version_mismatch, Version, Versions}, Name} | Acc] - end - end - end, - [], - RequiredVersions), - case Problems of - [] -> ok; - _ -> {error, Problems} - end. - -is_version_supported("", _) -> true; -is_version_supported("0.0.0", _) -> true; -is_version_supported(_Version, []) -> true; -is_version_supported(VersionFull, ExpectedVersions) -> - %% Pre-release version should be supported in plugins, - %% therefore preview part should be removed - Version = remove_version_preview_part(VersionFull), - case lists:any(fun(ExpectedVersion) -> - rabbit_misc:strict_version_minor_equivalent(ExpectedVersion, - Version) - andalso - rabbit_misc:version_compare(ExpectedVersion, Version, lte) - end, - ExpectedVersions) of - true -> true; - false -> false - end. - -remove_version_preview_part(Version) -> - {Ver, _Preview} = rabbit_semver:parse(Version), - iolist_to_binary(rabbit_semver:format({Ver, {[], []}})). - -clean_plugins(Plugins) -> - ExpandDir = plugins_expand_dir(), - [clean_plugin(Plugin, ExpandDir) || Plugin <- Plugins]. - -clean_plugin(Plugin, ExpandDir) -> - {ok, Mods} = application:get_key(Plugin, modules), - application:unload(Plugin), - [begin - code:soft_purge(Mod), - code:delete(Mod), - false = code:is_loaded(Mod) - end || Mod <- Mods], - delete_recursively(rabbit_misc:format("~s/~s", [ExpandDir, Plugin])). - -prepare_dir_plugin(PluginAppDescPath) -> - PluginEbinDir = filename:dirname(PluginAppDescPath), - Plugin = filename:basename(PluginAppDescPath, ".app"), - code:add_patha(PluginEbinDir), - case filelib:wildcard(PluginEbinDir++ "/*.beam") of - [] -> - ok; - [BeamPath | _] -> - Module = list_to_atom(filename:basename(BeamPath, ".beam")), - case code:ensure_loaded(Module) of - {module, _} -> - ok; - {error, badfile} -> - rabbit_log:error("Failed to enable plugin \"~s\": " - "it may have been built with an " - "incompatible (more recent?) " - "version of Erlang~n", [Plugin]), - throw({plugin_built_with_incompatible_erlang, Plugin}); - Error -> - throw({plugin_module_unloadable, Plugin, Error}) - end - end. - -%%---------------------------------------------------------------------------- - -delete_recursively(Fn) -> - case rabbit_file:recursive_delete([Fn]) of - ok -> ok; - {error, {Path, E}} -> {error, {cannot_delete, Path, E}} - end. - -find_unzipped_app_file(ExpandDir, Files) -> - StripComponents = length(filename:split(ExpandDir)), - [ X || X <- Files, - [_AppName, "ebin", MaybeAppFile] <- - [lists:nthtail(StripComponents, filename:split(X))], - lists:suffix(".app", MaybeAppFile) - ]. - -prepare_plugin(#plugin{type = ez, name = Name, location = Location}, ExpandDir) -> - case zip:unzip(Location, [{cwd, ExpandDir}]) of - {ok, Files} -> - case find_unzipped_app_file(ExpandDir, Files) of - [PluginAppDescPath|_] -> - prepare_dir_plugin(PluginAppDescPath); - _ -> - rabbit_log:error("Plugin archive '~s' doesn't contain an .app file~n", [Location]), - throw({app_file_missing, Name, Location}) - end; - {error, Reason} -> - rabbit_log:error("Could not unzip plugin archive '~s': ~p~n", [Location, Reason]), - throw({failed_to_unzip_plugin, Name, Location, Reason}) - end; -prepare_plugin(#plugin{type = dir, location = Location, name = Name}, - _ExpandDir) -> - case filelib:wildcard(Location ++ "/ebin/*.app") of - [PluginAppDescPath|_] -> - prepare_dir_plugin(PluginAppDescPath); - _ -> - rabbit_log:error("Plugin directory '~s' doesn't contain an .app file~n", [Location]), - throw({app_file_missing, Name, Location}) - end. - -plugin_info({ez, EZ}) -> - case read_app_file(EZ) of - {application, Name, Props} -> mkplugin(Name, Props, ez, EZ); - {error, Reason} -> {error, EZ, Reason} - end; -plugin_info({app, App}) -> - case rabbit_file:read_term_file(App) of - {ok, [{application, Name, Props}]} -> - mkplugin(Name, Props, dir, - filename:absname( - filename:dirname(filename:dirname(App)))); - {error, Reason} -> - {error, App, {invalid_app, Reason}} - end. - -mkplugin(Name, Props, Type, Location) -> - Version = proplists:get_value(vsn, Props, "0"), - Description = proplists:get_value(description, Props, ""), - Dependencies = proplists:get_value(applications, Props, []), - BrokerVersions = proplists:get_value(broker_version_requirements, Props, []), - DepsVersions = proplists:get_value(dependency_version_requirements, Props, []), - #plugin{name = Name, version = Version, description = Description, - dependencies = Dependencies, location = Location, type = Type, - broker_version_requirements = BrokerVersions, - dependency_version_requirements = DepsVersions}. - -read_app_file(EZ) -> - case zip:list_dir(EZ) of - {ok, [_|ZippedFiles]} -> - case find_app_files(ZippedFiles) of - [AppPath|_] -> - {ok, [{AppPath, AppFile}]} = - zip:extract(EZ, [{file_list, [AppPath]}, memory]), - parse_binary(AppFile); - [] -> - {error, no_app_file} - end; - {error, Reason} -> - {error, {invalid_ez, Reason}} - end. - -find_app_files(ZippedFiles) -> - {ok, RE} = re:compile("^.*/ebin/.*.app$"), - [Path || {zip_file, Path, _, _, _, _} <- ZippedFiles, - re:run(Path, RE, [{capture, none}]) =:= match]. - -parse_binary(Bin) -> - try - {ok, Ts, _} = erl_scan:string(binary_to_list(Bin)), - {ok, Term} = erl_parse:parse_term(Ts), - Term - catch - Err -> {error, {invalid_app, Err}} - end. - -plugin_names(Plugins) -> - [Name || #plugin{name = Name} <- Plugins]. - -lookup_plugins(Names, AllPlugins) -> - %% Preserve order of Names - lists:map( - fun(Name) -> - lists:keyfind(Name, #plugin.name, AllPlugins) - end, - Names). - -%% Split PATH-like value into its components. -split_path(PathString) -> - Delimiters = case os:type() of - {unix, _} -> ":"; - {win32, _} -> ";" - end, - string:tokens(PathString, Delimiters). - -%% Search for files using glob in a given dir. Returns full filenames of those files. -full_path_wildcard(Glob, Dir) -> - [filename:join([Dir, File]) || File <- filelib:wildcard(Glob, Dir)]. - -%% Returns list off all .ez files in a given set of directories -list_ezs([]) -> - []; -list_ezs([Dir|Rest]) -> - [{ez, EZ} || EZ <- full_path_wildcard("*.ez", Dir)] ++ list_ezs(Rest). - -%% Returns list of all files that look like OTP applications in a -%% given set of directories. -list_free_apps([]) -> - []; -list_free_apps([Dir|Rest]) -> - [{app, App} || App <- full_path_wildcard("*/ebin/*.app", Dir)] - ++ list_free_apps(Rest). - -compare_by_name_and_version(#plugin{name = Name, version = VersionA}, - #plugin{name = Name, version = VersionB}) -> - rabbit_semver:lte(VersionA, VersionB); -compare_by_name_and_version(#plugin{name = NameA}, - #plugin{name = NameB}) -> - NameA =< NameB. - --spec discover_plugins([Directory]) -> {[#plugin{}], [Problem]} when - Directory :: file:name(), - Problem :: {file:name(), term()}. -discover_plugins(PluginsDirs) -> - EZs = list_ezs(PluginsDirs), - FreeApps = list_free_apps(PluginsDirs), - read_plugins_info(EZs ++ FreeApps, {[], []}). - -read_plugins_info([], Acc) -> - Acc; -read_plugins_info([Path|Paths], {Plugins, Problems}) -> - case plugin_info(Path) of - #plugin{} = Plugin -> - read_plugins_info(Paths, {[Plugin|Plugins], Problems}); - {error, Location, Reason} -> - read_plugins_info(Paths, {Plugins, [{Location, Reason}|Problems]}) - end. - -remove_duplicate_plugins(Plugins) -> - %% Reverse order ensures that if there are several versions of the - %% same plugin, the most recent one comes first. - Sorted = lists:reverse( - lists:sort(fun compare_by_name_and_version/2, Plugins)), - remove_duplicate_plugins(Sorted, {[], []}). - -remove_duplicate_plugins([], Acc) -> - Acc; -remove_duplicate_plugins([Best = #plugin{name = Name}, Offender = #plugin{name = Name} | Rest], - {Plugins0, Problems0}) -> - Problems1 = [{Offender#plugin.location, duplicate_plugin}|Problems0], - remove_duplicate_plugins([Best|Rest], {Plugins0, Problems1}); -remove_duplicate_plugins([Plugin|Rest], {Plugins0, Problems0}) -> - Plugins1 = [Plugin|Plugins0], - remove_duplicate_plugins(Rest, {Plugins1, Problems0}). - -maybe_keep_required_deps(true, Plugins) -> - Plugins; -maybe_keep_required_deps(false, Plugins) -> - RabbitDeps = list_all_deps([rabbit]), - lists:filter(fun - (#plugin{name = Name}) -> - not lists:member(Name, RabbitDeps); - (Name) when is_atom(Name) -> - not lists:member(Name, RabbitDeps) - end, - Plugins). - -list_all_deps(Applications) -> - list_all_deps(Applications, []). - -list_all_deps([Application | Applications], Deps) -> - %% We load the application to be sure we can get the "applications" key. - %% This is required for rabbitmq-plugins for instance. - application:load(Application), - NewDeps = [Application | Deps], - case application:get_key(Application, applications) of - {ok, ApplicationDeps} -> - RemainingApplications0 = ApplicationDeps ++ Applications, - RemainingApplications = RemainingApplications0 -- NewDeps, - list_all_deps(RemainingApplications, NewDeps); - undefined -> - list_all_deps(Applications, NewDeps) - end; -list_all_deps([], Deps) -> - Deps. - -remove_plugins(Plugins) -> - %% We want to filter out all Erlang applications in the plugins - %% directories which are not actual RabbitMQ plugin. - %% - %% A RabbitMQ plugin must depend on `rabbit`. We also want to keep - %% all applications they depend on, except Erlang/OTP applications. - %% In the end, we will skip: - %% * Erlang/OTP applications - %% * All applications which do not depend on `rabbit` and which - %% are not direct or indirect dependencies of plugins. - ActualPlugins = [Plugin - || #plugin{dependencies = Deps} = Plugin <- Plugins, - lists:member(rabbit, Deps)], - %% As said above, we want to keep all non-plugins which are - %% dependencies of plugins. - PluginDeps = lists:usort( - lists:flatten( - [resolve_deps(Plugins, Plugin) - || Plugin <- ActualPlugins])), - lists:filter( - fun(#plugin{name = Name} = Plugin) -> - IsOTPApp = is_plugin_provided_by_otp(Plugin), - IsAPlugin = - lists:member(Plugin, ActualPlugins) orelse - lists:member(Name, PluginDeps), - if - IsOTPApp -> - rabbit_log:debug( - "Plugins discovery: " - "ignoring ~s, Erlang/OTP application", - [Name]); - not IsAPlugin -> - rabbit_log:debug( - "Plugins discovery: " - "ignoring ~s, not a RabbitMQ plugin", - [Name]); - true -> - ok - end, - not (IsOTPApp orelse not IsAPlugin) - end, Plugins). - -resolve_deps(Plugins, #plugin{dependencies = Deps}) -> - IndirectDeps = [case lists:keyfind(Dep, #plugin.name, Plugins) of - false -> []; - DepPlugin -> resolve_deps(Plugins, DepPlugin) - end - || Dep <- Deps], - Deps ++ IndirectDeps. - -maybe_report_plugin_loading_problems([]) -> - ok; -maybe_report_plugin_loading_problems(Problems) -> - io:format(standard_error, - "Problem reading some plugins: ~p~n", - [Problems]). diff --git a/src/rabbit_policies.erl b/src/rabbit_policies.erl deleted file mode 100644 index 54e4d2c03e..0000000000 --- a/src/rabbit_policies.erl +++ /dev/null @@ -1,179 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_policies). - -%% Provides built-in policy parameter -%% validation functions. - --behaviour(rabbit_policy_validator). --behaviour(rabbit_policy_merge_strategy). - --include("rabbit.hrl"). - --export([register/0, validate_policy/1, merge_policy_value/3]). - --rabbit_boot_step({?MODULE, - [{description, "internal policies"}, - {mfa, {rabbit_policies, register, []}}, - {requires, rabbit_registry}, - {enables, recovery}]}). - -register() -> - %% Note: there are more validators registered from other modules, - %% such as rabbit_mirror_queue_misc - [rabbit_registry:register(Class, Name, ?MODULE) || - {Class, Name} <- [{policy_validator, <<"alternate-exchange">>}, - {policy_validator, <<"dead-letter-exchange">>}, - {policy_validator, <<"dead-letter-routing-key">>}, - {policy_validator, <<"message-ttl">>}, - {policy_validator, <<"expires">>}, - {policy_validator, <<"max-length">>}, - {policy_validator, <<"max-length-bytes">>}, - {policy_validator, <<"max-in-memory-length">>}, - {policy_validator, <<"max-in-memory-bytes">>}, - {policy_validator, <<"queue-mode">>}, - {policy_validator, <<"overflow">>}, - {policy_validator, <<"delivery-limit">>}, - {policy_validator, <<"max-age">>}, - {policy_validator, <<"max-segment-size">>}, - {policy_validator, <<"queue-leader-locator">>}, - {policy_validator, <<"initial-cluster-size">>}, - {operator_policy_validator, <<"expires">>}, - {operator_policy_validator, <<"message-ttl">>}, - {operator_policy_validator, <<"max-length">>}, - {operator_policy_validator, <<"max-length-bytes">>}, - {operator_policy_validator, <<"max-in-memory-length">>}, - {operator_policy_validator, <<"max-in-memory-bytes">>}, - {operator_policy_validator, <<"delivery-limit">>}, - {policy_merge_strategy, <<"expires">>}, - {policy_merge_strategy, <<"message-ttl">>}, - {policy_merge_strategy, <<"max-length">>}, - {policy_merge_strategy, <<"max-length-bytes">>}, - {policy_merge_strategy, <<"max-in-memory-length">>}, - {policy_merge_strategy, <<"max-in-memory-bytes">>}, - {policy_merge_strategy, <<"delivery-limit">>}]], - ok. - --spec validate_policy([{binary(), term()}]) -> rabbit_policy_validator:validate_results(). - -validate_policy(Terms) -> - lists:foldl(fun ({Key, Value}, ok) -> validate_policy0(Key, Value); - (_, Error) -> Error - end, ok, Terms). - -validate_policy0(<<"alternate-exchange">>, Value) - when is_binary(Value) -> - ok; -validate_policy0(<<"alternate-exchange">>, Value) -> - {error, "~p is not a valid alternate exchange name", [Value]}; - -validate_policy0(<<"dead-letter-exchange">>, Value) - when is_binary(Value) -> - ok; -validate_policy0(<<"dead-letter-exchange">>, Value) -> - {error, "~p is not a valid dead letter exchange name", [Value]}; - -validate_policy0(<<"dead-letter-routing-key">>, Value) - when is_binary(Value) -> - ok; -validate_policy0(<<"dead-letter-routing-key">>, Value) -> - {error, "~p is not a valid dead letter routing key", [Value]}; - -validate_policy0(<<"message-ttl">>, Value) - when is_integer(Value), Value >= 0 -> - ok; -validate_policy0(<<"message-ttl">>, Value) -> - {error, "~p is not a valid message TTL", [Value]}; - -validate_policy0(<<"expires">>, Value) - when is_integer(Value), Value >= 1 -> - ok; -validate_policy0(<<"expires">>, Value) -> - {error, "~p is not a valid queue expiry", [Value]}; - -validate_policy0(<<"max-length">>, Value) - when is_integer(Value), Value >= 0 -> - ok; -validate_policy0(<<"max-length">>, Value) -> - {error, "~p is not a valid maximum length", [Value]}; - -validate_policy0(<<"max-length-bytes">>, Value) - when is_integer(Value), Value >= 0 -> - ok; -validate_policy0(<<"max-length-bytes">>, Value) -> - {error, "~p is not a valid maximum length in bytes", [Value]}; - -validate_policy0(<<"max-in-memory-length">>, Value) - when is_integer(Value), Value >= 0 -> - ok; -validate_policy0(<<"max-in-memory-length">>, Value) -> - {error, "~p is not a valid maximum memory in bytes", [Value]}; - -validate_policy0(<<"max-in-memory-bytes">>, Value) - when is_integer(Value), Value >= 0 -> - ok; -validate_policy0(<<"max-in-memory-bytes">>, Value) -> - {error, "~p is not a valid maximum memory in bytes", [Value]}; - -validate_policy0(<<"queue-mode">>, <<"default">>) -> - ok; -validate_policy0(<<"queue-mode">>, <<"lazy">>) -> - ok; -validate_policy0(<<"queue-mode">>, Value) -> - {error, "~p is not a valid queue-mode value", [Value]}; -validate_policy0(<<"overflow">>, <<"drop-head">>) -> - ok; -validate_policy0(<<"overflow">>, <<"reject-publish">>) -> - ok; -validate_policy0(<<"overflow">>, <<"reject-publish-dlx">>) -> - ok; -validate_policy0(<<"overflow">>, Value) -> - {error, "~p is not a valid overflow value", [Value]}; - -validate_policy0(<<"delivery-limit">>, Value) - when is_integer(Value), Value >= 0 -> - ok; -validate_policy0(<<"delivery-limit">>, Value) -> - {error, "~p is not a valid delivery limit", [Value]}; - -validate_policy0(<<"max-age">>, Value) -> - case rabbit_amqqueue:check_max_age(Value) of - {error, _} -> - {error, "~p is not a valid max age", [Value]}; - _ -> - ok - end; - -validate_policy0(<<"queue-leader-locator">>, <<"client-local">>) -> - ok; -validate_policy0(<<"queue-leader-locator">>, <<"random">>) -> - ok; -validate_policy0(<<"queue-leader-locator">>, <<"least-leaders">>) -> - ok; -validate_policy0(<<"queue-leader-locator">>, Value) -> - {error, "~p is not a valid queue leader locator value", [Value]}; - -validate_policy0(<<"initial-cluster-size">>, Value) - when is_integer(Value), Value >= 0 -> - ok; -validate_policy0(<<"initial-cluster-size">>, Value) -> - {error, "~p is not a valid cluster size", [Value]}; - -validate_policy0(<<"max-segment-size">>, Value) - when is_integer(Value), Value >= 0 -> - ok; -validate_policy0(<<"max-segment-size">>, Value) -> - {error, "~p is not a valid segment size", [Value]}. - -merge_policy_value(<<"message-ttl">>, Val, OpVal) -> min(Val, OpVal); -merge_policy_value(<<"max-length">>, Val, OpVal) -> min(Val, OpVal); -merge_policy_value(<<"max-length-bytes">>, Val, OpVal) -> min(Val, OpVal); -merge_policy_value(<<"max-in-memory-length">>, Val, OpVal) -> min(Val, OpVal); -merge_policy_value(<<"max-in-memory-bytes">>, Val, OpVal) -> min(Val, OpVal); -merge_policy_value(<<"expires">>, Val, OpVal) -> min(Val, OpVal); -merge_policy_value(<<"delivery-limit">>, Val, OpVal) -> min(Val, OpVal). diff --git a/src/rabbit_policy.erl b/src/rabbit_policy.erl deleted file mode 100644 index 44807de97d..0000000000 --- a/src/rabbit_policy.erl +++ /dev/null @@ -1,557 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_policy). - -%% Policies is a way to apply optional arguments ("x-args") -%% to exchanges and queues in bulk, using name matching. -%% -%% Only one policy can apply to a given queue or exchange -%% at a time. Priorities help determine what policy should -%% take precedence. -%% -%% Policies build on runtime parameters. Policy-driven parameters -%% are well known and therefore validated. -%% -%% See also: -%% -%% * rabbit_runtime_parameters -%% * rabbit_policies -%% * rabbit_registry - -%% TODO specs - --behaviour(rabbit_runtime_parameter). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --import(rabbit_misc, [pget/2, pget/3]). - --export([register/0]). --export([invalidate/0, recover/0]). --export([name/1, name_op/1, effective_definition/1, merge_operator_definitions/2, get/2, get_arg/3, set/1]). --export([validate/5, notify/5, notify_clear/4]). --export([parse_set/7, set/7, delete/3, lookup/2, list/0, list/1, - list_formatted/1, list_formatted/3, info_keys/0]). --export([parse_set_op/7, set_op/7, delete_op/3, lookup_op/2, list_op/0, list_op/1, - list_formatted_op/1, list_formatted_op/3]). - --rabbit_boot_step({?MODULE, - [{description, "policy parameters"}, - {mfa, {rabbit_policy, register, []}}, - {requires, rabbit_registry}, - {enables, recovery}]}). - -register() -> - rabbit_registry:register(runtime_parameter, <<"policy">>, ?MODULE), - rabbit_registry:register(runtime_parameter, <<"operator_policy">>, ?MODULE). - -name(Q) when ?is_amqqueue(Q) -> - Policy = amqqueue:get_policy(Q), - name0(Policy); -name(#exchange{policy = Policy}) -> name0(Policy). - -name_op(Q) when ?is_amqqueue(Q) -> - OpPolicy = amqqueue:get_operator_policy(Q), - name0(OpPolicy); -name_op(#exchange{operator_policy = Policy}) -> name0(Policy). - -name0(undefined) -> none; -name0(Policy) -> pget(name, Policy). - -effective_definition(Q) when ?is_amqqueue(Q) -> - Policy = amqqueue:get_policy(Q), - OpPolicy = amqqueue:get_operator_policy(Q), - merge_operator_definitions(Policy, OpPolicy); -effective_definition(#exchange{policy = Policy, operator_policy = OpPolicy}) -> - merge_operator_definitions(Policy, OpPolicy). - -merge_operator_definitions(undefined, undefined) -> undefined; -merge_operator_definitions(Policy, undefined) -> pget(definition, Policy); -merge_operator_definitions(undefined, OpPolicy) -> pget(definition, OpPolicy); -merge_operator_definitions(Policy, OpPolicy) -> - OpDefinition = rabbit_data_coercion:to_map(pget(definition, OpPolicy, [])), - Definition = rabbit_data_coercion:to_map(pget(definition, Policy, [])), - Keys = maps:keys(Definition), - OpKeys = maps:keys(OpDefinition), - lists:map(fun(Key) -> - case {maps:get(Key, Definition, undefined), maps:get(Key, OpDefinition, undefined)} of - {Val, undefined} -> {Key, Val}; - {undefined, OpVal} -> {Key, OpVal}; - {Val, OpVal} -> {Key, merge_policy_value(Key, Val, OpVal)} - end - end, - lists:umerge(Keys, OpKeys)). - -set(Q0) when ?is_amqqueue(Q0) -> - Name = amqqueue:get_name(Q0), - Policy = match(Name), - OpPolicy = match_op(Name), - Q1 = amqqueue:set_policy(Q0, Policy), - Q2 = amqqueue:set_operator_policy(Q1, OpPolicy), - Q2; -set(X = #exchange{name = Name}) -> - X#exchange{policy = match(Name), operator_policy = match_op(Name)}. - -match(Name = #resource{virtual_host = VHost}) -> - match(Name, list(VHost)). - -match_op(Name = #resource{virtual_host = VHost}) -> - match(Name, list_op(VHost)). - -get(Name, Q) when ?is_amqqueue(Q) -> - Policy = amqqueue:get_policy(Q), - OpPolicy = amqqueue:get_operator_policy(Q), - get0(Name, Policy, OpPolicy); -get(Name, #exchange{policy = Policy, operator_policy = OpPolicy}) -> - get0(Name, Policy, OpPolicy); - -%% Caution - SLOW. -get(Name, EntityName = #resource{virtual_host = VHost}) -> - get0(Name, - match(EntityName, list(VHost)), - match(EntityName, list_op(VHost))). - -get0(_Name, undefined, undefined) -> undefined; -get0(Name, undefined, OpPolicy) -> pget(Name, pget(definition, OpPolicy, [])); -get0(Name, Policy, undefined) -> pget(Name, pget(definition, Policy, [])); -get0(Name, Policy, OpPolicy) -> - OpDefinition = pget(definition, OpPolicy, []), - Definition = pget(definition, Policy, []), - case {pget(Name, Definition), pget(Name, OpDefinition)} of - {undefined, undefined} -> undefined; - {Val, undefined} -> Val; - {undefined, Val} -> Val; - {Val, OpVal} -> merge_policy_value(Name, Val, OpVal) - end. - -merge_policy_value(Name, PolicyVal, OpVal) -> - case policy_merge_strategy(Name) of - {ok, Module} -> Module:merge_policy_value(Name, PolicyVal, OpVal); - {error, not_found} -> rabbit_policies:merge_policy_value(Name, PolicyVal, OpVal) - end. - -policy_merge_strategy(Name) -> - case rabbit_registry:binary_to_type(rabbit_data_coercion:to_binary(Name)) of - {error, not_found} -> - {error, not_found}; - T -> - rabbit_registry:lookup_module(policy_merge_strategy, T) - end. - -%% Many heads for optimisation -get_arg(_AName, _PName, #exchange{arguments = [], policy = undefined}) -> - undefined; -get_arg(_AName, PName, X = #exchange{arguments = []}) -> - get(PName, X); -get_arg(AName, PName, X = #exchange{arguments = Args}) -> - case rabbit_misc:table_lookup(Args, AName) of - undefined -> get(PName, X); - {_Type, Arg} -> Arg - end. - -%%---------------------------------------------------------------------------- - -%% Gets called during upgrades - therefore must not assume anything about the -%% state of Mnesia -invalidate() -> - rabbit_file:write_file(invalid_file(), <<"">>). - -recover() -> - case rabbit_file:is_file(invalid_file()) of - true -> recover0(), - rabbit_file:delete(invalid_file()); - false -> ok - end. - -%% To get here we have to have just completed an Mnesia upgrade - i.e. we are -%% the first node starting. So we can rewrite the whole database. Note that -%% recovery has not yet happened; we must work with the rabbit_durable_<thing> -%% variants. -recover0() -> - Xs = mnesia:dirty_match_object(rabbit_durable_exchange, #exchange{_ = '_'}), - Qs = rabbit_amqqueue:list_with_possible_retry( - fun() -> - mnesia:dirty_match_object( - rabbit_durable_queue, amqqueue:pattern_match_all()) - end), - Policies = list(), - OpPolicies = list_op(), - [rabbit_misc:execute_mnesia_transaction( - fun () -> - mnesia:write( - rabbit_durable_exchange, - rabbit_exchange_decorator:set( - X#exchange{policy = match(Name, Policies), - operator_policy = match(Name, OpPolicies)}), - write) - end) || X = #exchange{name = Name} <- Xs], - [begin - QName = amqqueue:get_name(Q0), - Policy1 = match(QName, Policies), - Q1 = amqqueue:set_policy(Q0, Policy1), - OpPolicy1 = match(QName, OpPolicies), - Q2 = amqqueue:set_operator_policy(Q1, OpPolicy1), - Q3 = rabbit_queue_decorator:set(Q2), - ?try_mnesia_tx_or_upgrade_amqqueue_and_retry( - rabbit_misc:execute_mnesia_transaction( - fun () -> - mnesia:write(rabbit_durable_queue, Q3, write) - end), - begin - Q4 = amqqueue:upgrade(Q3), - rabbit_misc:execute_mnesia_transaction( - fun () -> - mnesia:write(rabbit_durable_queue, Q4, write) - end) - end) - end || Q0 <- Qs], - ok. - -invalid_file() -> - filename:join(rabbit_mnesia:dir(), "policies_are_invalid"). - -%%---------------------------------------------------------------------------- - -parse_set_op(VHost, Name, Pattern, Definition, Priority, ApplyTo, ActingUser) -> - parse_set(<<"operator_policy">>, VHost, Name, Pattern, Definition, Priority, - ApplyTo, ActingUser). - -parse_set(VHost, Name, Pattern, Definition, Priority, ApplyTo, ActingUser) -> - parse_set(<<"policy">>, VHost, Name, Pattern, Definition, Priority, ApplyTo, - ActingUser). - -parse_set(Type, VHost, Name, Pattern, Definition, Priority, ApplyTo, ActingUser) -> - try rabbit_data_coercion:to_integer(Priority) of - Num -> parse_set0(Type, VHost, Name, Pattern, Definition, Num, ApplyTo, - ActingUser) - catch - error:badarg -> {error, "~p priority must be a number", [Priority]} - end. - -parse_set0(Type, VHost, Name, Pattern, Defn, Priority, ApplyTo, ActingUser) -> - case rabbit_json:try_decode(Defn) of - {ok, Term} -> - R = set0(Type, VHost, Name, - [{<<"pattern">>, Pattern}, - {<<"definition">>, maps:to_list(Term)}, - {<<"priority">>, Priority}, - {<<"apply-to">>, ApplyTo}], - ActingUser), - rabbit_log:info("Successfully set policy '~s' matching ~s names in virtual host '~s' using pattern '~s'", - [Name, ApplyTo, VHost, Pattern]), - R; - {error, Reason} -> - {error_string, - rabbit_misc:format("JSON decoding error. Reason: ~ts", [Reason])} - end. - -set_op(VHost, Name, Pattern, Definition, Priority, ApplyTo, ActingUser) -> - set(<<"operator_policy">>, VHost, Name, Pattern, Definition, Priority, ApplyTo, ActingUser). - -set(VHost, Name, Pattern, Definition, Priority, ApplyTo, ActingUser) -> - set(<<"policy">>, VHost, Name, Pattern, Definition, Priority, ApplyTo, ActingUser). - -set(Type, VHost, Name, Pattern, Definition, Priority, ApplyTo, ActingUser) -> - PolicyProps = [{<<"pattern">>, Pattern}, - {<<"definition">>, Definition}, - {<<"priority">>, case Priority of - undefined -> 0; - _ -> Priority - end}, - {<<"apply-to">>, case ApplyTo of - undefined -> <<"all">>; - _ -> ApplyTo - end}], - set0(Type, VHost, Name, PolicyProps, ActingUser). - -set0(Type, VHost, Name, Term, ActingUser) -> - rabbit_runtime_parameters:set_any(VHost, Type, Name, Term, ActingUser). - -delete_op(VHost, Name, ActingUser) -> - rabbit_runtime_parameters:clear_any(VHost, <<"operator_policy">>, Name, ActingUser). - -delete(VHost, Name, ActingUser) -> - rabbit_runtime_parameters:clear_any(VHost, <<"policy">>, Name, ActingUser). - -lookup_op(VHost, Name) -> - case rabbit_runtime_parameters:lookup(VHost, <<"operator_policy">>, Name) of - not_found -> not_found; - P -> p(P, fun ident/1) - end. - -lookup(VHost, Name) -> - case rabbit_runtime_parameters:lookup(VHost, <<"policy">>, Name) of - not_found -> not_found; - P -> p(P, fun ident/1) - end. - -list_op() -> - list_op('_'). - -list_op(VHost) -> - list0_op(VHost, fun ident/1). - -list_formatted_op(VHost) -> - order_policies(list0_op(VHost, fun rabbit_json:encode/1)). - -list_formatted_op(VHost, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map(AggregatorPid, Ref, - fun(P) -> P end, list_formatted_op(VHost)). - -list0_op(VHost, DefnFun) -> - [p(P, DefnFun) - || P <- rabbit_runtime_parameters:list(VHost, <<"operator_policy">>)]. - - -list() -> - list('_'). - -list(VHost) -> - list0(VHost, fun ident/1). - -list_formatted(VHost) -> - order_policies(list0(VHost, fun rabbit_json:encode/1)). - -list_formatted(VHost, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map(AggregatorPid, Ref, - fun(P) -> P end, list_formatted(VHost)). - -list0(VHost, DefnFun) -> - [p(P, DefnFun) || P <- rabbit_runtime_parameters:list(VHost, <<"policy">>)]. - -order_policies(PropList) -> - lists:sort(fun (A, B) -> not sort_pred(A, B) end, PropList). - -p(Parameter, DefnFun) -> - Value = pget(value, Parameter), - [{vhost, pget(vhost, Parameter)}, - {name, pget(name, Parameter)}, - {pattern, pget(<<"pattern">>, Value)}, - {'apply-to', pget(<<"apply-to">>, Value)}, - {definition, DefnFun(pget(<<"definition">>, Value))}, - {priority, pget(<<"priority">>, Value)}]. - -ident(X) -> X. - -info_keys() -> [vhost, name, 'apply-to', pattern, definition, priority]. - -%%---------------------------------------------------------------------------- - -validate(_VHost, <<"policy">>, Name, Term, _User) -> - rabbit_parameter_validation:proplist( - Name, policy_validation(), Term); -validate(_VHost, <<"operator_policy">>, Name, Term, _User) -> - rabbit_parameter_validation:proplist( - Name, operator_policy_validation(), Term). - -notify(VHost, <<"policy">>, Name, Term, ActingUser) -> - rabbit_event:notify(policy_set, [{name, Name}, {vhost, VHost}, - {user_who_performed_action, ActingUser} | Term]), - update_policies(VHost); -notify(VHost, <<"operator_policy">>, Name, Term, ActingUser) -> - rabbit_event:notify(policy_set, [{name, Name}, {vhost, VHost}, - {user_who_performed_action, ActingUser} | Term]), - update_policies(VHost). - -notify_clear(VHost, <<"policy">>, Name, ActingUser) -> - rabbit_event:notify(policy_cleared, [{name, Name}, {vhost, VHost}, - {user_who_performed_action, ActingUser}]), - update_policies(VHost); -notify_clear(VHost, <<"operator_policy">>, Name, ActingUser) -> - rabbit_event:notify(operator_policy_cleared, - [{name, Name}, {vhost, VHost}, - {user_who_performed_action, ActingUser}]), - update_policies(VHost). - -%%---------------------------------------------------------------------------- - -%% [1] We need to prevent this from becoming O(n^2) in a similar -%% manner to rabbit_binding:remove_for_{source,destination}. So see -%% the comment in rabbit_binding:lock_route_tables/0 for more rationale. -%% [2] We could be here in a post-tx fun after the vhost has been -%% deleted; in which case it's fine to do nothing. -update_policies(VHost) -> - Tabs = [rabbit_queue, rabbit_durable_queue, - rabbit_exchange, rabbit_durable_exchange], - {Xs, Qs} = rabbit_misc:execute_mnesia_transaction( - fun() -> - [mnesia:lock({table, T}, write) || T <- Tabs], %% [1] - case catch {list(VHost), list_op(VHost)} of - {'EXIT', {throw, {error, {no_such_vhost, _}}}} -> - {[], []}; %% [2] - {'EXIT', Exit} -> - exit(Exit); - {Policies, OpPolicies} -> - {[update_exchange(X, Policies, OpPolicies) || - X <- rabbit_exchange:list(VHost)], - [update_queue(Q, Policies, OpPolicies) || - Q <- rabbit_amqqueue:list(VHost)]} - end - end), - [catch notify(X) || X <- Xs], - [catch notify(Q) || Q <- Qs], - ok. - -update_exchange(X = #exchange{name = XName, - policy = OldPolicy, - operator_policy = OldOpPolicy}, - Policies, OpPolicies) -> - case {match(XName, Policies), match(XName, OpPolicies)} of - {OldPolicy, OldOpPolicy} -> no_change; - {NewPolicy, NewOpPolicy} -> - NewExchange = rabbit_exchange:update( - XName, - fun(X0) -> - rabbit_exchange_decorator:set( - X0 #exchange{policy = NewPolicy, - operator_policy = NewOpPolicy}) - end), - case NewExchange of - #exchange{} = X1 -> {X, X1}; - not_found -> {X, X } - end - end. - -update_queue(Q0, Policies, OpPolicies) when ?is_amqqueue(Q0) -> - QName = amqqueue:get_name(Q0), - OldPolicy = amqqueue:get_policy(Q0), - OldOpPolicy = amqqueue:get_operator_policy(Q0), - case {match(QName, Policies), match(QName, OpPolicies)} of - {OldPolicy, OldOpPolicy} -> no_change; - {NewPolicy, NewOpPolicy} -> - F = fun (QFun0) -> - QFun1 = amqqueue:set_policy(QFun0, NewPolicy), - QFun2 = amqqueue:set_operator_policy(QFun1, NewOpPolicy), - NewPolicyVersion = amqqueue:get_policy_version(QFun2) + 1, - QFun3 = amqqueue:set_policy_version(QFun2, NewPolicyVersion), - rabbit_queue_decorator:set(QFun3) - end, - NewQueue = rabbit_amqqueue:update(QName, F), - case NewQueue of - Q1 when ?is_amqqueue(Q1) -> - {Q0, Q1}; - not_found -> - {Q0, Q0} - end - end. - -notify(no_change)-> - ok; -notify({X1 = #exchange{}, X2 = #exchange{}}) -> - rabbit_exchange:policy_changed(X1, X2); -notify({Q1, Q2}) when ?is_amqqueue(Q1), ?is_amqqueue(Q2) -> - rabbit_amqqueue:policy_changed(Q1, Q2). - -match(Name, Policies) -> - case match_all(Name, Policies) of - [] -> undefined; - [Policy | _] -> Policy - end. - -match_all(Name, Policies) -> - lists:sort(fun sort_pred/2, [P || P <- Policies, matches(Name, P)]). - -matches(#resource{name = Name, kind = Kind, virtual_host = VHost} = Resource, Policy) -> - matches_type(Kind, pget('apply-to', Policy)) andalso - is_applicable(Resource, pget(definition, Policy)) andalso - match =:= re:run(Name, pget(pattern, Policy), [{capture, none}]) andalso - VHost =:= pget(vhost, Policy). - -matches_type(exchange, <<"exchanges">>) -> true; -matches_type(queue, <<"queues">>) -> true; -matches_type(exchange, <<"all">>) -> true; -matches_type(queue, <<"all">>) -> true; -matches_type(_, _) -> false. - -sort_pred(A, B) -> pget(priority, A) >= pget(priority, B). - -is_applicable(#resource{kind = queue} = Resource, Policy) -> - rabbit_amqqueue:is_policy_applicable(Resource, to_list(Policy)); -is_applicable(_, _) -> - true. - -to_list(L) when is_list(L) -> - L; -to_list(M) when is_map(M) -> - maps:to_list(M). - -%%---------------------------------------------------------------------------- - -operator_policy_validation() -> - [{<<"priority">>, fun rabbit_parameter_validation:number/2, mandatory}, - {<<"pattern">>, fun rabbit_parameter_validation:regex/2, mandatory}, - {<<"apply-to">>, fun apply_to_validation/2, optional}, - {<<"definition">>, fun validation_op/2, mandatory}]. - -policy_validation() -> - [{<<"priority">>, fun rabbit_parameter_validation:number/2, mandatory}, - {<<"pattern">>, fun rabbit_parameter_validation:regex/2, mandatory}, - {<<"apply-to">>, fun apply_to_validation/2, optional}, - {<<"definition">>, fun validation/2, mandatory}]. - -validation_op(Name, Terms) -> - validation(Name, Terms, operator_policy_validator). - -validation(Name, Terms) -> - validation(Name, Terms, policy_validator). - -validation(_Name, [], _Validator) -> - {error, "no policy provided", []}; -validation(Name, Terms0, Validator) when is_map(Terms0) -> - Terms = maps:to_list(Terms0), - validation(Name, Terms, Validator); -validation(_Name, Terms, Validator) when is_list(Terms) -> - {Keys, Modules} = lists:unzip( - rabbit_registry:lookup_all(Validator)), - [] = dups(Keys), %% ASSERTION - Validators = lists:zipwith(fun (M, K) -> {M, a2b(K)} end, Modules, Keys), - case is_proplist(Terms) of - true -> {TermKeys, _} = lists:unzip(Terms), - case dups(TermKeys) of - [] -> validation0(Validators, Terms); - Dup -> {error, "~p duplicate keys not allowed", [Dup]} - end; - false -> {error, "definition must be a dictionary: ~p", [Terms]} - end; -validation(Name, Term, Validator) -> - {error, "parse error while reading policy ~s: ~p. Validator: ~p.", - [Name, Term, Validator]}. - -validation0(Validators, Terms) -> - case lists:foldl( - fun (Mod, {ok, TermsLeft}) -> - ModKeys = proplists:get_all_values(Mod, Validators), - case [T || {Key, _} = T <- TermsLeft, - lists:member(Key, ModKeys)] of - [] -> {ok, TermsLeft}; - Scope -> {Mod:validate_policy(Scope), TermsLeft -- Scope} - end; - (_, Acc) -> - Acc - end, {ok, Terms}, proplists:get_keys(Validators)) of - {ok, []} -> - ok; - {ok, Unvalidated} -> - {error, "~p are not recognised policy settings", [Unvalidated]}; - {Error, _} -> - Error - end. - -a2b(A) -> list_to_binary(atom_to_list(A)). - -dups(L) -> L -- lists:usort(L). - -is_proplist(L) -> length(L) =:= length([I || I = {_, _} <- L]). - -apply_to_validation(_Name, <<"all">>) -> ok; -apply_to_validation(_Name, <<"exchanges">>) -> ok; -apply_to_validation(_Name, <<"queues">>) -> ok; -apply_to_validation(_Name, Term) -> - {error, "apply-to '~s' unrecognised; should be 'queues', 'exchanges' " - "or 'all'", [Term]}. diff --git a/src/rabbit_policy_merge_strategy.erl b/src/rabbit_policy_merge_strategy.erl deleted file mode 100644 index f2b79e5862..0000000000 --- a/src/rabbit_policy_merge_strategy.erl +++ /dev/null @@ -1,19 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_policy_merge_strategy). - --behaviour(rabbit_registry_class). - --export([added_to_rabbit_registry/2, removed_from_rabbit_registry/1]). - --callback merge_policy_value(binary(), Value, Value) -> - Value - when Value :: term(). - -added_to_rabbit_registry(_Type, _ModuleName) -> ok. -removed_from_rabbit_registry(_Type) -> ok. diff --git a/src/rabbit_prelaunch_cluster.erl b/src/rabbit_prelaunch_cluster.erl deleted file mode 100644 index 9d3cda99e3..0000000000 --- a/src/rabbit_prelaunch_cluster.erl +++ /dev/null @@ -1,22 +0,0 @@ --module(rabbit_prelaunch_cluster). - --export([setup/1]). - -setup(Context) -> - rabbit_log_prelaunch:debug(""), - rabbit_log_prelaunch:debug("== Clustering =="), - rabbit_log_prelaunch:debug("Preparing cluster status files"), - rabbit_node_monitor:prepare_cluster_status_files(), - case Context of - #{initial_pass := true} -> - rabbit_log_prelaunch:debug("Upgrading Mnesia schema"), - ok = rabbit_upgrade:maybe_upgrade_mnesia(); - _ -> - ok - end, - %% It's important that the consistency check happens after - %% the upgrade, since if we are a secondary node the - %% primary node will have forgotten us - rabbit_log_prelaunch:debug("Checking cluster consistency"), - rabbit_mnesia:check_cluster_consistency(), - ok. diff --git a/src/rabbit_prelaunch_enabled_plugins_file.erl b/src/rabbit_prelaunch_enabled_plugins_file.erl deleted file mode 100644 index 57fe32f8e6..0000000000 --- a/src/rabbit_prelaunch_enabled_plugins_file.erl +++ /dev/null @@ -1,53 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_prelaunch_enabled_plugins_file). - --include_lib("rabbit_common/include/rabbit.hrl"). - --export([setup/1]). - -setup(Context) -> - rabbit_log_prelaunch:debug(""), - rabbit_log_prelaunch:debug("== Enabled plugins file =="), - update_enabled_plugins_file(Context). - -%% ------------------------------------------------------------------- -%% `enabled_plugins` file content initialization. -%% ------------------------------------------------------------------- - -update_enabled_plugins_file(#{enabled_plugins := undefined}) -> - ok; -update_enabled_plugins_file(#{enabled_plugins := all, - plugins_path := Path} = Context) -> - List = [P#plugin.name || P <- rabbit_plugins:list(Path)], - do_update_enabled_plugins_file(Context, List); -update_enabled_plugins_file(#{enabled_plugins := List} = Context) -> - do_update_enabled_plugins_file(Context, List). - -do_update_enabled_plugins_file(#{enabled_plugins_file := File}, List) -> - SortedList = lists:usort(List), - case SortedList of - [] -> - rabbit_log_prelaunch:debug("Marking all plugins as disabled"); - _ -> - rabbit_log_prelaunch:debug( - "Marking the following plugins as enabled:"), - [rabbit_log_prelaunch:debug(" - ~s", [P]) || P <- SortedList] - end, - Content = io_lib:format("~p.~n", [SortedList]), - case file:write_file(File, Content) of - ok -> - rabbit_log_prelaunch:debug("Wrote plugins file: ~ts", [File]), - ok; - {error, Reason} -> - rabbit_log_prelaunch:error( - "Failed to update enabled plugins file \"~ts\" " - "from $RABBITMQ_ENABLED_PLUGINS: ~ts", - [File, file:format_error(Reason)]), - throw({error, failed_to_update_enabled_plugins_file}) - end. diff --git a/src/rabbit_prelaunch_feature_flags.erl b/src/rabbit_prelaunch_feature_flags.erl deleted file mode 100644 index cd7b276f4c..0000000000 --- a/src/rabbit_prelaunch_feature_flags.erl +++ /dev/null @@ -1,32 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_prelaunch_feature_flags). - --export([setup/1]). - -setup(#{feature_flags_file := FFFile}) -> - rabbit_log_prelaunch:debug(""), - rabbit_log_prelaunch:debug("== Feature flags =="), - case filelib:ensure_dir(FFFile) of - ok -> - rabbit_log_prelaunch:debug("Initializing feature flags registry"), - case rabbit_feature_flags:initialize_registry() of - ok -> - ok; - {error, Reason} -> - rabbit_log_prelaunch:error( - "Failed to initialize feature flags registry: ~p", - [Reason]), - throw({error, failed_to_initialize_feature_flags_registry}) - end; - {error, Reason} -> - rabbit_log_prelaunch:error( - "Failed to create feature flags file \"~ts\" directory: ~ts", - [FFFile, file:format_error(Reason)]), - throw({error, failed_to_create_feature_flags_file_directory}) - end. diff --git a/src/rabbit_prelaunch_logging.erl b/src/rabbit_prelaunch_logging.erl deleted file mode 100644 index 6e3f040ec5..0000000000 --- a/src/rabbit_prelaunch_logging.erl +++ /dev/null @@ -1,75 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_prelaunch_logging). - --export([setup/1]). - -setup(Context) -> - rabbit_log_prelaunch:debug(""), - rabbit_log_prelaunch:debug("== Logging =="), - ok = set_ERL_CRASH_DUMP_envvar(Context), - ok = configure_lager(Context). - -set_ERL_CRASH_DUMP_envvar(#{log_base_dir := LogBaseDir}) -> - case os:getenv("ERL_CRASH_DUMP") of - false -> - ErlCrashDump = filename:join(LogBaseDir, "erl_crash.dump"), - rabbit_log_prelaunch:debug( - "Setting $ERL_CRASH_DUMP environment variable to \"~ts\"", - [ErlCrashDump]), - os:putenv("ERL_CRASH_DUMP", ErlCrashDump), - ok; - ErlCrashDump -> - rabbit_log_prelaunch:debug( - "$ERL_CRASH_DUMP environment variable already set to \"~ts\"", - [ErlCrashDump]), - ok - end. - -configure_lager(#{log_base_dir := LogBaseDir, - main_log_file := MainLog, - upgrade_log_file := UpgradeLog} = Context) -> - {SaslErrorLogger, - MainLagerHandler, - UpgradeLagerHandler} = case MainLog of - "-" -> - %% Log to STDOUT. - rabbit_log_prelaunch:debug( - "Logging to stdout"), - {tty, - tty, - tty}; - _ -> - rabbit_log_prelaunch:debug( - "Logging to:"), - [rabbit_log_prelaunch:debug( - " - ~ts", [Log]) - || Log <- [MainLog, UpgradeLog]], - %% Log to file. - {false, - MainLog, - UpgradeLog} - end, - - ok = application:set_env(lager, crash_log, "log/crash.log"), - - Fun = fun({App, Var, Value}) -> - case application:get_env(App, Var) of - undefined -> ok = application:set_env(App, Var, Value); - _ -> ok - end - end, - Vars = [{sasl, sasl_error_logger, SaslErrorLogger}, - {rabbit, lager_log_root, LogBaseDir}, - {rabbit, lager_default_file, MainLagerHandler}, - {rabbit, lager_upgrade_file, UpgradeLagerHandler}], - lists:foreach(Fun, Vars), - - ok = rabbit_lager:start_logger(), - - ok = rabbit_prelaunch_early_logging:setup_early_logging(Context, false). diff --git a/src/rabbit_prequeue.erl b/src/rabbit_prequeue.erl deleted file mode 100644 index b5af8927c7..0000000000 --- a/src/rabbit_prequeue.erl +++ /dev/null @@ -1,100 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2010-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_prequeue). - -%% This is the initial gen_server that all queue processes start off -%% as. It handles the decision as to whether we need to start a new -%% mirror, a new master/unmirrored, or whether we are restarting (and -%% if so, as what). Thus a crashing queue process can restart from here -%% and always do the right thing. - --export([start_link/3]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - --behaviour(gen_server2). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - -%%---------------------------------------------------------------------------- - --export_type([start_mode/0]). - --type start_mode() :: 'declare' | 'recovery' | 'slave'. - -%%---------------------------------------------------------------------------- - --spec start_link(amqqueue:amqqueue(), start_mode(), pid()) - -> rabbit_types:ok_pid_or_error(). - -start_link(Q, StartMode, Marker) -> - gen_server2:start_link(?MODULE, {Q, StartMode, Marker}, []). - -%%---------------------------------------------------------------------------- - -init({Q, StartMode, Marker}) -> - init(Q, case {is_process_alive(Marker), StartMode} of - {true, slave} -> slave; - {true, _} -> master; - {false, _} -> restart - end). - -init(Q, master) -> rabbit_amqqueue_process:init(Q); -init(Q, slave) -> rabbit_mirror_queue_slave:init(Q); - -init(Q0, restart) when ?is_amqqueue(Q0) -> - QueueName = amqqueue:get_name(Q0), - {ok, Q1} = rabbit_amqqueue:lookup(QueueName), - QPid = amqqueue:get_pid(Q1), - SPids = amqqueue:get_slave_pids(Q1), - LocalOrMasterDown = node(QPid) =:= node() - orelse not rabbit_mnesia:on_running_node(QPid), - Slaves = [SPid || SPid <- SPids, rabbit_mnesia:is_process_alive(SPid)], - case rabbit_mnesia:is_process_alive(QPid) of - true -> false = LocalOrMasterDown, %% assertion - rabbit_mirror_queue_slave:go(self(), async), - rabbit_mirror_queue_slave:init(Q1); %% [1] - false -> case LocalOrMasterDown andalso Slaves =:= [] of - true -> crash_restart(Q1); %% [2] - false -> timer:sleep(25), - init(Q1, restart) %% [3] - end - end. -%% [1] There is a master on another node. Regardless of whether we -%% were originally a master or a mirror, we are now a new slave. -%% -%% [2] Nothing is alive. We are the last best hope. Try to restart as a master. -%% -%% [3] The current master is dead but either there are alive mirrors to -%% take over or it's all happening on a different node anyway. This is -%% not a stable situation. Sleep and wait for somebody else to make a -%% move. - -crash_restart(Q0) when ?is_amqqueue(Q0) -> - QueueName = amqqueue:get_name(Q0), - rabbit_log:error("Restarting crashed ~s.~n", [rabbit_misc:rs(QueueName)]), - gen_server2:cast(self(), init), - Q1 = amqqueue:set_pid(Q0, self()), - rabbit_amqqueue_process:init(Q1). - -%%---------------------------------------------------------------------------- - -%% This gen_server2 always hands over to some other module at the end -%% of init/1. --spec handle_call(_, _, _) -> no_return(). -handle_call(_Msg, _From, _State) -> exit(unreachable). --spec handle_cast(_, _) -> no_return(). -handle_cast(_Msg, _State) -> exit(unreachable). --spec handle_info(_, _) -> no_return(). -handle_info(_Msg, _State) -> exit(unreachable). --spec terminate(_, _) -> no_return(). -terminate(_Reason, _State) -> exit(unreachable). --spec code_change(_, _, _) -> no_return(). -code_change(_OldVsn, _State, _Extra) -> exit(unreachable). diff --git a/src/rabbit_priority_queue.erl b/src/rabbit_priority_queue.erl deleted file mode 100644 index 4b41b8dfbd..0000000000 --- a/src/rabbit_priority_queue.erl +++ /dev/null @@ -1,688 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2015-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_priority_queue). - --include_lib("rabbit_common/include/rabbit.hrl"). --include_lib("rabbit_common/include/rabbit_framing.hrl"). --include("amqqueue.hrl"). - --behaviour(rabbit_backing_queue). - -%% enabled unconditionally. Disabling priority queuing after -%% it has been enabled is dangerous. --rabbit_boot_step({?MODULE, - [{description, "enable priority queue"}, - {mfa, {?MODULE, enable, []}}, - {requires, pre_boot}, - {enables, kernel_ready}]}). - --export([enable/0]). - --export([start/2, stop/1]). - --export([init/3, terminate/2, delete_and_terminate/2, delete_crashed/1, - purge/1, purge_acks/1, - publish/6, publish_delivered/5, discard/4, drain_confirmed/1, - batch_publish/4, batch_publish_delivered/4, - dropwhile/2, fetchwhile/4, fetch/2, drop/2, ack/2, requeue/2, - ackfold/4, fold/3, len/1, is_empty/1, depth/1, - set_ram_duration_target/2, ram_duration/1, needs_timeout/1, timeout/1, - handle_pre_hibernate/1, resume/1, msg_rates/1, - info/2, invoke/3, is_duplicate/2, set_queue_mode/2, - zip_msgs_and_acks/4, handle_info/2]). - --record(state, {bq, bqss, max_priority}). --record(passthrough, {bq, bqs}). - -%% See 'note on suffixes' below --define(passthrough1(F), State#passthrough{bqs = BQ:F}). --define(passthrough2(F), - {Res, BQS1} = BQ:F, {Res, State#passthrough{bqs = BQS1}}). --define(passthrough3(F), - {Res1, Res2, BQS1} = BQ:F, {Res1, Res2, State#passthrough{bqs = BQS1}}). - -%% This module adds support for priority queues. -%% -%% Priority queues have one backing queue per priority. Backing queue functions -%% then produce a list of results for each BQ and fold over them, sorting -%% by priority. -%% -%%For queues that do not -%% have priorities enabled, the functions in this module delegate to -%% their "regular" backing queue module counterparts. See the `passthrough` -%% record and passthrough{1,2,3} macros. -%% -%% Delivery to consumers happens by first "running" the queue with -%% the highest priority until there are no more messages to deliver, -%% then the next one, and so on. This offers good prioritisation -%% but may result in lower priority messages not being delivered -%% when there's a high ingress rate of messages with higher priority. - -enable() -> - {ok, RealBQ} = application:get_env(rabbit, backing_queue_module), - case RealBQ of - ?MODULE -> ok; - _ -> rabbit_log:info("Priority queues enabled, real BQ is ~s~n", - [RealBQ]), - application:set_env( - rabbitmq_priority_queue, backing_queue_module, RealBQ), - application:set_env(rabbit, backing_queue_module, ?MODULE) - end. - -%%---------------------------------------------------------------------------- - -start(VHost, QNames) -> - BQ = bq(), - %% TODO this expand-collapse dance is a bit ridiculous but it's what - %% rabbit_amqqueue:recover/0 expects. We could probably simplify - %% this if we rejigged recovery a bit. - {DupNames, ExpNames} = expand_queues(QNames), - case BQ:start(VHost, ExpNames) of - {ok, ExpRecovery} -> - {ok, collapse_recovery(QNames, DupNames, ExpRecovery)}; - Else -> - Else - end. - -stop(VHost) -> - BQ = bq(), - BQ:stop(VHost). - -%%---------------------------------------------------------------------------- - -mutate_name(P, Q) when ?is_amqqueue(Q) -> - Res0 = #resource{name = QNameBin0} = amqqueue:get_name(Q), - QNameBin1 = mutate_name_bin(P, QNameBin0), - Res1 = Res0#resource{name = QNameBin1}, - amqqueue:set_name(Q, Res1). - -mutate_name_bin(P, NameBin) -> - <<NameBin/binary, 0, P:8>>. - -expand_queues(QNames) -> - lists:unzip( - lists:append([expand_queue(QName) || QName <- QNames])). - -expand_queue(QName = #resource{name = QNameBin}) -> - {ok, Q} = rabbit_misc:dirty_read({rabbit_durable_queue, QName}), - case priorities(Q) of - none -> [{QName, QName}]; - Ps -> [{QName, QName#resource{name = mutate_name_bin(P, QNameBin)}} - || P <- Ps] - end. - -collapse_recovery(QNames, DupNames, Recovery) -> - NameToTerms = lists:foldl(fun({Name, RecTerm}, Dict) -> - dict:append(Name, RecTerm, Dict) - end, dict:new(), lists:zip(DupNames, Recovery)), - [dict:fetch(Name, NameToTerms) || Name <- QNames]. - -priorities(Q) when ?is_amqqueue(Q) -> - Args = amqqueue:get_arguments(Q), - Ints = [long, short, signedint, byte, unsignedbyte, unsignedshort, unsignedint], - case rabbit_misc:table_lookup(Args, <<"x-max-priority">>) of - {Type, RequestedMax} -> - case lists:member(Type, Ints) of - false -> none; - true -> - Max = min(RequestedMax, ?MAX_SUPPORTED_PRIORITY), - lists:reverse(lists:seq(0, Max)) - end; - _ -> none - end. - -%%---------------------------------------------------------------------------- - -init(Q, Recover, AsyncCallback) -> - BQ = bq(), - case priorities(Q) of - none -> RealRecover = case Recover of - [R] -> R; %% [0] - R -> R - end, - #passthrough{bq = BQ, - bqs = BQ:init(Q, RealRecover, AsyncCallback)}; - Ps -> Init = fun (P, Term) -> - BQ:init( - mutate_name(P, Q), Term, - fun (M, F) -> AsyncCallback(M, {P, F}) end) - end, - BQSs = case have_recovery_terms(Recover) of - false -> [{P, Init(P, Recover)} || P <- Ps]; - _ -> PsTerms = lists:zip(Ps, Recover), - [{P, Init(P, Term)} || {P, Term} <- PsTerms] - end, - #state{bq = BQ, - bqss = BQSs, - max_priority = hd(Ps)} - end. -%% [0] collapse_recovery has the effect of making a list of recovery -%% terms in priority order, even for non priority queues. It's easier -%% to do that and "unwrap" in init/3 than to have collapse_recovery be -%% aware of non-priority queues. - -have_recovery_terms(new) -> false; -have_recovery_terms(non_clean_shutdown) -> false; -have_recovery_terms(_) -> true. - -terminate(Reason, State = #state{bq = BQ}) -> - foreach1(fun (_P, BQSN) -> BQ:terminate(Reason, BQSN) end, State); -terminate(Reason, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(terminate(Reason, BQS)). - -delete_and_terminate(Reason, State = #state{bq = BQ}) -> - foreach1(fun (_P, BQSN) -> - BQ:delete_and_terminate(Reason, BQSN) - end, State); -delete_and_terminate(Reason, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(delete_and_terminate(Reason, BQS)). - -delete_crashed(Q) -> - BQ = bq(), - case priorities(Q) of - none -> BQ:delete_crashed(Q); - Ps -> [BQ:delete_crashed(mutate_name(P, Q)) || P <- Ps] - end. - -purge(State = #state{bq = BQ}) -> - fold_add2(fun (_P, BQSN) -> BQ:purge(BQSN) end, State); -purge(State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(purge(BQS)). - -purge_acks(State = #state{bq = BQ}) -> - foreach1(fun (_P, BQSN) -> BQ:purge_acks(BQSN) end, State); -purge_acks(State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(purge_acks(BQS)). - -publish(Msg, MsgProps, IsDelivered, ChPid, Flow, State = #state{bq = BQ}) -> - pick1(fun (_P, BQSN) -> - BQ:publish(Msg, MsgProps, IsDelivered, ChPid, Flow, BQSN) - end, Msg, State); -publish(Msg, MsgProps, IsDelivered, ChPid, Flow, - State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(publish(Msg, MsgProps, IsDelivered, ChPid, Flow, BQS)). - -batch_publish(Publishes, ChPid, Flow, State = #state{bq = BQ, bqss = [{MaxP, _} |_]}) -> - PubMap = partition_publish_batch(Publishes, MaxP), - lists:foldl( - fun ({Priority, Pubs}, St) -> - pick1(fun (_P, BQSN) -> - BQ:batch_publish(Pubs, ChPid, Flow, BQSN) - end, Priority, St) - end, State, maps:to_list(PubMap)); -batch_publish(Publishes, ChPid, Flow, - State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(batch_publish(Publishes, ChPid, Flow, BQS)). - -publish_delivered(Msg, MsgProps, ChPid, Flow, State = #state{bq = BQ}) -> - pick2(fun (P, BQSN) -> - {AckTag, BQSN1} = BQ:publish_delivered( - Msg, MsgProps, ChPid, Flow, BQSN), - {{P, AckTag}, BQSN1} - end, Msg, State); -publish_delivered(Msg, MsgProps, ChPid, Flow, - State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(publish_delivered(Msg, MsgProps, ChPid, Flow, BQS)). - -batch_publish_delivered(Publishes, ChPid, Flow, State = #state{bq = BQ, bqss = [{MaxP, _} |_]}) -> - PubMap = partition_publish_delivered_batch(Publishes, MaxP), - {PrioritiesAndAcks, State1} = - lists:foldl( - fun ({Priority, Pubs}, {PriosAndAcks, St}) -> - {PriosAndAcks1, St1} = - pick2(fun (P, BQSN) -> - {AckTags, BQSN1} = - BQ:batch_publish_delivered( - Pubs, ChPid, Flow, BQSN), - {priority_on_acktags(P, AckTags), BQSN1} - end, Priority, St), - {[PriosAndAcks1 | PriosAndAcks], St1} - end, {[], State}, maps:to_list(PubMap)), - {lists:reverse(PrioritiesAndAcks), State1}; -batch_publish_delivered(Publishes, ChPid, Flow, - State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(batch_publish_delivered(Publishes, ChPid, Flow, BQS)). - -%% TODO this is a hack. The BQ api does not give us enough information -%% here - if we had the Msg we could look at its priority and forward -%% to the appropriate sub-BQ. But we don't so we are stuck. -%% -%% But fortunately VQ ignores discard/4, so we can too, *assuming we -%% are talking to VQ*. discard/4 is used by HA, but that's "above" us -%% (if in use) so we don't break that either, just some hypothetical -%% alternate BQ implementation. -discard(_MsgId, _ChPid, _Flow, State = #state{}) -> - State; - %% We should have something a bit like this here: - %% pick1(fun (_P, BQSN) -> - %% BQ:discard(MsgId, ChPid, Flow, BQSN) - %% end, Msg, State); -discard(MsgId, ChPid, Flow, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(discard(MsgId, ChPid, Flow, BQS)). - -drain_confirmed(State = #state{bq = BQ}) -> - fold_append2(fun (_P, BQSN) -> BQ:drain_confirmed(BQSN) end, State); -drain_confirmed(State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(drain_confirmed(BQS)). - -dropwhile(Pred, State = #state{bq = BQ}) -> - find2(fun (_P, BQSN) -> BQ:dropwhile(Pred, BQSN) end, undefined, State); -dropwhile(Pred, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(dropwhile(Pred, BQS)). - -%% TODO this is a bit nasty. In the one place where fetchwhile/4 is -%% actually used the accumulator is a list of acktags, which of course -%% we need to mutate - so we do that although we are encoding an -%% assumption here. -fetchwhile(Pred, Fun, Acc, State = #state{bq = BQ}) -> - findfold3( - fun (P, BQSN, AccN) -> - {Res, AccN1, BQSN1} = BQ:fetchwhile(Pred, Fun, AccN, BQSN), - {Res, priority_on_acktags(P, AccN1), BQSN1} - end, Acc, undefined, State); -fetchwhile(Pred, Fun, Acc, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough3(fetchwhile(Pred, Fun, Acc, BQS)). - -fetch(AckRequired, State = #state{bq = BQ}) -> - find2( - fun (P, BQSN) -> - case BQ:fetch(AckRequired, BQSN) of - {empty, BQSN1} -> {empty, BQSN1}; - {{Msg, Del, ATag}, BQSN1} -> {{Msg, Del, {P, ATag}}, BQSN1} - end - end, empty, State); -fetch(AckRequired, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(fetch(AckRequired, BQS)). - -drop(AckRequired, State = #state{bq = BQ}) -> - find2(fun (P, BQSN) -> - case BQ:drop(AckRequired, BQSN) of - {empty, BQSN1} -> {empty, BQSN1}; - {{MsgId, AckTag}, BQSN1} -> {{MsgId, {P, AckTag}}, BQSN1} - end - end, empty, State); -drop(AckRequired, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(drop(AckRequired, BQS)). - -ack(AckTags, State = #state{bq = BQ}) -> - fold_by_acktags2(fun (AckTagsN, BQSN) -> - BQ:ack(AckTagsN, BQSN) - end, AckTags, State); -ack(AckTags, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(ack(AckTags, BQS)). - -requeue(AckTags, State = #state{bq = BQ}) -> - fold_by_acktags2(fun (AckTagsN, BQSN) -> - BQ:requeue(AckTagsN, BQSN) - end, AckTags, State); -requeue(AckTags, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(requeue(AckTags, BQS)). - -%% Similar problem to fetchwhile/4 -ackfold(MsgFun, Acc, State = #state{bq = BQ}, AckTags) -> - AckTagsByPriority = partition_acktags(AckTags), - fold2( - fun (P, BQSN, AccN) -> - case maps:find(P, AckTagsByPriority) of - {ok, ATagsN} -> {AccN1, BQSN1} = - BQ:ackfold(MsgFun, AccN, BQSN, ATagsN), - {priority_on_acktags(P, AccN1), BQSN1}; - error -> {AccN, BQSN} - end - end, Acc, State); -ackfold(MsgFun, Acc, State = #passthrough{bq = BQ, bqs = BQS}, AckTags) -> - ?passthrough2(ackfold(MsgFun, Acc, BQS, AckTags)). - -fold(Fun, Acc, State = #state{bq = BQ}) -> - fold2(fun (_P, BQSN, AccN) -> BQ:fold(Fun, AccN, BQSN) end, Acc, State); -fold(Fun, Acc, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(fold(Fun, Acc, BQS)). - -len(#state{bq = BQ, bqss = BQSs}) -> - add0(fun (_P, BQSN) -> BQ:len(BQSN) end, BQSs); -len(#passthrough{bq = BQ, bqs = BQS}) -> - BQ:len(BQS). - -is_empty(#state{bq = BQ, bqss = BQSs}) -> - all0(fun (_P, BQSN) -> BQ:is_empty(BQSN) end, BQSs); -is_empty(#passthrough{bq = BQ, bqs = BQS}) -> - BQ:is_empty(BQS). - -depth(#state{bq = BQ, bqss = BQSs}) -> - add0(fun (_P, BQSN) -> BQ:depth(BQSN) end, BQSs); -depth(#passthrough{bq = BQ, bqs = BQS}) -> - BQ:depth(BQS). - -set_ram_duration_target(DurationTarget, State = #state{bq = BQ}) -> - foreach1(fun (_P, BQSN) -> - BQ:set_ram_duration_target(DurationTarget, BQSN) - end, State); -set_ram_duration_target(DurationTarget, - State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(set_ram_duration_target(DurationTarget, BQS)). - -ram_duration(State = #state{bq = BQ}) -> - fold_min2(fun (_P, BQSN) -> BQ:ram_duration(BQSN) end, State); -ram_duration(State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(ram_duration(BQS)). - -needs_timeout(#state{bq = BQ, bqss = BQSs}) -> - fold0(fun (_P, _BQSN, timed) -> timed; - (_P, BQSN, idle) -> case BQ:needs_timeout(BQSN) of - timed -> timed; - _ -> idle - end; - (_P, BQSN, false) -> BQ:needs_timeout(BQSN) - end, false, BQSs); -needs_timeout(#passthrough{bq = BQ, bqs = BQS}) -> - BQ:needs_timeout(BQS). - -timeout(State = #state{bq = BQ}) -> - foreach1(fun (_P, BQSN) -> BQ:timeout(BQSN) end, State); -timeout(State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(timeout(BQS)). - -handle_pre_hibernate(State = #state{bq = BQ}) -> - foreach1(fun (_P, BQSN) -> - BQ:handle_pre_hibernate(BQSN) - end, State); -handle_pre_hibernate(State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(handle_pre_hibernate(BQS)). - -handle_info(Msg, State = #state{bq = BQ}) -> - foreach1(fun (_P, BQSN) -> BQ:handle_info(Msg, BQSN) end, State); -handle_info(Msg, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(handle_info(Msg, BQS)). - -resume(State = #state{bq = BQ}) -> - foreach1(fun (_P, BQSN) -> BQ:resume(BQSN) end, State); -resume(State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(resume(BQS)). - -msg_rates(#state{bq = BQ, bqss = BQSs}) -> - fold0(fun(_P, BQSN, {InN, OutN}) -> - {In, Out} = BQ:msg_rates(BQSN), - {InN + In, OutN + Out} - end, {0.0, 0.0}, BQSs); -msg_rates(#passthrough{bq = BQ, bqs = BQS}) -> - BQ:msg_rates(BQS). - -info(backing_queue_status, #state{bq = BQ, bqss = BQSs}) -> - fold0(fun (P, BQSN, Acc) -> - combine_status(P, BQ:info(backing_queue_status, BQSN), Acc) - end, nothing, BQSs); -info(head_message_timestamp, #state{bq = BQ, bqss = BQSs}) -> - find_head_message_timestamp(BQ, BQSs, ''); -info(Item, #state{bq = BQ, bqss = BQSs}) -> - fold0(fun (_P, BQSN, Acc) -> - Acc + BQ:info(Item, BQSN) - end, 0, BQSs); -info(Item, #passthrough{bq = BQ, bqs = BQS}) -> - BQ:info(Item, BQS). - -invoke(Mod, {P, Fun}, State = #state{bq = BQ}) -> - pick1(fun (_P, BQSN) -> BQ:invoke(Mod, Fun, BQSN) end, P, State); -invoke(Mod, Fun, State = #state{bq = BQ, max_priority = P}) -> - pick1(fun (_P, BQSN) -> BQ:invoke(Mod, Fun, BQSN) end, P, State); -invoke(Mod, Fun, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(invoke(Mod, Fun, BQS)). - -is_duplicate(Msg, State = #state{bq = BQ}) -> - pick2(fun (_P, BQSN) -> BQ:is_duplicate(Msg, BQSN) end, Msg, State); -is_duplicate(Msg, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough2(is_duplicate(Msg, BQS)). - -set_queue_mode(Mode, State = #state{bq = BQ}) -> - foreach1(fun (_P, BQSN) -> BQ:set_queue_mode(Mode, BQSN) end, State); -set_queue_mode(Mode, State = #passthrough{bq = BQ, bqs = BQS}) -> - ?passthrough1(set_queue_mode(Mode, BQS)). - -zip_msgs_and_acks(Msgs, AckTags, Accumulator, #state{bqss = [{MaxP, _} |_]}) -> - MsgsByPriority = partition_publish_delivered_batch(Msgs, MaxP), - lists:foldl(fun (Acks, MAs) -> - {P, _AckTag} = hd(Acks), - Pubs = maps:get(P, MsgsByPriority), - MAs0 = zip_msgs_and_acks(Pubs, Acks), - MAs ++ MAs0 - end, Accumulator, AckTags); -zip_msgs_and_acks(Msgs, AckTags, Accumulator, - #passthrough{bq = BQ, bqs = BQS}) -> - BQ:zip_msgs_and_acks(Msgs, AckTags, Accumulator, BQS). - -%%---------------------------------------------------------------------------- - -bq() -> - {ok, RealBQ} = application:get_env( - rabbitmq_priority_queue, backing_queue_module), - RealBQ. - -%% Note on suffixes: Many utility functions here have suffixes telling -%% you the arity of the return type of the BQ function they are -%% designed to work with. -%% -%% 0 - BQ function returns a value and does not modify state -%% 1 - BQ function just returns a new state -%% 2 - BQ function returns a 2-tuple of {Result, NewState} -%% 3 - BQ function returns a 3-tuple of {Result1, Result2, NewState} - -%% Fold over results -fold0(Fun, Acc, [{P, BQSN} | Rest]) -> fold0(Fun, Fun(P, BQSN, Acc), Rest); -fold0(_Fun, Acc, []) -> Acc. - -%% Do all BQs match? -all0(Pred, BQSs) -> fold0(fun (_P, _BQSN, false) -> false; - (P, BQSN, true) -> Pred(P, BQSN) - end, true, BQSs). - -%% Sum results -add0(Fun, BQSs) -> fold0(fun (P, BQSN, Acc) -> Acc + Fun(P, BQSN) end, 0, BQSs). - -%% Apply for all states -foreach1(Fun, State = #state{bqss = BQSs}) -> - a(State#state{bqss = foreach1(Fun, BQSs, [])}). -foreach1(Fun, [{Priority, BQSN} | Rest], BQSAcc) -> - BQSN1 = Fun(Priority, BQSN), - foreach1(Fun, Rest, [{Priority, BQSN1} | BQSAcc]); -foreach1(_Fun, [], BQSAcc) -> - lists:reverse(BQSAcc). - -%% For a given thing, just go to its BQ -pick1(Fun, Prioritisable, #state{bqss = BQSs} = State) -> - {P, BQSN} = priority_bq(Prioritisable, BQSs), - a(State#state{bqss = bq_store(P, Fun(P, BQSN), BQSs)}). - -%% Fold over results -fold2(Fun, Acc, State = #state{bqss = BQSs}) -> - {Res, BQSs1} = fold2(Fun, Acc, BQSs, []), - {Res, a(State#state{bqss = BQSs1})}. - -fold2(Fun, Acc, [{P, BQSN} | Rest], BQSAcc) -> - {Acc1, BQSN1} = Fun(P, BQSN, Acc), - fold2(Fun, Acc1, Rest, [{P, BQSN1} | BQSAcc]); -fold2(_Fun, Acc, [], BQSAcc) -> - {Acc, lists:reverse(BQSAcc)}. - -%% Fold over results assuming results are lists and we want to append them -fold_append2(Fun, State) -> - fold2(fun (P, BQSN, Acc) -> - {Res, BQSN1} = Fun(P, BQSN), - {Res ++ Acc, BQSN1} - end, [], State). - -%% Fold over results assuming results are numbers and we want to sum them -fold_add2(Fun, State) -> - fold2(fun (P, BQSN, Acc) -> - {Res, BQSN1} = Fun(P, BQSN), - {add_maybe_infinity(Res, Acc), BQSN1} - end, 0, State). - -%% Fold over results assuming results are numbers and we want the minimum -fold_min2(Fun, State) -> - fold2(fun (P, BQSN, Acc) -> - {Res, BQSN1} = Fun(P, BQSN), - {erlang:min(Res, Acc), BQSN1} - end, infinity, State). - -%% Fold over results assuming results are lists and we want to append -%% them, and also that we have some AckTags we want to pass in to each -%% invocation. -fold_by_acktags2(Fun, AckTags, State) -> - AckTagsByPriority = partition_acktags(AckTags), - fold_append2(fun (P, BQSN) -> - case maps:find(P, AckTagsByPriority) of - {ok, AckTagsN} -> Fun(AckTagsN, BQSN); - error -> {[], BQSN} - end - end, State). - -%% For a given thing, just go to its BQ -pick2(Fun, Prioritisable, #state{bqss = BQSs} = State) -> - {P, BQSN} = priority_bq(Prioritisable, BQSs), - {Res, BQSN1} = Fun(P, BQSN), - {Res, a(State#state{bqss = bq_store(P, BQSN1, BQSs)})}. - -%% Run through BQs in priority order until one does not return -%% {NotFound, NewState} or we have gone through them all. -find2(Fun, NotFound, State = #state{bqss = BQSs}) -> - {Res, BQSs1} = find2(Fun, NotFound, BQSs, []), - {Res, a(State#state{bqss = BQSs1})}. -find2(Fun, NotFound, [{P, BQSN} | Rest], BQSAcc) -> - case Fun(P, BQSN) of - {NotFound, BQSN1} -> find2(Fun, NotFound, Rest, [{P, BQSN1} | BQSAcc]); - {Res, BQSN1} -> {Res, lists:reverse([{P, BQSN1} | BQSAcc]) ++ Rest} - end; -find2(_Fun, NotFound, [], BQSAcc) -> - {NotFound, lists:reverse(BQSAcc)}. - -%% Run through BQs in priority order like find2 but also folding as we go. -findfold3(Fun, Acc, NotFound, State = #state{bqss = BQSs}) -> - {Res, Acc1, BQSs1} = findfold3(Fun, Acc, NotFound, BQSs, []), - {Res, Acc1, a(State#state{bqss = BQSs1})}. -findfold3(Fun, Acc, NotFound, [{P, BQSN} | Rest], BQSAcc) -> - case Fun(P, BQSN, Acc) of - {NotFound, Acc1, BQSN1} -> - findfold3(Fun, Acc1, NotFound, Rest, [{P, BQSN1} | BQSAcc]); - {Res, Acc1, BQSN1} -> - {Res, Acc1, lists:reverse([{P, BQSN1} | BQSAcc]) ++ Rest} - end; -findfold3(_Fun, Acc, NotFound, [], BQSAcc) -> - {NotFound, Acc, lists:reverse(BQSAcc)}. - -bq_fetch(P, []) -> exit({not_found, P}); -bq_fetch(P, [{P, BQSN} | _]) -> {P, BQSN}; -bq_fetch(P, [{_, _BQSN} | T]) -> bq_fetch(P, T). - -bq_store(P, BQS, BQSs) -> - [{PN, case PN of - P -> BQS; - _ -> BQSN - end} || {PN, BQSN} <- BQSs]. - -%% -a(State = #state{bqss = BQSs}) -> - Ps = [P || {P, _} <- BQSs], - case lists:reverse(lists:usort(Ps)) of - Ps -> State; - _ -> exit({bad_order, Ps}) - end. - -%%---------------------------------------------------------------------------- -partition_publish_batch(Publishes, MaxP) -> - partition_publishes( - Publishes, fun ({Msg, _, _}) -> Msg end, MaxP). - -partition_publish_delivered_batch(Publishes, MaxP) -> - partition_publishes( - Publishes, fun ({Msg, _}) -> Msg end, MaxP). - -partition_publishes(Publishes, ExtractMsg, MaxP) -> - Partitioned = - lists:foldl(fun (Pub, Dict) -> - Msg = ExtractMsg(Pub), - rabbit_misc:maps_cons(priority(Msg, MaxP), Pub, Dict) - end, maps:new(), Publishes), - maps:map(fun (_P, RevPubs) -> - lists:reverse(RevPubs) - end, Partitioned). - - -priority_bq(Priority, [{MaxP, _} | _] = BQSs) -> - bq_fetch(priority(Priority, MaxP), BQSs). - -%% Messages with a priority which is higher than the queue's maximum are treated -%% as if they were published with the maximum priority. -priority(undefined, _MaxP) -> - 0; -priority(Priority, MaxP) when is_integer(Priority), Priority =< MaxP -> - Priority; -priority(Priority, MaxP) when is_integer(Priority), Priority > MaxP -> - MaxP; -priority(#basic_message{content = Content}, MaxP) -> - priority(rabbit_binary_parser:ensure_content_decoded(Content), MaxP); -priority(#content{properties = Props}, MaxP) -> - #'P_basic'{priority = Priority0} = Props, - priority(Priority0, MaxP). - -add_maybe_infinity(infinity, _) -> infinity; -add_maybe_infinity(_, infinity) -> infinity; -add_maybe_infinity(A, B) -> A + B. - -partition_acktags(AckTags) -> partition_acktags(AckTags, maps:new()). - -partition_acktags([], Partitioned) -> - maps:map(fun (_P, RevAckTags) -> - lists:reverse(RevAckTags) - end, Partitioned); -partition_acktags([{P, AckTag} | Rest], Partitioned) -> - partition_acktags(Rest, rabbit_misc:maps_cons(P, AckTag, Partitioned)). - -priority_on_acktags(P, AckTags) -> - [case Tag of - _ when is_integer(Tag) -> {P, Tag}; - _ -> Tag - end || Tag <- AckTags]. - -combine_status(P, New, nothing) -> - [{priority_lengths, [{P, proplists:get_value(len, New)}]} | New]; -combine_status(P, New, Old) -> - Combined = [{K, cse(V, proplists:get_value(K, Old))} || {K, V} <- New], - Lens = [{P, proplists:get_value(len, New)} | - proplists:get_value(priority_lengths, Old)], - [{priority_lengths, Lens} | Combined]. - -cse(infinity, _) -> infinity; -cse(_, infinity) -> infinity; -%% queue modes -cse(_, default) -> default; -cse(default, _) -> default; -cse(_, lazy) -> lazy; -cse(lazy, _) -> lazy; -%% numerical stats -cse(A, B) when is_number(A) -> A + B; -cse({delta, _, _, _, _}, _) -> {delta, todo, todo, todo, todo}; -cse(_, _) -> undefined. - -%% When asked about 'head_message_timestamp' fro this priority queue, we -%% walk all the backing queues, starting by the highest priority. Once a -%% backing queue having messages (ready or unacknowledged) is found, its -%% 'head_message_timestamp' is returned even if it is null. - -find_head_message_timestamp(BQ, [{_, BQSN} | Rest], Timestamp) -> - MsgCount = BQ:len(BQSN) + BQ:info(messages_unacknowledged_ram, BQSN), - if - MsgCount =/= 0 -> BQ:info(head_message_timestamp, BQSN); - true -> find_head_message_timestamp(BQ, Rest, Timestamp) - end; -find_head_message_timestamp(_, [], Timestamp) -> - Timestamp. - -zip_msgs_and_acks(Pubs, AckTags) -> - lists:zipwith( - fun ({#basic_message{ id = Id }, _Props}, AckTag) -> - {Id, AckTag} - end, Pubs, AckTags). diff --git a/src/rabbit_queue_consumers.erl b/src/rabbit_queue_consumers.erl deleted file mode 100644 index 4f826f72e8..0000000000 --- a/src/rabbit_queue_consumers.erl +++ /dev/null @@ -1,568 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_queue_consumers). - --export([new/0, max_active_priority/1, inactive/1, all/1, all/3, count/0, - unacknowledged_message_count/0, add/10, remove/3, erase_ch/2, - send_drained/0, deliver/5, record_ack/3, subtract_acks/3, - possibly_unblock/3, - resume_fun/0, notify_sent_fun/1, activate_limit_fun/0, - credit/6, utilisation/1, is_same/3, get_consumer/1, get/3, - consumer_tag/1, get_infos/1]). - -%%---------------------------------------------------------------------------- - --define(QUEUE, lqueue). - --define(UNSENT_MESSAGE_LIMIT, 200). - -%% Utilisation average calculations are all in μs. --define(USE_AVG_HALF_LIFE, 1000000.0). - --record(state, {consumers, use}). - --record(consumer, {tag, ack_required, prefetch, args, user}). - -%% These are held in our process dictionary --record(cr, {ch_pid, - monitor_ref, - acktags, - consumer_count, - %% Queue of {ChPid, #consumer{}} for consumers which have - %% been blocked (rate/prefetch limited) for any reason - blocked_consumers, - %% The limiter itself - limiter, - %% Internal flow control for queue -> writer - unsent_message_count}). - -%%---------------------------------------------------------------------------- - --type time_micros() :: non_neg_integer(). --type ratio() :: float(). --type state() :: #state{consumers ::priority_queue:q(), - use :: {'inactive', - time_micros(), time_micros(), ratio()} | - {'active', time_micros(), ratio()}}. --type consumer() :: #consumer{tag::rabbit_types:ctag(), ack_required::boolean(), - prefetch::non_neg_integer(), args::rabbit_framing:amqp_table(), - user::rabbit_types:username()}. --type ch() :: pid(). --type ack() :: non_neg_integer(). --type cr_fun() :: fun ((#cr{}) -> #cr{}). --type fetch_result() :: {rabbit_types:basic_message(), boolean(), ack()}. - -%%---------------------------------------------------------------------------- - --spec new() -> state(). - -new() -> #state{consumers = priority_queue:new(), - use = {active, - erlang:monotonic_time(micro_seconds), - 1.0}}. - --spec max_active_priority(state()) -> integer() | 'infinity' | 'empty'. - -max_active_priority(#state{consumers = Consumers}) -> - priority_queue:highest(Consumers). - --spec inactive(state()) -> boolean(). - -inactive(#state{consumers = Consumers}) -> - priority_queue:is_empty(Consumers). - --spec all(state()) -> [{ch(), rabbit_types:ctag(), boolean(), - non_neg_integer(), boolean(), atom(), - rabbit_framing:amqp_table(), rabbit_types:username()}]. - -all(State) -> - all(State, none, false). - -all(#state{consumers = Consumers}, SingleActiveConsumer, SingleActiveConsumerOn) -> - lists:foldl(fun (C, Acc) -> consumers(C#cr.blocked_consumers, SingleActiveConsumer, SingleActiveConsumerOn, Acc) end, - consumers(Consumers, SingleActiveConsumer, SingleActiveConsumerOn, []), all_ch_record()). - -consumers(Consumers, SingleActiveConsumer, SingleActiveConsumerOn, Acc) -> - ActiveActivityStatusFun = case SingleActiveConsumerOn of - true -> - fun({ChPid, Consumer}) -> - case SingleActiveConsumer of - {ChPid, Consumer} -> - {true, single_active}; - _ -> - {false, waiting} - end - end; - false -> - fun(_) -> {true, up} end - end, - priority_queue:fold( - fun ({ChPid, Consumer}, _P, Acc1) -> - #consumer{tag = CTag, ack_required = Ack, prefetch = Prefetch, - args = Args, user = Username} = Consumer, - {Active, ActivityStatus} = ActiveActivityStatusFun({ChPid, Consumer}), - [{ChPid, CTag, Ack, Prefetch, Active, ActivityStatus, Args, Username} | Acc1] - end, Acc, Consumers). - --spec count() -> non_neg_integer(). - -count() -> lists:sum([Count || #cr{consumer_count = Count} <- all_ch_record()]). - --spec unacknowledged_message_count() -> non_neg_integer(). - -unacknowledged_message_count() -> - lists:sum([?QUEUE:len(C#cr.acktags) || C <- all_ch_record()]). - --spec add(ch(), rabbit_types:ctag(), boolean(), pid(), boolean(), - non_neg_integer(), rabbit_framing:amqp_table(), boolean(), - rabbit_types:username(), state()) - -> state(). - -add(ChPid, CTag, NoAck, LimiterPid, LimiterActive, Prefetch, Args, IsEmpty, - Username, State = #state{consumers = Consumers, - use = CUInfo}) -> - C = #cr{consumer_count = Count, - limiter = Limiter} = ch_record(ChPid, LimiterPid), - Limiter1 = case LimiterActive of - true -> rabbit_limiter:activate(Limiter); - false -> Limiter - end, - C1 = C#cr{consumer_count = Count + 1, limiter = Limiter1}, - update_ch_record( - case parse_credit_args(Prefetch, Args) of - {0, auto} -> C1; - {_Credit, auto} when NoAck -> C1; - {Credit, Mode} -> credit_and_drain( - C1, CTag, Credit, Mode, IsEmpty) - end), - Consumer = #consumer{tag = CTag, - ack_required = not NoAck, - prefetch = Prefetch, - args = Args, - user = Username}, - State#state{consumers = add_consumer({ChPid, Consumer}, Consumers), - use = update_use(CUInfo, active)}. - --spec remove(ch(), rabbit_types:ctag(), state()) -> - 'not_found' | state(). - -remove(ChPid, CTag, State = #state{consumers = Consumers}) -> - case lookup_ch(ChPid) of - not_found -> - not_found; - C = #cr{consumer_count = Count, - limiter = Limiter, - blocked_consumers = Blocked} -> - Blocked1 = remove_consumer(ChPid, CTag, Blocked), - Limiter1 = case Count of - 1 -> rabbit_limiter:deactivate(Limiter); - _ -> Limiter - end, - Limiter2 = rabbit_limiter:forget_consumer(Limiter1, CTag), - update_ch_record(C#cr{consumer_count = Count - 1, - limiter = Limiter2, - blocked_consumers = Blocked1}), - State#state{consumers = - remove_consumer(ChPid, CTag, Consumers)} - end. - --spec erase_ch(ch(), state()) -> - 'not_found' | {[ack()], [rabbit_types:ctag()], - state()}. - -erase_ch(ChPid, State = #state{consumers = Consumers}) -> - case lookup_ch(ChPid) of - not_found -> - not_found; - C = #cr{ch_pid = ChPid, - acktags = ChAckTags, - blocked_consumers = BlockedQ} -> - All = priority_queue:join(Consumers, BlockedQ), - ok = erase_ch_record(C), - Filtered = priority_queue:filter(chan_pred(ChPid, true), All), - {[AckTag || {AckTag, _CTag} <- ?QUEUE:to_list(ChAckTags)], - tags(priority_queue:to_list(Filtered)), - State#state{consumers = remove_consumers(ChPid, Consumers)}} - end. - --spec send_drained() -> 'ok'. - -send_drained() -> [update_ch_record(send_drained(C)) || C <- all_ch_record()], - ok. - --spec deliver(fun ((boolean()) -> {fetch_result(), T}), - rabbit_amqqueue:name(), state(), boolean(), - none | {ch(), rabbit_types:ctag()} | {ch(), consumer()}) -> - {'delivered', boolean(), T, state()} | - {'undelivered', boolean(), state()}. - -deliver(FetchFun, QName, State, SingleActiveConsumerIsOn, ActiveConsumer) -> - deliver(FetchFun, QName, false, State, SingleActiveConsumerIsOn, ActiveConsumer). - -deliver(_FetchFun, _QName, false, State, true, none) -> - {undelivered, false, - State#state{use = update_use(State#state.use, inactive)}}; -deliver(FetchFun, QName, false, State = #state{consumers = Consumers}, true, SingleActiveConsumer) -> - {ChPid, Consumer} = SingleActiveConsumer, - %% blocked (rate/prefetch limited) consumers are removed from the queue state, but not the exclusive_consumer field, - %% so we need to do this check to avoid adding the exclusive consumer to the channel record - %% over and over - case is_blocked(SingleActiveConsumer) of - true -> - {undelivered, false, - State#state{use = update_use(State#state.use, inactive)}}; - false -> - case deliver_to_consumer(FetchFun, SingleActiveConsumer, QName) of - {delivered, R} -> - {delivered, false, R, State}; - undelivered -> - {ChPid, Consumer} = SingleActiveConsumer, - Consumers1 = remove_consumer(ChPid, Consumer#consumer.tag, Consumers), - {undelivered, true, - State#state{consumers = Consumers1, use = update_use(State#state.use, inactive)}} - end - end; -deliver(FetchFun, QName, ConsumersChanged, - State = #state{consumers = Consumers}, false, _SingleActiveConsumer) -> - case priority_queue:out_p(Consumers) of - {empty, _} -> - {undelivered, ConsumersChanged, - State#state{use = update_use(State#state.use, inactive)}}; - {{value, QEntry, Priority}, Tail} -> - case deliver_to_consumer(FetchFun, QEntry, QName) of - {delivered, R} -> - {delivered, ConsumersChanged, R, - State#state{consumers = priority_queue:in(QEntry, Priority, - Tail)}}; - undelivered -> - deliver(FetchFun, QName, true, - State#state{consumers = Tail}, false, _SingleActiveConsumer) - end - end. - -deliver_to_consumer(FetchFun, E = {ChPid, Consumer}, QName) -> - C = lookup_ch(ChPid), - case is_ch_blocked(C) of - true -> - block_consumer(C, E), - undelivered; - false -> case rabbit_limiter:can_send(C#cr.limiter, - Consumer#consumer.ack_required, - Consumer#consumer.tag) of - {suspend, Limiter} -> - block_consumer(C#cr{limiter = Limiter}, E), - undelivered; - {continue, Limiter} -> - {delivered, deliver_to_consumer( - FetchFun, Consumer, - C#cr{limiter = Limiter}, QName)} - end - end. - -deliver_to_consumer(FetchFun, - #consumer{tag = CTag, - ack_required = AckRequired}, - C = #cr{ch_pid = ChPid, - acktags = ChAckTags, - unsent_message_count = Count}, - QName) -> - {{Message, IsDelivered, AckTag}, R} = FetchFun(AckRequired), - rabbit_channel:deliver(ChPid, CTag, AckRequired, - {QName, self(), AckTag, IsDelivered, Message}), - ChAckTags1 = case AckRequired of - true -> ?QUEUE:in({AckTag, CTag}, ChAckTags); - false -> ChAckTags - end, - update_ch_record(C#cr{acktags = ChAckTags1, - unsent_message_count = Count + 1}), - R. - -is_blocked(Consumer = {ChPid, _C}) -> - #cr{blocked_consumers = BlockedConsumers} = lookup_ch(ChPid), - priority_queue:member(Consumer, BlockedConsumers). - --spec record_ack(ch(), pid(), ack()) -> 'ok'. - -record_ack(ChPid, LimiterPid, AckTag) -> - C = #cr{acktags = ChAckTags} = ch_record(ChPid, LimiterPid), - update_ch_record(C#cr{acktags = ?QUEUE:in({AckTag, none}, ChAckTags)}), - ok. - --spec subtract_acks(ch(), [ack()], state()) -> - 'not_found' | 'unchanged' | {'unblocked', state()}. - -subtract_acks(ChPid, AckTags, State) -> - case lookup_ch(ChPid) of - not_found -> - not_found; - C = #cr{acktags = ChAckTags, limiter = Lim} -> - {CTagCounts, AckTags2} = subtract_acks( - AckTags, [], maps:new(), ChAckTags), - {Unblocked, Lim2} = - maps:fold( - fun (CTag, Count, {UnblockedN, LimN}) -> - {Unblocked1, LimN1} = - rabbit_limiter:ack_from_queue(LimN, CTag, Count), - {UnblockedN orelse Unblocked1, LimN1} - end, {false, Lim}, CTagCounts), - C2 = C#cr{acktags = AckTags2, limiter = Lim2}, - case Unblocked of - true -> unblock(C2, State); - false -> update_ch_record(C2), - unchanged - end - end. - -subtract_acks([], [], CTagCounts, AckQ) -> - {CTagCounts, AckQ}; -subtract_acks([], Prefix, CTagCounts, AckQ) -> - {CTagCounts, ?QUEUE:join(?QUEUE:from_list(lists:reverse(Prefix)), AckQ)}; -subtract_acks([T | TL] = AckTags, Prefix, CTagCounts, AckQ) -> - case ?QUEUE:out(AckQ) of - {{value, {T, CTag}}, QTail} -> - subtract_acks(TL, Prefix, - maps:update_with(CTag, fun (Old) -> Old + 1 end, 1, CTagCounts), QTail); - {{value, V}, QTail} -> - subtract_acks(AckTags, [V | Prefix], CTagCounts, QTail); - {empty, _} -> - subtract_acks([], Prefix, CTagCounts, AckQ) - end. - --spec possibly_unblock(cr_fun(), ch(), state()) -> - 'unchanged' | {'unblocked', state()}. - -possibly_unblock(Update, ChPid, State) -> - case lookup_ch(ChPid) of - not_found -> unchanged; - C -> C1 = Update(C), - case is_ch_blocked(C) andalso not is_ch_blocked(C1) of - false -> update_ch_record(C1), - unchanged; - true -> unblock(C1, State) - end - end. - -unblock(C = #cr{blocked_consumers = BlockedQ, limiter = Limiter}, - State = #state{consumers = Consumers, use = Use}) -> - case lists:partition( - fun({_P, {_ChPid, #consumer{tag = CTag}}}) -> - rabbit_limiter:is_consumer_blocked(Limiter, CTag) - end, priority_queue:to_list(BlockedQ)) of - {_, []} -> - update_ch_record(C), - unchanged; - {Blocked, Unblocked} -> - BlockedQ1 = priority_queue:from_list(Blocked), - UnblockedQ = priority_queue:from_list(Unblocked), - update_ch_record(C#cr{blocked_consumers = BlockedQ1}), - {unblocked, - State#state{consumers = priority_queue:join(Consumers, UnblockedQ), - use = update_use(Use, active)}} - end. - --spec resume_fun() -> cr_fun(). - -resume_fun() -> - fun (C = #cr{limiter = Limiter}) -> - C#cr{limiter = rabbit_limiter:resume(Limiter)} - end. - --spec notify_sent_fun(non_neg_integer()) -> cr_fun(). - -notify_sent_fun(Credit) -> - fun (C = #cr{unsent_message_count = Count}) -> - C#cr{unsent_message_count = Count - Credit} - end. - --spec activate_limit_fun() -> cr_fun(). - -activate_limit_fun() -> - fun (C = #cr{limiter = Limiter}) -> - C#cr{limiter = rabbit_limiter:activate(Limiter)} - end. - --spec credit(boolean(), integer(), boolean(), ch(), rabbit_types:ctag(), - state()) -> 'unchanged' | {'unblocked', state()}. - -credit(IsEmpty, Credit, Drain, ChPid, CTag, State) -> - case lookup_ch(ChPid) of - not_found -> - unchanged; - #cr{limiter = Limiter} = C -> - C1 = #cr{limiter = Limiter1} = - credit_and_drain(C, CTag, Credit, drain_mode(Drain), IsEmpty), - case is_ch_blocked(C1) orelse - (not rabbit_limiter:is_consumer_blocked(Limiter, CTag)) orelse - rabbit_limiter:is_consumer_blocked(Limiter1, CTag) of - true -> update_ch_record(C1), - unchanged; - false -> unblock(C1, State) - end - end. - -drain_mode(true) -> drain; -drain_mode(false) -> manual. - --spec utilisation(state()) -> ratio(). - -utilisation(#state{use = {active, Since, Avg}}) -> - use_avg(erlang:monotonic_time(micro_seconds) - Since, 0, Avg); -utilisation(#state{use = {inactive, Since, Active, Avg}}) -> - use_avg(Active, erlang:monotonic_time(micro_seconds) - Since, Avg). - -is_same(ChPid, ConsumerTag, {ChPid, #consumer{tag = ConsumerTag}}) -> - true; -is_same(_ChPid, _ConsumerTag, _Consumer) -> - false. - -get_consumer(#state{consumers = Consumers}) -> - case priority_queue:out_p(Consumers) of - {{value, Consumer, _Priority}, _Tail} -> Consumer; - {empty, _} -> undefined - end. - --spec get(ch(), rabbit_types:ctag(), state()) -> undefined | consumer(). - -get(ChPid, ConsumerTag, #state{consumers = Consumers}) -> - Consumers1 = priority_queue:filter(fun ({CP, #consumer{tag = CT}}) -> - (CP == ChPid) and (CT == ConsumerTag) - end, Consumers), - case priority_queue:out_p(Consumers1) of - {empty, _} -> undefined; - {{value, Consumer, _Priority}, _Tail} -> Consumer - end. - --spec get_infos(consumer()) -> term(). - -get_infos(Consumer) -> - {Consumer#consumer.tag,Consumer#consumer.ack_required, - Consumer#consumer.prefetch, Consumer#consumer.args}. - --spec consumer_tag(consumer()) -> rabbit_types:ctag(). - -consumer_tag(#consumer{tag = CTag}) -> - CTag. - - - -%%---------------------------------------------------------------------------- - -parse_credit_args(Default, Args) -> - case rabbit_misc:table_lookup(Args, <<"x-credit">>) of - {table, T} -> case {rabbit_misc:table_lookup(T, <<"credit">>), - rabbit_misc:table_lookup(T, <<"drain">>)} of - {{long, C}, {bool, D}} -> {C, drain_mode(D)}; - _ -> {Default, auto} - end; - undefined -> {Default, auto} - end. - -lookup_ch(ChPid) -> - case get({ch, ChPid}) of - undefined -> not_found; - C -> C - end. - -ch_record(ChPid, LimiterPid) -> - Key = {ch, ChPid}, - case get(Key) of - undefined -> MonitorRef = erlang:monitor(process, ChPid), - Limiter = rabbit_limiter:client(LimiterPid), - C = #cr{ch_pid = ChPid, - monitor_ref = MonitorRef, - acktags = ?QUEUE:new(), - consumer_count = 0, - blocked_consumers = priority_queue:new(), - limiter = Limiter, - unsent_message_count = 0}, - put(Key, C), - C; - C = #cr{} -> C - end. - -update_ch_record(C = #cr{consumer_count = ConsumerCount, - acktags = ChAckTags, - unsent_message_count = UnsentMessageCount}) -> - case {?QUEUE:is_empty(ChAckTags), ConsumerCount, UnsentMessageCount} of - {true, 0, 0} -> ok = erase_ch_record(C); - _ -> ok = store_ch_record(C) - end, - C. - -store_ch_record(C = #cr{ch_pid = ChPid}) -> - put({ch, ChPid}, C), - ok. - -erase_ch_record(#cr{ch_pid = ChPid, monitor_ref = MonitorRef}) -> - erlang:demonitor(MonitorRef), - erase({ch, ChPid}), - ok. - -all_ch_record() -> [C || {{ch, _}, C} <- get()]. - -block_consumer(C = #cr{blocked_consumers = Blocked}, QEntry) -> - update_ch_record(C#cr{blocked_consumers = add_consumer(QEntry, Blocked)}). - -is_ch_blocked(#cr{unsent_message_count = Count, limiter = Limiter}) -> - Count >= ?UNSENT_MESSAGE_LIMIT orelse rabbit_limiter:is_suspended(Limiter). - -send_drained(C = #cr{ch_pid = ChPid, limiter = Limiter}) -> - case rabbit_limiter:drained(Limiter) of - {[], Limiter} -> C; - {CTagCredit, Limiter2} -> rabbit_channel:send_drained( - ChPid, CTagCredit), - C#cr{limiter = Limiter2} - end. - -credit_and_drain(C = #cr{ch_pid = ChPid, limiter = Limiter}, - CTag, Credit, Mode, IsEmpty) -> - case rabbit_limiter:credit(Limiter, CTag, Credit, Mode, IsEmpty) of - {true, Limiter1} -> rabbit_channel:send_drained(ChPid, - [{CTag, Credit}]), - C#cr{limiter = Limiter1}; - {false, Limiter1} -> C#cr{limiter = Limiter1} - end. - -tags(CList) -> [CTag || {_P, {_ChPid, #consumer{tag = CTag}}} <- CList]. - -add_consumer({ChPid, Consumer = #consumer{args = Args}}, Queue) -> - Priority = case rabbit_misc:table_lookup(Args, <<"x-priority">>) of - {_, P} -> P; - _ -> 0 - end, - priority_queue:in({ChPid, Consumer}, Priority, Queue). - -remove_consumer(ChPid, CTag, Queue) -> - priority_queue:filter(fun ({CP, #consumer{tag = CT}}) -> - (CP /= ChPid) or (CT /= CTag) - end, Queue). - -remove_consumers(ChPid, Queue) -> - priority_queue:filter(chan_pred(ChPid, false), Queue). - -chan_pred(ChPid, Want) -> - fun ({CP, _Consumer}) when CP =:= ChPid -> Want; - (_) -> not Want - end. - -update_use({inactive, _, _, _} = CUInfo, inactive) -> - CUInfo; -update_use({active, _, _} = CUInfo, active) -> - CUInfo; -update_use({active, Since, Avg}, inactive) -> - Now = erlang:monotonic_time(micro_seconds), - {inactive, Now, Now - Since, Avg}; -update_use({inactive, Since, Active, Avg}, active) -> - Now = erlang:monotonic_time(micro_seconds), - {active, Now, use_avg(Active, Now - Since, Avg)}. - -use_avg(0, 0, Avg) -> - Avg; -use_avg(Active, Inactive, Avg) -> - Time = Inactive + Active, - rabbit_misc:moving_average(Time, ?USE_AVG_HALF_LIFE, Active / Time, Avg). diff --git a/src/rabbit_queue_decorator.erl b/src/rabbit_queue_decorator.erl deleted file mode 100644 index cbb50456c1..0000000000 --- a/src/rabbit_queue_decorator.erl +++ /dev/null @@ -1,72 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_queue_decorator). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --export([select/1, set/1, register/2, unregister/1]). - --behaviour(rabbit_registry_class). - --export([added_to_rabbit_registry/2, removed_from_rabbit_registry/1]). - -%%---------------------------------------------------------------------------- - --callback startup(amqqueue:amqqueue()) -> 'ok'. - --callback shutdown(amqqueue:amqqueue()) -> 'ok'. - --callback policy_changed(amqqueue:amqqueue(), amqqueue:amqqueue()) -> - 'ok'. - --callback active_for(amqqueue:amqqueue()) -> boolean(). - -%% called with Queue, MaxActivePriority, IsEmpty --callback consumer_state_changed( - amqqueue:amqqueue(), integer(), boolean()) -> 'ok'. - -%%---------------------------------------------------------------------------- - -added_to_rabbit_registry(_Type, _ModuleName) -> ok. -removed_from_rabbit_registry(_Type) -> ok. - -select(Modules) -> - [M || M <- Modules, code:which(M) =/= non_existing]. - -set(Q) when ?is_amqqueue(Q) -> - Decorators = [D || D <- list(), D:active_for(Q)], - amqqueue:set_decorators(Q, Decorators). - -list() -> [M || {_, M} <- rabbit_registry:lookup_all(queue_decorator)]. - -register(TypeName, ModuleName) -> - rabbit_registry:register(queue_decorator, TypeName, ModuleName), - [maybe_recover(Q) || Q <- rabbit_amqqueue:list()], - ok. - -unregister(TypeName) -> - rabbit_registry:unregister(queue_decorator, TypeName), - [maybe_recover(Q) || Q <- rabbit_amqqueue:list()], - ok. - -maybe_recover(Q0) when ?is_amqqueue(Q0) -> - Name = amqqueue:get_name(Q0), - Decs0 = amqqueue:get_decorators(Q0), - Q1 = set(Q0), - Decs1 = amqqueue:get_decorators(Q1), - Old = lists:sort(select(Decs0)), - New = lists:sort(select(Decs1)), - case New of - Old -> - ok; - _ -> - %% TODO LRB JSP 160169569 should startup be passed Q1 here? - [M:startup(Q0) || M <- New -- Old], - rabbit_amqqueue:update_decorators(Name) - end. diff --git a/src/rabbit_queue_index.erl b/src/rabbit_queue_index.erl deleted file mode 100644 index faab4380b5..0000000000 --- a/src/rabbit_queue_index.erl +++ /dev/null @@ -1,1521 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_queue_index). - --export([erase/1, init/3, reset_state/1, recover/6, - terminate/3, delete_and_terminate/1, - pre_publish/7, flush_pre_publish_cache/2, - publish/6, deliver/2, ack/2, sync/1, needs_sync/1, flush/1, - read/3, next_segment_boundary/1, bounds/1, start/2, stop/1]). - --export([add_queue_ttl/0, avoid_zeroes/0, store_msg_size/0, store_msg/0]). --export([scan_queue_segments/3, scan_queue_segments/4]). - -%% Migrates from global to per-vhost message stores --export([move_to_per_vhost_stores/1, - update_recovery_term/2, - read_global_recovery_terms/1, - cleanup_global_recovery_terms/0]). - --define(CLEAN_FILENAME, "clean.dot"). - -%%---------------------------------------------------------------------------- - -%% The queue index is responsible for recording the order of messages -%% within a queue on disk. As such it contains records of messages -%% being published, delivered and acknowledged. The publish record -%% includes the sequence ID, message ID and a small quantity of -%% metadata about the message; the delivery and acknowledgement -%% records just contain the sequence ID. A publish record may also -%% contain the complete message if provided to publish/5; this allows -%% the message store to be avoided altogether for small messages. In -%% either case the publish record is stored in memory in the same -%% serialised format it will take on disk. -%% -%% Because of the fact that the queue can decide at any point to send -%% a queue entry to disk, you can not rely on publishes appearing in -%% order. The only thing you can rely on is a message being published, -%% then delivered, then ack'd. -%% -%% In order to be able to clean up ack'd messages, we write to segment -%% files. These files have a fixed number of entries: ?SEGMENT_ENTRY_COUNT -%% publishes, delivers and acknowledgements. They are numbered, and so -%% it is known that the 0th segment contains messages 0 -> -%% ?SEGMENT_ENTRY_COUNT - 1, the 1st segment contains messages -%% ?SEGMENT_ENTRY_COUNT -> 2*?SEGMENT_ENTRY_COUNT - 1 and so on. As -%% such, in the segment files, we only refer to message sequence ids -%% by the LSBs as SeqId rem ?SEGMENT_ENTRY_COUNT. This gives them a -%% fixed size. -%% -%% However, transient messages which are not sent to disk at any point -%% will cause gaps to appear in segment files. Therefore, we delete a -%% segment file whenever the number of publishes == number of acks -%% (note that although it is not fully enforced, it is assumed that a -%% message will never be ackd before it is delivered, thus this test -%% also implies == number of delivers). In practise, this does not -%% cause disk churn in the pathological case because of the journal -%% and caching (see below). -%% -%% Because of the fact that publishes, delivers and acks can occur all -%% over, we wish to avoid lots of seeking. Therefore we have a fixed -%% sized journal to which all actions are appended. When the number of -%% entries in this journal reaches max_journal_entries, the journal -%% entries are scattered out to their relevant files, and the journal -%% is truncated to zero size. Note that entries in the journal must -%% carry the full sequence id, thus the format of entries in the -%% journal is different to that in the segments. -%% -%% The journal is also kept fully in memory, pre-segmented: the state -%% contains a mapping from segment numbers to state-per-segment (this -%% state is held for all segments which have been "seen": thus a -%% segment which has been read but has no pending entries in the -%% journal is still held in this mapping. Also note that a map is -%% used for this mapping, not an array because with an array, you will -%% always have entries from 0). Actions are stored directly in this -%% state. Thus at the point of flushing the journal, firstly no -%% reading from disk is necessary, but secondly if the known number of -%% acks and publishes in a segment are equal, given the known state of -%% the segment file combined with the journal, no writing needs to be -%% done to the segment file either (in fact it is deleted if it exists -%% at all). This is safe given that the set of acks is a subset of the -%% set of publishes. When it is necessary to sync messages, it is -%% sufficient to fsync on the journal: when entries are distributed -%% from the journal to segment files, those segments appended to are -%% fsync'd prior to the journal being truncated. -%% -%% This module is also responsible for scanning the queue index files -%% and seeding the message store on start up. -%% -%% Note that in general, the representation of a message's state as -%% the tuple: {('no_pub'|{IsPersistent, Bin, MsgBin}), -%% ('del'|'no_del'), ('ack'|'no_ack')} is richer than strictly -%% necessary for most operations. However, for startup, and to ensure -%% the safe and correct combination of journal entries with entries -%% read from the segment on disk, this richer representation vastly -%% simplifies and clarifies the code. -%% -%% For notes on Clean Shutdown and startup, see documentation in -%% rabbit_variable_queue. -%% -%%---------------------------------------------------------------------------- - -%% ---- Journal details ---- - --define(JOURNAL_FILENAME, "journal.jif"). --define(QUEUE_NAME_STUB_FILE, ".queue_name"). - --define(PUB_PERSIST_JPREFIX, 2#00). --define(PUB_TRANS_JPREFIX, 2#01). --define(DEL_JPREFIX, 2#10). --define(ACK_JPREFIX, 2#11). --define(JPREFIX_BITS, 2). --define(SEQ_BYTES, 8). --define(SEQ_BITS, ((?SEQ_BYTES * 8) - ?JPREFIX_BITS)). - -%% ---- Segment details ---- - --define(SEGMENT_EXTENSION, ".idx"). - -%% TODO: The segment size would be configurable, but deriving all the -%% other values is quite hairy and quite possibly noticeably less -%% efficient, depending on how clever the compiler is when it comes to -%% binary generation/matching with constant vs variable lengths. - --define(REL_SEQ_BITS, 14). -%% calculated as trunc(math:pow(2,?REL_SEQ_BITS))). --define(SEGMENT_ENTRY_COUNT, 16384). - -%% seq only is binary 01 followed by 14 bits of rel seq id -%% (range: 0 - 16383) --define(REL_SEQ_ONLY_PREFIX, 01). --define(REL_SEQ_ONLY_PREFIX_BITS, 2). --define(REL_SEQ_ONLY_RECORD_BYTES, 2). - -%% publish record is binary 1 followed by a bit for is_persistent, -%% then 14 bits of rel seq id, 64 bits for message expiry, 32 bits of -%% size and then 128 bits of md5sum msg id. --define(PUB_PREFIX, 1). --define(PUB_PREFIX_BITS, 1). - --define(EXPIRY_BYTES, 8). --define(EXPIRY_BITS, (?EXPIRY_BYTES * 8)). --define(NO_EXPIRY, 0). - --define(MSG_ID_BYTES, 16). %% md5sum is 128 bit or 16 bytes --define(MSG_ID_BITS, (?MSG_ID_BYTES * 8)). - -%% This is the size of the message body content, for stats --define(SIZE_BYTES, 4). --define(SIZE_BITS, (?SIZE_BYTES * 8)). - -%% This is the size of the message record embedded in the queue -%% index. If 0, the message can be found in the message store. --define(EMBEDDED_SIZE_BYTES, 4). --define(EMBEDDED_SIZE_BITS, (?EMBEDDED_SIZE_BYTES * 8)). - -%% 16 bytes for md5sum + 8 for expiry --define(PUB_RECORD_BODY_BYTES, (?MSG_ID_BYTES + ?EXPIRY_BYTES + ?SIZE_BYTES)). -%% + 4 for size --define(PUB_RECORD_SIZE_BYTES, (?PUB_RECORD_BODY_BYTES + ?EMBEDDED_SIZE_BYTES)). - -%% + 2 for seq, bits and prefix --define(PUB_RECORD_PREFIX_BYTES, 2). - -%% ---- misc ---- - --define(PUB, {_, _, _}). %% {IsPersistent, Bin, MsgBin} - --define(READ_MODE, [binary, raw, read]). --define(WRITE_MODE, [write | ?READ_MODE]). - -%%---------------------------------------------------------------------------- - --record(qistate, { - %% queue directory where segment and journal files are stored - dir, - %% map of #segment records - segments, - %% journal file handle obtained from/used by file_handle_cache - journal_handle, - %% how many not yet flushed entries are there - dirty_count, - %% this many not yet flushed journal entries will force a flush - max_journal_entries, - %% callback function invoked when a message is "handled" - %% by the index and potentially can be confirmed to the publisher - on_sync, - on_sync_msg, - %% set of IDs of unconfirmed [to publishers] messages - unconfirmed, - unconfirmed_msg, - %% optimisation - pre_publish_cache, - %% optimisation - delivered_cache, - %% queue name resource record - queue_name}). - --record(segment, { - %% segment ID (an integer) - num, - %% segment file path (see also ?SEGMENT_EXTENSION) - path, - %% index operation log entries in this segment - journal_entries, - entries_to_segment, - %% counter of unacknowledged messages - unacked -}). - --include("rabbit.hrl"). - -%%---------------------------------------------------------------------------- - --rabbit_upgrade({add_queue_ttl, local, []}). --rabbit_upgrade({avoid_zeroes, local, [add_queue_ttl]}). --rabbit_upgrade({store_msg_size, local, [avoid_zeroes]}). --rabbit_upgrade({store_msg, local, [store_msg_size]}). - --type hdl() :: ('undefined' | any()). --type segment() :: ('undefined' | - #segment { num :: non_neg_integer(), - path :: file:filename(), - journal_entries :: array:array(), - entries_to_segment :: array:array(), - unacked :: non_neg_integer() - }). --type seq_id() :: integer(). --type seg_map() :: {map(), [segment()]}. --type on_sync_fun() :: fun ((gb_sets:set()) -> ok). --type qistate() :: #qistate { dir :: file:filename(), - segments :: 'undefined' | seg_map(), - journal_handle :: hdl(), - dirty_count :: integer(), - max_journal_entries :: non_neg_integer(), - on_sync :: on_sync_fun(), - on_sync_msg :: on_sync_fun(), - unconfirmed :: gb_sets:set(), - unconfirmed_msg :: gb_sets:set(), - pre_publish_cache :: list(), - delivered_cache :: list() - }. --type contains_predicate() :: fun ((rabbit_types:msg_id()) -> boolean()). --type walker(A) :: fun ((A) -> 'finished' | - {rabbit_types:msg_id(), non_neg_integer(), A}). --type shutdown_terms() :: [term()] | 'non_clean_shutdown'. - -%%---------------------------------------------------------------------------- -%% public API -%%---------------------------------------------------------------------------- - --spec erase(rabbit_amqqueue:name()) -> 'ok'. - -erase(#resource{ virtual_host = VHost } = Name) -> - VHostDir = rabbit_vhost:msg_store_dir_path(VHost), - #qistate { dir = Dir } = blank_state(VHostDir, Name), - erase_index_dir(Dir). - -%% used during variable queue purge when there are no pending acks - --spec reset_state(qistate()) -> qistate(). - -reset_state(#qistate{ queue_name = Name, - dir = Dir, - on_sync = OnSyncFun, - on_sync_msg = OnSyncMsgFun, - journal_handle = JournalHdl }) -> - ok = case JournalHdl of - undefined -> ok; - _ -> file_handle_cache:close(JournalHdl) - end, - ok = erase_index_dir(Dir), - blank_state_name_dir_funs(Name, Dir, OnSyncFun, OnSyncMsgFun). - --spec init(rabbit_amqqueue:name(), - on_sync_fun(), on_sync_fun()) -> qistate(). - -init(#resource{ virtual_host = VHost } = Name, OnSyncFun, OnSyncMsgFun) -> - VHostDir = rabbit_vhost:msg_store_dir_path(VHost), - State = #qistate { dir = Dir } = blank_state(VHostDir, Name), - false = rabbit_file:is_file(Dir), %% is_file == is file or dir - State#qistate{on_sync = OnSyncFun, - on_sync_msg = OnSyncMsgFun}. - --spec recover(rabbit_amqqueue:name(), shutdown_terms(), boolean(), - contains_predicate(), - on_sync_fun(), on_sync_fun()) -> - {'undefined' | non_neg_integer(), - 'undefined' | non_neg_integer(), qistate()}. - -recover(#resource{ virtual_host = VHost } = Name, Terms, MsgStoreRecovered, - ContainsCheckFun, OnSyncFun, OnSyncMsgFun) -> - VHostDir = rabbit_vhost:msg_store_dir_path(VHost), - State = blank_state(VHostDir, Name), - State1 = State #qistate{on_sync = OnSyncFun, - on_sync_msg = OnSyncMsgFun}, - CleanShutdown = Terms /= non_clean_shutdown, - case CleanShutdown andalso MsgStoreRecovered of - true -> RecoveredCounts = proplists:get_value(segments, Terms, []), - init_clean(RecoveredCounts, State1); - false -> init_dirty(CleanShutdown, ContainsCheckFun, State1) - end. - --spec terminate(rabbit_types:vhost(), [any()], qistate()) -> qistate(). - -terminate(VHost, Terms, State = #qistate { dir = Dir }) -> - {SegmentCounts, State1} = terminate(State), - rabbit_recovery_terms:store(VHost, filename:basename(Dir), - [{segments, SegmentCounts} | Terms]), - State1. - --spec delete_and_terminate(qistate()) -> qistate(). - -delete_and_terminate(State) -> - {_SegmentCounts, State1 = #qistate { dir = Dir }} = terminate(State), - ok = rabbit_file:recursive_delete([Dir]), - State1. - -pre_publish(MsgOrId, SeqId, MsgProps, IsPersistent, IsDelivered, JournalSizeHint, - State = #qistate{pre_publish_cache = PPC, - delivered_cache = DC}) -> - State1 = maybe_needs_confirming(MsgProps, MsgOrId, State), - - {Bin, MsgBin} = create_pub_record_body(MsgOrId, MsgProps), - - PPC1 = - [[<<(case IsPersistent of - true -> ?PUB_PERSIST_JPREFIX; - false -> ?PUB_TRANS_JPREFIX - end):?JPREFIX_BITS, - SeqId:?SEQ_BITS, Bin/binary, - (size(MsgBin)):?EMBEDDED_SIZE_BITS>>, MsgBin] | PPC], - - DC1 = - case IsDelivered of - true -> - [SeqId | DC]; - false -> - DC - end, - - State2 = add_to_journal(SeqId, {IsPersistent, Bin, MsgBin}, State1), - maybe_flush_pre_publish_cache( - JournalSizeHint, - State2#qistate{pre_publish_cache = PPC1, - delivered_cache = DC1}). - -%% pre_publish_cache is the entry with most elements when compared to -%% delivered_cache so we only check the former in the guard. -maybe_flush_pre_publish_cache(JournalSizeHint, - #qistate{pre_publish_cache = PPC} = State) - when length(PPC) >= ?SEGMENT_ENTRY_COUNT -> - flush_pre_publish_cache(JournalSizeHint, State); -maybe_flush_pre_publish_cache(_JournalSizeHint, State) -> - State. - -flush_pre_publish_cache(JournalSizeHint, State) -> - State1 = flush_pre_publish_cache(State), - State2 = flush_delivered_cache(State1), - maybe_flush_journal(JournalSizeHint, State2). - -flush_pre_publish_cache(#qistate{pre_publish_cache = []} = State) -> - State; -flush_pre_publish_cache(State = #qistate{pre_publish_cache = PPC}) -> - {JournalHdl, State1} = get_journal_handle(State), - file_handle_cache_stats:update(queue_index_journal_write), - ok = file_handle_cache:append(JournalHdl, lists:reverse(PPC)), - State1#qistate{pre_publish_cache = []}. - -flush_delivered_cache(#qistate{delivered_cache = []} = State) -> - State; -flush_delivered_cache(State = #qistate{delivered_cache = DC}) -> - State1 = deliver(lists:reverse(DC), State), - State1#qistate{delivered_cache = []}. - --spec publish(rabbit_types:msg_id(), seq_id(), - rabbit_types:message_properties(), boolean(), - non_neg_integer(), qistate()) -> qistate(). - -publish(MsgOrId, SeqId, MsgProps, IsPersistent, JournalSizeHint, State) -> - {JournalHdl, State1} = - get_journal_handle( - maybe_needs_confirming(MsgProps, MsgOrId, State)), - file_handle_cache_stats:update(queue_index_journal_write), - {Bin, MsgBin} = create_pub_record_body(MsgOrId, MsgProps), - ok = file_handle_cache:append( - JournalHdl, [<<(case IsPersistent of - true -> ?PUB_PERSIST_JPREFIX; - false -> ?PUB_TRANS_JPREFIX - end):?JPREFIX_BITS, - SeqId:?SEQ_BITS, Bin/binary, - (size(MsgBin)):?EMBEDDED_SIZE_BITS>>, MsgBin]), - maybe_flush_journal( - JournalSizeHint, - add_to_journal(SeqId, {IsPersistent, Bin, MsgBin}, State1)). - -maybe_needs_confirming(MsgProps, MsgOrId, - State = #qistate{unconfirmed = UC, - unconfirmed_msg = UCM}) -> - MsgId = case MsgOrId of - #basic_message{id = Id} -> Id; - Id when is_binary(Id) -> Id - end, - ?MSG_ID_BYTES = size(MsgId), - case {MsgProps#message_properties.needs_confirming, MsgOrId} of - {true, MsgId} -> UC1 = gb_sets:add_element(MsgId, UC), - State#qistate{unconfirmed = UC1}; - {true, _} -> UCM1 = gb_sets:add_element(MsgId, UCM), - State#qistate{unconfirmed_msg = UCM1}; - {false, _} -> State - end. - --spec deliver([seq_id()], qistate()) -> qistate(). - -deliver(SeqIds, State) -> - deliver_or_ack(del, SeqIds, State). - --spec ack([seq_id()], qistate()) -> qistate(). - -ack(SeqIds, State) -> - deliver_or_ack(ack, SeqIds, State). - -%% This is called when there are outstanding confirms or when the -%% queue is idle and the journal needs syncing (see needs_sync/1). - --spec sync(qistate()) -> qistate(). - -sync(State = #qistate { journal_handle = undefined }) -> - State; -sync(State = #qistate { journal_handle = JournalHdl }) -> - ok = file_handle_cache:sync(JournalHdl), - notify_sync(State). - --spec needs_sync(qistate()) -> 'confirms' | 'other' | 'false'. - -needs_sync(#qistate{journal_handle = undefined}) -> - false; -needs_sync(#qistate{journal_handle = JournalHdl, - unconfirmed = UC, - unconfirmed_msg = UCM}) -> - case gb_sets:is_empty(UC) andalso gb_sets:is_empty(UCM) of - true -> case file_handle_cache:needs_sync(JournalHdl) of - true -> other; - false -> false - end; - false -> confirms - end. - --spec flush(qistate()) -> qistate(). - -flush(State = #qistate { dirty_count = 0 }) -> State; -flush(State) -> flush_journal(State). - --spec read(seq_id(), seq_id(), qistate()) -> - {[{rabbit_types:msg_id(), seq_id(), - rabbit_types:message_properties(), - boolean(), boolean()}], qistate()}. - -read(StartEnd, StartEnd, State) -> - {[], State}; -read(Start, End, State = #qistate { segments = Segments, - dir = Dir }) when Start =< End -> - %% Start is inclusive, End is exclusive. - LowerB = {StartSeg, _StartRelSeq} = seq_id_to_seg_and_rel_seq_id(Start), - UpperB = {EndSeg, _EndRelSeq} = seq_id_to_seg_and_rel_seq_id(End - 1), - {Messages, Segments1} = - lists:foldr(fun (Seg, Acc) -> - read_bounded_segment(Seg, LowerB, UpperB, Acc, Dir) - end, {[], Segments}, lists:seq(StartSeg, EndSeg)), - {Messages, State #qistate { segments = Segments1 }}. - --spec next_segment_boundary(seq_id()) -> seq_id(). - -next_segment_boundary(SeqId) -> - {Seg, _RelSeq} = seq_id_to_seg_and_rel_seq_id(SeqId), - reconstruct_seq_id(Seg + 1, 0). - --spec bounds(qistate()) -> - {non_neg_integer(), non_neg_integer(), qistate()}. - -bounds(State = #qistate { segments = Segments }) -> - %% This is not particularly efficient, but only gets invoked on - %% queue initialisation. - SegNums = lists:sort(segment_nums(Segments)), - %% Don't bother trying to figure out the lowest seq_id, merely the - %% seq_id of the start of the lowest segment. That seq_id may not - %% actually exist, but that's fine. The important thing is that - %% the segment exists and the seq_id reported is on a segment - %% boundary. - %% - %% We also don't really care about the max seq_id. Just start the - %% next segment: it makes life much easier. - %% - %% SegNums is sorted, ascending. - {LowSeqId, NextSeqId} = - case SegNums of - [] -> {0, 0}; - [MinSeg|_] -> {reconstruct_seq_id(MinSeg, 0), - reconstruct_seq_id(1 + lists:last(SegNums), 0)} - end, - {LowSeqId, NextSeqId, State}. - --spec start(rabbit_types:vhost(), [rabbit_amqqueue:name()]) -> {[[any()]], {walker(A), A}}. - -start(VHost, DurableQueueNames) -> - ok = rabbit_recovery_terms:start(VHost), - {DurableTerms, DurableDirectories} = - lists:foldl( - fun(QName, {RecoveryTerms, ValidDirectories}) -> - DirName = queue_name_to_dir_name(QName), - RecoveryInfo = case rabbit_recovery_terms:read(VHost, DirName) of - {error, _} -> non_clean_shutdown; - {ok, Terms} -> Terms - end, - {[RecoveryInfo | RecoveryTerms], - sets:add_element(DirName, ValidDirectories)} - end, {[], sets:new()}, DurableQueueNames), - %% Any queue directory we've not been asked to recover is considered garbage - rabbit_file:recursive_delete( - [DirName || - DirName <- all_queue_directory_names(VHost), - not sets:is_element(filename:basename(DirName), DurableDirectories)]), - rabbit_recovery_terms:clear(VHost), - - %% The backing queue interface requires that the queue recovery terms - %% which come back from start/1 are in the same order as DurableQueueNames - OrderedTerms = lists:reverse(DurableTerms), - {OrderedTerms, {fun queue_index_walker/1, {start, DurableQueueNames}}}. - - -stop(VHost) -> rabbit_recovery_terms:stop(VHost). - -all_queue_directory_names(VHost) -> - filelib:wildcard(filename:join([rabbit_vhost:msg_store_dir_path(VHost), - "queues", "*"])). - -all_queue_directory_names() -> - filelib:wildcard(filename:join([rabbit_vhost:msg_store_dir_wildcard(), - "queues", "*"])). - -%%---------------------------------------------------------------------------- -%% startup and shutdown -%%---------------------------------------------------------------------------- - -erase_index_dir(Dir) -> - case rabbit_file:is_dir(Dir) of - true -> rabbit_file:recursive_delete([Dir]); - false -> ok - end. - -blank_state(VHostDir, QueueName) -> - Dir = queue_dir(VHostDir, QueueName), - blank_state_name_dir_funs(QueueName, - Dir, - fun (_) -> ok end, - fun (_) -> ok end). - -queue_dir(VHostDir, QueueName) -> - %% Queue directory is - %% {node_database_dir}/msg_stores/vhosts/{vhost}/queues/{queue} - QueueDir = queue_name_to_dir_name(QueueName), - filename:join([VHostDir, "queues", QueueDir]). - -queue_name_to_dir_name(#resource { kind = queue, - virtual_host = VHost, - name = QName }) -> - <<Num:128>> = erlang:md5(<<"queue", VHost/binary, QName/binary>>), - rabbit_misc:format("~.36B", [Num]). - -queue_name_to_dir_name_legacy(Name = #resource { kind = queue }) -> - <<Num:128>> = erlang:md5(term_to_binary_compat:term_to_binary_1(Name)), - rabbit_misc:format("~.36B", [Num]). - -queues_base_dir() -> - rabbit_mnesia:dir(). - -blank_state_name_dir_funs(Name, Dir, OnSyncFun, OnSyncMsgFun) -> - {ok, MaxJournal} = - application:get_env(rabbit, queue_index_max_journal_entries), - #qistate { dir = Dir, - segments = segments_new(), - journal_handle = undefined, - dirty_count = 0, - max_journal_entries = MaxJournal, - on_sync = OnSyncFun, - on_sync_msg = OnSyncMsgFun, - unconfirmed = gb_sets:new(), - unconfirmed_msg = gb_sets:new(), - pre_publish_cache = [], - delivered_cache = [], - queue_name = Name }. - -init_clean(RecoveredCounts, State) -> - %% Load the journal. Since this is a clean recovery this (almost) - %% gets us back to where we were on shutdown. - State1 = #qistate { dir = Dir, segments = Segments } = load_journal(State), - %% The journal loading only creates records for segments touched - %% by the journal, and the counts are based on the journal entries - %% only. We need *complete* counts for *all* segments. By an - %% amazing coincidence we stored that information on shutdown. - Segments1 = - lists:foldl( - fun ({Seg, UnackedCount}, SegmentsN) -> - Segment = segment_find_or_new(Seg, Dir, SegmentsN), - segment_store(Segment #segment { unacked = UnackedCount }, - SegmentsN) - end, Segments, RecoveredCounts), - %% the counts above include transient messages, which would be the - %% wrong thing to return - {undefined, undefined, State1 # qistate { segments = Segments1 }}. - -init_dirty(CleanShutdown, ContainsCheckFun, State) -> - %% Recover the journal completely. This will also load segments - %% which have entries in the journal and remove duplicates. The - %% counts will correctly reflect the combination of the segment - %% and the journal. - State1 = #qistate { dir = Dir, segments = Segments } = - recover_journal(State), - {Segments1, Count, Bytes, DirtyCount} = - %% Load each segment in turn and filter out messages that are - %% not in the msg_store, by adding acks to the journal. These - %% acks only go to the RAM journal as it doesn't matter if we - %% lose them. Also mark delivered if not clean shutdown. Also - %% find the number of unacked messages. Also accumulate the - %% dirty count here, so we can call maybe_flush_journal below - %% and avoid unnecessary file system operations. - lists:foldl( - fun (Seg, {Segments2, CountAcc, BytesAcc, DirtyCount}) -> - {{Segment = #segment { unacked = UnackedCount }, Dirty}, - UnackedBytes} = - recover_segment(ContainsCheckFun, CleanShutdown, - segment_find_or_new(Seg, Dir, Segments2), - State1#qistate.max_journal_entries), - {segment_store(Segment, Segments2), - CountAcc + UnackedCount, - BytesAcc + UnackedBytes, DirtyCount + Dirty} - end, {Segments, 0, 0, 0}, all_segment_nums(State1)), - State2 = maybe_flush_journal(State1 #qistate { segments = Segments1, - dirty_count = DirtyCount }), - {Count, Bytes, State2}. - -terminate(State = #qistate { journal_handle = JournalHdl, - segments = Segments }) -> - ok = case JournalHdl of - undefined -> ok; - _ -> file_handle_cache:close(JournalHdl) - end, - SegmentCounts = - segment_fold( - fun (#segment { num = Seg, unacked = UnackedCount }, Acc) -> - [{Seg, UnackedCount} | Acc] - end, [], Segments), - {SegmentCounts, State #qistate { journal_handle = undefined, - segments = undefined }}. - -recover_segment(ContainsCheckFun, CleanShutdown, - Segment = #segment { journal_entries = JEntries }, MaxJournal) -> - {SegEntries, UnackedCount} = load_segment(false, Segment), - {SegEntries1, UnackedCountDelta} = - segment_plus_journal(SegEntries, JEntries), - array:sparse_foldl( - fun (RelSeq, {{IsPersistent, Bin, MsgBin}, Del, no_ack}, - {SegmentAndDirtyCount, Bytes}) -> - {MsgOrId, MsgProps} = parse_pub_record_body(Bin, MsgBin), - {recover_message(ContainsCheckFun(MsgOrId), CleanShutdown, - Del, RelSeq, SegmentAndDirtyCount, MaxJournal), - Bytes + case IsPersistent of - true -> MsgProps#message_properties.size; - false -> 0 - end} - end, - {{Segment #segment { unacked = UnackedCount + UnackedCountDelta }, 0}, 0}, - SegEntries1). - -recover_message( true, true, _Del, _RelSeq, SegmentAndDirtyCount, _MaxJournal) -> - SegmentAndDirtyCount; -recover_message( true, false, del, _RelSeq, SegmentAndDirtyCount, _MaxJournal) -> - SegmentAndDirtyCount; -recover_message( true, false, no_del, RelSeq, {Segment, _DirtyCount}, MaxJournal) -> - %% force to flush the segment - {add_to_journal(RelSeq, del, Segment), MaxJournal + 1}; -recover_message(false, _, del, RelSeq, {Segment, DirtyCount}, _MaxJournal) -> - {add_to_journal(RelSeq, ack, Segment), DirtyCount + 1}; -recover_message(false, _, no_del, RelSeq, {Segment, DirtyCount}, _MaxJournal) -> - {add_to_journal(RelSeq, ack, - add_to_journal(RelSeq, del, Segment)), - DirtyCount + 2}. - -%%---------------------------------------------------------------------------- -%% msg store startup delta function -%%---------------------------------------------------------------------------- - -queue_index_walker({start, DurableQueues}) when is_list(DurableQueues) -> - {ok, Gatherer} = gatherer:start_link(), - [begin - ok = gatherer:fork(Gatherer), - ok = worker_pool:submit_async( - fun () -> link(Gatherer), - ok = queue_index_walker_reader(QueueName, Gatherer), - unlink(Gatherer), - ok - end) - end || QueueName <- DurableQueues], - queue_index_walker({next, Gatherer}); - -queue_index_walker({next, Gatherer}) when is_pid(Gatherer) -> - case gatherer:out(Gatherer) of - empty -> - ok = gatherer:stop(Gatherer), - finished; - {value, {MsgId, Count}} -> - {MsgId, Count, {next, Gatherer}} - end. - -queue_index_walker_reader(QueueName, Gatherer) -> - ok = scan_queue_segments( - fun (_SeqId, MsgId, _MsgProps, true, _IsDelivered, no_ack, ok) - when is_binary(MsgId) -> - gatherer:sync_in(Gatherer, {MsgId, 1}); - (_SeqId, _MsgId, _MsgProps, _IsPersistent, _IsDelivered, - _IsAcked, Acc) -> - Acc - end, ok, QueueName), - ok = gatherer:finish(Gatherer). - -scan_queue_segments(Fun, Acc, #resource{ virtual_host = VHost } = QueueName) -> - VHostDir = rabbit_vhost:msg_store_dir_path(VHost), - scan_queue_segments(Fun, Acc, VHostDir, QueueName). - -scan_queue_segments(Fun, Acc, VHostDir, QueueName) -> - State = #qistate { segments = Segments, dir = Dir } = - recover_journal(blank_state(VHostDir, QueueName)), - Result = lists:foldr( - fun (Seg, AccN) -> - segment_entries_foldr( - fun (RelSeq, {{MsgOrId, MsgProps, IsPersistent}, - IsDelivered, IsAcked}, AccM) -> - Fun(reconstruct_seq_id(Seg, RelSeq), MsgOrId, MsgProps, - IsPersistent, IsDelivered, IsAcked, AccM) - end, AccN, segment_find_or_new(Seg, Dir, Segments)) - end, Acc, all_segment_nums(State)), - {_SegmentCounts, _State} = terminate(State), - Result. - -%%---------------------------------------------------------------------------- -%% expiry/binary manipulation -%%---------------------------------------------------------------------------- - -create_pub_record_body(MsgOrId, #message_properties { expiry = Expiry, - size = Size }) -> - ExpiryBin = expiry_to_binary(Expiry), - case MsgOrId of - MsgId when is_binary(MsgId) -> - {<<MsgId/binary, ExpiryBin/binary, Size:?SIZE_BITS>>, <<>>}; - #basic_message{id = MsgId} -> - MsgBin = term_to_binary(MsgOrId), - {<<MsgId/binary, ExpiryBin/binary, Size:?SIZE_BITS>>, MsgBin} - end. - -expiry_to_binary(undefined) -> <<?NO_EXPIRY:?EXPIRY_BITS>>; -expiry_to_binary(Expiry) -> <<Expiry:?EXPIRY_BITS>>. - -parse_pub_record_body(<<MsgIdNum:?MSG_ID_BITS, Expiry:?EXPIRY_BITS, - Size:?SIZE_BITS>>, MsgBin) -> - %% work around for binary data fragmentation. See - %% rabbit_msg_file:read_next/2 - <<MsgId:?MSG_ID_BYTES/binary>> = <<MsgIdNum:?MSG_ID_BITS>>, - Props = #message_properties{expiry = case Expiry of - ?NO_EXPIRY -> undefined; - X -> X - end, - size = Size}, - case MsgBin of - <<>> -> {MsgId, Props}; - _ -> Msg = #basic_message{id = MsgId} = binary_to_term(MsgBin), - {Msg, Props} - end. - -%%---------------------------------------------------------------------------- -%% journal manipulation -%%---------------------------------------------------------------------------- - -add_to_journal(SeqId, Action, State = #qistate { dirty_count = DCount, - segments = Segments, - dir = Dir }) -> - {Seg, RelSeq} = seq_id_to_seg_and_rel_seq_id(SeqId), - Segment = segment_find_or_new(Seg, Dir, Segments), - Segment1 = add_to_journal(RelSeq, Action, Segment), - State #qistate { dirty_count = DCount + 1, - segments = segment_store(Segment1, Segments) }; - -add_to_journal(RelSeq, Action, - Segment = #segment { journal_entries = JEntries, - entries_to_segment = EToSeg, - unacked = UnackedCount }) -> - - {Fun, Entry} = action_to_entry(RelSeq, Action, JEntries), - - {JEntries1, EToSeg1} = - case Fun of - set -> - {array:set(RelSeq, Entry, JEntries), - array:set(RelSeq, entry_to_segment(RelSeq, Entry, []), - EToSeg)}; - reset -> - {array:reset(RelSeq, JEntries), - array:reset(RelSeq, EToSeg)} - end, - - Segment #segment { - journal_entries = JEntries1, - entries_to_segment = EToSeg1, - unacked = UnackedCount + case Action of - ?PUB -> +1; - del -> 0; - ack -> -1 - end}. - -action_to_entry(RelSeq, Action, JEntries) -> - case array:get(RelSeq, JEntries) of - undefined -> - {set, - case Action of - ?PUB -> {Action, no_del, no_ack}; - del -> {no_pub, del, no_ack}; - ack -> {no_pub, no_del, ack} - end}; - ({Pub, no_del, no_ack}) when Action == del -> - {set, {Pub, del, no_ack}}; - ({no_pub, del, no_ack}) when Action == ack -> - {set, {no_pub, del, ack}}; - ({?PUB, del, no_ack}) when Action == ack -> - {reset, none} - end. - -maybe_flush_journal(State) -> - maybe_flush_journal(infinity, State). - -maybe_flush_journal(Hint, State = #qistate { dirty_count = DCount, - max_journal_entries = MaxJournal }) - when DCount > MaxJournal orelse (Hint =/= infinity andalso DCount > Hint) -> - flush_journal(State); -maybe_flush_journal(_Hint, State) -> - State. - -flush_journal(State = #qistate { segments = Segments }) -> - Segments1 = - segment_fold( - fun (#segment { unacked = 0, path = Path }, SegmentsN) -> - case rabbit_file:is_file(Path) of - true -> ok = rabbit_file:delete(Path); - false -> ok - end, - SegmentsN; - (#segment {} = Segment, SegmentsN) -> - segment_store(append_journal_to_segment(Segment), SegmentsN) - end, segments_new(), Segments), - {JournalHdl, State1} = - get_journal_handle(State #qistate { segments = Segments1 }), - ok = file_handle_cache:clear(JournalHdl), - notify_sync(State1 #qistate { dirty_count = 0 }). - -append_journal_to_segment(#segment { journal_entries = JEntries, - entries_to_segment = EToSeg, - path = Path } = Segment) -> - case array:sparse_size(JEntries) of - 0 -> Segment; - _ -> - file_handle_cache_stats:update(queue_index_write), - - {ok, Hdl} = file_handle_cache:open_with_absolute_path( - Path, ?WRITE_MODE, - [{write_buffer, infinity}]), - %% the file_handle_cache also does a list reverse, so this - %% might not be required here, but before we were doing a - %% sparse_foldr, a lists:reverse/1 seems to be the correct - %% thing to do for now. - file_handle_cache:append(Hdl, lists:reverse(array:to_list(EToSeg))), - ok = file_handle_cache:close(Hdl), - Segment #segment { journal_entries = array_new(), - entries_to_segment = array_new([]) } - end. - -get_journal_handle(State = #qistate { journal_handle = undefined, - dir = Dir, - queue_name = Name }) -> - Path = filename:join(Dir, ?JOURNAL_FILENAME), - ok = rabbit_file:ensure_dir(Path), - ok = ensure_queue_name_stub_file(Dir, Name), - {ok, Hdl} = file_handle_cache:open_with_absolute_path( - Path, ?WRITE_MODE, [{write_buffer, infinity}]), - {Hdl, State #qistate { journal_handle = Hdl }}; -get_journal_handle(State = #qistate { journal_handle = Hdl }) -> - {Hdl, State}. - -%% Loading Journal. This isn't idempotent and will mess up the counts -%% if you call it more than once on the same state. Assumes the counts -%% are 0 to start with. -load_journal(State = #qistate { dir = Dir }) -> - Path = filename:join(Dir, ?JOURNAL_FILENAME), - case rabbit_file:is_file(Path) of - true -> {JournalHdl, State1} = get_journal_handle(State), - Size = rabbit_file:file_size(Path), - {ok, 0} = file_handle_cache:position(JournalHdl, 0), - {ok, JournalBin} = file_handle_cache:read(JournalHdl, Size), - parse_journal_entries(JournalBin, State1); - false -> State - end. - -%% ditto -recover_journal(State) -> - State1 = #qistate { segments = Segments } = load_journal(State), - Segments1 = - segment_map( - fun (Segment = #segment { journal_entries = JEntries, - entries_to_segment = EToSeg, - unacked = UnackedCountInJournal }) -> - %% We want to keep ack'd entries in so that we can - %% remove them if duplicates are in the journal. The - %% counts here are purely from the segment itself. - {SegEntries, UnackedCountInSeg} = load_segment(true, Segment), - {JEntries1, EToSeg1, UnackedCountDuplicates} = - journal_minus_segment(JEntries, EToSeg, SegEntries), - Segment #segment { journal_entries = JEntries1, - entries_to_segment = EToSeg1, - unacked = (UnackedCountInJournal + - UnackedCountInSeg - - UnackedCountDuplicates) } - end, Segments), - State1 #qistate { segments = Segments1 }. - -parse_journal_entries(<<?DEL_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS, - Rest/binary>>, State) -> - parse_journal_entries(Rest, add_to_journal(SeqId, del, State)); - -parse_journal_entries(<<?ACK_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS, - Rest/binary>>, State) -> - parse_journal_entries(Rest, add_to_journal(SeqId, ack, State)); -parse_journal_entries(<<0:?JPREFIX_BITS, 0:?SEQ_BITS, - 0:?PUB_RECORD_SIZE_BYTES/unit:8, _/binary>>, State) -> - %% Journal entry composed only of zeroes was probably - %% produced during a dirty shutdown so stop reading - State; -parse_journal_entries(<<Prefix:?JPREFIX_BITS, SeqId:?SEQ_BITS, - Bin:?PUB_RECORD_BODY_BYTES/binary, - MsgSize:?EMBEDDED_SIZE_BITS, MsgBin:MsgSize/binary, - Rest/binary>>, State) -> - IsPersistent = case Prefix of - ?PUB_PERSIST_JPREFIX -> true; - ?PUB_TRANS_JPREFIX -> false - end, - parse_journal_entries( - Rest, add_to_journal(SeqId, {IsPersistent, Bin, MsgBin}, State)); -parse_journal_entries(_ErrOrEoF, State) -> - State. - -deliver_or_ack(_Kind, [], State) -> - State; -deliver_or_ack(Kind, SeqIds, State) -> - JPrefix = case Kind of ack -> ?ACK_JPREFIX; del -> ?DEL_JPREFIX end, - {JournalHdl, State1} = get_journal_handle(State), - file_handle_cache_stats:update(queue_index_journal_write), - ok = file_handle_cache:append( - JournalHdl, - [<<JPrefix:?JPREFIX_BITS, SeqId:?SEQ_BITS>> || SeqId <- SeqIds]), - maybe_flush_journal(lists:foldl(fun (SeqId, StateN) -> - add_to_journal(SeqId, Kind, StateN) - end, State1, SeqIds)). - -notify_sync(State = #qistate{unconfirmed = UC, - unconfirmed_msg = UCM, - on_sync = OnSyncFun, - on_sync_msg = OnSyncMsgFun}) -> - State1 = case gb_sets:is_empty(UC) of - true -> State; - false -> OnSyncFun(UC), - State#qistate{unconfirmed = gb_sets:new()} - end, - case gb_sets:is_empty(UCM) of - true -> State1; - false -> OnSyncMsgFun(UCM), - State1#qistate{unconfirmed_msg = gb_sets:new()} - end. - -%%---------------------------------------------------------------------------- -%% segment manipulation -%%---------------------------------------------------------------------------- - -seq_id_to_seg_and_rel_seq_id(SeqId) -> - { SeqId div ?SEGMENT_ENTRY_COUNT, SeqId rem ?SEGMENT_ENTRY_COUNT }. - -reconstruct_seq_id(Seg, RelSeq) -> - (Seg * ?SEGMENT_ENTRY_COUNT) + RelSeq. - -all_segment_nums(#qistate { dir = Dir, segments = Segments }) -> - lists:sort( - sets:to_list( - lists:foldl( - fun (SegName, Set) -> - sets:add_element( - list_to_integer( - lists:takewhile(fun (C) -> $0 =< C andalso C =< $9 end, - SegName)), Set) - end, sets:from_list(segment_nums(Segments)), - rabbit_file:wildcard(".*\\" ++ ?SEGMENT_EXTENSION, Dir)))). - -segment_find_or_new(Seg, Dir, Segments) -> - case segment_find(Seg, Segments) of - {ok, Segment} -> Segment; - error -> SegName = integer_to_list(Seg) ++ ?SEGMENT_EXTENSION, - Path = filename:join(Dir, SegName), - #segment { num = Seg, - path = Path, - journal_entries = array_new(), - entries_to_segment = array_new([]), - unacked = 0 } - end. - -segment_find(Seg, {_Segments, [Segment = #segment { num = Seg } |_]}) -> - {ok, Segment}; %% 1 or (2, matches head) -segment_find(Seg, {_Segments, [_, Segment = #segment { num = Seg }]}) -> - {ok, Segment}; %% 2, matches tail -segment_find(Seg, {Segments, _}) -> %% no match - maps:find(Seg, Segments). - -segment_store(Segment = #segment { num = Seg }, %% 1 or (2, matches head) - {Segments, [#segment { num = Seg } | Tail]}) -> - {Segments, [Segment | Tail]}; -segment_store(Segment = #segment { num = Seg }, %% 2, matches tail - {Segments, [SegmentA, #segment { num = Seg }]}) -> - {Segments, [Segment, SegmentA]}; -segment_store(Segment = #segment { num = Seg }, {Segments, []}) -> - {maps:remove(Seg, Segments), [Segment]}; -segment_store(Segment = #segment { num = Seg }, {Segments, [SegmentA]}) -> - {maps:remove(Seg, Segments), [Segment, SegmentA]}; -segment_store(Segment = #segment { num = Seg }, - {Segments, [SegmentA, SegmentB]}) -> - {maps:put(SegmentB#segment.num, SegmentB, maps:remove(Seg, Segments)), - [Segment, SegmentA]}. - -segment_fold(Fun, Acc, {Segments, CachedSegments}) -> - maps:fold(fun (_Seg, Segment, Acc1) -> Fun(Segment, Acc1) end, - lists:foldl(Fun, Acc, CachedSegments), Segments). - -segment_map(Fun, {Segments, CachedSegments}) -> - {maps:map(fun (_Seg, Segment) -> Fun(Segment) end, Segments), - lists:map(Fun, CachedSegments)}. - -segment_nums({Segments, CachedSegments}) -> - lists:map(fun (#segment { num = Num }) -> Num end, CachedSegments) ++ - maps:keys(Segments). - -segments_new() -> - {#{}, []}. - -entry_to_segment(_RelSeq, {?PUB, del, ack}, Initial) -> - Initial; -entry_to_segment(RelSeq, {Pub, Del, Ack}, Initial) -> - %% NB: we are assembling the segment in reverse order here, so - %% del/ack comes first. - Buf1 = case {Del, Ack} of - {no_del, no_ack} -> - Initial; - _ -> - Binary = <<?REL_SEQ_ONLY_PREFIX:?REL_SEQ_ONLY_PREFIX_BITS, - RelSeq:?REL_SEQ_BITS>>, - case {Del, Ack} of - {del, ack} -> [[Binary, Binary] | Initial]; - _ -> [Binary | Initial] - end - end, - case Pub of - no_pub -> - Buf1; - {IsPersistent, Bin, MsgBin} -> - [[<<?PUB_PREFIX:?PUB_PREFIX_BITS, - (bool_to_int(IsPersistent)):1, - RelSeq:?REL_SEQ_BITS, Bin/binary, - (size(MsgBin)):?EMBEDDED_SIZE_BITS>>, MsgBin] | Buf1] - end. - -read_bounded_segment(Seg, {StartSeg, StartRelSeq}, {EndSeg, EndRelSeq}, - {Messages, Segments}, Dir) -> - Segment = segment_find_or_new(Seg, Dir, Segments), - {segment_entries_foldr( - fun (RelSeq, {{MsgOrId, MsgProps, IsPersistent}, IsDelivered, no_ack}, - Acc) - when (Seg > StartSeg orelse StartRelSeq =< RelSeq) andalso - (Seg < EndSeg orelse EndRelSeq >= RelSeq) -> - [{MsgOrId, reconstruct_seq_id(StartSeg, RelSeq), MsgProps, - IsPersistent, IsDelivered == del} | Acc]; - (_RelSeq, _Value, Acc) -> - Acc - end, Messages, Segment), - segment_store(Segment, Segments)}. - -segment_entries_foldr(Fun, Init, - Segment = #segment { journal_entries = JEntries }) -> - {SegEntries, _UnackedCount} = load_segment(false, Segment), - {SegEntries1, _UnackedCountD} = segment_plus_journal(SegEntries, JEntries), - array:sparse_foldr( - fun (RelSeq, {{IsPersistent, Bin, MsgBin}, Del, Ack}, Acc) -> - {MsgOrId, MsgProps} = parse_pub_record_body(Bin, MsgBin), - Fun(RelSeq, {{MsgOrId, MsgProps, IsPersistent}, Del, Ack}, Acc) - end, Init, SegEntries1). - -%% Loading segments -%% -%% Does not do any combining with the journal at all. -load_segment(KeepAcked, #segment { path = Path }) -> - Empty = {array_new(), 0}, - case rabbit_file:is_file(Path) of - false -> Empty; - true -> Size = rabbit_file:file_size(Path), - file_handle_cache_stats:update(queue_index_read), - {ok, Hdl} = file_handle_cache:open_with_absolute_path( - Path, ?READ_MODE, []), - {ok, 0} = file_handle_cache:position(Hdl, bof), - {ok, SegBin} = file_handle_cache:read(Hdl, Size), - ok = file_handle_cache:close(Hdl), - Res = parse_segment_entries(SegBin, KeepAcked, Empty), - Res - end. - -parse_segment_entries(<<?PUB_PREFIX:?PUB_PREFIX_BITS, - IsPersistNum:1, RelSeq:?REL_SEQ_BITS, Rest/binary>>, - KeepAcked, Acc) -> - parse_segment_publish_entry( - Rest, 1 == IsPersistNum, RelSeq, KeepAcked, Acc); -parse_segment_entries(<<?REL_SEQ_ONLY_PREFIX:?REL_SEQ_ONLY_PREFIX_BITS, - RelSeq:?REL_SEQ_BITS, Rest/binary>>, KeepAcked, Acc) -> - parse_segment_entries( - Rest, KeepAcked, add_segment_relseq_entry(KeepAcked, RelSeq, Acc)); -parse_segment_entries(<<>>, _KeepAcked, Acc) -> - Acc. - -parse_segment_publish_entry(<<Bin:?PUB_RECORD_BODY_BYTES/binary, - MsgSize:?EMBEDDED_SIZE_BITS, - MsgBin:MsgSize/binary, Rest/binary>>, - IsPersistent, RelSeq, KeepAcked, - {SegEntries, Unacked}) -> - Obj = {{IsPersistent, Bin, MsgBin}, no_del, no_ack}, - SegEntries1 = array:set(RelSeq, Obj, SegEntries), - parse_segment_entries(Rest, KeepAcked, {SegEntries1, Unacked + 1}); -parse_segment_publish_entry(Rest, _IsPersistent, _RelSeq, KeepAcked, Acc) -> - parse_segment_entries(Rest, KeepAcked, Acc). - -add_segment_relseq_entry(KeepAcked, RelSeq, {SegEntries, Unacked}) -> - case array:get(RelSeq, SegEntries) of - {Pub, no_del, no_ack} -> - {array:set(RelSeq, {Pub, del, no_ack}, SegEntries), Unacked}; - {Pub, del, no_ack} when KeepAcked -> - {array:set(RelSeq, {Pub, del, ack}, SegEntries), Unacked - 1}; - {_Pub, del, no_ack} -> - {array:reset(RelSeq, SegEntries), Unacked - 1} - end. - -array_new() -> - array_new(undefined). - -array_new(Default) -> - array:new([{default, Default}, fixed, {size, ?SEGMENT_ENTRY_COUNT}]). - -bool_to_int(true ) -> 1; -bool_to_int(false) -> 0. - -%%---------------------------------------------------------------------------- -%% journal & segment combination -%%---------------------------------------------------------------------------- - -%% Combine what we have just read from a segment file with what we're -%% holding for that segment in memory. There must be no duplicates. -segment_plus_journal(SegEntries, JEntries) -> - array:sparse_foldl( - fun (RelSeq, JObj, {SegEntriesOut, AdditionalUnacked}) -> - SegEntry = array:get(RelSeq, SegEntriesOut), - {Obj, AdditionalUnackedDelta} = - segment_plus_journal1(SegEntry, JObj), - {case Obj of - undefined -> array:reset(RelSeq, SegEntriesOut); - _ -> array:set(RelSeq, Obj, SegEntriesOut) - end, - AdditionalUnacked + AdditionalUnackedDelta} - end, {SegEntries, 0}, JEntries). - -%% Here, the result is a tuple with the first element containing the -%% item which we may be adding to (for items only in the journal), -%% modifying in (bits in both), or, when returning 'undefined', -%% erasing from (ack in journal, not segment) the segment array. The -%% other element of the tuple is the delta for AdditionalUnacked. -segment_plus_journal1(undefined, {?PUB, no_del, no_ack} = Obj) -> - {Obj, 1}; -segment_plus_journal1(undefined, {?PUB, del, no_ack} = Obj) -> - {Obj, 1}; -segment_plus_journal1(undefined, {?PUB, del, ack}) -> - {undefined, 0}; - -segment_plus_journal1({?PUB = Pub, no_del, no_ack}, {no_pub, del, no_ack}) -> - {{Pub, del, no_ack}, 0}; -segment_plus_journal1({?PUB, no_del, no_ack}, {no_pub, del, ack}) -> - {undefined, -1}; -segment_plus_journal1({?PUB, del, no_ack}, {no_pub, no_del, ack}) -> - {undefined, -1}. - -%% Remove from the journal entries for a segment, items that are -%% duplicates of entries found in the segment itself. Used on start up -%% to clean up the journal. -%% -%% We need to update the entries_to_segment since they are just a -%% cache of what's on the journal. -journal_minus_segment(JEntries, EToSeg, SegEntries) -> - array:sparse_foldl( - fun (RelSeq, JObj, {JEntriesOut, EToSegOut, UnackedRemoved}) -> - SegEntry = array:get(RelSeq, SegEntries), - {Obj, UnackedRemovedDelta} = - journal_minus_segment1(JObj, SegEntry), - {JEntriesOut1, EToSegOut1} = - case Obj of - keep -> - {JEntriesOut, EToSegOut}; - undefined -> - {array:reset(RelSeq, JEntriesOut), - array:reset(RelSeq, EToSegOut)}; - _ -> - {array:set(RelSeq, Obj, JEntriesOut), - array:set(RelSeq, entry_to_segment(RelSeq, Obj, []), - EToSegOut)} - end, - {JEntriesOut1, EToSegOut1, UnackedRemoved + UnackedRemovedDelta} - end, {JEntries, EToSeg, 0}, JEntries). - -%% Here, the result is a tuple with the first element containing the -%% item we are adding to or modifying in the (initially fresh) journal -%% array. If the item is 'undefined' we leave the journal array -%% alone. The other element of the tuple is the deltas for -%% UnackedRemoved. - -%% Both the same. Must be at least the publish -journal_minus_segment1({?PUB, _Del, no_ack} = Obj, Obj) -> - {undefined, 1}; -journal_minus_segment1({?PUB, _Del, ack} = Obj, Obj) -> - {undefined, 0}; - -%% Just publish in journal -journal_minus_segment1({?PUB, no_del, no_ack}, undefined) -> - {keep, 0}; - -%% Publish and deliver in journal -journal_minus_segment1({?PUB, del, no_ack}, undefined) -> - {keep, 0}; -journal_minus_segment1({?PUB = Pub, del, no_ack}, {Pub, no_del, no_ack}) -> - {{no_pub, del, no_ack}, 1}; - -%% Publish, deliver and ack in journal -journal_minus_segment1({?PUB, del, ack}, undefined) -> - {keep, 0}; -journal_minus_segment1({?PUB = Pub, del, ack}, {Pub, no_del, no_ack}) -> - {{no_pub, del, ack}, 1}; -journal_minus_segment1({?PUB = Pub, del, ack}, {Pub, del, no_ack}) -> - {{no_pub, no_del, ack}, 1}; - -%% Just deliver in journal -journal_minus_segment1({no_pub, del, no_ack}, {?PUB, no_del, no_ack}) -> - {keep, 0}; -journal_minus_segment1({no_pub, del, no_ack}, {?PUB, del, no_ack}) -> - {undefined, 0}; - -%% Just ack in journal -journal_minus_segment1({no_pub, no_del, ack}, {?PUB, del, no_ack}) -> - {keep, 0}; -journal_minus_segment1({no_pub, no_del, ack}, {?PUB, del, ack}) -> - {undefined, -1}; - -%% Deliver and ack in journal -journal_minus_segment1({no_pub, del, ack}, {?PUB, no_del, no_ack}) -> - {keep, 0}; -journal_minus_segment1({no_pub, del, ack}, {?PUB, del, no_ack}) -> - {{no_pub, no_del, ack}, 0}; -journal_minus_segment1({no_pub, del, ack}, {?PUB, del, ack}) -> - {undefined, -1}; - -%% Missing segment. If flush_journal/1 is interrupted after deleting -%% the segment but before truncating the journal we can get these -%% cases: a delivery and an acknowledgement in the journal, or just an -%% acknowledgement in the journal, but with no segment. In both cases -%% we have really forgotten the message; so ignore what's in the -%% journal. -journal_minus_segment1({no_pub, no_del, ack}, undefined) -> - {undefined, 0}; -journal_minus_segment1({no_pub, del, ack}, undefined) -> - {undefined, 0}. - -%%---------------------------------------------------------------------------- -%% upgrade -%%---------------------------------------------------------------------------- - --spec add_queue_ttl() -> 'ok'. - -add_queue_ttl() -> - foreach_queue_index({fun add_queue_ttl_journal/1, - fun add_queue_ttl_segment/1}). - -add_queue_ttl_journal(<<?DEL_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS, - Rest/binary>>) -> - {<<?DEL_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS>>, Rest}; -add_queue_ttl_journal(<<?ACK_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS, - Rest/binary>>) -> - {<<?ACK_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS>>, Rest}; -add_queue_ttl_journal(<<Prefix:?JPREFIX_BITS, SeqId:?SEQ_BITS, - MsgId:?MSG_ID_BYTES/binary, Rest/binary>>) -> - {[<<Prefix:?JPREFIX_BITS, SeqId:?SEQ_BITS>>, MsgId, - expiry_to_binary(undefined)], Rest}; -add_queue_ttl_journal(_) -> - stop. - -add_queue_ttl_segment(<<?PUB_PREFIX:?PUB_PREFIX_BITS, IsPersistentNum:1, - RelSeq:?REL_SEQ_BITS, MsgId:?MSG_ID_BYTES/binary, - Rest/binary>>) -> - {[<<?PUB_PREFIX:?PUB_PREFIX_BITS, IsPersistentNum:1, RelSeq:?REL_SEQ_BITS>>, - MsgId, expiry_to_binary(undefined)], Rest}; -add_queue_ttl_segment(<<?REL_SEQ_ONLY_PREFIX:?REL_SEQ_ONLY_PREFIX_BITS, - RelSeq:?REL_SEQ_BITS, Rest/binary>>) -> - {<<?REL_SEQ_ONLY_PREFIX:?REL_SEQ_ONLY_PREFIX_BITS, RelSeq:?REL_SEQ_BITS>>, - Rest}; -add_queue_ttl_segment(_) -> - stop. - -avoid_zeroes() -> - foreach_queue_index({none, fun avoid_zeroes_segment/1}). - -avoid_zeroes_segment(<<?PUB_PREFIX:?PUB_PREFIX_BITS, IsPersistentNum:1, - RelSeq:?REL_SEQ_BITS, MsgId:?MSG_ID_BITS, - Expiry:?EXPIRY_BITS, Rest/binary>>) -> - {<<?PUB_PREFIX:?PUB_PREFIX_BITS, IsPersistentNum:1, RelSeq:?REL_SEQ_BITS, - MsgId:?MSG_ID_BITS, Expiry:?EXPIRY_BITS>>, Rest}; -avoid_zeroes_segment(<<0:?REL_SEQ_ONLY_PREFIX_BITS, - RelSeq:?REL_SEQ_BITS, Rest/binary>>) -> - {<<?REL_SEQ_ONLY_PREFIX:?REL_SEQ_ONLY_PREFIX_BITS, RelSeq:?REL_SEQ_BITS>>, - Rest}; -avoid_zeroes_segment(_) -> - stop. - -%% At upgrade time we just define every message's size as 0 - that -%% will save us a load of faff with the message store, and means we -%% can actually use the clean recovery terms in VQ. It does mean we -%% don't count message bodies from before the migration, but we can -%% live with that. -store_msg_size() -> - foreach_queue_index({fun store_msg_size_journal/1, - fun store_msg_size_segment/1}). - -store_msg_size_journal(<<?DEL_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS, - Rest/binary>>) -> - {<<?DEL_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS>>, Rest}; -store_msg_size_journal(<<?ACK_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS, - Rest/binary>>) -> - {<<?ACK_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS>>, Rest}; -store_msg_size_journal(<<Prefix:?JPREFIX_BITS, SeqId:?SEQ_BITS, - MsgId:?MSG_ID_BITS, Expiry:?EXPIRY_BITS, - Rest/binary>>) -> - {<<Prefix:?JPREFIX_BITS, SeqId:?SEQ_BITS, MsgId:?MSG_ID_BITS, - Expiry:?EXPIRY_BITS, 0:?SIZE_BITS>>, Rest}; -store_msg_size_journal(_) -> - stop. - -store_msg_size_segment(<<?PUB_PREFIX:?PUB_PREFIX_BITS, IsPersistentNum:1, - RelSeq:?REL_SEQ_BITS, MsgId:?MSG_ID_BITS, - Expiry:?EXPIRY_BITS, Rest/binary>>) -> - {<<?PUB_PREFIX:?PUB_PREFIX_BITS, IsPersistentNum:1, RelSeq:?REL_SEQ_BITS, - MsgId:?MSG_ID_BITS, Expiry:?EXPIRY_BITS, 0:?SIZE_BITS>>, Rest}; -store_msg_size_segment(<<?REL_SEQ_ONLY_PREFIX:?REL_SEQ_ONLY_PREFIX_BITS, - RelSeq:?REL_SEQ_BITS, Rest/binary>>) -> - {<<?REL_SEQ_ONLY_PREFIX:?REL_SEQ_ONLY_PREFIX_BITS, RelSeq:?REL_SEQ_BITS>>, - Rest}; -store_msg_size_segment(_) -> - stop. - -store_msg() -> - foreach_queue_index({fun store_msg_journal/1, - fun store_msg_segment/1}). - -store_msg_journal(<<?DEL_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS, - Rest/binary>>) -> - {<<?DEL_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS>>, Rest}; -store_msg_journal(<<?ACK_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS, - Rest/binary>>) -> - {<<?ACK_JPREFIX:?JPREFIX_BITS, SeqId:?SEQ_BITS>>, Rest}; -store_msg_journal(<<Prefix:?JPREFIX_BITS, SeqId:?SEQ_BITS, - MsgId:?MSG_ID_BITS, Expiry:?EXPIRY_BITS, Size:?SIZE_BITS, - Rest/binary>>) -> - {<<Prefix:?JPREFIX_BITS, SeqId:?SEQ_BITS, MsgId:?MSG_ID_BITS, - Expiry:?EXPIRY_BITS, Size:?SIZE_BITS, - 0:?EMBEDDED_SIZE_BITS>>, Rest}; -store_msg_journal(_) -> - stop. - -store_msg_segment(<<?PUB_PREFIX:?PUB_PREFIX_BITS, IsPersistentNum:1, - RelSeq:?REL_SEQ_BITS, MsgId:?MSG_ID_BITS, - Expiry:?EXPIRY_BITS, Size:?SIZE_BITS, Rest/binary>>) -> - {<<?PUB_PREFIX:?PUB_PREFIX_BITS, IsPersistentNum:1, RelSeq:?REL_SEQ_BITS, - MsgId:?MSG_ID_BITS, Expiry:?EXPIRY_BITS, Size:?SIZE_BITS, - 0:?EMBEDDED_SIZE_BITS>>, Rest}; -store_msg_segment(<<?REL_SEQ_ONLY_PREFIX:?REL_SEQ_ONLY_PREFIX_BITS, - RelSeq:?REL_SEQ_BITS, Rest/binary>>) -> - {<<?REL_SEQ_ONLY_PREFIX:?REL_SEQ_ONLY_PREFIX_BITS, RelSeq:?REL_SEQ_BITS>>, - Rest}; -store_msg_segment(_) -> - stop. - - - -%%---------------------------------------------------------------------------- -%% Migration functions -%%---------------------------------------------------------------------------- - -foreach_queue_index(Funs) -> - QueueDirNames = all_queue_directory_names(), - {ok, Gatherer} = gatherer:start_link(), - [begin - ok = gatherer:fork(Gatherer), - ok = worker_pool:submit_async( - fun () -> - transform_queue(QueueDirName, Gatherer, Funs) - end) - end || QueueDirName <- QueueDirNames], - empty = gatherer:out(Gatherer), - ok = gatherer:stop(Gatherer). - -transform_queue(Dir, Gatherer, {JournalFun, SegmentFun}) -> - ok = transform_file(filename:join(Dir, ?JOURNAL_FILENAME), JournalFun), - [ok = transform_file(filename:join(Dir, Seg), SegmentFun) - || Seg <- rabbit_file:wildcard(".*\\" ++ ?SEGMENT_EXTENSION, Dir)], - ok = gatherer:finish(Gatherer). - -transform_file(_Path, none) -> - ok; -transform_file(Path, Fun) when is_function(Fun)-> - PathTmp = Path ++ ".upgrade", - case rabbit_file:file_size(Path) of - 0 -> ok; - Size -> {ok, PathTmpHdl} = - file_handle_cache:open_with_absolute_path( - PathTmp, ?WRITE_MODE, - [{write_buffer, infinity}]), - - {ok, PathHdl} = file_handle_cache:open_with_absolute_path( - Path, ?READ_MODE, [{read_buffer, Size}]), - {ok, Content} = file_handle_cache:read(PathHdl, Size), - ok = file_handle_cache:close(PathHdl), - - ok = drive_transform_fun(Fun, PathTmpHdl, Content), - - ok = file_handle_cache:close(PathTmpHdl), - ok = rabbit_file:rename(PathTmp, Path) - end. - -drive_transform_fun(Fun, Hdl, Contents) -> - case Fun(Contents) of - stop -> ok; - {Output, Contents1} -> ok = file_handle_cache:append(Hdl, Output), - drive_transform_fun(Fun, Hdl, Contents1) - end. - -move_to_per_vhost_stores(#resource{virtual_host = VHost} = QueueName) -> - OldQueueDir = filename:join([queues_base_dir(), "queues", - queue_name_to_dir_name_legacy(QueueName)]), - VHostDir = rabbit_vhost:msg_store_dir_path(VHost), - NewQueueDir = queue_dir(VHostDir, QueueName), - rabbit_log_upgrade:info("About to migrate queue directory '~s' to '~s'", - [OldQueueDir, NewQueueDir]), - case rabbit_file:is_dir(OldQueueDir) of - true -> - ok = rabbit_file:ensure_dir(NewQueueDir), - ok = rabbit_file:rename(OldQueueDir, NewQueueDir), - ok = ensure_queue_name_stub_file(NewQueueDir, QueueName); - false -> - Msg = "Queue index directory '~s' not found for ~s~n", - Args = [OldQueueDir, rabbit_misc:rs(QueueName)], - rabbit_log_upgrade:error(Msg, Args), - rabbit_log:error(Msg, Args) - end, - ok. - -ensure_queue_name_stub_file(Dir, #resource{virtual_host = VHost, name = QName}) -> - QueueNameFile = filename:join(Dir, ?QUEUE_NAME_STUB_FILE), - file:write_file(QueueNameFile, <<"VHOST: ", VHost/binary, "\n", - "QUEUE: ", QName/binary, "\n">>). - -read_global_recovery_terms(DurableQueueNames) -> - ok = rabbit_recovery_terms:open_global_table(), - - DurableTerms = - lists:foldl( - fun(QName, RecoveryTerms) -> - DirName = queue_name_to_dir_name_legacy(QName), - RecoveryInfo = case rabbit_recovery_terms:read_global(DirName) of - {error, _} -> non_clean_shutdown; - {ok, Terms} -> Terms - end, - [RecoveryInfo | RecoveryTerms] - end, [], DurableQueueNames), - - ok = rabbit_recovery_terms:close_global_table(), - %% The backing queue interface requires that the queue recovery terms - %% which come back from start/1 are in the same order as DurableQueueNames - OrderedTerms = lists:reverse(DurableTerms), - {OrderedTerms, {fun queue_index_walker/1, {start, DurableQueueNames}}}. - -cleanup_global_recovery_terms() -> - rabbit_file:recursive_delete([filename:join([queues_base_dir(), "queues"])]), - rabbit_recovery_terms:delete_global_table(), - ok. - - -update_recovery_term(#resource{virtual_host = VHost} = QueueName, Term) -> - Key = queue_name_to_dir_name(QueueName), - rabbit_recovery_terms:store(VHost, Key, Term). diff --git a/src/rabbit_queue_location_client_local.erl b/src/rabbit_queue_location_client_local.erl deleted file mode 100644 index 2df1608534..0000000000 --- a/src/rabbit_queue_location_client_local.erl +++ /dev/null @@ -1,39 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_queue_location_client_local). --behaviour(rabbit_queue_master_locator). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --export([description/0, queue_master_location/1]). - --rabbit_boot_step({?MODULE, - [{description, "locate queue master client local"}, - {mfa, {rabbit_registry, register, - [queue_master_locator, - <<"client-local">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - - -%%--------------------------------------------------------------------------- -%% Queue Master Location Callbacks -%%--------------------------------------------------------------------------- - -description() -> - [{description, <<"Locate queue master node as the client local node">>}]. - -queue_master_location(Q) when ?is_amqqueue(Q) -> - %% unlike with other locator strategies we do not check node maintenance - %% status for two reasons: - %% - %% * nodes in maintenance mode will drop their client connections - %% * with other strategies, if no nodes are available, the current node - %% is returned but this strategy already does just that - {ok, node()}. diff --git a/src/rabbit_queue_location_min_masters.erl b/src/rabbit_queue_location_min_masters.erl deleted file mode 100644 index 6535f082fe..0000000000 --- a/src/rabbit_queue_location_min_masters.erl +++ /dev/null @@ -1,70 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_queue_location_min_masters). --behaviour(rabbit_queue_master_locator). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --export([description/0, queue_master_location/1]). - --rabbit_boot_step({?MODULE, - [{description, "locate queue master min bound queues"}, - {mfa, {rabbit_registry, register, - [queue_master_locator, - <<"min-masters">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -%%--------------------------------------------------------------------------- -%% Queue Master Location Callbacks -%%--------------------------------------------------------------------------- - -description() -> - [{description, - <<"Locate queue master node from cluster node with least bound queues">>}]. - -queue_master_location(Q) when ?is_amqqueue(Q) -> - Cluster = rabbit_queue_master_location_misc:all_nodes(Q), - QueueNames = rabbit_amqqueue:list_names(), - MastersPerNode0 = lists:foldl( - fun(#resource{virtual_host = VHost, name = QueueName}, NodeMasters) -> - case rabbit_queue_master_location_misc:lookup_master(QueueName, VHost) of - {ok, Master} when is_atom(Master) -> - case maps:is_key(Master, NodeMasters) of - true -> maps:update_with(Master, - fun(N) -> N + 1 end, - NodeMasters); - false -> NodeMasters - end; - _ -> NodeMasters - end - end, - maps:from_list([{N, 0} || N <- Cluster]), - QueueNames), - - MastersPerNode = maps:filter(fun (Node, _N) -> - not rabbit_maintenance:is_being_drained_local_read(Node) - end, MastersPerNode0), - - case map_size(MastersPerNode) > 0 of - true -> - {MinNode, _NMasters} = maps:fold( - fun(Node, NMasters, init) -> - {Node, NMasters}; - (Node, NMasters, {MinNode, MinMasters}) -> - case NMasters < MinMasters of - true -> {Node, NMasters}; - false -> {MinNode, MinMasters} - end - end, - init, MastersPerNode), - {ok, MinNode}; - false -> - undefined - end. diff --git a/src/rabbit_queue_location_random.erl b/src/rabbit_queue_location_random.erl deleted file mode 100644 index 7232fc6703..0000000000 --- a/src/rabbit_queue_location_random.erl +++ /dev/null @@ -1,42 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_queue_location_random). --behaviour(rabbit_queue_master_locator). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --export([description/0, queue_master_location/1]). - --rabbit_boot_step({?MODULE, - [{description, "locate queue master random"}, - {mfa, {rabbit_registry, register, - [queue_master_locator, - <<"random">>, ?MODULE]}}, - {requires, rabbit_registry}, - {enables, kernel_ready}]}). - -%%--------------------------------------------------------------------------- -%% Queue Master Location Callbacks -%%--------------------------------------------------------------------------- - -description() -> - [{description, - <<"Locate queue master node from cluster in a random manner">>}]. - -queue_master_location(Q) when ?is_amqqueue(Q) -> - Cluster0 = rabbit_queue_master_location_misc:all_nodes(Q), - Cluster = rabbit_maintenance:filter_out_drained_nodes_local_read(Cluster0), - case Cluster of - [] -> - undefined; - Candidates when is_list(Candidates) -> - RandomPos = erlang:phash2(erlang:monotonic_time(), length(Candidates)), - MasterNode = lists:nth(RandomPos + 1, Candidates), - {ok, MasterNode} - end. diff --git a/src/rabbit_queue_location_validator.erl b/src/rabbit_queue_location_validator.erl deleted file mode 100644 index bf41be622c..0000000000 --- a/src/rabbit_queue_location_validator.erl +++ /dev/null @@ -1,67 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_queue_location_validator). --behaviour(rabbit_policy_validator). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --export([validate_policy/1, validate_strategy/1]). - --rabbit_boot_step({?MODULE, - [{description, "Queue location policy validation"}, - {mfa, {rabbit_registry, register, - [policy_validator, - <<"queue-master-locator">>, - ?MODULE]}}, - {requires, rabbit_registry}, - {enables, recovery}]}). - -validate_policy(KeyList) -> - case proplists:lookup(<<"queue-master-locator">> , KeyList) of - {_, Strategy} -> case validate_strategy(Strategy) of - {error, _, _} = Er -> Er; - _ -> ok - end; - _ -> {error, "queue-master-locator undefined"} - end. - -validate_strategy(Strategy) -> - case module(Strategy) of - R = {ok, _M} -> R; - _ -> - {error, "~p invalid queue-master-locator value", [Strategy]} - end. - -policy(Policy, Q) -> - case rabbit_policy:get(Policy, Q) of - undefined -> none; - P -> P - end. - -module(Q) when ?is_amqqueue(Q) -> - case policy(<<"queue-master-locator">>, Q) of - undefined -> no_location_strategy; - Mode -> module(Mode) - end; -module(Strategy) when is_binary(Strategy) -> - case rabbit_registry:binary_to_type(Strategy) of - {error, not_found} -> no_location_strategy; - T -> - case rabbit_registry:lookup_module(queue_master_locator, T) of - {ok, Module} -> - case code:which(Module) of - non_existing -> no_location_strategy; - _ -> {ok, Module} - end; - _ -> - no_location_strategy - end - end; -module(Strategy) -> - module(rabbit_data_coercion:to_binary(Strategy)). diff --git a/src/rabbit_queue_master_location_misc.erl b/src/rabbit_queue_master_location_misc.erl deleted file mode 100644 index 37698e184f..0000000000 --- a/src/rabbit_queue_master_location_misc.erl +++ /dev/null @@ -1,108 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_queue_master_location_misc). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("amqqueue.hrl"). - --export([lookup_master/2, - lookup_queue/2, - get_location/1, - get_location_mod_by_config/1, - get_location_mod_by_args/1, - get_location_mod_by_policy/1, - all_nodes/1]). - --spec lookup_master(binary(), binary()) -> {ok, node()} | {error, not_found}. -lookup_master(QueueNameBin, VHostPath) when is_binary(QueueNameBin), - is_binary(VHostPath) -> - QueueR = rabbit_misc:r(VHostPath, queue, QueueNameBin), - case rabbit_amqqueue:lookup(QueueR) of - {ok, Queue} when ?amqqueue_has_valid_pid(Queue) -> - Pid = amqqueue:get_pid(Queue), - {ok, node(Pid)}; - Error -> Error - end. - -lookup_queue(QueueNameBin, VHostPath) when is_binary(QueueNameBin), - is_binary(VHostPath) -> - QueueR = rabbit_misc:r(VHostPath, queue, QueueNameBin), - case rabbit_amqqueue:lookup(QueueR) of - Reply = {ok, Queue} when ?is_amqqueue(Queue) -> - Reply; - Error -> - Error - end. - -get_location(Queue) when ?is_amqqueue(Queue) -> - Reply1 = case get_location_mod_by_args(Queue) of - _Err1 = {error, _} -> - case get_location_mod_by_policy(Queue) of - _Err2 = {error, _} -> - case get_location_mod_by_config(Queue) of - Err3 = {error, _} -> Err3; - Reply0 = {ok, _Module} -> Reply0 - end; - Reply0 = {ok, _Module} -> Reply0 - end; - Reply0 = {ok, _Module} -> Reply0 - end, - - case Reply1 of - {ok, CB} -> CB:queue_master_location(Queue); - Error -> Error - end. - -get_location_mod_by_args(Queue) when ?is_amqqueue(Queue) -> - Args = amqqueue:get_arguments(Queue), - case rabbit_misc:table_lookup(Args, <<"x-queue-master-locator">>) of - {_Type, Strategy} -> - case rabbit_queue_location_validator:validate_strategy(Strategy) of - Reply = {ok, _CB} -> Reply; - Error -> Error - end; - _ -> {error, "x-queue-master-locator undefined"} - end. - -get_location_mod_by_policy(Queue) when ?is_amqqueue(Queue) -> - case rabbit_policy:get(<<"queue-master-locator">> , Queue) of - undefined -> {error, "queue-master-locator policy undefined"}; - Strategy -> - case rabbit_queue_location_validator:validate_strategy(Strategy) of - Reply = {ok, _CB} -> Reply; - Error -> Error - end - end. - -get_location_mod_by_config(Queue) when ?is_amqqueue(Queue) -> - case application:get_env(rabbit, queue_master_locator) of - {ok, Strategy} -> - case rabbit_queue_location_validator:validate_strategy(Strategy) of - Reply = {ok, _CB} -> Reply; - Error -> Error - end; - _ -> {error, "queue_master_locator undefined"} - end. - -all_nodes(Queue) when ?is_amqqueue(Queue) -> - handle_is_mirrored_ha_nodes(rabbit_mirror_queue_misc:is_mirrored_ha_nodes(Queue), Queue). - -handle_is_mirrored_ha_nodes(false, _Queue) -> - % Note: ha-mode is NOT 'nodes' - it is either exactly or all, which means - % that any node in the cluster is eligible to be the new queue master node - rabbit_nodes:all_running(); -handle_is_mirrored_ha_nodes(true, Queue) -> - % Note: ha-mode is 'nodes', which explicitly specifies allowed nodes. - % We must use suggested_queue_nodes to get that list of nodes as the - % starting point for finding the queue master location - handle_suggested_queue_nodes(rabbit_mirror_queue_misc:suggested_queue_nodes(Queue)). - -handle_suggested_queue_nodes({_MNode, []}) -> - rabbit_nodes:all_running(); -handle_suggested_queue_nodes({MNode, SNodes}) -> - [MNode | SNodes]. diff --git a/src/rabbit_queue_master_locator.erl b/src/rabbit_queue_master_locator.erl deleted file mode 100644 index ff2e30f587..0000000000 --- a/src/rabbit_queue_master_locator.erl +++ /dev/null @@ -1,19 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_queue_master_locator). - --behaviour(rabbit_registry_class). - --export([added_to_rabbit_registry/2, removed_from_rabbit_registry/1]). - --callback description() -> [proplists:property()]. --callback queue_master_location(amqqueue:amqqueue()) -> - {'ok', node()} | {'error', term()}. - -added_to_rabbit_registry(_Type, _ModuleName) -> ok. -removed_from_rabbit_registry(_Type) -> ok. diff --git a/src/rabbit_queue_type.erl b/src/rabbit_queue_type.erl deleted file mode 100644 index 4e59b6a7c0..0000000000 --- a/src/rabbit_queue_type.erl +++ /dev/null @@ -1,581 +0,0 @@ --module(rabbit_queue_type). --include("amqqueue.hrl"). --include_lib("rabbit_common/include/resource.hrl"). - --export([ - init/0, - close/1, - discover/1, - default/0, - is_enabled/1, - declare/2, - delete/4, - is_recoverable/1, - recover/2, - purge/1, - policy_changed/1, - stat/1, - remove/2, - info/2, - state_info/1, - info_down/2, - info_down/3, - %% stateful client API - new/2, - consume/3, - cancel/5, - handle_down/3, - handle_event/3, - module/2, - deliver/3, - settle/5, - credit/5, - dequeue/5, - fold_state/3, - is_policy_applicable/2, - is_server_named_allowed/1 - ]). - -%% gah what is a good identity of a classic queue including all replicas --type queue_name() :: rabbit_types:r(queue). --type queue_ref() :: queue_name() | atom(). --type queue_state() :: term(). --type msg_tag() :: term(). - --define(STATE, ?MODULE). - -%% Recoverable slaves shouldn't really be a generic one, but let's keep it here until -%% mirrored queues are deprecated. --define(DOWN_KEYS, [name, durable, auto_delete, arguments, pid, recoverable_slaves, type, state]). - --define(QREF(QueueReference), - (is_tuple(QueueReference) andalso element(1, QueueReference) == resource) - orelse is_atom(QueueReference)). -%% anything that the host process needs to do on behalf of the queue type -%% session, like knowing when to notify on monitor down --type action() :: - {monitor, Pid :: pid(), queue_ref()} | - %% indicate to the queue type module that a message has been delivered - %% fully to the queue - {settled, Success :: boolean(), [msg_tag()]} | - {deliver, rabbit_types:ctag(), boolean(), [rabbit_amqqueue:qmsg()]}. - --type actions() :: [action()]. - --type event() :: - {down, pid(), Info :: term()} | - term(). - --record(ctx, {module :: module(), - name :: queue_name(), - %% "publisher confirm queue accounting" - %% queue type implementation should emit a: - %% {settle, Success :: boolean(), msg_tag()} - %% to either settle or reject the delivery of a - %% message to the queue instance - %% The queue type module will then emit a {confirm | reject, [msg_tag()} - %% action to the channel or channel like process when a msg_tag - %% has reached its conclusion - state :: queue_state()}). - - --record(?STATE, {ctxs = #{} :: #{queue_ref() => #ctx{} | queue_ref()}, - monitor_registry = #{} :: #{pid() => queue_ref()} - }). - --opaque state() :: #?STATE{}. - --type consume_spec() :: #{no_ack := boolean(), - channel_pid := pid(), - limiter_pid => pid(), - limiter_active => boolean(), - prefetch_count => non_neg_integer(), - consumer_tag := rabbit_types:ctag(), - exclusive_consume => boolean(), - args => rabbit_framing:amqp_table(), - ok_msg := term(), - acting_user := rabbit_types:username()}. - - - -% copied from rabbit_amqqueue --type absent_reason() :: 'nodedown' | 'crashed' | stopped | timeout. - --type settle_op() :: 'complete' | 'requeue' | 'discard'. - --export_type([state/0, - consume_spec/0, - action/0, - actions/0, - settle_op/0]). - -%% is the queue type feature enabled --callback is_enabled() -> boolean(). - --callback declare(amqqueue:amqqueue(), node()) -> - {'new' | 'existing' | 'owner_died', amqqueue:amqqueue()} | - {'absent', amqqueue:amqqueue(), absent_reason()} | - {'protocol_error', Type :: atom(), Reason :: string(), Args :: term()}. - --callback delete(amqqueue:amqqueue(), - boolean(), - boolean(), - rabbit_types:username()) -> - rabbit_types:ok(non_neg_integer()) | - rabbit_types:error(in_use | not_empty) | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. - --callback recover(rabbit_types:vhost(), [amqqueue:amqqueue()]) -> - {Recovered :: [amqqueue:amqqueue()], - Failed :: [amqqueue:amqqueue()]}. - -%% checks if the queue should be recovered --callback is_recoverable(amqqueue:amqqueue()) -> - boolean(). - --callback purge(amqqueue:amqqueue()) -> - {ok, non_neg_integer()} | {error, term()}. - --callback policy_changed(amqqueue:amqqueue()) -> ok. - -%% stateful -%% intitialise and return a queue type specific session context --callback init(amqqueue:amqqueue()) -> queue_state(). - --callback close(queue_state()) -> ok. -%% update the queue type state from amqqrecord --callback update(amqqueue:amqqueue(), queue_state()) -> queue_state(). - --callback consume(amqqueue:amqqueue(), - consume_spec(), - queue_state()) -> - {ok, queue_state(), actions()} | {error, term()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. - --callback cancel(amqqueue:amqqueue(), - rabbit_types:ctag(), - term(), - rabbit_types:username(), - queue_state()) -> - {ok, queue_state()} | {error, term()}. - -%% any async events returned from the queue system should be processed through -%% this --callback handle_event(Event :: event(), - queue_state()) -> - {ok, queue_state(), actions()} | {error, term()} | eol | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. - --callback deliver([{amqqueue:amqqueue(), queue_state()}], - Delivery :: term()) -> - {[{amqqueue:amqqueue(), queue_state()}], actions()}. - --callback settle(settle_op(), rabbit_types:ctag(), [non_neg_integer()], queue_state()) -> - {queue_state(), actions()} | - {'protocol_error', Type :: atom(), Reason :: string(), Args :: term()}. - --callback credit(rabbit_types:ctag(), - non_neg_integer(), Drain :: boolean(), queue_state()) -> - {queue_state(), actions()}. - --callback dequeue(NoAck :: boolean(), LimiterPid :: pid(), - rabbit_types:ctag(), queue_state()) -> - {ok, Count :: non_neg_integer(), rabbit_amqqueue:qmsg(), queue_state()} | - {empty, queue_state()} | - {error, term()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. - -%% return a map of state summary information --callback state_info(queue_state()) -> - #{atom() := term()}. - -%% general queue info --callback info(amqqueue:amqqueue(), all_keys | rabbit_types:info_keys()) -> - rabbit_types:infos(). - --callback stat(amqqueue:amqqueue()) -> - {'ok', non_neg_integer(), non_neg_integer()}. - --callback capabilities() -> - #{atom() := term()}. - -%% TODO: this should be controlled by a registry that is populated on boot -discover(<<"quorum">>) -> - rabbit_quorum_queue; -discover(<<"classic">>) -> - rabbit_classic_queue; -discover(<<"stream">>) -> - rabbit_stream_queue. - -default() -> - rabbit_classic_queue. - --spec is_enabled(module()) -> boolean(). -is_enabled(Type) -> - Type:is_enabled(). - --spec declare(amqqueue:amqqueue(), node()) -> - {'new' | 'existing' | 'owner_died', amqqueue:amqqueue()} | - {'absent', amqqueue:amqqueue(), absent_reason()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -declare(Q, Node) -> - Mod = amqqueue:get_type(Q), - Mod:declare(Q, Node). - --spec delete(amqqueue:amqqueue(), boolean(), - boolean(), rabbit_types:username()) -> - rabbit_types:ok(non_neg_integer()) | - rabbit_types:error(in_use | not_empty) | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -delete(Q, IfUnused, IfEmpty, ActingUser) -> - Mod = amqqueue:get_type(Q), - Mod:delete(Q, IfUnused, IfEmpty, ActingUser). - --spec purge(amqqueue:amqqueue()) -> - {'ok', non_neg_integer()} | {error, term()}. -purge(Q) -> - Mod = amqqueue:get_type(Q), - Mod:purge(Q). - --spec policy_changed(amqqueue:amqqueue()) -> 'ok'. -policy_changed(Q) -> - Mod = amqqueue:get_type(Q), - Mod:policy_changed(Q). - --spec stat(amqqueue:amqqueue()) -> - {'ok', non_neg_integer(), non_neg_integer()}. -stat(Q) -> - Mod = amqqueue:get_type(Q), - Mod:stat(Q). - --spec remove(queue_ref(), state()) -> state(). -remove(QRef, #?STATE{ctxs = Ctxs0} = State) -> - case maps:take(QRef, Ctxs0) of - error -> - State; - {_, Ctxs} -> - State#?STATE{ctxs = Ctxs} - end. - --spec info(amqqueue:amqqueue(), all_keys | rabbit_types:info_keys()) -> - rabbit_types:infos(). -info(Q, Items) when ?amqqueue_state_is(Q, crashed) -> - info_down(Q, Items, crashed); -info(Q, Items) when ?amqqueue_state_is(Q, stopped) -> - info_down(Q, Items, stopped); -info(Q, Items) -> - Mod = amqqueue:get_type(Q), - Mod:info(Q, Items). - -fold_state(Fun, Acc, #?STATE{ctxs = Ctxs}) -> - maps:fold(Fun, Acc, Ctxs). - -state_info(#ctx{state = S, - module = Mod}) -> - Mod:state_info(S); -state_info(_) -> - #{}. - -down_keys() -> ?DOWN_KEYS. - -info_down(Q, DownReason) -> - info_down(Q, down_keys(), DownReason). - -info_down(Q, all_keys, DownReason) -> - info_down(Q, down_keys(), DownReason); -info_down(Q, Items, DownReason) -> - [{Item, i_down(Item, Q, DownReason)} || Item <- Items]. - -i_down(name, Q, _) -> amqqueue:get_name(Q); -i_down(durable, Q, _) -> amqqueue:is_durable(Q); -i_down(auto_delete, Q, _) -> amqqueue:is_auto_delete(Q); -i_down(arguments, Q, _) -> amqqueue:get_arguments(Q); -i_down(pid, Q, _) -> amqqueue:get_pid(Q); -i_down(recoverable_slaves, Q, _) -> amqqueue:get_recoverable_slaves(Q); -i_down(type, Q, _) -> amqqueue:get_type(Q); -i_down(state, _Q, DownReason) -> DownReason; -i_down(_K, _Q, _DownReason) -> ''. - -is_policy_applicable(Q, Policy) -> - Mod = amqqueue:get_type(Q), - Capabilities = Mod:capabilities(), - Applicable = maps:get(policies, Capabilities, []), - lists:all(fun({P, _}) -> - lists:member(P, Applicable) - end, Policy). - -is_server_named_allowed(Type) -> - Capabilities = Type:capabilities(), - maps:get(server_named, Capabilities, false). - --spec init() -> state(). -init() -> - #?STATE{}. - --spec close(state()) -> ok. -close(#?STATE{ctxs = Contexts}) -> - _ = maps:map( - fun (_, #ctx{module = Mod, - state = S}) -> - ok = Mod:close(S) - end, Contexts), - ok. - --spec new(amqqueue:amqqueue(), state()) -> state(). -new(Q, State) when ?is_amqqueue(Q) -> - Ctx = get_ctx(Q, State), - set_ctx(Q, Ctx, State). - --spec consume(amqqueue:amqqueue(), consume_spec(), state()) -> - {ok, state(), actions()} | {error, term()}. -consume(Q, Spec, State) -> - #ctx{state = CtxState0} = Ctx = get_ctx(Q, State), - Mod = amqqueue:get_type(Q), - case Mod:consume(Q, Spec, CtxState0) of - {ok, CtxState, Actions} -> - return_ok(set_ctx(Q, Ctx#ctx{state = CtxState}, State), Actions); - Err -> - Err - end. - -%% TODO switch to cancel spec api --spec cancel(amqqueue:amqqueue(), - rabbit_types:ctag(), - term(), - rabbit_types:username(), - state()) -> - {ok, state()} | {error, term()}. -cancel(Q, Tag, OkMsg, ActiveUser, Ctxs) -> - #ctx{state = State0} = Ctx = get_ctx(Q, Ctxs), - Mod = amqqueue:get_type(Q), - case Mod:cancel(Q, Tag, OkMsg, ActiveUser, State0) of - {ok, State} -> - {ok, set_ctx(Q, Ctx#ctx{state = State}, Ctxs)}; - Err -> - Err - end. - --spec is_recoverable(amqqueue:amqqueue()) -> - boolean(). -is_recoverable(Q) -> - Mod = amqqueue:get_type(Q), - Mod:is_recoverable(Q). - --spec recover(rabbit_types:vhost(), [amqqueue:amqqueue()]) -> - {Recovered :: [amqqueue:amqqueue()], - Failed :: [amqqueue:amqqueue()]}. -recover(VHost, Qs) -> - ByType = lists:foldl( - fun (Q, Acc) -> - T = amqqueue:get_type(Q), - maps:update_with(T, fun (X) -> - [Q | X] - end, Acc) - %% TODO resolve all registered queue types from registry - end, #{rabbit_classic_queue => [], - rabbit_quorum_queue => [], - rabbit_stream_queue => []}, Qs), - maps:fold(fun (Mod, Queues, {R0, F0}) -> - {R, F} = Mod:recover(VHost, Queues), - {R0 ++ R, F0 ++ F} - end, {[], []}, ByType). - --spec handle_down(pid(), term(), state()) -> - {ok, state(), actions()} | {eol, queue_ref()} | {error, term()}. -handle_down(Pid, Info, #?STATE{monitor_registry = Reg0} = State0) -> - %% lookup queue ref in monitor registry - case maps:take(Pid, Reg0) of - {QRef, Reg} -> - case handle_event(QRef, {down, Pid, Info}, State0) of - {ok, State, Actions} -> - {ok, State#?STATE{monitor_registry = Reg}, Actions}; - eol -> - {eol, QRef}; - Err -> - Err - end; - error -> - {ok, State0, []} - end. - -%% messages sent from queues --spec handle_event(queue_ref(), term(), state()) -> - {ok, state(), actions()} | eol | {error, term()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -handle_event(QRef, Evt, Ctxs) -> - %% events can arrive after a queue state has been cleared up - %% so need to be defensive here - case get_ctx(QRef, Ctxs, undefined) of - #ctx{module = Mod, - state = State0} = Ctx -> - case Mod:handle_event(Evt, State0) of - {ok, State, Actions} -> - return_ok(set_ctx(QRef, Ctx#ctx{state = State}, Ctxs), Actions); - Err -> - Err - end; - undefined -> - {ok, Ctxs, []} - end. - --spec module(queue_ref(), state()) -> - {ok, module()} | {error, not_found}. -module(QRef, Ctxs) -> - %% events can arrive after a queue state has been cleared up - %% so need to be defensive here - case get_ctx(QRef, Ctxs, undefined) of - #ctx{module = Mod} -> - {ok, Mod}; - undefined -> - {error, not_found} - end. - --spec deliver([amqqueue:amqqueue()], Delivery :: term(), - stateless | state()) -> - {ok, state(), actions()}. -deliver(Qs, Delivery, stateless) -> - _ = lists:map(fun(Q) -> - Mod = amqqueue:get_type(Q), - _ = Mod:deliver([{Q, stateless}], Delivery) - end, Qs), - {ok, stateless, []}; -deliver(Qs, Delivery, #?STATE{} = State0) -> - %% sort by queue type - then dispatch each group - ByType = lists:foldl( - fun (Q, Acc) -> - T = amqqueue:get_type(Q), - Ctx = get_ctx(Q, State0), - maps:update_with( - T, fun (A) -> - [{Q, Ctx#ctx.state} | A] - end, [{Q, Ctx#ctx.state}], Acc) - end, #{}, Qs), - %%% dispatch each group to queue type interface? - {Xs, Actions} = maps:fold(fun(Mod, QSs, {X0, A0}) -> - {X, A} = Mod:deliver(QSs, Delivery), - {X0 ++ X, A0 ++ A} - end, {[], []}, ByType), - State = lists:foldl( - fun({Q, S}, Acc) -> - Ctx = get_ctx(Q, Acc), - set_ctx(qref(Q), Ctx#ctx{state = S}, Acc) - end, State0, Xs), - return_ok(State, Actions). - - --spec settle(queue_ref(), settle_op(), rabbit_types:ctag(), - [non_neg_integer()], state()) -> - {ok, state(), actions()} | - {'protocol_error', Type :: atom(), Reason :: string(), Args :: term()}. -settle(QRef, Op, CTag, MsgIds, Ctxs) - when ?QREF(QRef) -> - case get_ctx(QRef, Ctxs, undefined) of - undefined -> - %% if we receive a settlement and there is no queue state it means - %% the queue was deleted with active consumers - {ok, Ctxs, []}; - #ctx{state = State0, - module = Mod} = Ctx -> - case Mod:settle(Op, CTag, MsgIds, State0) of - {State, Actions} -> - {ok, set_ctx(QRef, Ctx#ctx{state = State}, Ctxs), Actions}; - Err -> - Err - end - end. - --spec credit(amqqueue:amqqueue() | queue_ref(), - rabbit_types:ctag(), non_neg_integer(), - boolean(), state()) -> {ok, state(), actions()}. -credit(Q, CTag, Credit, Drain, Ctxs) -> - #ctx{state = State0, - module = Mod} = Ctx = get_ctx(Q, Ctxs), - {State, Actions} = Mod:credit(CTag, Credit, Drain, State0), - {ok, set_ctx(Q, Ctx#ctx{state = State}, Ctxs), Actions}. - --spec dequeue(amqqueue:amqqueue(), boolean(), - pid(), rabbit_types:ctag(), state()) -> - {ok, non_neg_integer(), term(), state()} | - {empty, state()}. -dequeue(Q, NoAck, LimiterPid, CTag, Ctxs) -> - #ctx{state = State0} = Ctx = get_ctx(Q, Ctxs), - Mod = amqqueue:get_type(Q), - case Mod:dequeue(NoAck, LimiterPid, CTag, State0) of - {ok, Num, Msg, State} -> - {ok, Num, Msg, set_ctx(Q, Ctx#ctx{state = State}, Ctxs)}; - {empty, State} -> - {empty, set_ctx(Q, Ctx#ctx{state = State}, Ctxs)}; - {error, _} = Err -> - Err; - {protocol_error, _, _, _} = Err -> - Err - end. - -get_ctx(Q, #?STATE{ctxs = Contexts}) when ?is_amqqueue(Q) -> - Ref = qref(Q), - case Contexts of - #{Ref := #ctx{module = Mod, - state = State} = Ctx} -> - Ctx#ctx{state = Mod:update(Q, State)}; - _ -> - %% not found - initialize - Mod = amqqueue:get_type(Q), - Name = amqqueue:get_name(Q), - #ctx{module = Mod, - name = Name, - state = Mod:init(Q)} - end; -get_ctx(QRef, Contexts) when ?QREF(QRef) -> - case get_ctx(QRef, Contexts, undefined) of - undefined -> - exit({queue_context_not_found, QRef}); - Ctx -> - Ctx - end. - -get_ctx(QRef, #?STATE{ctxs = Contexts}, Default) -> - Ref = qref(QRef), - %% if we use a QRef it should always be initialised - case maps:get(Ref, Contexts, undefined) of - #ctx{} = Ctx -> - Ctx; - undefined -> - Default - end. - -set_ctx(Q, Ctx, #?STATE{ctxs = Contexts} = State) - when ?is_amqqueue(Q) -> - Ref = qref(Q), - State#?STATE{ctxs = maps:put(Ref, Ctx, Contexts)}; -set_ctx(QRef, Ctx, #?STATE{ctxs = Contexts} = State) -> - Ref = qref(QRef), - State#?STATE{ctxs = maps:put(Ref, Ctx, Contexts)}. - -qref(#resource{kind = queue} = QName) -> - QName; -qref(Q) when ?is_amqqueue(Q) -> - amqqueue:get_name(Q). - -return_ok(State0, []) -> - {ok, State0, []}; -return_ok(State0, Actions0) -> - {State, Actions} = - lists:foldl( - fun({monitor, Pid, QRef}, - {#?STATE{monitor_registry = M0} = S0, A0}) -> - case M0 of - #{Pid := QRef} -> - %% already monitored by the qref - {S0, A0}; - #{Pid := _} -> - %% TODO: allow multiple Qrefs to monitor the same pid - exit(return_ok_duplicate_monitored_pid); - _ -> - _ = erlang:monitor(process, Pid), - M = M0#{Pid => QRef}, - {S0#?STATE{monitor_registry = M}, A0} - end; - (Act, {S, A0}) -> - {S, [Act | A0]} - end, {State0, []}, Actions0), - {ok, State, lists:reverse(Actions)}. diff --git a/src/rabbit_queue_type_util.erl b/src/rabbit_queue_type_util.erl deleted file mode 100644 index e417cb13c4..0000000000 --- a/src/rabbit_queue_type_util.erl +++ /dev/null @@ -1,74 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License -%% at https://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and -%% limitations under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% The Initial Developer of the Original Code is GoPivotal, Inc. -%% Copyright (c) 2018-2020 Pivotal Software, Inc. All rights reserved. -%% - --module(rabbit_queue_type_util). - --export([args_policy_lookup/3, - qname_to_internal_name/1, - check_auto_delete/1, - check_exclusive/1, - check_non_durable/1, - run_checks/2]). - --include("rabbit.hrl"). --include("amqqueue.hrl"). - -args_policy_lookup(Name, Resolve, Q) when ?is_amqqueue(Q) -> - Args = amqqueue:get_arguments(Q), - AName = <<"x-", Name/binary>>, - case {rabbit_policy:get(Name, Q), rabbit_misc:table_lookup(Args, AName)} of - {undefined, undefined} -> undefined; - {undefined, {_Type, Val}} -> Val; - {Val, undefined} -> Val; - {PolVal, {_Type, ArgVal}} -> Resolve(PolVal, ArgVal) - end. - -%% TODO escape hack -qname_to_internal_name(#resource{virtual_host = <<"/">>, name = Name}) -> - erlang:binary_to_atom(<<"%2F_", Name/binary>>, utf8); -qname_to_internal_name(#resource{virtual_host = VHost, name = Name}) -> - erlang:binary_to_atom(<<VHost/binary, "_", Name/binary>>, utf8). - -check_auto_delete(Q) when ?amqqueue_is_auto_delete(Q) -> - Name = amqqueue:get_name(Q), - {protocol_error, precondition_failed, "invalid property 'auto-delete' for ~s", - [rabbit_misc:rs(Name)]}; -check_auto_delete(_) -> - ok. - -check_exclusive(Q) when ?amqqueue_exclusive_owner_is(Q, none) -> - ok; -check_exclusive(Q) when ?is_amqqueue(Q) -> - Name = amqqueue:get_name(Q), - {protocol_error, precondition_failed, "invalid property 'exclusive-owner' for ~s", - [rabbit_misc:rs(Name)]}. - -check_non_durable(Q) when ?amqqueue_is_durable(Q) -> - ok; -check_non_durable(Q) when not ?amqqueue_is_durable(Q) -> - Name = amqqueue:get_name(Q), - {protocol_error, precondition_failed, "invalid property 'non-durable' for ~s", - [rabbit_misc:rs(Name)]}. - -run_checks([], _) -> - ok; -run_checks([C | Checks], Q) -> - case C(Q) of - ok -> - run_checks(Checks, Q); - Err -> - Err - end. diff --git a/src/rabbit_quorum_memory_manager.erl b/src/rabbit_quorum_memory_manager.erl deleted file mode 100644 index 94c2ef6b4b..0000000000 --- a/src/rabbit_quorum_memory_manager.erl +++ /dev/null @@ -1,67 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% --module(rabbit_quorum_memory_manager). - --include_lib("rabbit_common/include/rabbit.hrl"). - --export([init/1, handle_call/2, handle_event/2, handle_info/2, - terminate/2, code_change/3]). --export([register/0, unregister/0]). - --record(state, {last_roll_over, - interval}). - --rabbit_boot_step({rabbit_quorum_memory_manager, - [{description, "quorum memory manager"}, - {mfa, {?MODULE, register, []}}, - {cleanup, {?MODULE, unregister, []}}, - {requires, rabbit_event}, - {enables, recovery}]}). - -register() -> - gen_event:add_handler(rabbit_alarm, ?MODULE, []). - -unregister() -> - gen_event:delete_handler(rabbit_alarm, ?MODULE, []). - -init([]) -> - {ok, #state{interval = interval()}}. - -handle_call( _, State) -> - {ok, ok, State}. - -handle_event({set_alarm, {{resource_limit, memory, Node}, []}}, - #state{last_roll_over = undefined} = State) when Node == node() -> - {ok, force_roll_over(State)}; -handle_event({set_alarm, {{resource_limit, memory, Node}, []}}, - #state{last_roll_over = Last, interval = Interval } = State) - when Node == node() -> - Now = erlang:system_time(millisecond), - case Now > (Last + Interval) of - true -> - {ok, force_roll_over(State)}; - false -> - {ok, State} - end; -handle_event(_, State) -> - {ok, State}. - -handle_info(_, State) -> - {ok, State}. - -terminate(_, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -force_roll_over(State) -> - ra_log_wal:force_roll_over(ra_log_wal), - State#state{last_roll_over = erlang:system_time(millisecond)}. - -interval() -> - application:get_env(rabbit, min_wal_roll_over_interval, 20000). diff --git a/src/rabbit_quorum_queue.erl b/src/rabbit_quorum_queue.erl deleted file mode 100644 index a51fc3f43e..0000000000 --- a/src/rabbit_quorum_queue.erl +++ /dev/null @@ -1,1496 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_quorum_queue). - --behaviour(rabbit_queue_type). - --export([init/1, - close/1, - update/2, - handle_event/2]). --export([is_recoverable/1, recover/2, stop/1, delete/4, delete_immediately/2]). --export([state_info/1, info/2, stat/1, infos/1]). --export([settle/4, dequeue/4, consume/3, cancel/5]). --export([credit/4]). --export([purge/1]). --export([stateless_deliver/2, deliver/3, deliver/2]). --export([dead_letter_publish/4]). --export([queue_name/1]). --export([cluster_state/1, status/2]). --export([update_consumer_handler/8, update_consumer/9]). --export([cancel_consumer_handler/2, cancel_consumer/3]). --export([become_leader/2, handle_tick/3, spawn_deleter/1]). --export([rpc_delete_metrics/1]). --export([format/1]). --export([open_files/1]). --export([peek/2, peek/3]). --export([add_member/4]). --export([delete_member/3]). --export([requeue/3]). --export([policy_changed/1]). --export([format_ra_event/3]). --export([cleanup_data_dir/0]). --export([shrink_all/1, - grow/4]). --export([transfer_leadership/2, get_replicas/1, queue_length/1]). --export([file_handle_leader_reservation/1, file_handle_other_reservation/0]). --export([file_handle_release_reservation/0]). --export([list_with_minimum_quorum/0, list_with_minimum_quorum_for_cli/0, - filter_quorum_critical/1, filter_quorum_critical/2, - all_replica_states/0]). --export([capabilities/0]). --export([repair_amqqueue_nodes/1, - repair_amqqueue_nodes/2 - ]). --export([reclaim_memory/2]). - --export([is_enabled/0, - declare/2]). - --import(rabbit_queue_type_util, [args_policy_lookup/3, - qname_to_internal_name/1]). - --include_lib("stdlib/include/qlc.hrl"). --include("rabbit.hrl"). --include("amqqueue.hrl"). - --type msg_id() :: non_neg_integer(). --type qmsg() :: {rabbit_types:r('queue'), pid(), msg_id(), boolean(), rabbit_types:message()}. - --define(STATISTICS_KEYS, - [policy, - operator_policy, - effective_policy_definition, - consumers, - memory, - state, - garbage_collection, - leader, - online, - members, - open_files, - single_active_consumer_pid, - single_active_consumer_ctag, - messages_ram, - message_bytes_ram - ]). - --define(INFO_KEYS, [name, durable, auto_delete, arguments, pid, messages, messages_ready, - messages_unacknowledged, local_state, type] ++ ?STATISTICS_KEYS). - --define(RPC_TIMEOUT, 1000). --define(TICK_TIMEOUT, 5000). %% the ra server tick time --define(DELETE_TIMEOUT, 5000). --define(ADD_MEMBER_TIMEOUT, 5000). - -%%----------- rabbit_queue_type --------------------------------------------- - --spec is_enabled() -> boolean(). -is_enabled() -> - rabbit_feature_flags:is_enabled(quorum_queue). - -%%---------------------------------------------------------------------------- - --spec init(amqqueue:amqqueue()) -> rabbit_fifo_client:state(). -init(Q) when ?is_amqqueue(Q) -> - {ok, SoftLimit} = application:get_env(rabbit, quorum_commands_soft_limit), - %% This lookup could potentially return an {error, not_found}, but we do not - %% know what to do if the queue has `disappeared`. Let it crash. - {Name, _LeaderNode} = Leader = amqqueue:get_pid(Q), - Nodes = get_nodes(Q), - QName = amqqueue:get_name(Q), - %% Ensure the leader is listed first - Servers0 = [{Name, N} || N <- Nodes], - Servers = [Leader | lists:delete(Leader, Servers0)], - rabbit_fifo_client:init(QName, Servers, SoftLimit, - fun() -> credit_flow:block(Name) end, - fun() -> credit_flow:unblock(Name), ok end). - --spec close(rabbit_fifo_client:state()) -> ok. -close(_State) -> - ok. - --spec update(amqqueue:amqqueue(), rabbit_fifo_client:state()) -> - rabbit_fifo_client:state(). -update(Q, State) when ?amqqueue_is_quorum(Q) -> - %% QQ state maintains it's own updates - State. - --spec handle_event({amqqueue:ra_server_id(), any()}, - rabbit_fifo_client:state()) -> - {ok, rabbit_fifo_client:state(), rabbit_queue_type:actions()} | - eol | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -handle_event({From, Evt}, QState) -> - rabbit_fifo_client:handle_ra_event(From, Evt, QState). - --spec declare(amqqueue:amqqueue(), node()) -> - {new | existing, amqqueue:amqqueue()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -declare(Q, _Node) when ?amqqueue_is_quorum(Q) -> - case rabbit_queue_type_util:run_checks( - [fun rabbit_queue_type_util:check_auto_delete/1, - fun rabbit_queue_type_util:check_exclusive/1, - fun rabbit_queue_type_util:check_non_durable/1], - Q) of - ok -> - start_cluster(Q); - Err -> - Err - end. - -start_cluster(Q) -> - QName = amqqueue:get_name(Q), - Durable = amqqueue:is_durable(Q), - AutoDelete = amqqueue:is_auto_delete(Q), - Arguments = amqqueue:get_arguments(Q), - Opts = amqqueue:get_options(Q), - ActingUser = maps:get(user, Opts, ?UNKNOWN_USER), - QuorumSize = get_default_quorum_initial_group_size(Arguments), - RaName = qname_to_internal_name(QName), - Id = {RaName, node()}, - Nodes = select_quorum_nodes(QuorumSize, rabbit_mnesia:cluster_nodes(all)), - NewQ0 = amqqueue:set_pid(Q, Id), - NewQ1 = amqqueue:set_type_state(NewQ0, #{nodes => Nodes}), - case rabbit_amqqueue:internal_declare(NewQ1, false) of - {created, NewQ} -> - TickTimeout = application:get_env(rabbit, quorum_tick_interval, ?TICK_TIMEOUT), - RaConfs = [make_ra_conf(NewQ, ServerId, TickTimeout) - || ServerId <- members(NewQ)], - case ra:start_cluster(RaConfs) of - {ok, _, _} -> - %% TODO: handle error - what should be done if the - %% config cannot be updated - ok = rabbit_fifo_client:update_machine_state(Id, - ra_machine_config(NewQ)), - %% force a policy change to ensure the latest config is - %% updated even when running the machine version from 0 - rabbit_event:notify(queue_created, - [{name, QName}, - {durable, Durable}, - {auto_delete, AutoDelete}, - {arguments, Arguments}, - {user_who_performed_action, - ActingUser}]), - {new, NewQ}; - {error, Error} -> - _ = rabbit_amqqueue:internal_delete(QName, ActingUser), - {protocol_error, internal_error, - "Cannot declare a queue '~s' on node '~s': ~255p", - [rabbit_misc:rs(QName), node(), Error]} - end; - {existing, _} = Ex -> - Ex - end. - -ra_machine(Q) -> - {module, rabbit_fifo, ra_machine_config(Q)}. - -ra_machine_config(Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - {Name, _} = amqqueue:get_pid(Q), - %% take the minimum value of the policy and the queue arg if present - MaxLength = args_policy_lookup(<<"max-length">>, fun min/2, Q), - %% prefer the policy defined strategy if available - Overflow = args_policy_lookup(<<"overflow">>, fun (A, _B) -> A end , Q), - MaxBytes = args_policy_lookup(<<"max-length-bytes">>, fun min/2, Q), - MaxMemoryLength = args_policy_lookup(<<"max-in-memory-length">>, fun min/2, Q), - MaxMemoryBytes = args_policy_lookup(<<"max-in-memory-bytes">>, fun min/2, Q), - DeliveryLimit = args_policy_lookup(<<"delivery-limit">>, fun min/2, Q), - Expires = args_policy_lookup(<<"expires">>, - fun (A, _B) -> A end, - Q), - #{name => Name, - queue_resource => QName, - dead_letter_handler => dlx_mfa(Q), - become_leader_handler => {?MODULE, become_leader, [QName]}, - max_length => MaxLength, - max_bytes => MaxBytes, - max_in_memory_length => MaxMemoryLength, - max_in_memory_bytes => MaxMemoryBytes, - single_active_consumer_on => single_active_consumer_on(Q), - delivery_limit => DeliveryLimit, - overflow_strategy => overflow(Overflow, drop_head, QName), - created => erlang:system_time(millisecond), - expires => Expires - }. - -single_active_consumer_on(Q) -> - QArguments = amqqueue:get_arguments(Q), - case rabbit_misc:table_lookup(QArguments, <<"x-single-active-consumer">>) of - {bool, true} -> true; - _ -> false - end. - -update_consumer_handler(QName, {ConsumerTag, ChPid}, Exclusive, AckRequired, Prefetch, Active, ActivityStatus, Args) -> - local_or_remote_handler(ChPid, rabbit_quorum_queue, update_consumer, - [QName, ChPid, ConsumerTag, Exclusive, AckRequired, Prefetch, Active, ActivityStatus, Args]). - -update_consumer(QName, ChPid, ConsumerTag, Exclusive, AckRequired, Prefetch, Active, ActivityStatus, Args) -> - catch rabbit_core_metrics:consumer_updated(ChPid, ConsumerTag, Exclusive, AckRequired, - QName, Prefetch, Active, ActivityStatus, Args). - -cancel_consumer_handler(QName, {ConsumerTag, ChPid}) -> - local_or_remote_handler(ChPid, rabbit_quorum_queue, cancel_consumer, - [QName, ChPid, ConsumerTag]). - -cancel_consumer(QName, ChPid, ConsumerTag) -> - catch rabbit_core_metrics:consumer_deleted(ChPid, ConsumerTag, QName), - emit_consumer_deleted(ChPid, ConsumerTag, QName, ?INTERNAL_USER). - -local_or_remote_handler(ChPid, Module, Function, Args) -> - Node = node(ChPid), - case Node == node() of - true -> - erlang:apply(Module, Function, Args); - false -> - %% this could potentially block for a while if the node is - %% in disconnected state or tcp buffers are full - rpc:cast(Node, Module, Function, Args) - end. - -become_leader(QName, Name) -> - Fun = fun (Q1) -> - amqqueue:set_state( - amqqueue:set_pid(Q1, {Name, node()}), - live) - end, - %% as this function is called synchronously when a ra node becomes leader - %% we need to ensure there is no chance of blocking as else the ra node - %% may not be able to establish it's leadership - spawn(fun() -> - rabbit_misc:execute_mnesia_transaction( - fun() -> - rabbit_amqqueue:update(QName, Fun) - end), - case rabbit_amqqueue:lookup(QName) of - {ok, Q0} when ?is_amqqueue(Q0) -> - Nodes = get_nodes(Q0), - [rpc:call(Node, ?MODULE, rpc_delete_metrics, - [QName], ?RPC_TIMEOUT) - || Node <- Nodes, Node =/= node()]; - _ -> - ok - end - end). - --spec all_replica_states() -> {node(), #{atom() => atom()}}. -all_replica_states() -> - Rows = ets:tab2list(ra_state), - {node(), maps:from_list(Rows)}. - --spec list_with_minimum_quorum() -> [amqqueue:amqqueue()]. -list_with_minimum_quorum() -> - filter_quorum_critical( - rabbit_amqqueue:list_local_quorum_queues()). - --spec list_with_minimum_quorum_for_cli() -> [#{binary() => term()}]. -list_with_minimum_quorum_for_cli() -> - QQs = list_with_minimum_quorum(), - [begin - #resource{name = Name} = amqqueue:get_name(Q), - #{ - <<"readable_name">> => rabbit_data_coercion:to_binary(rabbit_misc:rs(amqqueue:get_name(Q))), - <<"name">> => Name, - <<"virtual_host">> => amqqueue:get_vhost(Q), - <<"type">> => <<"quorum">> - } - end || Q <- QQs]. - --spec filter_quorum_critical([amqqueue:amqqueue()]) -> [amqqueue:amqqueue()]. -filter_quorum_critical(Queues) -> - %% Example map of QQ replica states: - %% #{rabbit@warp10 => - %% #{'%2F_qq.636' => leader,'%2F_qq.243' => leader, - %% '%2F_qq.1939' => leader,'%2F_qq.1150' => leader, - %% '%2F_qq.1109' => leader,'%2F_qq.1654' => leader, - %% '%2F_qq.1679' => leader,'%2F_qq.1003' => leader, - %% '%2F_qq.1593' => leader,'%2F_qq.1765' => leader, - %% '%2F_qq.933' => leader,'%2F_qq.38' => leader, - %% '%2F_qq.1357' => leader,'%2F_qq.1345' => leader, - %% '%2F_qq.1694' => leader,'%2F_qq.994' => leader, - %% '%2F_qq.490' => leader,'%2F_qq.1704' => leader, - %% '%2F_qq.58' => leader,'%2F_qq.564' => leader, - %% '%2F_qq.683' => leader,'%2F_qq.386' => leader, - %% '%2F_qq.753' => leader,'%2F_qq.6' => leader, - %% '%2F_qq.1590' => leader,'%2F_qq.1363' => leader, - %% '%2F_qq.882' => leader,'%2F_qq.1161' => leader,...}} - ReplicaStates = maps:from_list( - rabbit_misc:append_rpc_all_nodes(rabbit_nodes:all_running(), - ?MODULE, all_replica_states, [])), - filter_quorum_critical(Queues, ReplicaStates). - --spec filter_quorum_critical([amqqueue:amqqueue()], #{node() => #{atom() => atom()}}) -> [amqqueue:amqqueue()]. - -filter_quorum_critical(Queues, ReplicaStates) -> - lists:filter(fun (Q) -> - MemberNodes = rabbit_amqqueue:get_quorum_nodes(Q), - {Name, _Node} = amqqueue:get_pid(Q), - AllUp = lists:filter(fun (N) -> - {Name, _} = amqqueue:get_pid(Q), - case maps:get(N, ReplicaStates, undefined) of - #{Name := State} when State =:= follower orelse State =:= leader -> - true; - _ -> false - end - end, MemberNodes), - MinQuorum = length(MemberNodes) div 2 + 1, - length(AllUp) =< MinQuorum - end, Queues). - -capabilities() -> - #{policies => [<<"max-length">>, <<"max-length-bytes">>, <<"overflow">>, - <<"expires">>, <<"max-in-memory-length">>, <<"max-in-memory-bytes">>, - <<"delivery-limit">>, <<"dead-letter-exchange">>, <<"dead-letter-routing-key">>], - queue_arguments => [<<"x-expires">>, <<"x-dead-letter-exchange">>, - <<"x-dead-letter-routing-key">>, <<"x-max-length">>, - <<"x-max-length-bytes">>, <<"x-max-in-memory-length">>, - <<"x-max-in-memory-bytes">>, <<"x-overflow">>, - <<"x-single-active-consumer">>, <<"x-queue-type">>, - <<"x-quorum-initial-group-size">>, <<"x-delivery-limit">>], - consumer_arguments => [<<"x-priority">>, <<"x-credit">>], - server_named => false}. - -rpc_delete_metrics(QName) -> - ets:delete(queue_coarse_metrics, QName), - ets:delete(queue_metrics, QName), - ok. - -spawn_deleter(QName) -> - spawn(fun () -> - {ok, Q} = rabbit_amqqueue:lookup(QName), - delete(Q, false, false, <<"expired">>) - end). - -handle_tick(QName, - {Name, MR, MU, M, C, MsgBytesReady, MsgBytesUnack}, - Nodes) -> - %% this makes calls to remote processes so cannot be run inside the - %% ra server - Self = self(), - _ = spawn(fun() -> - R = reductions(Name), - rabbit_core_metrics:queue_stats(QName, MR, MU, M, R), - Util = case C of - 0 -> 0; - _ -> rabbit_fifo:usage(Name) - end, - Infos = [{consumers, C}, - {consumer_utilisation, Util}, - {message_bytes_ready, MsgBytesReady}, - {message_bytes_unacknowledged, MsgBytesUnack}, - {message_bytes, MsgBytesReady + MsgBytesUnack}, - {message_bytes_persistent, MsgBytesReady + MsgBytesUnack}, - {messages_persistent, M} - - | infos(QName, ?STATISTICS_KEYS -- [consumers])], - rabbit_core_metrics:queue_stats(QName, Infos), - rabbit_event:notify(queue_stats, - Infos ++ [{name, QName}, - {messages, M}, - {messages_ready, MR}, - {messages_unacknowledged, MU}, - {reductions, R}]), - ok = repair_leader_record(QName, Self), - ExpectedNodes = rabbit_mnesia:cluster_nodes(all), - case Nodes -- ExpectedNodes of - [] -> - ok; - Stale -> - rabbit_log:info("~s: stale nodes detected. Purging ~w~n", - [rabbit_misc:rs(QName), Stale]), - %% pipeline purge command - {ok, Q} = rabbit_amqqueue:lookup(QName), - ok = ra:pipeline_command(amqqueue:get_pid(Q), - rabbit_fifo:make_purge_nodes(Stale)), - - ok - end - end), - ok. - -repair_leader_record(QName, Self) -> - {ok, Q} = rabbit_amqqueue:lookup(QName), - Node = node(), - case amqqueue:get_pid(Q) of - {_, Node} -> - %% it's ok - we don't need to do anything - ok; - _ -> - rabbit_log:debug("~s: repairing leader record", - [rabbit_misc:rs(QName)]), - {_, Name} = erlang:process_info(Self, registered_name), - become_leader(QName, Name) - end, - ok. - -repair_amqqueue_nodes(VHost, QueueName) -> - QName = #resource{virtual_host = VHost, name = QueueName, kind = queue}, - repair_amqqueue_nodes(QName). - --spec repair_amqqueue_nodes(rabbit_types:r('queue') | amqqueue:amqqueue()) -> - ok | repaired. -repair_amqqueue_nodes(QName = #resource{}) -> - {ok, Q0} = rabbit_amqqueue:lookup(QName), - repair_amqqueue_nodes(Q0); -repair_amqqueue_nodes(Q0) -> - QName = amqqueue:get_name(Q0), - Leader = amqqueue:get_pid(Q0), - {ok, Members, _} = ra:members(Leader), - RaNodes = [N || {_, N} <- Members], - #{nodes := Nodes} = amqqueue:get_type_state(Q0), - case lists:sort(RaNodes) =:= lists:sort(Nodes) of - true -> - %% up to date - ok; - false -> - %% update amqqueue record - Fun = fun (Q) -> - TS0 = amqqueue:get_type_state(Q), - TS = TS0#{nodes => RaNodes}, - amqqueue:set_type_state(Q, TS) - end, - rabbit_misc:execute_mnesia_transaction( - fun() -> - rabbit_amqqueue:update(QName, Fun) - end), - repaired - end. - -reductions(Name) -> - try - {reductions, R} = process_info(whereis(Name), reductions), - R - catch - error:badarg -> - 0 - end. - -is_recoverable(Q) -> - Node = node(), - Nodes = get_nodes(Q), - lists:member(Node, Nodes). - --spec recover(binary(), [amqqueue:amqqueue()]) -> - {[amqqueue:amqqueue()], [amqqueue:amqqueue()]}. -recover(_Vhost, Queues) -> - lists:foldl( - fun (Q0, {R0, F0}) -> - {Name, _} = amqqueue:get_pid(Q0), - QName = amqqueue:get_name(Q0), - Nodes = get_nodes(Q0), - Formatter = {?MODULE, format_ra_event, [QName]}, - Res = case ra:restart_server({Name, node()}, - #{ra_event_formatter => Formatter}) of - ok -> - % queue was restarted, good - ok; - {error, Err1} - when Err1 == not_started orelse - Err1 == name_not_registered -> - % queue was never started on this node - % so needs to be started from scratch. - Machine = ra_machine(Q0), - RaNodes = [{Name, Node} || Node <- Nodes], - case ra:start_server(Name, {Name, node()}, Machine, RaNodes) of - ok -> ok; - Err2 -> - rabbit_log:warning("recover: quorum queue ~w could not" - " be started ~w", [Name, Err2]), - fail - end; - {error, {already_started, _}} -> - %% this is fine and can happen if a vhost crashes and performs - %% recovery whilst the ra application and servers are still - %% running - ok; - Err -> - %% catch all clause to avoid causing the vhost not to start - rabbit_log:warning("recover: quorum queue ~w could not be " - "restarted ~w", [Name, Err]), - fail - end, - %% we have to ensure the quorum queue is - %% present in the rabbit_queue table and not just in - %% rabbit_durable_queue - %% So many code paths are dependent on this. - {ok, Q} = rabbit_amqqueue:ensure_rabbit_queue_record_is_initialized(Q0), - case Res of - ok -> - {[Q | R0], F0}; - fail -> - {R0, [Q | F0]} - end - end, {[], []}, Queues). - --spec stop(rabbit_types:vhost()) -> ok. -stop(VHost) -> - _ = [begin - Pid = amqqueue:get_pid(Q), - ra:stop_server(Pid) - end || Q <- find_quorum_queues(VHost)], - ok. - --spec delete(amqqueue:amqqueue(), - boolean(), boolean(), - rabbit_types:username()) -> - {ok, QLen :: non_neg_integer()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -delete(Q, true, _IfEmpty, _ActingUser) when ?amqqueue_is_quorum(Q) -> - {protocol_error, not_implemented, - "cannot delete ~s. queue.delete operations with if-unused flag set are not supported by quorum queues", - [rabbit_misc:rs(amqqueue:get_name(Q))]}; -delete(Q, _IfUnused, true, _ActingUser) when ?amqqueue_is_quorum(Q) -> - {protocol_error, not_implemented, - "cannot delete ~s. queue.delete operations with if-empty flag set are not supported by quorum queues", - [rabbit_misc:rs(amqqueue:get_name(Q))]}; -delete(Q, _IfUnused, _IfEmpty, ActingUser) when ?amqqueue_is_quorum(Q) -> - {Name, _} = amqqueue:get_pid(Q), - QName = amqqueue:get_name(Q), - QNodes = get_nodes(Q), - %% TODO Quorum queue needs to support consumer tracking for IfUnused - Timeout = ?DELETE_TIMEOUT, - {ok, ReadyMsgs, _} = stat(Q), - Servers = [{Name, Node} || Node <- QNodes], - case ra:delete_cluster(Servers, Timeout) of - {ok, {_, LeaderNode} = Leader} -> - MRef = erlang:monitor(process, Leader), - receive - {'DOWN', MRef, process, _, _} -> - ok - after Timeout -> - ok = force_delete_queue(Servers) - end, - ok = delete_queue_data(QName, ActingUser), - rpc:call(LeaderNode, rabbit_core_metrics, queue_deleted, [QName], - ?RPC_TIMEOUT), - {ok, ReadyMsgs}; - {error, {no_more_servers_to_try, Errs}} -> - case lists:all(fun({{error, noproc}, _}) -> true; - (_) -> false - end, Errs) of - true -> - %% If all ra nodes were already down, the delete - %% has succeed - delete_queue_data(QName, ActingUser), - {ok, ReadyMsgs}; - false -> - %% attempt forced deletion of all servers - rabbit_log:warning( - "Could not delete quorum queue '~s', not enough nodes " - " online to reach a quorum: ~255p." - " Attempting force delete.", - [rabbit_misc:rs(QName), Errs]), - ok = force_delete_queue(Servers), - delete_queue_data(QName, ActingUser), - {ok, ReadyMsgs} - end - end. - -force_delete_queue(Servers) -> - [begin - case catch(ra:force_delete_server(S)) of - ok -> ok; - Err -> - rabbit_log:warning( - "Force delete of ~w failed with: ~w" - "This may require manual data clean up~n", - [S, Err]), - ok - end - end || S <- Servers], - ok. - -delete_queue_data(QName, ActingUser) -> - _ = rabbit_amqqueue:internal_delete(QName, ActingUser), - ok. - - -delete_immediately(Resource, {_Name, _} = QPid) -> - _ = rabbit_amqqueue:internal_delete(Resource, ?INTERNAL_USER), - {ok, _} = ra:delete_cluster([QPid]), - rabbit_core_metrics:queue_deleted(Resource), - ok. - -settle(complete, CTag, MsgIds, QState) -> - rabbit_fifo_client:settle(quorum_ctag(CTag), MsgIds, QState); -settle(requeue, CTag, MsgIds, QState) -> - rabbit_fifo_client:return(quorum_ctag(CTag), MsgIds, QState); -settle(discard, CTag, MsgIds, QState) -> - rabbit_fifo_client:discard(quorum_ctag(CTag), MsgIds, QState). - -credit(CTag, Credit, Drain, QState) -> - rabbit_fifo_client:credit(quorum_ctag(CTag), Credit, Drain, QState). - --spec dequeue(NoAck :: boolean(), pid(), - rabbit_types:ctag(), rabbit_fifo_client:state()) -> - {empty, rabbit_fifo_client:state()} | - {ok, QLen :: non_neg_integer(), qmsg(), rabbit_fifo_client:state()} | - {error, term()}. -dequeue(NoAck, _LimiterPid, CTag0, QState0) -> - CTag = quorum_ctag(CTag0), - Settlement = case NoAck of - true -> - settled; - false -> - unsettled - end, - rabbit_fifo_client:dequeue(CTag, Settlement, QState0). - --spec consume(amqqueue:amqqueue(), - rabbit_queue_type:consume_spec(), - rabbit_fifo_client:state()) -> - {ok, rabbit_fifo_client:state(), rabbit_queue_type:actions()} | - {error, global_qos_not_supported_for_queue_type}. -consume(Q, #{limiter_active := true}, _State) - when ?amqqueue_is_quorum(Q) -> - {error, global_qos_not_supported_for_queue_type}; -consume(Q, Spec, QState0) when ?amqqueue_is_quorum(Q) -> - #{no_ack := NoAck, - channel_pid := ChPid, - prefetch_count := ConsumerPrefetchCount, - consumer_tag := ConsumerTag0, - exclusive_consume := ExclusiveConsume, - args := Args, - ok_msg := OkMsg, - acting_user := ActingUser} = Spec, - %% TODO: validate consumer arguments - %% currently quorum queues do not support any arguments - QName = amqqueue:get_name(Q), - QPid = amqqueue:get_pid(Q), - maybe_send_reply(ChPid, OkMsg), - ConsumerTag = quorum_ctag(ConsumerTag0), - %% A prefetch count of 0 means no limitation, - %% let's make it into something large for ra - Prefetch = case ConsumerPrefetchCount of - 0 -> 2000; - Other -> Other - end, - %% consumer info is used to describe the consumer properties - AckRequired = not NoAck, - ConsumerMeta = #{ack => AckRequired, - prefetch => ConsumerPrefetchCount, - args => Args, - username => ActingUser}, - {ok, QState} = rabbit_fifo_client:checkout(ConsumerTag, - Prefetch, - ConsumerMeta, - QState0), - case ra:local_query(QPid, - fun rabbit_fifo:query_single_active_consumer/1) of - {ok, {_, SacResult}, _} -> - SingleActiveConsumerOn = single_active_consumer_on(Q), - {IsSingleActiveConsumer, ActivityStatus} = case {SingleActiveConsumerOn, SacResult} of - {false, _} -> - {true, up}; - {true, {value, {ConsumerTag, ChPid}}} -> - {true, single_active}; - _ -> - {false, waiting} - end, - rabbit_core_metrics:consumer_created( - ChPid, ConsumerTag, ExclusiveConsume, - AckRequired, QName, - ConsumerPrefetchCount, IsSingleActiveConsumer, - ActivityStatus, Args), - emit_consumer_created(ChPid, ConsumerTag, ExclusiveConsume, - AckRequired, QName, Prefetch, - Args, none, ActingUser), - {ok, QState, []}; - {error, Error} -> - Error; - {timeout, _} -> - {error, timeout} - end. - -% -spec basic_cancel(rabbit_types:ctag(), ChPid :: pid(), any(), rabbit_fifo_client:state()) -> -% {'ok', rabbit_fifo_client:state()}. - -cancel(_Q, ConsumerTag, OkMsg, _ActingUser, State) -> - maybe_send_reply(self(), OkMsg), - rabbit_fifo_client:cancel_checkout(quorum_ctag(ConsumerTag), State). - -emit_consumer_created(ChPid, CTag, Exclusive, AckRequired, QName, PrefetchCount, Args, Ref, ActingUser) -> - rabbit_event:notify(consumer_created, - [{consumer_tag, CTag}, - {exclusive, Exclusive}, - {ack_required, AckRequired}, - {channel, ChPid}, - {queue, QName}, - {prefetch_count, PrefetchCount}, - {arguments, Args}, - {user_who_performed_action, ActingUser}], - Ref). - -emit_consumer_deleted(ChPid, ConsumerTag, QName, ActingUser) -> - rabbit_event:notify(consumer_deleted, - [{consumer_tag, ConsumerTag}, - {channel, ChPid}, - {queue, QName}, - {user_who_performed_action, ActingUser}]). - --spec stateless_deliver(amqqueue:ra_server_id(), rabbit_types:delivery()) -> 'ok'. - -stateless_deliver(ServerId, Delivery) -> - ok = rabbit_fifo_client:untracked_enqueue([ServerId], - Delivery#delivery.message). - --spec deliver(Confirm :: boolean(), rabbit_types:delivery(), - rabbit_fifo_client:state()) -> - {ok | slow, rabbit_fifo_client:state()} | - {reject_publish, rabbit_fifo_client:state()}. -deliver(false, Delivery, QState0) -> - case rabbit_fifo_client:enqueue(Delivery#delivery.message, QState0) of - {ok, _} = Res -> Res; - {slow, _} = Res -> Res; - {reject_publish, State} -> - {ok, State} - end; -deliver(true, Delivery, QState0) -> - rabbit_fifo_client:enqueue(Delivery#delivery.msg_seq_no, - Delivery#delivery.message, QState0). - -deliver(QSs, #delivery{confirm = Confirm} = Delivery) -> - lists:foldl( - fun({Q, stateless}, {Qs, Actions}) -> - QRef = amqqueue:get_pid(Q), - ok = rabbit_fifo_client:untracked_enqueue( - [QRef], Delivery#delivery.message), - {Qs, Actions}; - ({Q, S0}, {Qs, Actions}) -> - case deliver(Confirm, Delivery, S0) of - {reject_publish, S} -> - Seq = Delivery#delivery.msg_seq_no, - QName = rabbit_fifo_client:cluster_name(S), - {[{Q, S} | Qs], [{rejected, QName, [Seq]} | Actions]}; - {_, S} -> - {[{Q, S} | Qs], Actions} - end - end, {[], []}, QSs). - - -state_info(S) -> - #{pending_raft_commands => rabbit_fifo_client:pending_size(S)}. - - - --spec infos(rabbit_types:r('queue')) -> rabbit_types:infos(). -infos(QName) -> - infos(QName, ?STATISTICS_KEYS). - -infos(QName, Keys) -> - case rabbit_amqqueue:lookup(QName) of - {ok, Q} -> - info(Q, Keys); - {error, not_found} -> - [] - end. - -info(Q, all_keys) -> - info(Q, ?INFO_KEYS); -info(Q, Items) -> - lists:foldr(fun(totals, Acc) -> - i_totals(Q) ++ Acc; - (type_specific, Acc) -> - format(Q) ++ Acc; - (Item, Acc) -> - [{Item, i(Item, Q)} | Acc] - end, [], Items). - --spec stat(amqqueue:amqqueue()) -> - {'ok', non_neg_integer(), non_neg_integer()}. -stat(Q) when ?is_amqqueue(Q) -> - %% same short default timeout as in rabbit_fifo_client:stat/1 - stat(Q, 250). - --spec stat(amqqueue:amqqueue(), non_neg_integer()) -> {'ok', non_neg_integer(), non_neg_integer()}. - -stat(Q, Timeout) when ?is_amqqueue(Q) -> - Leader = amqqueue:get_pid(Q), - try - case rabbit_fifo_client:stat(Leader, Timeout) of - {ok, _, _} = Success -> Success; - {error, _} -> {ok, 0, 0}; - {timeout, _} -> {ok, 0, 0} - end - catch - _:_ -> - %% Leader is not available, cluster might be in minority - {ok, 0, 0} - end. - --spec purge(amqqueue:amqqueue()) -> - {ok, non_neg_integer()}. -purge(Q) when ?is_amqqueue(Q) -> - Node = amqqueue:get_pid(Q), - rabbit_fifo_client:purge(Node). - -requeue(ConsumerTag, MsgIds, QState) -> - rabbit_fifo_client:return(quorum_ctag(ConsumerTag), MsgIds, QState). - -cleanup_data_dir() -> - Names = [begin - {Name, _} = amqqueue:get_pid(Q), - Name - end - || Q <- rabbit_amqqueue:list_by_type(?MODULE), - lists:member(node(), get_nodes(Q))], - NoQQClusters = rabbit_ra_registry:list_not_quorum_clusters(), - Registered = ra_directory:list_registered(), - Running = Names ++ NoQQClusters, - _ = [maybe_delete_data_dir(UId) || {Name, UId} <- Registered, - not lists:member(Name, Running)], - ok. - -maybe_delete_data_dir(UId) -> - Dir = ra_env:server_data_dir(UId), - {ok, Config} = ra_log:read_config(Dir), - case maps:get(machine, Config) of - {module, rabbit_fifo, _} -> - ra_lib:recursive_delete(Dir), - ra_directory:unregister_name(UId); - _ -> - ok - end. - -policy_changed(Q) -> - QPid = amqqueue:get_pid(Q), - _ = rabbit_fifo_client:update_machine_state(QPid, ra_machine_config(Q)), - ok. - --spec cluster_state(Name :: atom()) -> 'down' | 'recovering' | 'running'. - -cluster_state(Name) -> - case whereis(Name) of - undefined -> down; - _ -> - case ets:lookup(ra_state, Name) of - [{_, recover}] -> recovering; - _ -> running - end - end. - --spec status(rabbit_types:vhost(), Name :: rabbit_misc:resource_name()) -> - [[{binary(), term()}]] | {error, term()}. -status(Vhost, QueueName) -> - %% Handle not found queues - QName = #resource{virtual_host = Vhost, name = QueueName, kind = queue}, - RName = qname_to_internal_name(QName), - case rabbit_amqqueue:lookup(QName) of - {ok, Q} when ?amqqueue_is_classic(Q) -> - {error, classic_queue_not_supported}; - {ok, Q} when ?amqqueue_is_quorum(Q) -> - Nodes = get_nodes(Q), - [begin - case get_sys_status({RName, N}) of - {ok, Sys} -> - {_, M} = lists:keyfind(ra_server_state, 1, Sys), - {_, RaftState} = lists:keyfind(raft_state, 1, Sys), - #{commit_index := Commit, - machine_version := MacVer, - current_term := Term, - log := #{last_index := Last, - snapshot_index := SnapIdx}} = M, - [{<<"Node Name">>, N}, - {<<"Raft State">>, RaftState}, - {<<"Log Index">>, Last}, - {<<"Commit Index">>, Commit}, - {<<"Snapshot Index">>, SnapIdx}, - {<<"Term">>, Term}, - {<<"Machine Version">>, MacVer} - ]; - {error, Err} -> - [{<<"Node Name">>, N}, - {<<"Raft State">>, Err}, - {<<"Log Index">>, <<>>}, - {<<"Commit Index">>, <<>>}, - {<<"Snapshot Index">>, <<>>}, - {<<"Term">>, <<>>}, - {<<"Machine Version">>, <<>>} - ] - end - end || N <- Nodes]; - {error, not_found} = E -> - E - end. - -get_sys_status(Proc) -> - try lists:nth(5, element(4, sys:get_status(Proc))) of - Sys -> {ok, Sys} - catch - _:Err when is_tuple(Err) -> - {error, element(1, Err)}; - _:_ -> - {error, other} - - end. - - -add_member(VHost, Name, Node, Timeout) -> - QName = #resource{virtual_host = VHost, name = Name, kind = queue}, - case rabbit_amqqueue:lookup(QName) of - {ok, Q} when ?amqqueue_is_classic(Q) -> - {error, classic_queue_not_supported}; - {ok, Q} when ?amqqueue_is_quorum(Q) -> - QNodes = get_nodes(Q), - case lists:member(Node, rabbit_nodes:all_running()) of - false -> - {error, node_not_running}; - true -> - case lists:member(Node, QNodes) of - true -> - %% idempotent by design - ok; - false -> - add_member(Q, Node, Timeout) - end - end; - {error, not_found} = E -> - E - end. - -add_member(Q, Node, Timeout) when ?amqqueue_is_quorum(Q) -> - {RaName, _} = amqqueue:get_pid(Q), - QName = amqqueue:get_name(Q), - %% TODO parallel calls might crash this, or add a duplicate in quorum_nodes - ServerId = {RaName, Node}, - Members = members(Q), - TickTimeout = application:get_env(rabbit, quorum_tick_interval, - ?TICK_TIMEOUT), - Conf = make_ra_conf(Q, ServerId, TickTimeout), - case ra:start_server(Conf) of - ok -> - case ra:add_member(Members, ServerId, Timeout) of - {ok, _, Leader} -> - Fun = fun(Q1) -> - Q2 = update_type_state( - Q1, fun(#{nodes := Nodes} = Ts) -> - Ts#{nodes => [Node | Nodes]} - end), - amqqueue:set_pid(Q2, Leader) - end, - rabbit_misc:execute_mnesia_transaction( - fun() -> rabbit_amqqueue:update(QName, Fun) end), - ok; - {timeout, _} -> - _ = ra:force_delete_server(ServerId), - _ = ra:remove_member(Members, ServerId), - {error, timeout}; - E -> - _ = ra:force_delete_server(ServerId), - E - end; - E -> - E - end. - -delete_member(VHost, Name, Node) -> - QName = #resource{virtual_host = VHost, name = Name, kind = queue}, - case rabbit_amqqueue:lookup(QName) of - {ok, Q} when ?amqqueue_is_classic(Q) -> - {error, classic_queue_not_supported}; - {ok, Q} when ?amqqueue_is_quorum(Q) -> - QNodes = get_nodes(Q), - case lists:member(Node, QNodes) of - false -> - %% idempotent by design - ok; - true -> - delete_member(Q, Node) - end; - {error, not_found} = E -> - E - end. - - -delete_member(Q, Node) when ?amqqueue_is_quorum(Q) -> - QName = amqqueue:get_name(Q), - {RaName, _} = amqqueue:get_pid(Q), - ServerId = {RaName, Node}, - case members(Q) of - [{_, Node}] -> - - %% deleting the last member is not allowed - {error, last_node}; - Members -> - case ra:remove_member(Members, ServerId) of - {ok, _, _Leader} -> - Fun = fun(Q1) -> - update_type_state( - Q1, - fun(#{nodes := Nodes} = Ts) -> - Ts#{nodes => lists:delete(Node, Nodes)} - end) - end, - rabbit_misc:execute_mnesia_transaction( - fun() -> rabbit_amqqueue:update(QName, Fun) end), - case ra:force_delete_server(ServerId) of - ok -> - ok; - {error, {badrpc, nodedown}} -> - ok; - {error, {badrpc, {'EXIT', {badarg, _}}}} -> - %% DETS/ETS tables can't be found, application isn't running - ok; - {error, _} = Err -> - Err; - Err -> - {error, Err} - end; - {timeout, _} -> - {error, timeout}; - E -> - E - end - end. - --spec shrink_all(node()) -> - [{rabbit_amqqueue:name(), - {ok, pos_integer()} | {error, pos_integer(), term()}}]. -shrink_all(Node) -> - [begin - QName = amqqueue:get_name(Q), - rabbit_log:info("~s: removing member (replica) on node ~w", - [rabbit_misc:rs(QName), Node]), - Size = length(get_nodes(Q)), - case delete_member(Q, Node) of - ok -> - {QName, {ok, Size-1}}; - {error, Err} -> - rabbit_log:warning("~s: failed to remove member (replica) on node ~w, error: ~w", - [rabbit_misc:rs(QName), Node, Err]), - {QName, {error, Size, Err}} - end - end || Q <- rabbit_amqqueue:list(), - amqqueue:get_type(Q) == ?MODULE, - lists:member(Node, get_nodes(Q))]. - --spec grow(node(), binary(), binary(), all | even) -> - [{rabbit_amqqueue:name(), - {ok, pos_integer()} | {error, pos_integer(), term()}}]. -grow(Node, VhostSpec, QueueSpec, Strategy) -> - Running = rabbit_nodes:all_running(), - [begin - Size = length(get_nodes(Q)), - QName = amqqueue:get_name(Q), - rabbit_log:info("~s: adding a new member (replica) on node ~w", - [rabbit_misc:rs(QName), Node]), - case add_member(Q, Node, ?ADD_MEMBER_TIMEOUT) of - ok -> - {QName, {ok, Size + 1}}; - {error, Err} -> - rabbit_log:warning( - "~s: failed to add member (replica) on node ~w, error: ~w", - [rabbit_misc:rs(QName), Node, Err]), - {QName, {error, Size, Err}} - end - end - || Q <- rabbit_amqqueue:list(), - amqqueue:get_type(Q) == ?MODULE, - %% don't add a member if there is already one on the node - not lists:member(Node, get_nodes(Q)), - %% node needs to be running - lists:member(Node, Running), - matches_strategy(Strategy, get_nodes(Q)), - is_match(amqqueue:get_vhost(Q), VhostSpec) andalso - is_match(get_resource_name(amqqueue:get_name(Q)), QueueSpec) ]. - -transfer_leadership(Q, Destination) -> - {RaName, _} = Pid = amqqueue:get_pid(Q), - case ra:transfer_leadership(Pid, {RaName, Destination}) of - ok -> - case ra:members(Pid) of - {_, _, {_, NewNode}} -> - {migrated, NewNode}; - {timeout, _} -> - {not_migrated, ra_members_timeout} - end; - already_leader -> - {not_migrated, already_leader}; - {error, Reason} -> - {not_migrated, Reason}; - {timeout, _} -> - %% TODO should we retry once? - {not_migrated, timeout} - end. - -queue_length(Q) -> - Name = amqqueue:get_name(Q), - case ets:lookup(ra_metrics, Name) of - [] -> 0; - [{_, _, SnapIdx, _, _, LastIdx, _}] -> LastIdx - SnapIdx - end. - -get_replicas(Q) -> - get_nodes(Q). - -get_resource_name(#resource{name = Name}) -> - Name. - -matches_strategy(all, _) -> true; -matches_strategy(even, Members) -> - length(Members) rem 2 == 0. - -is_match(Subj, E) -> - nomatch /= re:run(Subj, E). - -file_handle_leader_reservation(QName) -> - {ok, Q} = rabbit_amqqueue:lookup(QName), - ClusterSize = length(get_nodes(Q)), - file_handle_cache:set_reservation(2 + ClusterSize). - -file_handle_other_reservation() -> - file_handle_cache:set_reservation(2). - -file_handle_release_reservation() -> - file_handle_cache:release_reservation(). - --spec reclaim_memory(rabbit_types:vhost(), Name :: rabbit_misc:resource_name()) -> ok | {error, term()}. -reclaim_memory(Vhost, QueueName) -> - QName = #resource{virtual_host = Vhost, name = QueueName, kind = queue}, - case rabbit_amqqueue:lookup(QName) of - {ok, Q} when ?amqqueue_is_classic(Q) -> - {error, classic_queue_not_supported}; - {ok, Q} when ?amqqueue_is_quorum(Q) -> - ok = ra:pipeline_command(amqqueue:get_pid(Q), - rabbit_fifo:make_garbage_collection()); - {error, not_found} = E -> - E - end. - -%%---------------------------------------------------------------------------- -dlx_mfa(Q) -> - DLX = init_dlx(args_policy_lookup(<<"dead-letter-exchange">>, - fun res_arg/2, Q), Q), - DLXRKey = args_policy_lookup(<<"dead-letter-routing-key">>, - fun res_arg/2, Q), - {?MODULE, dead_letter_publish, [DLX, DLXRKey, amqqueue:get_name(Q)]}. - -init_dlx(undefined, _Q) -> - undefined; -init_dlx(DLX, Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - rabbit_misc:r(QName, exchange, DLX). - -res_arg(_PolVal, ArgVal) -> ArgVal. - -dead_letter_publish(undefined, _, _, _) -> - ok; -dead_letter_publish(X, RK, QName, ReasonMsgs) -> - case rabbit_exchange:lookup(X) of - {ok, Exchange} -> - [rabbit_dead_letter:publish(Msg, Reason, Exchange, RK, QName) - || {Reason, Msg} <- ReasonMsgs]; - {error, not_found} -> - ok - end. - -find_quorum_queues(VHost) -> - Node = node(), - mnesia:async_dirty( - fun () -> - qlc:e(qlc:q([Q || Q <- mnesia:table(rabbit_durable_queue), - ?amqqueue_is_quorum(Q), - amqqueue:get_vhost(Q) =:= VHost, - amqqueue:qnode(Q) == Node])) - end). - -i_totals(Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - case ets:lookup(queue_coarse_metrics, QName) of - [{_, MR, MU, M, _}] -> - [{messages_ready, MR}, - {messages_unacknowledged, MU}, - {messages, M}]; - [] -> - [{messages_ready, 0}, - {messages_unacknowledged, 0}, - {messages, 0}] - end. - -i(name, Q) when ?is_amqqueue(Q) -> amqqueue:get_name(Q); -i(durable, Q) when ?is_amqqueue(Q) -> amqqueue:is_durable(Q); -i(auto_delete, Q) when ?is_amqqueue(Q) -> amqqueue:is_auto_delete(Q); -i(arguments, Q) when ?is_amqqueue(Q) -> amqqueue:get_arguments(Q); -i(pid, Q) when ?is_amqqueue(Q) -> - {Name, _} = amqqueue:get_pid(Q), - whereis(Name); -i(messages, Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - quorum_messages(QName); -i(messages_ready, Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - case ets:lookup(queue_coarse_metrics, QName) of - [{_, MR, _, _, _}] -> - MR; - [] -> - 0 - end; -i(messages_unacknowledged, Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - case ets:lookup(queue_coarse_metrics, QName) of - [{_, _, MU, _, _}] -> - MU; - [] -> - 0 - end; -i(policy, Q) -> - case rabbit_policy:name(Q) of - none -> ''; - Policy -> Policy - end; -i(operator_policy, Q) -> - case rabbit_policy:name_op(Q) of - none -> ''; - Policy -> Policy - end; -i(effective_policy_definition, Q) -> - case rabbit_policy:effective_definition(Q) of - undefined -> []; - Def -> Def - end; -i(consumers, Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - case ets:lookup(queue_metrics, QName) of - [{_, M, _}] -> - proplists:get_value(consumers, M, 0); - [] -> - 0 - end; -i(memory, Q) when ?is_amqqueue(Q) -> - {Name, _} = amqqueue:get_pid(Q), - try - {memory, M} = process_info(whereis(Name), memory), - M - catch - error:badarg -> - 0 - end; -i(state, Q) when ?is_amqqueue(Q) -> - {Name, Node} = amqqueue:get_pid(Q), - %% Check against the leader or last known leader - case rpc:call(Node, ?MODULE, cluster_state, [Name], ?RPC_TIMEOUT) of - {badrpc, _} -> down; - State -> State - end; -i(local_state, Q) when ?is_amqqueue(Q) -> - {Name, _} = amqqueue:get_pid(Q), - case ets:lookup(ra_state, Name) of - [{_, State}] -> State; - _ -> not_member - end; -i(garbage_collection, Q) when ?is_amqqueue(Q) -> - {Name, _} = amqqueue:get_pid(Q), - try - rabbit_misc:get_gc_info(whereis(Name)) - catch - error:badarg -> - [] - end; -i(members, Q) when ?is_amqqueue(Q) -> - get_nodes(Q); -i(online, Q) -> online(Q); -i(leader, Q) -> leader(Q); -i(open_files, Q) when ?is_amqqueue(Q) -> - {Name, _} = amqqueue:get_pid(Q), - Nodes = get_nodes(Q), - {Data, _} = rpc:multicall(Nodes, ?MODULE, open_files, [Name]), - lists:flatten(Data); -i(single_active_consumer_pid, Q) when ?is_amqqueue(Q) -> - QPid = amqqueue:get_pid(Q), - case ra:local_query(QPid, fun rabbit_fifo:query_single_active_consumer/1) of - {ok, {_, {value, {_ConsumerTag, ChPid}}}, _} -> - ChPid; - {ok, _, _} -> - ''; - {error, _} -> - ''; - {timeout, _} -> - '' - end; -i(single_active_consumer_ctag, Q) when ?is_amqqueue(Q) -> - QPid = amqqueue:get_pid(Q), - case ra:local_query(QPid, - fun rabbit_fifo:query_single_active_consumer/1) of - {ok, {_, {value, {ConsumerTag, _ChPid}}}, _} -> - ConsumerTag; - {ok, _, _} -> - ''; - {error, _} -> - ''; - {timeout, _} -> - '' - end; -i(type, _) -> quorum; -i(messages_ram, Q) when ?is_amqqueue(Q) -> - QPid = amqqueue:get_pid(Q), - case ra:local_query(QPid, - fun rabbit_fifo:query_in_memory_usage/1) of - {ok, {_, {Length, _}}, _} -> - Length; - {error, _} -> - 0; - {timeout, _} -> - 0 - end; -i(message_bytes_ram, Q) when ?is_amqqueue(Q) -> - QPid = amqqueue:get_pid(Q), - case ra:local_query(QPid, - fun rabbit_fifo:query_in_memory_usage/1) of - {ok, {_, {_, Bytes}}, _} -> - Bytes; - {error, _} -> - 0; - {timeout, _} -> - 0 - end; -i(_K, _Q) -> ''. - -open_files(Name) -> - case whereis(Name) of - undefined -> {node(), 0}; - Pid -> case ets:lookup(ra_open_file_metrics, Pid) of - [] -> {node(), 0}; - [{_, Count}] -> {node(), Count} - end - end. - -leader(Q) when ?is_amqqueue(Q) -> - {Name, Leader} = amqqueue:get_pid(Q), - case is_process_alive(Name, Leader) of - true -> Leader; - false -> '' - end. - -peek(Vhost, Queue, Pos) -> - peek(Pos, rabbit_misc:r(Vhost, queue, Queue)). - -peek(Pos, #resource{} = QName) -> - case rabbit_amqqueue:lookup(QName) of - {ok, Q} -> - peek(Pos, Q); - Err -> - Err - end; -peek(Pos, Q) when ?is_amqqueue(Q) andalso ?amqqueue_is_quorum(Q) -> - LeaderPid = amqqueue:get_pid(Q), - case ra:aux_command(LeaderPid, {peek, Pos}) of - {ok, {MsgHeader, Msg0}} -> - Count = case MsgHeader of - #{delivery_count := C} -> C; - _ -> 0 - end, - Msg = rabbit_basic:add_header(<<"x-delivery-count">>, long, - Count, Msg0), - {ok, rabbit_basic:peek_fmt_message(Msg)}; - {error, Err} -> - {error, Err}; - Err -> - Err - end; -peek(_Pos, Q) when ?is_amqqueue(Q) andalso ?amqqueue_is_classic(Q) -> - {error, classic_queue_not_supported}. - -online(Q) when ?is_amqqueue(Q) -> - Nodes = get_nodes(Q), - {Name, _} = amqqueue:get_pid(Q), - [Node || Node <- Nodes, is_process_alive(Name, Node)]. - -format(Q) when ?is_amqqueue(Q) -> - Nodes = get_nodes(Q), - [{members, Nodes}, {online, online(Q)}, {leader, leader(Q)}]. - -is_process_alive(Name, Node) -> - erlang:is_pid(rpc:call(Node, erlang, whereis, [Name], ?RPC_TIMEOUT)). - --spec quorum_messages(rabbit_amqqueue:name()) -> non_neg_integer(). - -quorum_messages(QName) -> - case ets:lookup(queue_coarse_metrics, QName) of - [{_, _, _, M, _}] -> - M; - [] -> - 0 - end. - -quorum_ctag(Int) when is_integer(Int) -> - integer_to_binary(Int); -quorum_ctag(Other) -> - Other. - -maybe_send_reply(_ChPid, undefined) -> ok; -maybe_send_reply(ChPid, Msg) -> ok = rabbit_channel:send_command(ChPid, Msg). - -queue_name(RaFifoState) -> - rabbit_fifo_client:cluster_name(RaFifoState). - -get_default_quorum_initial_group_size(Arguments) -> - case rabbit_misc:table_lookup(Arguments, <<"x-quorum-initial-group-size">>) of - undefined -> application:get_env(rabbit, default_quorum_initial_group_size); - {_Type, Val} -> Val - end. - -select_quorum_nodes(Size, All) when length(All) =< Size -> - All; -select_quorum_nodes(Size, All) -> - Node = node(), - case lists:member(Node, All) of - true -> - select_quorum_nodes(Size - 1, lists:delete(Node, All), [Node]); - false -> - select_quorum_nodes(Size, All, []) - end. - -select_quorum_nodes(0, _, Selected) -> - Selected; -select_quorum_nodes(Size, Rest, Selected) -> - S = lists:nth(rand:uniform(length(Rest)), Rest), - select_quorum_nodes(Size - 1, lists:delete(S, Rest), [S | Selected]). - -%% member with the current leader first -members(Q) when ?amqqueue_is_quorum(Q) -> - {RaName, LeaderNode} = amqqueue:get_pid(Q), - Nodes = lists:delete(LeaderNode, get_nodes(Q)), - [{RaName, N} || N <- [LeaderNode | Nodes]]. - -format_ra_event(ServerId, Evt, QRef) -> - {'$gen_cast', {queue_event, QRef, {ServerId, Evt}}}. - -make_ra_conf(Q, ServerId, TickTimeout) -> - QName = amqqueue:get_name(Q), - RaMachine = ra_machine(Q), - [{ClusterName, _} | _] = Members = members(Q), - UId = ra:new_uid(ra_lib:to_binary(ClusterName)), - FName = rabbit_misc:rs(QName), - Formatter = {?MODULE, format_ra_event, [QName]}, - #{cluster_name => ClusterName, - id => ServerId, - uid => UId, - friendly_name => FName, - metrics_key => QName, - initial_members => Members, - log_init_args => #{uid => UId}, - tick_timeout => TickTimeout, - machine => RaMachine, - ra_event_formatter => Formatter}. - -get_nodes(Q) when ?is_amqqueue(Q) -> - #{nodes := Nodes} = amqqueue:get_type_state(Q), - Nodes. - -update_type_state(Q, Fun) when ?is_amqqueue(Q) -> - Ts = amqqueue:get_type_state(Q), - amqqueue:set_type_state(Q, Fun(Ts)). - -overflow(undefined, Def, _QName) -> Def; -overflow(<<"reject-publish">>, _Def, _QName) -> reject_publish; -overflow(<<"drop-head">>, _Def, _QName) -> drop_head; -overflow(<<"reject-publish-dlx">> = V, Def, QName) -> - rabbit_log:warning("Invalid overflow strategy ~p for quorum queue: ~p", - [V, rabbit_misc:rs(QName)]), - Def. diff --git a/src/rabbit_ra_registry.erl b/src/rabbit_ra_registry.erl deleted file mode 100644 index b02d89eda5..0000000000 --- a/src/rabbit_ra_registry.erl +++ /dev/null @@ -1,25 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License -%% at https://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and -%% limitations under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% The Initial Developer of the Original Code is GoPivotal, Inc. -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_ra_registry). - --export([list_not_quorum_clusters/0]). - -%% Not all ra clusters are quorum queues. We need to keep a list of these so we don't -%% take them into account in operations such as memory calculation and data cleanup. -%% Hardcoded atm -list_not_quorum_clusters() -> - [rabbit_stream_coordinator]. diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl deleted file mode 100644 index c91dbbc105..0000000000 --- a/src/rabbit_reader.erl +++ /dev/null @@ -1,1803 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_reader). - -%% Transitional step until we can require Erlang/OTP 21 and -%% use the now recommended try/catch syntax for obtaining the stack trace. --compile(nowarn_deprecated_function). - -%% This is an AMQP 0-9-1 connection implementation. If AMQP 1.0 plugin is enabled, -%% this module passes control of incoming AMQP 1.0 connections to it. -%% -%% Every connection (as in, a process using this module) -%% is a controlling process for a server socket. -%% -%% Connections have a number of responsibilities: -%% -%% * Performing protocol handshake -%% * Parsing incoming data and dispatching protocol methods -%% * Authenticating clients (with the help of authentication backends) -%% * Enforcing TCP backpressure (throttling clients) -%% * Enforcing connection limits, e.g. channel_max -%% * Channel management -%% * Setting up heartbeater and alarm notifications -%% * Emitting connection and network activity metric events -%% * Gracefully handling client disconnects, channel termination, etc -%% -%% and a few more. -%% -%% Every connection has -%% -%% * a queue collector which is responsible for keeping -%% track of exclusive queues on the connection and their cleanup. -%% * a heartbeater that's responsible for sending heartbeat frames to clients, -%% keeping track of the incoming ones and notifying connection about -%% heartbeat timeouts -%% * Stats timer, a timer that is used to periodically emit metric events -%% -%% Some dependencies are started under a separate supervisor to avoid deadlocks -%% during system shutdown. See rabbit_channel_sup:start_link/0 for details. -%% -%% Reader processes are special processes (in the OTP sense). - --include("rabbit_framing.hrl"). --include("rabbit.hrl"). - --export([start_link/2, info_keys/0, info/1, info/2, force_event_refresh/2, - shutdown/2]). - --export([system_continue/3, system_terminate/4, system_code_change/4]). - --export([init/3, mainloop/4, recvloop/4]). - --export([conserve_resources/3, server_properties/1]). - --define(NORMAL_TIMEOUT, 3). --define(CLOSING_TIMEOUT, 30). --define(CHANNEL_TERMINATION_TIMEOUT, 3). -%% we wait for this many seconds before closing TCP connection -%% with a client that failed to log in. Provides some relief -%% from connection storms and DoS. --define(SILENT_CLOSE_DELAY, 3). --define(CHANNEL_MIN, 1). - -%%-------------------------------------------------------------------------- - --record(v1, { - %% parent process - parent, - %% socket - sock, - %% connection state, see connection record - connection, - callback, - recv_len, - pending_recv, - %% pre_init | securing | running | blocking | blocked | closing | closed | {become, F} - connection_state, - %% see comment in rabbit_connection_sup:start_link/0 - helper_sup, - %% takes care of cleaning up exclusive queues, - %% see rabbit_queue_collector - queue_collector, - %% sends and receives heartbeat frames, - %% see rabbit_heartbeat - heartbeater, - %% timer used to emit statistics - stats_timer, - %% channel supervisor - channel_sup_sup_pid, - %% how many channels this connection has - channel_count, - %% throttling state, for both - %% credit- and resource-driven flow control - throttle, - proxy_socket}). - --record(throttle, { - %% never | timestamp() - last_blocked_at, - %% a set of the reasons why we are - %% blocked: {resource, memory}, {resource, disk}. - %% More reasons can be added in the future. - blocked_by, - %% true if received any publishes, false otherwise - %% note that this will also be true when connection is - %% already blocked - should_block, - %% true if we had we sent a connection.blocked, - %% false otherwise - connection_blocked_message_sent -}). - --define(STATISTICS_KEYS, [pid, recv_oct, recv_cnt, send_oct, send_cnt, - send_pend, state, channels, reductions, - garbage_collection]). - --define(SIMPLE_METRICS, [pid, recv_oct, send_oct, reductions]). --define(OTHER_METRICS, [recv_cnt, send_cnt, send_pend, state, channels, - garbage_collection]). - --define(CREATION_EVENT_KEYS, - [pid, name, port, peer_port, host, - peer_host, ssl, peer_cert_subject, peer_cert_issuer, - peer_cert_validity, auth_mechanism, ssl_protocol, - ssl_key_exchange, ssl_cipher, ssl_hash, protocol, user, vhost, - timeout, frame_max, channel_max, client_properties, connected_at, - node, user_who_performed_action]). - --define(INFO_KEYS, ?CREATION_EVENT_KEYS ++ ?STATISTICS_KEYS -- [pid]). - --define(AUTH_NOTIFICATION_INFO_KEYS, - [host, name, peer_host, peer_port, protocol, auth_mechanism, - ssl, ssl_protocol, ssl_cipher, peer_cert_issuer, peer_cert_subject, - peer_cert_validity]). - --define(IS_RUNNING(State), - (State#v1.connection_state =:= running orelse - State#v1.connection_state =:= blocked)). - --define(IS_STOPPING(State), - (State#v1.connection_state =:= closing orelse - State#v1.connection_state =:= closed)). - -%%-------------------------------------------------------------------------- - --type resource_alert() :: {WasAlarmSetForNode :: boolean(), - IsThereAnyAlarmsWithSameSourceInTheCluster :: boolean(), - NodeForWhichAlarmWasSetOrCleared :: node()}. - -%%-------------------------------------------------------------------------- - --spec start_link(pid(), any()) -> rabbit_types:ok(pid()). - -start_link(HelperSup, Ref) -> - Pid = proc_lib:spawn_link(?MODULE, init, [self(), HelperSup, Ref]), - - {ok, Pid}. - --spec shutdown(pid(), string()) -> 'ok'. - -shutdown(Pid, Explanation) -> - gen_server:call(Pid, {shutdown, Explanation}, infinity). - --spec init(pid(), pid(), any()) -> no_return(). - -init(Parent, HelperSup, Ref) -> - ?LG_PROCESS_TYPE(reader), - {ok, Sock} = rabbit_networking:handshake(Ref, - application:get_env(rabbit, proxy_protocol, false)), - Deb = sys:debug_options([]), - start_connection(Parent, HelperSup, Deb, Sock). - --spec system_continue(_,_,{[binary()], non_neg_integer(), #v1{}}) -> any(). - -system_continue(Parent, Deb, {Buf, BufLen, State}) -> - mainloop(Deb, Buf, BufLen, State#v1{parent = Parent}). - --spec system_terminate(_,_,_,_) -> no_return(). - -system_terminate(Reason, _Parent, _Deb, _State) -> - exit(Reason). - --spec system_code_change(_,_,_,_) -> {'ok',_}. - -system_code_change(Misc, _Module, _OldVsn, _Extra) -> - {ok, Misc}. - --spec info_keys() -> rabbit_types:info_keys(). - -info_keys() -> ?INFO_KEYS. - --spec info(pid()) -> rabbit_types:infos(). - -info(Pid) -> - gen_server:call(Pid, info, infinity). - --spec info(pid(), rabbit_types:info_keys()) -> rabbit_types:infos(). - -info(Pid, Items) -> - case gen_server:call(Pid, {info, Items}, infinity) of - {ok, Res} -> Res; - {error, Error} -> throw(Error) - end. - --spec force_event_refresh(pid(), reference()) -> 'ok'. - -% Note: https://www.pivotaltracker.com/story/show/166962656 -% This event is necessary for the stats timer to be initialized with -% the correct values once the management agent has started -force_event_refresh(Pid, Ref) -> - gen_server:cast(Pid, {force_event_refresh, Ref}). - --spec conserve_resources(pid(), atom(), resource_alert()) -> 'ok'. - -conserve_resources(Pid, Source, {_, Conserve, _}) -> - Pid ! {conserve_resources, Source, Conserve}, - ok. - --spec server_properties(rabbit_types:protocol()) -> - rabbit_framing:amqp_table(). - -server_properties(Protocol) -> - {ok, Product} = application:get_key(rabbit, description), - {ok, Version} = application:get_key(rabbit, vsn), - - %% Get any configuration-specified server properties - {ok, RawConfigServerProps} = application:get_env(rabbit, - server_properties), - - %% Normalize the simplified (2-tuple) and unsimplified (3-tuple) forms - %% from the config and merge them with the generated built-in properties - NormalizedConfigServerProps = - [{<<"capabilities">>, table, server_capabilities(Protocol)} | - [case X of - {KeyAtom, Value} -> {list_to_binary(atom_to_list(KeyAtom)), - longstr, - maybe_list_to_binary(Value)}; - {BinKey, Type, Value} -> {BinKey, Type, Value} - end || X <- RawConfigServerProps ++ - [{product, Product}, - {version, Version}, - {cluster_name, rabbit_nodes:cluster_name()}, - {platform, rabbit_misc:platform_and_version()}, - {copyright, ?COPYRIGHT_MESSAGE}, - {information, ?INFORMATION_MESSAGE}]]], - - %% Filter duplicated properties in favour of config file provided values - lists:usort(fun ({K1,_,_}, {K2,_,_}) -> K1 =< K2 end, - NormalizedConfigServerProps). - -maybe_list_to_binary(V) when is_binary(V) -> V; -maybe_list_to_binary(V) when is_list(V) -> list_to_binary(V). - -server_capabilities(rabbit_framing_amqp_0_9_1) -> - [{<<"publisher_confirms">>, bool, true}, - {<<"exchange_exchange_bindings">>, bool, true}, - {<<"basic.nack">>, bool, true}, - {<<"consumer_cancel_notify">>, bool, true}, - {<<"connection.blocked">>, bool, true}, - {<<"consumer_priorities">>, bool, true}, - {<<"authentication_failure_close">>, bool, true}, - {<<"per_consumer_qos">>, bool, true}, - {<<"direct_reply_to">>, bool, true}]; -server_capabilities(_) -> - []. - -%%-------------------------------------------------------------------------- - -socket_error(Reason) when is_atom(Reason) -> - rabbit_log_connection:error("Error on AMQP connection ~p: ~s~n", - [self(), rabbit_misc:format_inet_error(Reason)]); -socket_error(Reason) -> - Fmt = "Error on AMQP connection ~p:~n~p~n", - Args = [self(), Reason], - case Reason of - %% The socket was closed while upgrading to SSL. - %% This is presumably a TCP healthcheck, so don't log - %% it unless specified otherwise. - {ssl_upgrade_error, closed} -> - %% Lager sinks (rabbit_log_connection) - %% are handled by the lager parse_transform. - %% Hence have to define the loglevel as a function call. - rabbit_log_connection:debug(Fmt, Args); - _ -> - rabbit_log_connection:error(Fmt, Args) - end. - -inet_op(F) -> rabbit_misc:throw_on_error(inet_error, F). - -socket_op(Sock, Fun) -> - RealSocket = rabbit_net:unwrap_socket(Sock), - case Fun(Sock) of - {ok, Res} -> Res; - {error, Reason} -> socket_error(Reason), - rabbit_net:fast_close(RealSocket), - exit(normal) - end. - --spec start_connection(pid(), pid(), any(), rabbit_net:socket()) -> - no_return(). - -start_connection(Parent, HelperSup, Deb, Sock) -> - process_flag(trap_exit, true), - RealSocket = rabbit_net:unwrap_socket(Sock), - Name = case rabbit_net:connection_string(Sock, inbound) of - {ok, Str} -> list_to_binary(Str); - {error, enotconn} -> rabbit_net:fast_close(RealSocket), - exit(normal); - {error, Reason} -> socket_error(Reason), - rabbit_net:fast_close(RealSocket), - exit(normal) - end, - {ok, HandshakeTimeout} = application:get_env(rabbit, handshake_timeout), - InitialFrameMax = application:get_env(rabbit, initial_frame_max, ?FRAME_MIN_SIZE), - erlang:send_after(HandshakeTimeout, self(), handshake_timeout), - {PeerHost, PeerPort, Host, Port} = - socket_op(Sock, fun (S) -> rabbit_net:socket_ends(S, inbound) end), - ?store_proc_name(Name), - State = #v1{parent = Parent, - sock = RealSocket, - connection = #connection{ - name = Name, - log_name = Name, - host = Host, - peer_host = PeerHost, - port = Port, - peer_port = PeerPort, - protocol = none, - user = none, - timeout_sec = (HandshakeTimeout / 1000), - frame_max = InitialFrameMax, - vhost = none, - client_properties = none, - capabilities = [], - auth_mechanism = none, - auth_state = none, - connected_at = os:system_time( - milli_seconds)}, - callback = uninitialized_callback, - recv_len = 0, - pending_recv = false, - connection_state = pre_init, - queue_collector = undefined, %% started on tune-ok - helper_sup = HelperSup, - heartbeater = none, - channel_sup_sup_pid = none, - channel_count = 0, - throttle = #throttle{ - last_blocked_at = never, - should_block = false, - blocked_by = sets:new(), - connection_blocked_message_sent = false - }, - proxy_socket = rabbit_net:maybe_get_proxy_socket(Sock)}, - try - case run({?MODULE, recvloop, - [Deb, [], 0, switch_callback(rabbit_event:init_stats_timer( - State, #v1.stats_timer), - handshake, 8)]}) of - %% connection was closed cleanly by the client - #v1{connection = #connection{user = #user{username = Username}, - vhost = VHost}} -> - rabbit_log_connection:info("closing AMQP connection ~p (~s, vhost: '~s', user: '~s')~n", - [self(), dynamic_connection_name(Name), VHost, Username]); - %% just to be more defensive - _ -> - rabbit_log_connection:info("closing AMQP connection ~p (~s)~n", - [self(), dynamic_connection_name(Name)]) - end - catch - Ex -> - log_connection_exception(dynamic_connection_name(Name), Ex) - after - %% We don't call gen_tcp:close/1 here since it waits for - %% pending output to be sent, which results in unnecessary - %% delays. We could just terminate - the reader is the - %% controlling process and hence its termination will close - %% the socket. However, to keep the file_handle_cache - %% accounting as accurate as possible we ought to close the - %% socket w/o delay before termination. - rabbit_net:fast_close(RealSocket), - rabbit_networking:unregister_connection(self()), - rabbit_core_metrics:connection_closed(self()), - ClientProperties = case get(client_properties) of - undefined -> - []; - Properties -> - Properties - end, - EventProperties = [{name, Name}, - {pid, self()}, - {node, node()}, - {client_properties, ClientProperties}], - EventProperties1 = case get(connection_user_provided_name) of - undefined -> - EventProperties; - ConnectionUserProvidedName -> - [{user_provided_name, ConnectionUserProvidedName} | EventProperties] - end, - rabbit_event:notify(connection_closed, EventProperties1) - end, - done. - -log_connection_exception(Name, Ex) -> - Severity = case Ex of - connection_closed_with_no_data_received -> debug; - {connection_closed_abruptly, _} -> warning; - connection_closed_abruptly -> warning; - _ -> error - end, - log_connection_exception(Severity, Name, Ex). - -log_connection_exception(Severity, Name, {heartbeat_timeout, TimeoutSec}) -> - %% Long line to avoid extra spaces and line breaks in log - log_connection_exception_with_severity(Severity, - "closing AMQP connection ~p (~s):~n" - "missed heartbeats from client, timeout: ~ps~n", - [self(), Name, TimeoutSec]); -log_connection_exception(Severity, Name, {connection_closed_abruptly, - #v1{connection = #connection{user = #user{username = Username}, - vhost = VHost}}}) -> - log_connection_exception_with_severity(Severity, - "closing AMQP connection ~p (~s, vhost: '~s', user: '~s'):~nclient unexpectedly closed TCP connection~n", - [self(), Name, VHost, Username]); -%% when client abruptly closes connection before connection.open/authentication/authorization -%% succeeded, don't log username and vhost as 'none' -log_connection_exception(Severity, Name, {connection_closed_abruptly, _}) -> - log_connection_exception_with_severity(Severity, - "closing AMQP connection ~p (~s):~nclient unexpectedly closed TCP connection~n", - [self(), Name]); -%% failed connection.tune negotiations -log_connection_exception(Severity, Name, {handshake_error, tuning, _Channel, - {exit, #amqp_error{explanation = Explanation}, - _Method, _Stacktrace}}) -> - log_connection_exception_with_severity(Severity, - "closing AMQP connection ~p (~s):~nfailed to negotiate connection parameters: ~s~n", - [self(), Name, Explanation]); -%% old exception structure -log_connection_exception(Severity, Name, connection_closed_abruptly) -> - log_connection_exception_with_severity(Severity, - "closing AMQP connection ~p (~s):~n" - "client unexpectedly closed TCP connection~n", - [self(), Name]); -log_connection_exception(Severity, Name, Ex) -> - log_connection_exception_with_severity(Severity, - "closing AMQP connection ~p (~s):~n~p~n", - [self(), Name, Ex]). - -log_connection_exception_with_severity(Severity, Fmt, Args) -> - case Severity of - debug -> rabbit_log_connection:debug(Fmt, Args); - warning -> rabbit_log_connection:warning(Fmt, Args); - error -> rabbit_log_connection:error(Fmt, Args) - end. - -run({M, F, A}) -> - try apply(M, F, A) - catch {become, MFA} -> run(MFA) - end. - -recvloop(Deb, Buf, BufLen, State = #v1{pending_recv = true}) -> - mainloop(Deb, Buf, BufLen, State); -recvloop(Deb, Buf, BufLen, State = #v1{connection_state = blocked}) -> - mainloop(Deb, Buf, BufLen, State); -recvloop(Deb, Buf, BufLen, State = #v1{connection_state = {become, F}}) -> - throw({become, F(Deb, Buf, BufLen, State)}); -recvloop(Deb, Buf, BufLen, State = #v1{sock = Sock, recv_len = RecvLen}) - when BufLen < RecvLen -> - case rabbit_net:setopts(Sock, [{active, once}]) of - ok -> mainloop(Deb, Buf, BufLen, - State#v1{pending_recv = true}); - {error, Reason} -> stop(Reason, State) - end; -recvloop(Deb, [B], _BufLen, State) -> - {Rest, State1} = handle_input(State#v1.callback, B, State), - recvloop(Deb, [Rest], size(Rest), State1); -recvloop(Deb, Buf, BufLen, State = #v1{recv_len = RecvLen}) -> - {DataLRev, RestLRev} = binlist_split(BufLen - RecvLen, Buf, []), - Data = list_to_binary(lists:reverse(DataLRev)), - {<<>>, State1} = handle_input(State#v1.callback, Data, State), - recvloop(Deb, lists:reverse(RestLRev), BufLen - RecvLen, State1). - -binlist_split(0, L, Acc) -> - {L, Acc}; -binlist_split(Len, L, [Acc0|Acc]) when Len < 0 -> - {H, T} = split_binary(Acc0, -Len), - {[H|L], [T|Acc]}; -binlist_split(Len, [H|T], Acc) -> - binlist_split(Len - size(H), T, [H|Acc]). - --spec mainloop(_,[binary()], non_neg_integer(), #v1{}) -> any(). - -mainloop(Deb, Buf, BufLen, State = #v1{sock = Sock, - connection_state = CS, - connection = #connection{ - name = ConnName}}) -> - Recv = rabbit_net:recv(Sock), - case CS of - pre_init when Buf =:= [] -> - %% We only log incoming connections when either the - %% first byte was received or there was an error (eg. a - %% timeout). - %% - %% The goal is to not log TCP healthchecks (a connection - %% with no data received) unless specified otherwise. - Fmt = "accepting AMQP connection ~p (~s)~n", - Args = [self(), ConnName], - case Recv of - closed -> rabbit_log_connection:debug(Fmt, Args); - _ -> rabbit_log_connection:info(Fmt, Args) - end; - _ -> - ok - end, - case Recv of - {data, Data} -> - recvloop(Deb, [Data | Buf], BufLen + size(Data), - State#v1{pending_recv = false}); - closed when State#v1.connection_state =:= closed -> - State; - closed when CS =:= pre_init andalso Buf =:= [] -> - stop(tcp_healthcheck, State); - closed -> - stop(closed, State); - {other, {heartbeat_send_error, Reason}} -> - %% The only portable way to detect disconnect on blocked - %% connection is to wait for heartbeat send failure. - stop(Reason, State); - {error, Reason} -> - stop(Reason, State); - {other, {system, From, Request}} -> - sys:handle_system_msg(Request, From, State#v1.parent, - ?MODULE, Deb, {Buf, BufLen, State}); - {other, Other} -> - case handle_other(Other, State) of - stop -> State; - NewState -> recvloop(Deb, Buf, BufLen, NewState) - end - end. - --spec stop(_, #v1{}) -> no_return(). -stop(tcp_healthcheck, State) -> - %% The connection was closed before any packet was received. It's - %% probably a load-balancer healthcheck: don't consider this a - %% failure. - maybe_emit_stats(State), - throw(connection_closed_with_no_data_received); -stop(closed, State) -> - maybe_emit_stats(State), - throw({connection_closed_abruptly, State}); -stop(Reason, State) -> - maybe_emit_stats(State), - throw({inet_error, Reason}). - -handle_other({conserve_resources, Source, Conserve}, - State = #v1{throttle = Throttle = #throttle{blocked_by = Blockers}}) -> - Resource = {resource, Source}, - Blockers1 = case Conserve of - true -> sets:add_element(Resource, Blockers); - false -> sets:del_element(Resource, Blockers) - end, - control_throttle(State#v1{throttle = Throttle#throttle{blocked_by = Blockers1}}); -handle_other({channel_closing, ChPid}, State) -> - ok = rabbit_channel:ready_for_close(ChPid), - {_, State1} = channel_cleanup(ChPid, State), - maybe_close(control_throttle(State1)); -handle_other({'EXIT', Parent, normal}, State = #v1{parent = Parent}) -> - %% rabbitmq/rabbitmq-server#544 - %% The connection port process has exited due to the TCP socket being closed. - %% Handle this case in the same manner as receiving {error, closed} - stop(closed, State); -handle_other({'EXIT', Parent, Reason}, State = #v1{parent = Parent}) -> - Msg = io_lib:format("broker forced connection closure with reason '~w'", [Reason]), - terminate(Msg, State), - %% this is what we are expected to do according to - %% https://www.erlang.org/doc/man/sys.html - %% - %% If we wanted to be *really* nice we should wait for a while for - %% clients to close the socket at their end, just as we do in the - %% ordinary error case. However, since this termination is - %% initiated by our parent it is probably more important to exit - %% quickly. - maybe_emit_stats(State), - exit(Reason); -handle_other({channel_exit, _Channel, E = {writer, send_failed, _E}}, State) -> - maybe_emit_stats(State), - throw(E); -handle_other({channel_exit, Channel, Reason}, State) -> - handle_exception(State, Channel, Reason); -handle_other({'DOWN', _MRef, process, ChPid, Reason}, State) -> - handle_dependent_exit(ChPid, Reason, State); -handle_other(terminate_connection, State) -> - maybe_emit_stats(State), - stop; -handle_other(handshake_timeout, State) - when ?IS_RUNNING(State) orelse ?IS_STOPPING(State) -> - State; -handle_other(handshake_timeout, State) -> - maybe_emit_stats(State), - throw({handshake_timeout, State#v1.callback}); -handle_other(heartbeat_timeout, State = #v1{connection_state = closed}) -> - State; -handle_other(heartbeat_timeout, - State = #v1{connection = #connection{timeout_sec = T}}) -> - maybe_emit_stats(State), - throw({heartbeat_timeout, T}); -handle_other({'$gen_call', From, {shutdown, Explanation}}, State) -> - {ForceTermination, NewState} = terminate(Explanation, State), - gen_server:reply(From, ok), - case ForceTermination of - force -> stop; - normal -> NewState - end; -handle_other({'$gen_call', From, info}, State) -> - gen_server:reply(From, infos(?INFO_KEYS, State)), - State; -handle_other({'$gen_call', From, {info, Items}}, State) -> - gen_server:reply(From, try {ok, infos(Items, State)} - catch Error -> {error, Error} - end), - State; -handle_other({'$gen_cast', {force_event_refresh, Ref}}, State) - when ?IS_RUNNING(State) -> - rabbit_event:notify( - connection_created, - augment_infos_with_user_provided_connection_name( - [{type, network} | infos(?CREATION_EVENT_KEYS, State)], State), - Ref), - rabbit_event:init_stats_timer(State, #v1.stats_timer); -handle_other({'$gen_cast', {force_event_refresh, _Ref}}, State) -> - %% Ignore, we will emit a created event once we start running. - State; -handle_other(ensure_stats, State) -> - ensure_stats_timer(State); -handle_other(emit_stats, State) -> - emit_stats(State); -handle_other({bump_credit, Msg}, State) -> - %% Here we are receiving credit by some channel process. - credit_flow:handle_bump_msg(Msg), - control_throttle(State); -handle_other(Other, State) -> - %% internal error -> something worth dying for - maybe_emit_stats(State), - exit({unexpected_message, Other}). - -switch_callback(State, Callback, Length) -> - State#v1{callback = Callback, recv_len = Length}. - -terminate(Explanation, State) when ?IS_RUNNING(State) -> - {normal, handle_exception(State, 0, - rabbit_misc:amqp_error( - connection_forced, "~s", [Explanation], none))}; -terminate(_Explanation, State) -> - {force, State}. - -send_blocked(#v1{connection = #connection{protocol = Protocol, - capabilities = Capabilities}, - sock = Sock}, Reason) -> - case rabbit_misc:table_lookup(Capabilities, <<"connection.blocked">>) of - {bool, true} -> - - ok = send_on_channel0(Sock, #'connection.blocked'{reason = Reason}, - Protocol); - _ -> - ok - end. - -send_unblocked(#v1{connection = #connection{protocol = Protocol, - capabilities = Capabilities}, - sock = Sock}) -> - case rabbit_misc:table_lookup(Capabilities, <<"connection.blocked">>) of - {bool, true} -> - ok = send_on_channel0(Sock, #'connection.unblocked'{}, Protocol); - _ -> - ok - end. - -%%-------------------------------------------------------------------------- -%% error handling / termination - -close_connection(State = #v1{queue_collector = Collector, - connection = #connection{ - timeout_sec = TimeoutSec}}) -> - %% The spec says "Exclusive queues may only be accessed by the - %% current connection, and are deleted when that connection - %% closes." This does not strictly imply synchrony, but in - %% practice it seems to be what people assume. - clean_up_exclusive_queues(Collector), - %% We terminate the connection after the specified interval, but - %% no later than ?CLOSING_TIMEOUT seconds. - erlang:send_after((if TimeoutSec > 0 andalso - TimeoutSec < ?CLOSING_TIMEOUT -> TimeoutSec; - true -> ?CLOSING_TIMEOUT - end) * 1000, self(), terminate_connection), - State#v1{connection_state = closed}. - -%% queue collector will be undefined when connection -%% tuning was never performed or didn't finish. In such cases -%% there's also nothing to clean up. -clean_up_exclusive_queues(undefined) -> - ok; - -clean_up_exclusive_queues(Collector) -> - rabbit_queue_collector:delete_all(Collector). - -handle_dependent_exit(ChPid, Reason, State) -> - {Channel, State1} = channel_cleanup(ChPid, State), - case {Channel, termination_kind(Reason)} of - {undefined, controlled} -> State1; - {undefined, uncontrolled} -> handle_uncontrolled_channel_close(ChPid), - exit({abnormal_dependent_exit, - ChPid, Reason}); - {_, controlled} -> maybe_close(control_throttle(State1)); - {_, uncontrolled} -> handle_uncontrolled_channel_close(ChPid), - State2 = handle_exception( - State1, Channel, Reason), - maybe_close(control_throttle(State2)) - end. - -terminate_channels(#v1{channel_count = 0} = State) -> - State; -terminate_channels(#v1{channel_count = ChannelCount} = State) -> - lists:foreach(fun rabbit_channel:shutdown/1, all_channels()), - Timeout = 1000 * ?CHANNEL_TERMINATION_TIMEOUT * ChannelCount, - TimerRef = erlang:send_after(Timeout, self(), cancel_wait), - wait_for_channel_termination(ChannelCount, TimerRef, State). - -wait_for_channel_termination(0, TimerRef, State) -> - case erlang:cancel_timer(TimerRef) of - false -> receive - cancel_wait -> State - end; - _ -> State - end; -wait_for_channel_termination(N, TimerRef, - State = #v1{connection_state = CS, - connection = #connection{ - log_name = ConnName, - user = User, - vhost = VHost}, - sock = Sock}) -> - receive - {'DOWN', _MRef, process, ChPid, Reason} -> - {Channel, State1} = channel_cleanup(ChPid, State), - case {Channel, termination_kind(Reason)} of - {undefined, _} -> - exit({abnormal_dependent_exit, ChPid, Reason}); - {_, controlled} -> - wait_for_channel_termination(N-1, TimerRef, State1); - {_, uncontrolled} -> - rabbit_log_connection:error( - "Error on AMQP connection ~p (~s, vhost: '~s'," - " user: '~s', state: ~p), channel ~p:" - "error while terminating:~n~p~n", - [self(), ConnName, VHost, User#user.username, - CS, Channel, Reason]), - handle_uncontrolled_channel_close(ChPid), - wait_for_channel_termination(N-1, TimerRef, State1) - end; - {'EXIT', Sock, _Reason} -> - clean_up_all_channels(State), - exit(normal); - cancel_wait -> - exit(channel_termination_timeout) - end. - -maybe_close(State = #v1{connection_state = closing, - channel_count = 0, - connection = #connection{protocol = Protocol}, - sock = Sock}) -> - NewState = close_connection(State), - ok = send_on_channel0(Sock, #'connection.close_ok'{}, Protocol), - NewState; -maybe_close(State) -> - State. - -termination_kind(normal) -> controlled; -termination_kind(_) -> uncontrolled. - -format_hard_error(#amqp_error{name = N, explanation = E, method = M}) -> - io_lib:format("operation ~s caused a connection exception ~s: ~p", [M, N, E]); -format_hard_error(Reason) -> - case io_lib:deep_char_list(Reason) of - true -> Reason; - false -> rabbit_misc:format("~p", [Reason]) - end. - -log_hard_error(#v1{connection_state = CS, - connection = #connection{ - log_name = ConnName, - user = User, - vhost = VHost}}, Channel, Reason) -> - rabbit_log_connection:error( - "Error on AMQP connection ~p (~s, vhost: '~s'," - " user: '~s', state: ~p), channel ~p:~n ~s~n", - [self(), ConnName, VHost, User#user.username, CS, Channel, format_hard_error(Reason)]). - -handle_exception(State = #v1{connection_state = closed}, Channel, Reason) -> - log_hard_error(State, Channel, Reason), - State; -handle_exception(State = #v1{connection = #connection{protocol = Protocol}, - connection_state = CS}, - Channel, Reason) - when ?IS_RUNNING(State) orelse CS =:= closing -> - respond_and_close(State, Channel, Protocol, Reason, Reason); -%% authentication failure -handle_exception(State = #v1{connection = #connection{protocol = Protocol, - log_name = ConnName, - capabilities = Capabilities}, - connection_state = starting}, - Channel, Reason = #amqp_error{name = access_refused, - explanation = ErrMsg}) -> - rabbit_log_connection:error( - "Error on AMQP connection ~p (~s, state: ~p):~n~s~n", - [self(), ConnName, starting, ErrMsg]), - %% respect authentication failure notification capability - case rabbit_misc:table_lookup(Capabilities, - <<"authentication_failure_close">>) of - {bool, true} -> - send_error_on_channel0_and_close(Channel, Protocol, Reason, State); - _ -> - close_connection(terminate_channels(State)) - end; -%% when loopback-only user tries to connect from a non-local host -%% when user tries to access a vhost it has no permissions for -handle_exception(State = #v1{connection = #connection{protocol = Protocol, - log_name = ConnName, - user = User}, - connection_state = opening}, - Channel, Reason = #amqp_error{name = not_allowed, - explanation = ErrMsg}) -> - rabbit_log_connection:error( - "Error on AMQP connection ~p (~s, user: '~s', state: ~p):~n~s~n", - [self(), ConnName, User#user.username, opening, ErrMsg]), - send_error_on_channel0_and_close(Channel, Protocol, Reason, State); -handle_exception(State = #v1{connection = #connection{protocol = Protocol}, - connection_state = CS = opening}, - Channel, Reason = #amqp_error{}) -> - respond_and_close(State, Channel, Protocol, Reason, - {handshake_error, CS, Reason}); -%% when negotiation fails, e.g. due to channel_max being higher than the -%% maximum allowed limit -handle_exception(State = #v1{connection = #connection{protocol = Protocol, - log_name = ConnName, - user = User}, - connection_state = tuning}, - Channel, Reason = #amqp_error{name = not_allowed, - explanation = ErrMsg}) -> - rabbit_log_connection:error( - "Error on AMQP connection ~p (~s," - " user: '~s', state: ~p):~n~s~n", - [self(), ConnName, User#user.username, tuning, ErrMsg]), - send_error_on_channel0_and_close(Channel, Protocol, Reason, State); -handle_exception(State, Channel, Reason) -> - %% We don't trust the client at this point - force them to wait - %% for a bit so they can't DOS us with repeated failed logins etc. - timer:sleep(?SILENT_CLOSE_DELAY * 1000), - throw({handshake_error, State#v1.connection_state, Channel, Reason}). - -%% we've "lost sync" with the client and hence must not accept any -%% more input --spec fatal_frame_error(_, _, _, _, _) -> no_return(). -fatal_frame_error(Error, Type, Channel, Payload, State) -> - frame_error(Error, Type, Channel, Payload, State), - %% grace period to allow transmission of error - timer:sleep(?SILENT_CLOSE_DELAY * 1000), - throw(fatal_frame_error). - -frame_error(Error, Type, Channel, Payload, State) -> - {Str, Bin} = payload_snippet(Payload), - handle_exception(State, Channel, - rabbit_misc:amqp_error(frame_error, - "type ~p, ~s octets = ~p: ~p", - [Type, Str, Bin, Error], none)). - -unexpected_frame(Type, Channel, Payload, State) -> - {Str, Bin} = payload_snippet(Payload), - handle_exception(State, Channel, - rabbit_misc:amqp_error(unexpected_frame, - "type ~p, ~s octets = ~p", - [Type, Str, Bin], none)). - -payload_snippet(Payload) when size(Payload) =< 16 -> - {"all", Payload}; -payload_snippet(<<Snippet:16/binary, _/binary>>) -> - {"first 16", Snippet}. - -%%-------------------------------------------------------------------------- - -create_channel(_Channel, - #v1{channel_count = ChannelCount, - connection = #connection{channel_max = ChannelMax}}) - when ChannelMax /= 0 andalso ChannelCount >= ChannelMax -> - {error, rabbit_misc:amqp_error( - not_allowed, "number of channels opened (~w) has reached the " - "negotiated channel_max (~w)", - [ChannelCount, ChannelMax], 'none')}; -create_channel(Channel, - #v1{sock = Sock, - queue_collector = Collector, - channel_sup_sup_pid = ChanSupSup, - channel_count = ChannelCount, - connection = - #connection{name = Name, - protocol = Protocol, - frame_max = FrameMax, - vhost = VHost, - capabilities = Capabilities, - user = #user{username = Username} = User} - } = State) -> - case rabbit_auth_backend_internal:is_over_channel_limit(Username) of - false -> - {ok, _ChSupPid, {ChPid, AState}} = - rabbit_channel_sup_sup:start_channel( - ChanSupSup, {tcp, Sock, Channel, FrameMax, self(), Name, - Protocol, User, VHost, Capabilities, - Collector}), - MRef = erlang:monitor(process, ChPid), - put({ch_pid, ChPid}, {Channel, MRef}), - put({channel, Channel}, {ChPid, AState}), - {ok, {ChPid, AState}, State#v1{channel_count = ChannelCount + 1}}; - {true, Limit} -> - {error, rabbit_misc:amqp_error(not_allowed, - "number of channels opened for user '~s' has reached " - "the maximum allowed user limit of (~w)", - [Username, Limit], 'none')} - end. - -channel_cleanup(ChPid, State = #v1{channel_count = ChannelCount}) -> - case get({ch_pid, ChPid}) of - undefined -> {undefined, State}; - {Channel, MRef} -> credit_flow:peer_down(ChPid), - erase({channel, Channel}), - erase({ch_pid, ChPid}), - erlang:demonitor(MRef, [flush]), - {Channel, State#v1{channel_count = ChannelCount - 1}} - end. - -all_channels() -> [ChPid || {{ch_pid, ChPid}, _ChannelMRef} <- get()]. - -clean_up_all_channels(State) -> - CleanupFun = fun(ChPid) -> - channel_cleanup(ChPid, State) - end, - lists:foreach(CleanupFun, all_channels()). - -%%-------------------------------------------------------------------------- - -handle_frame(Type, 0, Payload, - State = #v1{connection = #connection{protocol = Protocol}}) - when ?IS_STOPPING(State) -> - case rabbit_command_assembler:analyze_frame(Type, Payload, Protocol) of - {method, MethodName, FieldsBin} -> - handle_method0(MethodName, FieldsBin, State); - _Other -> State - end; -handle_frame(Type, 0, Payload, - State = #v1{connection = #connection{protocol = Protocol}}) -> - case rabbit_command_assembler:analyze_frame(Type, Payload, Protocol) of - error -> frame_error(unknown_frame, Type, 0, Payload, State); - heartbeat -> State; - {method, MethodName, FieldsBin} -> - handle_method0(MethodName, FieldsBin, State); - _Other -> unexpected_frame(Type, 0, Payload, State) - end; -handle_frame(Type, Channel, Payload, - State = #v1{connection = #connection{protocol = Protocol}}) - when ?IS_RUNNING(State) -> - case rabbit_command_assembler:analyze_frame(Type, Payload, Protocol) of - error -> frame_error(unknown_frame, Type, Channel, Payload, State); - heartbeat -> unexpected_frame(Type, Channel, Payload, State); - Frame -> process_frame(Frame, Channel, State) - end; -handle_frame(_Type, _Channel, _Payload, State) when ?IS_STOPPING(State) -> - State; -handle_frame(Type, Channel, Payload, State) -> - unexpected_frame(Type, Channel, Payload, State). - -process_frame(Frame, Channel, State) -> - ChKey = {channel, Channel}, - case (case get(ChKey) of - undefined -> create_channel(Channel, State); - Other -> {ok, Other, State} - end) of - {error, Error} -> - handle_exception(State, Channel, Error); - {ok, {ChPid, AState}, State1} -> - case rabbit_command_assembler:process(Frame, AState) of - {ok, NewAState} -> - put(ChKey, {ChPid, NewAState}), - post_process_frame(Frame, ChPid, State1); - {ok, Method, NewAState} -> - rabbit_channel:do(ChPid, Method), - put(ChKey, {ChPid, NewAState}), - post_process_frame(Frame, ChPid, State1); - {ok, Method, Content, NewAState} -> - rabbit_channel:do_flow(ChPid, Method, Content), - put(ChKey, {ChPid, NewAState}), - post_process_frame(Frame, ChPid, control_throttle(State1)); - {error, Reason} -> - handle_exception(State1, Channel, Reason) - end - end. - -post_process_frame({method, 'channel.close_ok', _}, ChPid, State) -> - {_, State1} = channel_cleanup(ChPid, State), - %% This is not strictly necessary, but more obviously - %% correct. Also note that we do not need to call maybe_close/1 - %% since we cannot possibly be in the 'closing' state. - control_throttle(State1); -post_process_frame({content_header, _, _, _, _}, _ChPid, State) -> - publish_received(State); -post_process_frame({content_body, _}, _ChPid, State) -> - publish_received(State); -post_process_frame(_Frame, _ChPid, State) -> - State. - -%%-------------------------------------------------------------------------- - -%% We allow clients to exceed the frame size a little bit since quite -%% a few get it wrong - off-by 1 or 8 (empty frame size) are typical. --define(FRAME_SIZE_FUDGE, ?EMPTY_FRAME_SIZE). - -handle_input(frame_header, <<Type:8,Channel:16,PayloadSize:32, _/binary>>, - State = #v1{connection = #connection{frame_max = FrameMax}}) - when FrameMax /= 0 andalso - PayloadSize > FrameMax - ?EMPTY_FRAME_SIZE + ?FRAME_SIZE_FUDGE -> - fatal_frame_error( - {frame_too_large, PayloadSize, FrameMax - ?EMPTY_FRAME_SIZE}, - Type, Channel, <<>>, State); -handle_input(frame_header, <<Type:8,Channel:16,PayloadSize:32, - Payload:PayloadSize/binary, ?FRAME_END, - Rest/binary>>, - State) -> - {Rest, ensure_stats_timer(handle_frame(Type, Channel, Payload, State))}; -handle_input(frame_header, <<Type:8,Channel:16,PayloadSize:32, Rest/binary>>, - State) -> - {Rest, ensure_stats_timer( - switch_callback(State, - {frame_payload, Type, Channel, PayloadSize}, - PayloadSize + 1))}; -handle_input({frame_payload, Type, Channel, PayloadSize}, Data, State) -> - <<Payload:PayloadSize/binary, EndMarker, Rest/binary>> = Data, - case EndMarker of - ?FRAME_END -> State1 = handle_frame(Type, Channel, Payload, State), - {Rest, switch_callback(State1, frame_header, 7)}; - _ -> fatal_frame_error({invalid_frame_end_marker, EndMarker}, - Type, Channel, Payload, State) - end; -handle_input(handshake, <<"AMQP", A, B, C, D, Rest/binary>>, State) -> - {Rest, handshake({A, B, C, D}, State)}; -handle_input(handshake, <<Other:8/binary, _/binary>>, #v1{sock = Sock}) -> - refuse_connection(Sock, {bad_header, Other}); -handle_input(Callback, Data, _State) -> - throw({bad_input, Callback, Data}). - -%% The two rules pertaining to version negotiation: -%% -%% * If the server cannot support the protocol specified in the -%% protocol header, it MUST respond with a valid protocol header and -%% then close the socket connection. -%% -%% * The server MUST provide a protocol version that is lower than or -%% equal to that requested by the client in the protocol header. -handshake({0, 0, 9, 1}, State) -> - start_connection({0, 9, 1}, rabbit_framing_amqp_0_9_1, State); - -%% This is the protocol header for 0-9, which we can safely treat as -%% though it were 0-9-1. -handshake({1, 1, 0, 9}, State) -> - start_connection({0, 9, 0}, rabbit_framing_amqp_0_9_1, State); - -%% This is what most clients send for 0-8. The 0-8 spec, confusingly, -%% defines the version as 8-0. -handshake({1, 1, 8, 0}, State) -> - start_connection({8, 0, 0}, rabbit_framing_amqp_0_8, State); - -%% The 0-8 spec as on the AMQP web site actually has this as the -%% protocol header; some libraries e.g., py-amqplib, send it when they -%% want 0-8. -handshake({1, 1, 9, 1}, State) -> - start_connection({8, 0, 0}, rabbit_framing_amqp_0_8, State); - -%% ... and finally, the 1.0 spec is crystal clear! -handshake({Id, 1, 0, 0}, State) -> - become_1_0(Id, State); - -handshake(Vsn, #v1{sock = Sock}) -> - refuse_connection(Sock, {bad_version, Vsn}). - -%% Offer a protocol version to the client. Connection.start only -%% includes a major and minor version number, Luckily 0-9 and 0-9-1 -%% are similar enough that clients will be happy with either. -start_connection({ProtocolMajor, ProtocolMinor, _ProtocolRevision}, - Protocol, - State = #v1{sock = Sock, connection = Connection}) -> - rabbit_networking:register_connection(self()), - Start = #'connection.start'{ - version_major = ProtocolMajor, - version_minor = ProtocolMinor, - server_properties = server_properties(Protocol), - mechanisms = auth_mechanisms_binary(Sock), - locales = <<"en_US">> }, - ok = send_on_channel0(Sock, Start, Protocol), - switch_callback(State#v1{connection = Connection#connection{ - timeout_sec = ?NORMAL_TIMEOUT, - protocol = Protocol}, - connection_state = starting}, - frame_header, 7). - --spec refuse_connection(_, _, _) -> no_return(). -refuse_connection(Sock, Exception, {A, B, C, D}) -> - ok = inet_op(fun () -> rabbit_net:send(Sock, <<"AMQP",A,B,C,D>>) end), - throw(Exception). - --spec refuse_connection(rabbit_net:socket(), any()) -> no_return(). - -refuse_connection(Sock, Exception) -> - refuse_connection(Sock, Exception, {0, 0, 9, 1}). - -ensure_stats_timer(State = #v1{connection_state = running}) -> - rabbit_event:ensure_stats_timer(State, #v1.stats_timer, emit_stats); -ensure_stats_timer(State) -> - State. - -%%-------------------------------------------------------------------------- - -handle_method0(MethodName, FieldsBin, - State = #v1{connection = #connection{protocol = Protocol}}) -> - try - handle_method0(Protocol:decode_method_fields(MethodName, FieldsBin), - State) - catch throw:{inet_error, E} when E =:= closed; E =:= enotconn -> - maybe_emit_stats(State), - throw({connection_closed_abruptly, State}); - exit:#amqp_error{method = none} = Reason -> - handle_exception(State, 0, Reason#amqp_error{method = MethodName}); - Type:Reason:Stacktrace -> - handle_exception(State, 0, {Type, Reason, MethodName, Stacktrace}) - end. - -handle_method0(#'connection.start_ok'{mechanism = Mechanism, - response = Response, - client_properties = ClientProperties}, - State0 = #v1{connection_state = starting, - connection = Connection0, - sock = Sock}) -> - AuthMechanism = auth_mechanism_to_module(Mechanism, Sock), - Capabilities = - case rabbit_misc:table_lookup(ClientProperties, <<"capabilities">>) of - {table, Capabilities1} -> Capabilities1; - _ -> [] - end, - Connection1 = Connection0#connection{ - client_properties = ClientProperties, - capabilities = Capabilities, - auth_mechanism = {Mechanism, AuthMechanism}, - auth_state = AuthMechanism:init(Sock)}, - Connection2 = augment_connection_log_name(Connection1), - State = State0#v1{connection_state = securing, - connection = Connection2}, - % adding client properties to process dictionary to send them later - % in the connection_closed event - put(client_properties, ClientProperties), - case user_provided_connection_name(Connection2) of - undefined -> - undefined; - UserProvidedConnectionName -> - put(connection_user_provided_name, UserProvidedConnectionName) - end, - auth_phase(Response, State); - -handle_method0(#'connection.secure_ok'{response = Response}, - State = #v1{connection_state = securing}) -> - auth_phase(Response, State); - -handle_method0(#'connection.tune_ok'{frame_max = FrameMax, - channel_max = ChannelMax, - heartbeat = ClientHeartbeat}, - State = #v1{connection_state = tuning, - connection = Connection, - helper_sup = SupPid, - sock = Sock}) -> - ok = validate_negotiated_integer_value( - frame_max, ?FRAME_MIN_SIZE, FrameMax), - ok = validate_negotiated_integer_value( - channel_max, ?CHANNEL_MIN, ChannelMax), - {ok, Collector} = rabbit_connection_helper_sup:start_queue_collector( - SupPid, Connection#connection.name), - Frame = rabbit_binary_generator:build_heartbeat_frame(), - Parent = self(), - SendFun = - fun() -> - case catch rabbit_net:send(Sock, Frame) of - ok -> - ok; - {error, Reason} -> - Parent ! {heartbeat_send_error, Reason}; - Unexpected -> - Parent ! {heartbeat_send_error, Unexpected} - end, - ok - end, - ReceiveFun = fun() -> Parent ! heartbeat_timeout end, - Heartbeater = rabbit_heartbeat:start( - SupPid, Sock, Connection#connection.name, - ClientHeartbeat, SendFun, ClientHeartbeat, ReceiveFun), - State#v1{connection_state = opening, - connection = Connection#connection{ - frame_max = FrameMax, - channel_max = ChannelMax, - timeout_sec = ClientHeartbeat}, - queue_collector = Collector, - heartbeater = Heartbeater}; - -handle_method0(#'connection.open'{virtual_host = VHost}, - State = #v1{connection_state = opening, - connection = Connection = #connection{ - log_name = ConnName, - user = User = #user{username = Username}, - protocol = Protocol}, - helper_sup = SupPid, - sock = Sock, - throttle = Throttle}) -> - - ok = is_over_vhost_connection_limit(VHost, User), - ok = is_over_user_connection_limit(User), - ok = rabbit_access_control:check_vhost_access(User, VHost, {socket, Sock}, #{}), - ok = is_vhost_alive(VHost, User), - NewConnection = Connection#connection{vhost = VHost}, - ok = send_on_channel0(Sock, #'connection.open_ok'{}, Protocol), - - Alarms = rabbit_alarm:register(self(), {?MODULE, conserve_resources, []}), - BlockedBy = sets:from_list([{resource, Alarm} || Alarm <- Alarms]), - Throttle1 = Throttle#throttle{blocked_by = BlockedBy}, - - {ok, ChannelSupSupPid} = - rabbit_connection_helper_sup:start_channel_sup_sup(SupPid), - State1 = control_throttle( - State#v1{connection_state = running, - connection = NewConnection, - channel_sup_sup_pid = ChannelSupSupPid, - throttle = Throttle1}), - Infos = augment_infos_with_user_provided_connection_name( - [{type, network} | infos(?CREATION_EVENT_KEYS, State1)], - State1 - ), - rabbit_core_metrics:connection_created(proplists:get_value(pid, Infos), - Infos), - rabbit_event:notify(connection_created, Infos), - maybe_emit_stats(State1), - rabbit_log_connection:info( - "connection ~p (~s): " - "user '~s' authenticated and granted access to vhost '~s'~n", - [self(), dynamic_connection_name(ConnName), Username, VHost]), - State1; -handle_method0(#'connection.close'{}, State) when ?IS_RUNNING(State) -> - lists:foreach(fun rabbit_channel:shutdown/1, all_channels()), - maybe_close(State#v1{connection_state = closing}); -handle_method0(#'connection.close'{}, - State = #v1{connection = #connection{protocol = Protocol}, - sock = Sock}) - when ?IS_STOPPING(State) -> - %% We're already closed or closing, so we don't need to cleanup - %% anything. - ok = send_on_channel0(Sock, #'connection.close_ok'{}, Protocol), - State; -handle_method0(#'connection.close_ok'{}, - State = #v1{connection_state = closed}) -> - self() ! terminate_connection, - State; -handle_method0(#'connection.update_secret'{new_secret = NewSecret, reason = Reason}, - State = #v1{connection = - #connection{protocol = Protocol, - user = User = #user{username = Username}, - log_name = ConnName} = Conn, - sock = Sock}) when ?IS_RUNNING(State) -> - rabbit_log_connection:debug( - "connection ~p (~s) of user '~s': " - "asked to update secret, reason: ~s~n", - [self(), dynamic_connection_name(ConnName), Username, Reason]), - case rabbit_access_control:update_state(User, NewSecret) of - {ok, User1} -> - %% User/auth backend state has been updated. Now we can propagate it to channels - %% asynchronously and return. All the channels have to do is to update their - %% own state. - %% - %% Any secret update errors coming from the authz backend will be handled in the other branch. - %% Therefore we optimistically do no error handling here. MK. - lists:foreach(fun(Ch) -> - rabbit_log:debug("Updating user/auth backend state for channel ~p", [Ch]), - _ = rabbit_channel:update_user_state(Ch, User1) - end, all_channels()), - ok = send_on_channel0(Sock, #'connection.update_secret_ok'{}, Protocol), - rabbit_log_connection:info( - "connection ~p (~s): " - "user '~s' updated secret, reason: ~s~n", - [self(), dynamic_connection_name(ConnName), Username, Reason]), - State#v1{connection = Conn#connection{user = User1}}; - {refused, Message} -> - rabbit_log_connection:error("Secret update was refused for user '~p': ~p", - [Username, Message]), - rabbit_misc:protocol_error(not_allowed, "New secret was refused by one of the backends", []); - {error, Message} -> - rabbit_log_connection:error("Secret update for user '~p' failed: ~p", - [Username, Message]), - rabbit_misc:protocol_error(not_allowed, - "Secret update failed", []) - end; -handle_method0(_Method, State) when ?IS_STOPPING(State) -> - State; -handle_method0(_Method, #v1{connection_state = S}) -> - rabbit_misc:protocol_error( - channel_error, "unexpected method in connection state ~w", [S]). - -is_vhost_alive(VHostPath, User) -> - case rabbit_vhost_sup_sup:is_vhost_alive(VHostPath) of - true -> ok; - false -> - rabbit_misc:protocol_error(internal_error, - "access to vhost '~s' refused for user '~s': " - "vhost '~s' is down", - [VHostPath, User#user.username, VHostPath]) - end. - -is_over_vhost_connection_limit(VHostPath, User) -> - try rabbit_vhost_limit:is_over_connection_limit(VHostPath) of - false -> ok; - {true, Limit} -> rabbit_misc:protocol_error(not_allowed, - "access to vhost '~s' refused for user '~s': " - "connection limit (~p) is reached", - [VHostPath, User#user.username, Limit]) - catch - throw:{error, {no_such_vhost, VHostPath}} -> - rabbit_misc:protocol_error(not_allowed, "vhost ~s not found", [VHostPath]) - end. - -is_over_user_connection_limit(#user{username = Username}) -> - case rabbit_auth_backend_internal:is_over_connection_limit(Username) of - false -> ok; - {true, Limit} -> rabbit_misc:protocol_error(not_allowed, - "Connection refused for user '~s': " - "user connection limit (~p) is reached", - [Username, Limit]) - end. - -validate_negotiated_integer_value(Field, Min, ClientValue) -> - ServerValue = get_env(Field), - if ClientValue /= 0 andalso ClientValue < Min -> - fail_negotiation(Field, min, Min, ClientValue); - ServerValue /= 0 andalso (ClientValue =:= 0 orelse - ClientValue > ServerValue) -> - fail_negotiation(Field, max, ServerValue, ClientValue); - true -> - ok - end. - -%% keep dialyzer happy --spec fail_negotiation(atom(), 'min' | 'max', integer(), integer()) -> - no_return(). -fail_negotiation(Field, MinOrMax, ServerValue, ClientValue) -> - {S1, S2} = case MinOrMax of - min -> {lower, minimum}; - max -> {higher, maximum} - end, - ClientValueDetail = get_client_value_detail(Field, ClientValue), - rabbit_misc:protocol_error( - not_allowed, "negotiated ~w = ~w~s is ~w than the ~w allowed value (~w)", - [Field, ClientValue, ClientValueDetail, S1, S2, ServerValue], 'connection.tune'). - -get_env(Key) -> - {ok, Value} = application:get_env(rabbit, Key), - Value. - -send_on_channel0(Sock, Method, Protocol) -> - ok = rabbit_writer:internal_send_command(Sock, 0, Method, Protocol). - -auth_mechanism_to_module(TypeBin, Sock) -> - case rabbit_registry:binary_to_type(TypeBin) of - {error, not_found} -> - rabbit_misc:protocol_error( - command_invalid, "unknown authentication mechanism '~s'", - [TypeBin]); - T -> - case {lists:member(T, auth_mechanisms(Sock)), - rabbit_registry:lookup_module(auth_mechanism, T)} of - {true, {ok, Module}} -> - Module; - _ -> - rabbit_misc:protocol_error( - command_invalid, - "invalid authentication mechanism '~s'", [T]) - end - end. - -auth_mechanisms(Sock) -> - {ok, Configured} = application:get_env(auth_mechanisms), - [Name || {Name, Module} <- rabbit_registry:lookup_all(auth_mechanism), - Module:should_offer(Sock), lists:member(Name, Configured)]. - -auth_mechanisms_binary(Sock) -> - list_to_binary( - string:join([atom_to_list(A) || A <- auth_mechanisms(Sock)], " ")). - -auth_phase(Response, - State = #v1{connection = Connection = - #connection{protocol = Protocol, - auth_mechanism = {Name, AuthMechanism}, - auth_state = AuthState}, - sock = Sock}) -> - RemoteAddress = list_to_binary(inet:ntoa(Connection#connection.host)), - case AuthMechanism:handle_response(Response, AuthState) of - {refused, Username, Msg, Args} -> - rabbit_core_metrics:auth_attempt_failed(RemoteAddress, Username, amqp091), - auth_fail(Username, Msg, Args, Name, State); - {protocol_error, Msg, Args} -> - rabbit_core_metrics:auth_attempt_failed(RemoteAddress, <<>>, amqp091), - notify_auth_result(none, user_authentication_failure, - [{error, rabbit_misc:format(Msg, Args)}], - State), - rabbit_misc:protocol_error(syntax_error, Msg, Args); - {challenge, Challenge, AuthState1} -> - rabbit_core_metrics:auth_attempt_succeeded(RemoteAddress, <<>>, amqp091), - Secure = #'connection.secure'{challenge = Challenge}, - ok = send_on_channel0(Sock, Secure, Protocol), - State#v1{connection = Connection#connection{ - auth_state = AuthState1}}; - {ok, User = #user{username = Username}} -> - case rabbit_access_control:check_user_loopback(Username, Sock) of - ok -> - rabbit_core_metrics:auth_attempt_succeeded(RemoteAddress, Username, amqp091), - notify_auth_result(Username, user_authentication_success, - [], State); - not_allowed -> - rabbit_core_metrics:auth_attempt_failed(RemoteAddress, Username, amqp091), - auth_fail(Username, "user '~s' can only connect via " - "localhost", [Username], Name, State) - end, - Tune = #'connection.tune'{frame_max = get_env(frame_max), - channel_max = get_env(channel_max), - heartbeat = get_env(heartbeat)}, - ok = send_on_channel0(Sock, Tune, Protocol), - State#v1{connection_state = tuning, - connection = Connection#connection{user = User, - auth_state = none}} - end. - --spec auth_fail - (rabbit_types:username() | none, string(), [any()], binary(), #v1{}) -> - no_return(). - -auth_fail(Username, Msg, Args, AuthName, - State = #v1{connection = #connection{protocol = Protocol, - capabilities = Capabilities}}) -> - notify_auth_result(Username, user_authentication_failure, - [{error, rabbit_misc:format(Msg, Args)}], State), - AmqpError = rabbit_misc:amqp_error( - access_refused, "~s login refused: ~s", - [AuthName, io_lib:format(Msg, Args)], none), - case rabbit_misc:table_lookup(Capabilities, - <<"authentication_failure_close">>) of - {bool, true} -> - SafeMsg = io_lib:format( - "Login was refused using authentication " - "mechanism ~s. For details see the broker " - "logfile.", [AuthName]), - AmqpError1 = AmqpError#amqp_error{explanation = SafeMsg}, - {0, CloseMethod} = rabbit_binary_generator:map_exception( - 0, AmqpError1, Protocol), - ok = send_on_channel0(State#v1.sock, CloseMethod, Protocol); - _ -> ok - end, - rabbit_misc:protocol_error(AmqpError). - -notify_auth_result(Username, AuthResult, ExtraProps, State) -> - EventProps = [{connection_type, network}, - {name, case Username of none -> ''; _ -> Username end}] ++ - [case Item of - name -> {connection_name, i(name, State)}; - _ -> {Item, i(Item, State)} - end || Item <- ?AUTH_NOTIFICATION_INFO_KEYS] ++ - ExtraProps, - rabbit_event:notify(AuthResult, [P || {_, V} = P <- EventProps, V =/= '']). - -%%-------------------------------------------------------------------------- - -infos(Items, State) -> [{Item, i(Item, State)} || Item <- Items]. - -i(pid, #v1{}) -> self(); -i(node, #v1{}) -> node(); -i(SockStat, S) when SockStat =:= recv_oct; - SockStat =:= recv_cnt; - SockStat =:= send_oct; - SockStat =:= send_cnt; - SockStat =:= send_pend -> - socket_info(fun (Sock) -> rabbit_net:getstat(Sock, [SockStat]) end, - fun ([{_, I}]) -> I end, S); -i(ssl, #v1{sock = Sock}) -> rabbit_net:is_ssl(Sock); -i(ssl_protocol, S) -> ssl_info(fun ({P, _}) -> P end, S); -i(ssl_key_exchange, S) -> ssl_info(fun ({_, {K, _, _}}) -> K end, S); -i(ssl_cipher, S) -> ssl_info(fun ({_, {_, C, _}}) -> C end, S); -i(ssl_hash, S) -> ssl_info(fun ({_, {_, _, H}}) -> H end, S); -i(peer_cert_issuer, S) -> cert_info(fun rabbit_ssl:peer_cert_issuer/1, S); -i(peer_cert_subject, S) -> cert_info(fun rabbit_ssl:peer_cert_subject/1, S); -i(peer_cert_validity, S) -> cert_info(fun rabbit_ssl:peer_cert_validity/1, S); -i(channels, #v1{channel_count = ChannelCount}) -> ChannelCount; -i(state, #v1{connection_state = ConnectionState, - throttle = #throttle{blocked_by = Reasons, - last_blocked_at = T} = Throttle}) -> - %% not throttled by resource or other longer-term reasons - %% TODO: come up with a sensible function name - case sets:size(sets:del_element(flow, Reasons)) =:= 0 andalso - (credit_flow:blocked() %% throttled by flow now - orelse %% throttled by flow recently - (is_blocked_by_flow(Throttle) andalso T =/= never andalso - erlang:convert_time_unit(erlang:monotonic_time() - T, - native, - micro_seconds) < 5000000)) of - true -> flow; - false -> - case {has_reasons_to_block(Throttle), ConnectionState} of - %% blocked - {_, blocked} -> blocked; - %% not yet blocked (there were no publishes) - {true, running} -> blocking; - %% not blocked - {false, _} -> ConnectionState; - %% catch all to be defensive - _ -> ConnectionState - end - end; -i(garbage_collection, _State) -> - rabbit_misc:get_gc_info(self()); -i(reductions, _State) -> - {reductions, Reductions} = erlang:process_info(self(), reductions), - Reductions; -i(Item, #v1{connection = Conn}) -> ic(Item, Conn). - -ic(name, #connection{name = Name}) -> Name; -ic(host, #connection{host = Host}) -> Host; -ic(peer_host, #connection{peer_host = PeerHost}) -> PeerHost; -ic(port, #connection{port = Port}) -> Port; -ic(peer_port, #connection{peer_port = PeerPort}) -> PeerPort; -ic(protocol, #connection{protocol = none}) -> none; -ic(protocol, #connection{protocol = P}) -> P:version(); -ic(user, #connection{user = none}) -> ''; -ic(user, #connection{user = U}) -> U#user.username; -ic(user_who_performed_action, C) -> ic(user, C); -ic(vhost, #connection{vhost = VHost}) -> VHost; -ic(timeout, #connection{timeout_sec = Timeout}) -> Timeout; -ic(frame_max, #connection{frame_max = FrameMax}) -> FrameMax; -ic(channel_max, #connection{channel_max = ChMax}) -> ChMax; -ic(client_properties, #connection{client_properties = CP}) -> CP; -ic(auth_mechanism, #connection{auth_mechanism = none}) -> none; -ic(auth_mechanism, #connection{auth_mechanism = {Name, _Mod}}) -> Name; -ic(connected_at, #connection{connected_at = T}) -> T; -ic(Item, #connection{}) -> throw({bad_argument, Item}). - -socket_info(Get, Select, #v1{sock = Sock}) -> - case Get(Sock) of - {ok, T} -> case Select(T) of - N when is_number(N) -> N; - _ -> 0 - end; - {error, _} -> 0 - end. - -ssl_info(F, #v1{sock = Sock}) -> - case rabbit_net:ssl_info(Sock) of - nossl -> ''; - {error, _} -> ''; - {ok, Items} -> - P = proplists:get_value(protocol, Items), - #{cipher := C, - key_exchange := K, - mac := H} = proplists:get_value(selected_cipher_suite, Items), - F({P, {K, C, H}}) - end. - -cert_info(F, #v1{sock = Sock}) -> - case rabbit_net:peercert(Sock) of - nossl -> ''; - {error, _} -> ''; - {ok, Cert} -> list_to_binary(F(Cert)) - end. - -maybe_emit_stats(State) -> - rabbit_event:if_enabled(State, #v1.stats_timer, - fun() -> emit_stats(State) end). - -emit_stats(State) -> - [{_, Pid}, {_, Recv_oct}, {_, Send_oct}, {_, Reductions}] = I - = infos(?SIMPLE_METRICS, State), - Infos = infos(?OTHER_METRICS, State), - rabbit_core_metrics:connection_stats(Pid, Infos), - rabbit_core_metrics:connection_stats(Pid, Recv_oct, Send_oct, Reductions), - rabbit_event:notify(connection_stats, Infos ++ I), - State1 = rabbit_event:reset_stats_timer(State, #v1.stats_timer), - ensure_stats_timer(State1). - -%% 1.0 stub --spec become_1_0(non_neg_integer(), #v1{}) -> no_return(). - -become_1_0(Id, State = #v1{sock = Sock}) -> - case code:is_loaded(rabbit_amqp1_0_reader) of - false -> refuse_connection(Sock, amqp1_0_plugin_not_enabled); - _ -> Mode = case Id of - 0 -> amqp; - 3 -> sasl; - _ -> refuse_connection( - Sock, {unsupported_amqp1_0_protocol_id, Id}, - {3, 1, 0, 0}) - end, - F = fun (_Deb, Buf, BufLen, S) -> - {rabbit_amqp1_0_reader, init, - [Mode, pack_for_1_0(Buf, BufLen, S)]} - end, - State#v1{connection_state = {become, F}} - end. - -pack_for_1_0(Buf, BufLen, #v1{parent = Parent, - sock = Sock, - recv_len = RecvLen, - pending_recv = PendingRecv, - helper_sup = SupPid, - proxy_socket = ProxySocket}) -> - {Parent, Sock, RecvLen, PendingRecv, SupPid, Buf, BufLen, ProxySocket}. - -respond_and_close(State, Channel, Protocol, Reason, LogErr) -> - log_hard_error(State, Channel, LogErr), - send_error_on_channel0_and_close(Channel, Protocol, Reason, State). - -send_error_on_channel0_and_close(Channel, Protocol, Reason, State) -> - {0, CloseMethod} = - rabbit_binary_generator:map_exception(Channel, Reason, Protocol), - State1 = close_connection(terminate_channels(State)), - ok = send_on_channel0(State#v1.sock, CloseMethod, Protocol), - State1. - -%% -%% Publisher throttling -%% - -blocked_by_message(#throttle{blocked_by = Reasons}) -> - %% we don't want to report internal flow as a reason here since - %% it is entirely transient - Reasons1 = sets:del_element(flow, Reasons), - RStr = string:join([format_blocked_by(R) || R <- sets:to_list(Reasons1)], " & "), - list_to_binary(rabbit_misc:format("low on ~s", [RStr])). - -format_blocked_by({resource, memory}) -> "memory"; -format_blocked_by({resource, disk}) -> "disk"; -format_blocked_by({resource, disc}) -> "disk". - -update_last_blocked_at(Throttle) -> - Throttle#throttle{last_blocked_at = erlang:monotonic_time()}. - -connection_blocked_message_sent( - #throttle{connection_blocked_message_sent = BS}) -> BS. - -should_send_blocked(Throttle = #throttle{blocked_by = Reasons}) -> - should_block(Throttle) - andalso - sets:size(sets:del_element(flow, Reasons)) =/= 0 - andalso - not connection_blocked_message_sent(Throttle). - -should_send_unblocked(Throttle = #throttle{blocked_by = Reasons}) -> - connection_blocked_message_sent(Throttle) - andalso - sets:size(sets:del_element(flow, Reasons)) == 0. - -%% Returns true if we have a reason to block -%% this connection. -has_reasons_to_block(#throttle{blocked_by = Reasons}) -> - sets:size(Reasons) > 0. - -is_blocked_by_flow(#throttle{blocked_by = Reasons}) -> - sets:is_element(flow, Reasons). - -should_block(#throttle{should_block = Val}) -> Val. - -should_block_connection(Throttle) -> - should_block(Throttle) andalso has_reasons_to_block(Throttle). - -should_unblock_connection(Throttle) -> - not should_block_connection(Throttle). - -maybe_block(State = #v1{connection_state = CS, throttle = Throttle}) -> - case should_block_connection(Throttle) of - true -> - State1 = State#v1{connection_state = blocked, - throttle = update_last_blocked_at(Throttle)}, - case CS of - running -> - ok = rabbit_heartbeat:pause_monitor(State#v1.heartbeater); - _ -> ok - end, - maybe_send_blocked_or_unblocked(State1); - false -> State - end. - -maybe_unblock(State = #v1{throttle = Throttle}) -> - case should_unblock_connection(Throttle) of - true -> - ok = rabbit_heartbeat:resume_monitor(State#v1.heartbeater), - State1 = State#v1{connection_state = running, - throttle = Throttle#throttle{should_block = false}}, - maybe_send_unblocked(State1); - false -> State - end. - -maybe_send_unblocked(State = #v1{throttle = Throttle}) -> - case should_send_unblocked(Throttle) of - true -> - ok = send_unblocked(State), - State#v1{throttle = - Throttle#throttle{connection_blocked_message_sent = false}}; - false -> State - end. - -maybe_send_blocked_or_unblocked(State = #v1{throttle = Throttle}) -> - case should_send_blocked(Throttle) of - true -> - ok = send_blocked(State, blocked_by_message(Throttle)), - State#v1{throttle = - Throttle#throttle{connection_blocked_message_sent = true}}; - false -> maybe_send_unblocked(State) - end. - -publish_received(State = #v1{throttle = Throttle}) -> - case has_reasons_to_block(Throttle) of - false -> State; - true -> - Throttle1 = Throttle#throttle{should_block = true}, - maybe_block(State#v1{throttle = Throttle1}) - end. - -control_throttle(State = #v1{connection_state = CS, - throttle = #throttle{blocked_by = Reasons} = Throttle}) -> - Throttle1 = case credit_flow:blocked() of - true -> - Throttle#throttle{blocked_by = sets:add_element(flow, Reasons)}; - false -> - Throttle#throttle{blocked_by = sets:del_element(flow, Reasons)} - end, - State1 = State#v1{throttle = Throttle1}, - case CS of - running -> maybe_block(State1); - %% unblock or re-enable blocking - blocked -> maybe_block(maybe_unblock(State1)); - _ -> State1 - end. - -augment_connection_log_name(#connection{name = Name} = Connection) -> - case user_provided_connection_name(Connection) of - undefined -> - Connection; - UserSpecifiedName -> - LogName = <<Name/binary, " - ", UserSpecifiedName/binary>>, - rabbit_log_connection:info("Connection ~p (~s) has a client-provided name: ~s~n", [self(), Name, UserSpecifiedName]), - ?store_proc_name(LogName), - Connection#connection{log_name = LogName} - end. - -augment_infos_with_user_provided_connection_name(Infos, #v1{connection = Connection}) -> - case user_provided_connection_name(Connection) of - undefined -> - Infos; - UserProvidedConnectionName -> - [{user_provided_name, UserProvidedConnectionName} | Infos] - end. - -user_provided_connection_name(#connection{client_properties = ClientProperties}) -> - case rabbit_misc:table_lookup(ClientProperties, <<"connection_name">>) of - {longstr, UserSpecifiedName} -> - UserSpecifiedName; - _ -> - undefined - end. - -dynamic_connection_name(Default) -> - case rabbit_misc:get_proc_name() of - {ok, Name} -> - Name; - _ -> - Default - end. - -handle_uncontrolled_channel_close(ChPid) -> - rabbit_core_metrics:channel_closed(ChPid), - rabbit_event:notify(channel_closed, [{pid, ChPid}]). - --spec get_client_value_detail(atom(), integer()) -> string(). -get_client_value_detail(channel_max, 0) -> - " (no limit)"; -get_client_value_detail(_Field, _ClientValue) -> - "". diff --git a/src/rabbit_recovery_terms.erl b/src/rabbit_recovery_terms.erl deleted file mode 100644 index d89de9ece3..0000000000 --- a/src/rabbit_recovery_terms.erl +++ /dev/null @@ -1,240 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% We use a gen_server simply so that during the terminate/2 call -%% (i.e., during shutdown), we can sync/flush the dets table to disk. - --module(rabbit_recovery_terms). - --behaviour(gen_server). - --export([start/1, stop/1, store/3, read/2, clear/1]). - --export([start_link/1]). --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --export([upgrade_recovery_terms/0, persistent_bytes/0]). --export([open_global_table/0, close_global_table/0, - read_global/1, delete_global_table/0]). --export([open_table/1, close_table/1]). - --rabbit_upgrade({upgrade_recovery_terms, local, []}). --rabbit_upgrade({persistent_bytes, local, [upgrade_recovery_terms]}). - --include("rabbit.hrl"). - -%%---------------------------------------------------------------------------- - --spec start(rabbit_types:vhost()) -> rabbit_types:ok_or_error(term()). - -start(VHost) -> - case rabbit_vhost_sup_sup:get_vhost_sup(VHost) of - {ok, VHostSup} -> - {ok, _} = supervisor2:start_child( - VHostSup, - {?MODULE, - {?MODULE, start_link, [VHost]}, - transient, ?WORKER_WAIT, worker, - [?MODULE]}); - %% we can get here if a vhost is added and removed concurrently - %% e.g. some integration tests do it - {error, {no_such_vhost, VHost}} -> - rabbit_log:error("Failed to start a recovery terms manager for vhost ~s: vhost no longer exists!", - [VHost]) - end, - ok. - --spec stop(rabbit_types:vhost()) -> rabbit_types:ok_or_error(term()). - -stop(VHost) -> - case rabbit_vhost_sup_sup:get_vhost_sup(VHost) of - {ok, VHostSup} -> - case supervisor:terminate_child(VHostSup, ?MODULE) of - ok -> supervisor:delete_child(VHostSup, ?MODULE); - E -> E - end; - %% see start/1 - {error, {no_such_vhost, VHost}} -> - rabbit_log:error("Failed to stop a recovery terms manager for vhost ~s: vhost no longer exists!", - [VHost]), - - ok - end. - --spec store(rabbit_types:vhost(), file:filename(), term()) -> rabbit_types:ok_or_error(term()). - -store(VHost, DirBaseName, Terms) -> - dets:insert(VHost, {DirBaseName, Terms}). - --spec read(rabbit_types:vhost(), file:filename()) -> rabbit_types:ok_or_error2(term(), not_found). - -read(VHost, DirBaseName) -> - case dets:lookup(VHost, DirBaseName) of - [{_, Terms}] -> {ok, Terms}; - _ -> {error, not_found} - end. - --spec clear(rabbit_types:vhost()) -> 'ok'. - -clear(VHost) -> - try - dets:delete_all_objects(VHost) - %% see start/1 - catch _:badarg -> - rabbit_log:error("Failed to clear recovery terms for vhost ~s: table no longer exists!", - [VHost]), - ok - end, - flush(VHost). - -start_link(VHost) -> - gen_server:start_link(?MODULE, [VHost], []). - -%%---------------------------------------------------------------------------- - -upgrade_recovery_terms() -> - open_global_table(), - try - QueuesDir = filename:join(rabbit_mnesia:dir(), "queues"), - Dirs = case rabbit_file:list_dir(QueuesDir) of - {ok, Entries} -> Entries; - {error, _} -> [] - end, - [begin - File = filename:join([QueuesDir, Dir, "clean.dot"]), - case rabbit_file:read_term_file(File) of - {ok, Terms} -> ok = store_global_table(Dir, Terms); - {error, _} -> ok - end, - file:delete(File) - end || Dir <- Dirs], - ok - after - close_global_table() - end. - -persistent_bytes() -> dets_upgrade(fun persistent_bytes/1). -persistent_bytes(Props) -> Props ++ [{persistent_bytes, 0}]. - -dets_upgrade(Fun)-> - open_global_table(), - try - ok = dets:foldl(fun ({DirBaseName, Terms}, Acc) -> - store_global_table(DirBaseName, Fun(Terms)), - Acc - end, ok, ?MODULE), - ok - after - close_global_table() - end. - -open_global_table() -> - File = filename:join(rabbit_mnesia:dir(), "recovery.dets"), - {ok, _} = dets:open_file(?MODULE, [{file, File}, - {ram_file, true}, - {auto_save, infinity}]), - ok. - -close_global_table() -> - try - dets:sync(?MODULE), - dets:close(?MODULE) - %% see clear/1 - catch _:badarg -> - rabbit_log:error("Failed to clear global recovery terms: table no longer exists!", - []), - ok - end. - -store_global_table(DirBaseName, Terms) -> - dets:insert(?MODULE, {DirBaseName, Terms}). - -read_global(DirBaseName) -> - case dets:lookup(?MODULE, DirBaseName) of - [{_, Terms}] -> {ok, Terms}; - _ -> {error, not_found} - end. - -delete_global_table() -> - file:delete(filename:join(rabbit_mnesia:dir(), "recovery.dets")). - -%%---------------------------------------------------------------------------- - -init([VHost]) -> - process_flag(trap_exit, true), - open_table(VHost), - {ok, VHost}. - -handle_call(Msg, _, State) -> {stop, {unexpected_call, Msg}, State}. - -handle_cast(Msg, State) -> {stop, {unexpected_cast, Msg}, State}. - -handle_info(_Info, State) -> {noreply, State}. - -terminate(_Reason, VHost) -> - close_table(VHost). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%---------------------------------------------------------------------------- - --spec open_table(vhost:name()) -> rabbit_types:ok_or_error(any()). - -open_table(VHost) -> - open_table(VHost, 10). - --spec open_table(vhost:name(), non_neg_integer()) -> rabbit_types:ok_or_error(any()). - -open_table(VHost, RetriesLeft) -> - VHostDir = rabbit_vhost:msg_store_dir_path(VHost), - File = filename:join(VHostDir, "recovery.dets"), - Opts = [{file, File}, - {ram_file, true}, - {auto_save, infinity}], - case dets:open_file(VHost, Opts) of - {ok, _} -> ok; - {error, Error} -> - case RetriesLeft of - 0 -> - {error, Error}; - N when is_integer(N) -> - _ = file:delete(File), - %% Wait before retrying - DelayInMs = 1000, - rabbit_log:warning("Failed to open a recovery terms DETS file at ~p. Will delete it and retry in ~p ms (~p retries left)", - [File, DelayInMs, RetriesLeft]), - timer:sleep(DelayInMs), - open_table(VHost, RetriesLeft - 1) - end - end. - --spec flush(vhost:name()) -> rabbit_types:ok_or_error(any()). - -flush(VHost) -> - try - dets:sync(VHost) - %% see clear/1 - catch _:badarg -> - rabbit_log:error("Failed to sync recovery terms table for vhost ~s: the table no longer exists!", - [VHost]), - ok - end. - --spec close_table(vhost:name()) -> rabbit_types:ok_or_error(any()). - -close_table(VHost) -> - try - ok = flush(VHost), - ok = dets:close(VHost) - %% see clear/1 - catch _:badarg -> - rabbit_log:error("Failed to close recovery terms table for vhost ~s: the table no longer exists!", - [VHost]), - ok - end. diff --git a/src/rabbit_restartable_sup.erl b/src/rabbit_restartable_sup.erl deleted file mode 100644 index 46fcace99f..0000000000 --- a/src/rabbit_restartable_sup.erl +++ /dev/null @@ -1,33 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_restartable_sup). - --behaviour(supervisor2). - --export([start_link/3]). - --export([init/1]). - --include("rabbit.hrl"). - --define(DELAY, 2). - -%%---------------------------------------------------------------------------- - --spec start_link(atom(), rabbit_types:mfargs(), boolean()) -> - rabbit_types:ok_pid_or_error(). - -start_link(Name, {_M, _F, _A} = Fun, Delay) -> - supervisor2:start_link({local, Name}, ?MODULE, [Fun, Delay]). - -init([{Mod, _F, _A} = Fun, Delay]) -> - {ok, {{one_for_one, 10, 10}, - [{Mod, Fun, case Delay of - true -> {transient, 1}; - false -> transient - end, ?WORKER_WAIT, worker, [Mod]}]}}. diff --git a/src/rabbit_router.erl b/src/rabbit_router.erl deleted file mode 100644 index ed170bcd8e..0000000000 --- a/src/rabbit_router.erl +++ /dev/null @@ -1,65 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_router). --include_lib("stdlib/include/qlc.hrl"). --include("rabbit.hrl"). - --export([match_bindings/2, match_routing_key/2]). - -%%---------------------------------------------------------------------------- - --export_type([routing_key/0, match_result/0]). - --type routing_key() :: binary(). --type match_result() :: [rabbit_types:binding_destination()]. - --spec match_bindings(rabbit_types:binding_source(), - fun ((rabbit_types:binding()) -> boolean())) -> - match_result(). --spec match_routing_key(rabbit_types:binding_source(), - [routing_key()] | ['_']) -> - match_result(). - -%%---------------------------------------------------------------------------- - -match_bindings(SrcName, Match) -> - MatchHead = #route{binding = #binding{source = SrcName, - _ = '_'}}, - Routes = ets:select(rabbit_route, [{MatchHead, [], [['$_']]}]), - [Dest || [#route{binding = Binding = #binding{destination = Dest}}] <- - Routes, Match(Binding)]. - -match_routing_key(SrcName, [RoutingKey]) -> - find_routes(#route{binding = #binding{source = SrcName, - destination = '$1', - key = RoutingKey, - _ = '_'}}, - []); -match_routing_key(SrcName, [_|_] = RoutingKeys) -> - find_routes(#route{binding = #binding{source = SrcName, - destination = '$1', - key = '$2', - _ = '_'}}, - [list_to_tuple(['orelse' | [{'=:=', '$2', RKey} || - RKey <- RoutingKeys]])]). - -%%-------------------------------------------------------------------- - -%% Normally we'd call mnesia:dirty_select/2 here, but that is quite -%% expensive for the same reasons as above, and, additionally, due to -%% mnesia 'fixing' the table with ets:safe_fixtable/2, which is wholly -%% unnecessary. According to the ets docs (and the code in erl_db.c), -%% 'select' is safe anyway ("Functions that internally traverse over a -%% table, like select and match, will give the same guarantee as -%% safe_fixtable.") and, furthermore, even the lower level iterators -%% ('first' and 'next') are safe on ordered_set tables ("Note that for -%% tables of the ordered_set type, safe_fixtable/2 is not necessary as -%% calls to first/1 and next/2 will always succeed."), which -%% rabbit_route is. -find_routes(MatchHead, Conditions) -> - ets:select(rabbit_route, [{MatchHead, Conditions, ['$1']}]). diff --git a/src/rabbit_runtime_parameters.erl b/src/rabbit_runtime_parameters.erl deleted file mode 100644 index 1870b5dfa5..0000000000 --- a/src/rabbit_runtime_parameters.erl +++ /dev/null @@ -1,412 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_runtime_parameters). - -%% Runtime parameters are bits of configuration that are -%% set, as the name implies, at runtime and not in the config file. -%% -%% The benefits of storing some bits of configuration at runtime vary: -%% -%% * Some parameters are vhost-specific -%% * Others are specific to individual nodes -%% * ...or even queues, exchanges, etc -%% -%% The most obvious use case for runtime parameters is policies but -%% there are others: -%% -%% * Plugin-specific parameters that only make sense at runtime, -%% e.g. Federation and Shovel link settings -%% * Exchange and queue decorators -%% -%% Parameters are grouped by components, e.g. <<"policy">> or <<"shovel">>. -%% Components are mapped to modules that perform validation. -%% Runtime parameter values are then looked up by the modules that -%% need to use them. -%% -%% Parameters are stored in Mnesia and can be global. Their changes -%% are broadcasted over rabbit_event. -%% -%% Global parameters keys are atoms and values are JSON documents. -%% -%% See also: -%% -%% * rabbit_policies -%% * rabbit_policy -%% * rabbit_registry -%% * rabbit_event - --include("rabbit.hrl"). - --export([parse_set/5, set/5, set_any/5, clear/4, clear_any/4, list/0, list/1, - list_component/1, list/2, list_formatted/1, list_formatted/3, - lookup/3, value/3, value/4, info_keys/0, clear_component/2]). - --export([parse_set_global/3, set_global/3, value_global/1, value_global/2, - list_global/0, list_global_formatted/0, list_global_formatted/2, - lookup_global/1, global_info_keys/0, clear_global/2]). - -%%---------------------------------------------------------------------------- - --type ok_or_error_string() :: 'ok' | {'error_string', string()}. --type ok_thunk_or_error_string() :: ok_or_error_string() | fun(() -> 'ok'). - --spec parse_set(rabbit_types:vhost(), binary(), binary(), string(), - rabbit_types:user() | rabbit_types:username() | 'none') - -> ok_or_error_string(). --spec set(rabbit_types:vhost(), binary(), binary(), term(), - rabbit_types:user() | rabbit_types:username() | 'none') - -> ok_or_error_string(). --spec set_any(rabbit_types:vhost(), binary(), binary(), term(), - rabbit_types:user() | rabbit_types:username() | 'none') - -> ok_or_error_string(). --spec set_global(atom(), term(), rabbit_types:username()) -> 'ok'. --spec clear(rabbit_types:vhost(), binary(), binary(), rabbit_types:username()) - -> ok_thunk_or_error_string(). --spec clear_any(rabbit_types:vhost(), binary(), binary(), rabbit_types:username()) - -> ok_thunk_or_error_string(). --spec list() -> [rabbit_types:infos()]. --spec list(rabbit_types:vhost() | '_') -> [rabbit_types:infos()]. --spec list_component(binary()) -> [rabbit_types:infos()]. --spec list(rabbit_types:vhost() | '_', binary() | '_') - -> [rabbit_types:infos()]. --spec list_formatted(rabbit_types:vhost()) -> [rabbit_types:infos()]. --spec list_formatted(rabbit_types:vhost(), reference(), pid()) -> 'ok'. --spec lookup(rabbit_types:vhost(), binary(), binary()) - -> rabbit_types:infos() | 'not_found'. --spec value(rabbit_types:vhost(), binary(), binary()) -> term(). --spec value(rabbit_types:vhost(), binary(), binary(), term()) -> term(). --spec value_global(atom()) -> term() | 'not_found'. --spec value_global(atom(), term()) -> term(). --spec info_keys() -> rabbit_types:info_keys(). - -%%--------------------------------------------------------------------------- - --import(rabbit_misc, [pget/2]). - --define(TABLE, rabbit_runtime_parameters). - -%%--------------------------------------------------------------------------- - -parse_set(_, <<"policy">>, _, _, _) -> - {error_string, "policies may not be set using this method"}; -parse_set(VHost, Component, Name, String, User) -> - Definition = rabbit_data_coercion:to_binary(String), - case rabbit_json:try_decode(Definition) of - {ok, Term} when is_map(Term) -> set(VHost, Component, Name, maps:to_list(Term), User); - {ok, Term} -> set(VHost, Component, Name, Term, User); - {error, Reason} -> - {error_string, - rabbit_misc:format("JSON decoding error. Reason: ~ts", [Reason])} - end. - -set(_, <<"policy">>, _, _, _) -> - {error_string, "policies may not be set using this method"}; -set(VHost, Component, Name, Term, User) -> - set_any(VHost, Component, Name, Term, User). - -parse_set_global(Name, String, ActingUser) -> - Definition = rabbit_data_coercion:to_binary(String), - case rabbit_json:try_decode(Definition) of - {ok, Term} when is_map(Term) -> set_global(Name, maps:to_list(Term), ActingUser); - {ok, Term} -> set_global(Name, Term, ActingUser); - {error, Reason} -> - {error_string, - rabbit_misc:format("JSON decoding error. Reason: ~ts", [Reason])} - end. - -set_global(Name, Term, ActingUser) -> - NameAsAtom = rabbit_data_coercion:to_atom(Name), - rabbit_log:debug("Setting global parameter '~s' to ~p", [NameAsAtom, Term]), - mnesia_update(NameAsAtom, Term), - event_notify(parameter_set, none, global, [{name, NameAsAtom}, - {value, Term}, - {user_who_performed_action, ActingUser}]), - ok. - -format_error(L) -> - {error_string, rabbit_misc:format_many([{"Validation failed~n", []} | L])}. - -set_any(VHost, Component, Name, Term, User) -> - case set_any0(VHost, Component, Name, Term, User) of - ok -> ok; - {errors, L} -> format_error(L) - end. - -set_any0(VHost, Component, Name, Term, User) -> - rabbit_log:debug("Asked to set or update runtime parameter '~s' in vhost '~s' " - "for component '~s', value: ~p", - [Name, VHost, Component, Term]), - case lookup_component(Component) of - {ok, Mod} -> - case flatten_errors( - Mod:validate(VHost, Component, Name, Term, get_user(User))) of - ok -> - case mnesia_update(VHost, Component, Name, Term) of - {old, Term} -> - ok; - _ -> - ActingUser = get_username(User), - event_notify( - parameter_set, VHost, Component, - [{name, Name}, - {value, Term}, - {user_who_performed_action, ActingUser}]), - Mod:notify(VHost, Component, Name, Term, ActingUser) - end, - ok; - E -> - E - end; - E -> - E - end. - -%% Validate only an user record as expected by the API before #rabbitmq-event-exchange-10 -get_user(#user{} = User) -> - User; -get_user(_) -> - none. - -get_username(#user{username = Username}) -> - Username; -get_username(none) -> - ?INTERNAL_USER; -get_username(Any) -> - Any. - -mnesia_update(Key, Term) -> - rabbit_misc:execute_mnesia_transaction(mnesia_update_fun(Key, Term)). - -mnesia_update(VHost, Comp, Name, Term) -> - rabbit_misc:execute_mnesia_transaction( - rabbit_vhost:with(VHost, mnesia_update_fun({VHost, Comp, Name}, Term))). - -mnesia_update_fun(Key, Term) -> - fun () -> - Res = case mnesia:read(?TABLE, Key, read) of - [] -> new; - [Params] -> {old, Params#runtime_parameters.value} - end, - ok = mnesia:write(?TABLE, c(Key, Term), write), - Res - end. - -clear(_, <<"policy">> , _, _) -> - {error_string, "policies may not be cleared using this method"}; -clear(VHost, Component, Name, ActingUser) -> - clear_any(VHost, Component, Name, ActingUser). - -clear_global(Key, ActingUser) -> - KeyAsAtom = rabbit_data_coercion:to_atom(Key), - Notify = fun() -> - event_notify(parameter_set, none, global, - [{name, KeyAsAtom}, - {user_who_performed_action, ActingUser}]), - ok - end, - case value_global(KeyAsAtom) of - not_found -> - {error_string, "Parameter does not exist"}; - _ -> - F = fun () -> - ok = mnesia:delete(?TABLE, KeyAsAtom, write) - end, - ok = rabbit_misc:execute_mnesia_transaction(F), - case mnesia:is_transaction() of - true -> Notify; - false -> Notify() - end - end. - -clear_component(Component, ActingUser) -> - case list_component(Component) of - [] -> - ok; - Xs -> - [clear(pget(vhost, X), - pget(component, X), - pget(name, X), - ActingUser) || X <- Xs], - ok - end. - -clear_any(VHost, Component, Name, ActingUser) -> - Notify = fun () -> - case lookup_component(Component) of - {ok, Mod} -> event_notify( - parameter_cleared, VHost, Component, - [{name, Name}, - {user_who_performed_action, ActingUser}]), - Mod:notify_clear(VHost, Component, Name, ActingUser); - _ -> ok - end - end, - case lookup(VHost, Component, Name) of - not_found -> {error_string, "Parameter does not exist"}; - _ -> mnesia_clear(VHost, Component, Name), - case mnesia:is_transaction() of - true -> Notify; - false -> Notify() - end - end. - -mnesia_clear(VHost, Component, Name) -> - F = fun () -> - ok = mnesia:delete(?TABLE, {VHost, Component, Name}, write) - end, - ok = rabbit_misc:execute_mnesia_transaction(rabbit_vhost:with(VHost, F)). - -event_notify(_Event, _VHost, <<"policy">>, _Props) -> - ok; -event_notify(Event, none, Component, Props) -> - rabbit_event:notify(Event, [{component, Component} | Props]); -event_notify(Event, VHost, Component, Props) -> - rabbit_event:notify(Event, [{vhost, VHost}, - {component, Component} | Props]). - -list() -> - [p(P) || #runtime_parameters{ key = {_VHost, Comp, _Name}} = P <- - rabbit_misc:dirty_read_all(?TABLE), Comp /= <<"policy">>]. - -list(VHost) -> list(VHost, '_'). -list_component(Component) -> list('_', Component). - -%% Not dirty_match_object since that would not be transactional when used in a -%% tx context -list(VHost, Component) -> - mnesia:async_dirty( - fun () -> - case VHost of - '_' -> ok; - _ -> rabbit_vhost:assert(VHost) - end, - Match = #runtime_parameters{key = {VHost, Component, '_'}, - _ = '_'}, - [p(P) || #runtime_parameters{key = {_VHost, Comp, _Name}} = P <- - mnesia:match_object(?TABLE, Match, read), - Comp =/= <<"policy">> orelse Component =:= <<"policy">>] - end). - -list_global() -> - %% list only atom keys - mnesia:async_dirty( - fun () -> - Match = #runtime_parameters{key = '_', _ = '_'}, - [p(P) || P <- mnesia:match_object(?TABLE, Match, read), - is_atom(P#runtime_parameters.key)] - end). - -list_formatted(VHost) -> - [ format_parameter(info_keys(), P) || P <- list(VHost) ]. - -format_parameter(InfoKeys, P) -> - lists:foldr(fun - (value, Acc) -> - [{value, rabbit_json:encode(pget(value, P))} | Acc]; - (Key, Acc) -> - case lists:keyfind(Key, 1, P) of - false -> Acc; - {Key, Val} -> [{Key, Val} | Acc] - end - end, - [], InfoKeys). - -list_formatted(VHost, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map( - AggregatorPid, Ref, - fun(P) -> format_parameter(info_keys(), P) end, list(VHost)). - -list_global_formatted() -> - [ format_parameter(global_info_keys(), P) || P <- list_global() ]. - -list_global_formatted(Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map( - AggregatorPid, Ref, - fun(P) -> format_parameter(global_info_keys(), P) end, list_global()). - -lookup(VHost, Component, Name) -> - case lookup0({VHost, Component, Name}, rabbit_misc:const(not_found)) of - not_found -> not_found; - Params -> p(Params) - end. - -lookup_global(Name) -> - case lookup0(Name, rabbit_misc:const(not_found)) of - not_found -> not_found; - Params -> p(Params) - end. - -value(VHost, Comp, Name) -> value0({VHost, Comp, Name}). -value(VHost, Comp, Name, Def) -> value0({VHost, Comp, Name}, Def). - -value_global(Key) -> - value0(Key). - -value_global(Key, Default) -> - value0(Key, Default). - -value0(Key) -> - case lookup0(Key, rabbit_misc:const(not_found)) of - not_found -> not_found; - Params -> Params#runtime_parameters.value - end. - -value0(Key, Default) -> - Params = lookup0(Key, fun () -> lookup_missing(Key, Default) end), - Params#runtime_parameters.value. - -lookup0(Key, DefaultFun) -> - case mnesia:dirty_read(?TABLE, Key) of - [] -> DefaultFun(); - [R] -> R - end. - -lookup_missing(Key, Default) -> - rabbit_misc:execute_mnesia_transaction( - fun () -> - case mnesia:read(?TABLE, Key, read) of - [] -> Record = c(Key, Default), - mnesia:write(?TABLE, Record, write), - Record; - [R] -> R - end - end). - -c(Key, Default) -> - #runtime_parameters{key = Key, - value = Default}. - -p(#runtime_parameters{key = {VHost, Component, Name}, value = Value}) -> - [{vhost, VHost}, - {component, Component}, - {name, Name}, - {value, Value}]; - -p(#runtime_parameters{key = Key, value = Value}) when is_atom(Key) -> - [{name, Key}, - {value, Value}]. - -info_keys() -> [component, name, value]. - -global_info_keys() -> [name, value]. - -%%--------------------------------------------------------------------------- - -lookup_component(Component) -> - case rabbit_registry:lookup_module( - runtime_parameter, list_to_atom(binary_to_list(Component))) of - {error, not_found} -> {errors, - [{"component ~s not found", [Component]}]}; - {ok, Module} -> {ok, Module} - end. - -flatten_errors(L) -> - case [{F, A} || I <- lists:flatten([L]), {error, F, A} <- [I]] of - [] -> ok; - E -> {errors, E} - end. diff --git a/src/rabbit_ssl.erl b/src/rabbit_ssl.erl deleted file mode 100644 index 84670b0a19..0000000000 --- a/src/rabbit_ssl.erl +++ /dev/null @@ -1,195 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_ssl). - --include_lib("public_key/include/public_key.hrl"). - --export([peer_cert_issuer/1, peer_cert_subject/1, peer_cert_validity/1]). --export([peer_cert_subject_items/2, peer_cert_auth_name/1]). --export([cipher_suites_erlang/2, cipher_suites_erlang/1, - cipher_suites_openssl/2, cipher_suites_openssl/1, - cipher_suites/1]). - -%%-------------------------------------------------------------------------- - --export_type([certificate/0]). - -% Due to API differences between OTP releases. --dialyzer(no_missing_calls). --ignore_xref([{ssl_cipher_format, suite_legacy, 1}, - {ssl_cipher_format, suite, 1}, - {ssl_cipher_format, suite_to_str, 1}, - {ssl_cipher_format, erl_suite_definition, 1}, - {ssl_cipher_format, suite_map_to_openssl_str, 1}, - {ssl_cipher_format, suite_map_to_bin, 1}]). - --type certificate() :: rabbit_cert_info:certificate(). - --type cipher_suites_mode() :: default | all | anonymous. - --spec cipher_suites(cipher_suites_mode()) -> ssl:ciphers(). -cipher_suites(Mode) -> - Version = get_highest_protocol_version(), - ssl:cipher_suites(Mode, Version). - --spec cipher_suites_erlang(cipher_suites_mode()) -> - [ssl:old_cipher_suite()]. -cipher_suites_erlang(Mode) -> - Version = get_highest_protocol_version(), - cipher_suites_erlang(Mode, Version). - --spec cipher_suites_erlang(cipher_suites_mode(), - ssl:protocol_version() | tls_record:tls_version()) -> - [ssl:old_cipher_suite()]. -cipher_suites_erlang(Mode, Version) -> - [ format_cipher_erlang(C) - || C <- ssl:cipher_suites(Mode, Version) ]. - --spec cipher_suites_openssl(cipher_suites_mode()) -> - [ssl:old_cipher_suite()]. -cipher_suites_openssl(Mode) -> - Version = get_highest_protocol_version(), - cipher_suites_openssl(Mode, Version). - --spec cipher_suites_openssl(cipher_suites_mode(), - ssl:protocol_version() | tls_record:tls_version()) -> - [ssl:old_cipher_suite()]. -cipher_suites_openssl(Mode, Version) -> - lists:filtermap(fun(C) -> - OpenSSL = format_cipher_openssl(C), - case is_list(OpenSSL) of - true -> {true, OpenSSL}; - false -> false - end - end, - ssl:cipher_suites(Mode, Version)). - - -format_cipher_erlang(Cipher) -> - case erlang:function_exported(ssl_cipher_format, suite_map_to_bin, 1) of - true -> - format_cipher_erlang22(Cipher); - false -> - format_cipher_erlang21(Cipher) - end. - -format_cipher_erlang22(Cipher) -> - ssl_cipher_format:suite_legacy(ssl_cipher_format:suite_map_to_bin(Cipher)). - -format_cipher_erlang21(Cipher) -> - ssl_cipher_format:erl_suite_definition(ssl_cipher_format:suite(Cipher)). - - -format_cipher_openssl(Cipher) -> - case erlang:function_exported(ssl_cipher_format, suite_map_to_bin, 1) of - true -> - format_cipher_openssl22(Cipher); - false -> - format_cipher_openssl21(Cipher) - end. - -format_cipher_openssl22(Cipher) -> - ssl_cipher_format:suite_map_to_openssl_str(Cipher). - -format_cipher_openssl21(Cipher) -> - ssl_cipher_format:suite_to_str(Cipher). - --spec get_highest_protocol_version() -> tls_record:tls_atom_version(). -get_highest_protocol_version() -> - tls_record:protocol_version( - tls_record:highest_protocol_version([])). - -%%-------------------------------------------------------------------------- -%% High-level functions used by reader -%%-------------------------------------------------------------------------- - -%% Return a string describing the certificate's issuer. -peer_cert_issuer(Cert) -> - rabbit_cert_info:issuer(Cert). - -%% Return a string describing the certificate's subject, as per RFC4514. -peer_cert_subject(Cert) -> - rabbit_cert_info:subject(Cert). - -%% Return the parts of the certificate's subject. -peer_cert_subject_items(Cert, Type) -> - rabbit_cert_info:subject_items(Cert, Type). - -%% Filters certificate SAN extensions by (OTP) SAN type name. -peer_cert_subject_alternative_names(Cert, Type) -> - SANs = rabbit_cert_info:subject_alternative_names(Cert), - lists:filter(fun({Key, _}) -> Key =:= Type end, SANs). - -%% Return a string describing the certificate's validity. -peer_cert_validity(Cert) -> - rabbit_cert_info:validity(Cert). - -%% Extract a username from the certificate --spec peer_cert_auth_name - (certificate()) -> binary() | 'not_found' | 'unsafe'. - -peer_cert_auth_name(Cert) -> - {ok, Mode} = application:get_env(rabbit, ssl_cert_login_from), - peer_cert_auth_name(Mode, Cert). - -peer_cert_auth_name(distinguished_name, Cert) -> - case auth_config_sane() of - true -> iolist_to_binary(peer_cert_subject(Cert)); - false -> unsafe - end; - -peer_cert_auth_name(subject_alt_name, Cert) -> - peer_cert_auth_name(subject_alternative_name, Cert); - -peer_cert_auth_name(subject_alternative_name, Cert) -> - case auth_config_sane() of - true -> - Type = application:get_env(rabbit, ssl_cert_login_san_type, dns), - %% lists:nth/2 is 1-based - Index = application:get_env(rabbit, ssl_cert_login_san_index, 0) + 1, - OfType = peer_cert_subject_alternative_names(Cert, otp_san_type(Type)), - rabbit_log:debug("Peer certificate SANs of type ~s: ~p, index to use with lists:nth/2: ~b", [Type, OfType, Index]), - case length(OfType) of - 0 -> not_found; - N when N < Index -> not_found; - N when N >= Index -> - {_, Value} = lists:nth(Index, OfType), - rabbit_data_coercion:to_binary(Value) - end; - false -> unsafe - end; - -peer_cert_auth_name(common_name, Cert) -> - %% If there is more than one CN then we join them with "," in a - %% vaguely DN-like way. But this is more just so we do something - %% more intelligent than crashing, if you actually want to escape - %% things properly etc, use DN mode. - case auth_config_sane() of - true -> case peer_cert_subject_items(Cert, ?'id-at-commonName') of - not_found -> not_found; - CNs -> list_to_binary(string:join(CNs, ",")) - end; - false -> unsafe - end. - -auth_config_sane() -> - {ok, Opts} = application:get_env(rabbit, ssl_options), - case proplists:get_value(verify, Opts) of - verify_peer -> true; - V -> rabbit_log:warning("TLS peer verification (authentication) is " - "disabled, ssl_options.verify value used: ~p. " - "See https://www.rabbitmq.com/ssl.html#peer-verification to learn more.", [V]), - false - end. - -otp_san_type(dns) -> dNSName; -otp_san_type(ip) -> iPAddress; -otp_san_type(email) -> rfc822Name; -otp_san_type(uri) -> uniformResourceIdentifier; -otp_san_type(other_name) -> otherName; -otp_san_type(Other) -> Other. diff --git a/src/rabbit_stream_coordinator.erl b/src/rabbit_stream_coordinator.erl deleted file mode 100644 index 9e4890c894..0000000000 --- a/src/rabbit_stream_coordinator.erl +++ /dev/null @@ -1,949 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License -%% at https://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and -%% limitations under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% Copyright (c) 2012-2020 VMware, Inc. or its affiliates. All rights reserved. -%% --module(rabbit_stream_coordinator). - --behaviour(ra_machine). - --export([start/0]). --export([format_ra_event/2]). - --export([init/1, - apply/3, - state_enter/2, - init_aux/1, - handle_aux/6, - tick/2]). - --export([recover/0, - start_cluster/1, - delete_cluster/2, - add_replica/2, - delete_replica/2]). - --export([policy_changed/1]). - --export([phase_repair_mnesia/2, - phase_start_cluster/1, - phase_delete_cluster/2, - phase_check_quorum/1, - phase_start_new_leader/1, - phase_stop_replicas/1, - phase_start_replica/3, - phase_delete_replica/2]). - --export([log_overview/1]). - --define(STREAM_COORDINATOR_STARTUP, {stream_coordinator_startup, self()}). --define(TICK_TIMEOUT, 60000). --define(RESTART_TIMEOUT, 1000). --define(PHASE_RETRY_TIMEOUT, 10000). --define(CMD_TIMEOUT, 30000). - --record(?MODULE, {streams, monitors}). - -start() -> - Nodes = rabbit_mnesia:cluster_nodes(all), - ServerId = {?MODULE, node()}, - case ra:restart_server(ServerId) of - {error, Reason} when Reason == not_started orelse - Reason == name_not_registered -> - case ra:start_server(make_ra_conf(node(), Nodes)) of - ok -> - global:set_lock(?STREAM_COORDINATOR_STARTUP), - case find_members(Nodes) of - [] -> - %% We're the first (and maybe only) one - ra:trigger_election(ServerId); - Members -> - %% What to do if we get a timeout? - {ok, _, _} = ra:add_member(Members, ServerId, 30000) - end, - global:del_lock(?STREAM_COORDINATOR_STARTUP), - _ = ra:members(ServerId), - ok; - Error -> - exit(Error) - end; - ok -> - ok; - Error -> - exit(Error) - end. - -find_members([]) -> - []; -find_members([Node | Nodes]) -> - case ra:members({?MODULE, Node}) of - {_, Members, _} -> - Members; - {error, noproc} -> - find_members(Nodes); - {timeout, _} -> - %% not sure what to do here - find_members(Nodes) - end. - -recover() -> - ra:restart_server({?MODULE, node()}). - -start_cluster(Q) -> - process_command({start_cluster, #{queue => Q}}). - -delete_cluster(StreamId, ActingUser) -> - process_command({delete_cluster, #{stream_id => StreamId, acting_user => ActingUser}}). - -add_replica(StreamId, Node) -> - process_command({start_replica, #{stream_id => StreamId, node => Node, - retries => 1}}). - -policy_changed(StreamId) -> - process_command({policy_changed, #{stream_id => StreamId}}). - -delete_replica(StreamId, Node) -> - process_command({delete_replica, #{stream_id => StreamId, node => Node}}). - -process_command(Cmd) -> - global:set_lock(?STREAM_COORDINATOR_STARTUP), - Servers = ensure_coordinator_started(), - global:del_lock(?STREAM_COORDINATOR_STARTUP), - process_command(Servers, Cmd). - -process_command([], _Cmd) -> - {error, coordinator_unavailable}; -process_command([Server | Servers], {CmdName, _} = Cmd) -> - case ra:process_command(Server, Cmd, ?CMD_TIMEOUT) of - {timeout, _} -> - rabbit_log:warning("Coordinator timeout on server ~p when processing command ~p", - [Server, CmdName]), - process_command(Servers, Cmd); - {error, noproc} -> - process_command(Servers, Cmd); - Reply -> - Reply - end. - -ensure_coordinator_started() -> - Local = {?MODULE, node()}, - AllNodes = all_nodes(), - case ra:restart_server(Local) of - {error, Reason} when Reason == not_started orelse - Reason == name_not_registered -> - OtherNodes = all_nodes() -- [Local], - %% We can't use find_members/0 here as a process that timeouts means the cluster is up - case lists:filter(fun(N) -> global:whereis_name(N) =/= undefined end, OtherNodes) of - [] -> - start_coordinator_cluster(); - _ -> - OtherNodes - end; - ok -> - AllNodes; - {error, {already_started, _}} -> - AllNodes; - _ -> - AllNodes - end. - -start_coordinator_cluster() -> - Nodes = rabbit_mnesia:cluster_nodes(running), - case ra:start_cluster([make_ra_conf(Node, Nodes) || Node <- Nodes]) of - {ok, Started, _} -> - Started; - {error, cluster_not_formed} -> - rabbit_log:warning("Stream coordinator cluster not formed", []), - [] - end. - -all_nodes() -> - Nodes = rabbit_mnesia:cluster_nodes(running) -- [node()], - [{?MODULE, Node} || Node <- [node() | Nodes]]. - -init(_Conf) -> - #?MODULE{streams = #{}, - monitors = #{}}. - -apply(#{from := From}, {policy_changed, #{stream_id := StreamId}} = Cmd, - #?MODULE{streams = Streams0} = State) -> - case maps:get(StreamId, Streams0, undefined) of - undefined -> - {State, ok, []}; - #{conf := Conf, - state := running} -> - case rabbit_stream_queue:update_stream_conf(Conf) of - Conf -> - %% No changes, ensure we only trigger an election if it's a must - {State, ok, []}; - _ -> - {State, ok, [{mod_call, osiris_writer, stop, [Conf]}]} - end; - SState0 -> - Streams = maps:put(StreamId, add_pending_cmd(From, Cmd, SState0), Streams0), - {State#?MODULE{streams = Streams}, '$ra_no_reply', []} - - end; -apply(#{from := From}, {start_cluster, #{queue := Q}}, #?MODULE{streams = Streams} = State) -> - #{name := StreamId} = Conf0 = amqqueue:get_type_state(Q), - Conf = apply_leader_locator_strategy(Conf0, Streams), - case maps:is_key(StreamId, Streams) of - true -> - {State, '$ra_no_reply', wrap_reply(From, {error, already_started})}; - false -> - Phase = phase_start_cluster, - PhaseArgs = [amqqueue:set_type_state(Q, Conf)], - SState = #{state => start_cluster, - phase => Phase, - phase_args => PhaseArgs, - conf => Conf, - reply_to => From, - pending_cmds => [], - pending_replicas => []}, - rabbit_log:debug("rabbit_stream_coordinator: ~p entering phase_start_cluster", [StreamId]), - {State#?MODULE{streams = maps:put(StreamId, SState, Streams)}, '$ra_no_reply', - [{aux, {phase, StreamId, Phase, PhaseArgs}}]} - end; -apply(_Meta, {start_cluster_reply, Q}, #?MODULE{streams = Streams, - monitors = Monitors0} = State) -> - #{name := StreamId, - leader_pid := LeaderPid, - replica_pids := ReplicaPids} = Conf = amqqueue:get_type_state(Q), - SState0 = maps:get(StreamId, Streams), - Phase = phase_repair_mnesia, - PhaseArgs = [new, Q], - SState = SState0#{conf => Conf, - phase => Phase, - phase_args => PhaseArgs}, - Monitors = lists:foldl(fun(Pid, M) -> - maps:put(Pid, {StreamId, follower}, M) - end, maps:put(LeaderPid, {StreamId, leader}, Monitors0), ReplicaPids), - MonitorActions = [{monitor, process, Pid} || Pid <- ReplicaPids ++ [LeaderPid]], - rabbit_log:debug("rabbit_stream_coordinator: ~p entering ~p " - "after start_cluster_reply", [StreamId, Phase]), - {State#?MODULE{streams = maps:put(StreamId, SState, Streams), - monitors = Monitors}, ok, - MonitorActions ++ [{aux, {phase, StreamId, Phase, PhaseArgs}}]}; -apply(_Meta, {start_replica_failed, StreamId, Node, Retries, Reply}, - #?MODULE{streams = Streams0} = State) -> - rabbit_log:debug("rabbit_stream_coordinator: ~p start replica failed", [StreamId]), - case maps:get(StreamId, Streams0, undefined) of - undefined -> - {State, {error, not_found}, []}; - #{pending_replicas := Pending, - reply_to := From} = SState -> - Streams = Streams0#{StreamId => clear_stream_state(SState#{pending_replicas => - add_unique(Node, Pending)})}, - reply_and_run_pending( - From, StreamId, ok, Reply, - [{timer, {pipeline, - [{start_replica, #{stream_id => StreamId, - node => Node, - from => undefined, - retries => Retries + 1}}]}, - ?RESTART_TIMEOUT * Retries}], - State#?MODULE{streams = Streams}) - end; -apply(_Meta, {phase_finished, StreamId, Reply}, #?MODULE{streams = Streams0} = State) -> - rabbit_log:debug("rabbit_stream_coordinator: ~p phase finished", [StreamId]), - case maps:get(StreamId, Streams0, undefined) of - undefined -> - {State, {error, not_found}, []}; - #{reply_to := From} = SState -> - Streams = Streams0#{StreamId => clear_stream_state(SState)}, - reply_and_run_pending(From, StreamId, ok, Reply, [], State#?MODULE{streams = Streams}) - end; -apply(#{from := From}, {start_replica, #{stream_id := StreamId, node := Node, - retries := Retries}} = Cmd, - #?MODULE{streams = Streams0} = State) -> - case maps:get(StreamId, Streams0, undefined) of - undefined -> - case From of - undefined -> - {State, ok, []}; - _ -> - {State, '$ra_no_reply', wrap_reply(From, {error, not_found})} - end; - #{conf := Conf, - state := running} = SState0 -> - Phase = phase_start_replica, - PhaseArgs = [Node, Conf, Retries], - SState = update_stream_state(From, start_replica, Phase, PhaseArgs, SState0), - rabbit_log:debug("rabbit_stream_coordinator: ~p entering ~p on node ~p", - [StreamId, Phase, Node]), - {State#?MODULE{streams = Streams0#{StreamId => SState}}, '$ra_no_reply', - [{aux, {phase, StreamId, Phase, PhaseArgs}}]}; - SState0 -> - Streams = maps:put(StreamId, add_pending_cmd(From, Cmd, SState0), Streams0), - {State#?MODULE{streams = Streams}, '$ra_no_reply', []} - end; -apply(_Meta, {start_replica_reply, StreamId, Pid}, - #?MODULE{streams = Streams, monitors = Monitors0} = State) -> - case maps:get(StreamId, Streams, undefined) of - undefined -> - {State, {error, not_found}, []}; - #{conf := Conf0} = SState0 -> - #{replica_nodes := Replicas0, - replica_pids := ReplicaPids0} = Conf0, - {ReplicaPids, MaybePid} = delete_replica_pid(node(Pid), ReplicaPids0), - Conf = Conf0#{replica_pids => [Pid | ReplicaPids], - replica_nodes => add_unique(node(Pid), Replicas0)}, - Phase = phase_repair_mnesia, - PhaseArgs = [update, Conf], - rabbit_log:debug("rabbit_stream_coordinator: ~p entering ~p after start replica", [StreamId, Phase]), - #{pending_replicas := Pending} = SState0 = maps:get(StreamId, Streams), - SState = SState0#{conf => Conf, - phase => Phase, - phase_args => PhaseArgs, - pending_replicas => lists:delete(node(Pid), Pending)}, - Monitors1 = Monitors0#{Pid => {StreamId, follower}}, - Monitors = case MaybePid of - [P] -> maps:remove(P, Monitors1); - _ -> Monitors1 - end, - {State#?MODULE{streams = Streams#{StreamId => SState}, - monitors = Monitors}, ok, - [{monitor, process, Pid}, {aux, {phase, StreamId, Phase, PhaseArgs}}]} - end; -apply(#{from := From}, {delete_replica, #{stream_id := StreamId, node := Node}} = Cmd, - #?MODULE{streams = Streams0, - monitors = Monitors0} = State) -> - case maps:get(StreamId, Streams0, undefined) of - undefined -> - {State, '$ra_no_reply', wrap_reply(From, {error, not_found})}; - #{conf := Conf0, - state := running, - pending_replicas := Pending0} = SState0 -> - Replicas0 = maps:get(replica_nodes, Conf0), - ReplicaPids0 = maps:get(replica_pids, Conf0), - case lists:member(Node, Replicas0) of - false -> - reply_and_run_pending(From, StreamId, '$ra_no_reply', ok, [], State); - true -> - [Pid] = lists:filter(fun(P) -> node(P) == Node end, ReplicaPids0), - ReplicaPids = lists:delete(Pid, ReplicaPids0), - Replicas = lists:delete(Node, Replicas0), - Pending = lists:delete(Node, Pending0), - Conf = Conf0#{replica_pids => ReplicaPids, - replica_nodes => Replicas}, - Phase = phase_delete_replica, - PhaseArgs = [Node, Conf], - SState = update_stream_state(From, delete_replica, - Phase, PhaseArgs, - SState0#{conf => Conf0, - pending_replicas => Pending}), - Monitors = maps:remove(Pid, Monitors0), - rabbit_log:debug("rabbit_stream_coordinator: ~p entering ~p on node ~p", [StreamId, Phase, Node]), - {State#?MODULE{monitors = Monitors, - streams = Streams0#{StreamId => SState}}, - '$ra_no_reply', - [{demonitor, process, Pid}, - {aux, {phase, StreamId, Phase, PhaseArgs}}]} - end; - SState0 -> - Streams = maps:put(StreamId, add_pending_cmd(From, Cmd, SState0), Streams0), - {State#?MODULE{streams = Streams}, '$ra_no_reply', []} - end; -apply(#{from := From}, {delete_cluster, #{stream_id := StreamId, - acting_user := ActingUser}} = Cmd, - #?MODULE{streams = Streams0, monitors = Monitors0} = State) -> - case maps:get(StreamId, Streams0, undefined) of - undefined -> - {State, '$ra_no_reply', wrap_reply(From, {ok, 0})}; - #{conf := Conf, - state := running} = SState0 -> - ReplicaPids = maps:get(replica_pids, Conf), - LeaderPid = maps:get(leader_pid, Conf), - Monitors = lists:foldl(fun(Pid, M) -> - maps:remove(Pid, M) - end, Monitors0, ReplicaPids ++ [LeaderPid]), - Phase = phase_delete_cluster, - PhaseArgs = [Conf, ActingUser], - SState = update_stream_state(From, delete_cluster, Phase, PhaseArgs, SState0), - Demonitors = [{demonitor, process, Pid} || Pid <- [LeaderPid | ReplicaPids]], - rabbit_log:debug("rabbit_stream_coordinator: ~p entering ~p", - [StreamId, Phase]), - {State#?MODULE{monitors = Monitors, - streams = Streams0#{StreamId => SState}}, '$ra_no_reply', - Demonitors ++ [{aux, {phase, StreamId, Phase, PhaseArgs}}]}; - SState0 -> - Streams = maps:put(StreamId, add_pending_cmd(From, Cmd, SState0), Streams0), - {State#?MODULE{streams = Streams}, '$ra_no_reply', []} - end; -apply(_Meta, {delete_cluster_reply, StreamId}, #?MODULE{streams = Streams} = State0) -> - #{reply_to := From, - pending_cmds := Pending} = maps:get(StreamId, Streams), - State = State0#?MODULE{streams = maps:remove(StreamId, Streams)}, - rabbit_log:debug("rabbit_stream_coordinator: ~p finished delete_cluster_reply", - [StreamId]), - Actions = [{ra, pipeline_command, [{?MODULE, node()}, Cmd]} || Cmd <- Pending], - {State, ok, Actions ++ wrap_reply(From, {ok, 0})}; -apply(_Meta, {down, Pid, _Reason} = Cmd, #?MODULE{streams = Streams, - monitors = Monitors0} = State) -> - case maps:get(Pid, Monitors0, undefined) of - {StreamId, Role} -> - Monitors = maps:remove(Pid, Monitors0), - case maps:get(StreamId, Streams, undefined) of - #{state := delete_cluster} -> - {State#?MODULE{monitors = Monitors}, ok, []}; - undefined -> - {State#?MODULE{monitors = Monitors}, ok, []}; - #{state := running, - conf := #{replica_pids := Pids} = Conf0, - pending_cmds := Pending0} = SState0 -> - case Role of - leader -> - rabbit_log:info("rabbit_stream_coordinator: ~p leader is down, starting election", [StreamId]), - Phase = phase_stop_replicas, - PhaseArgs = [Conf0], - SState = update_stream_state(undefined, leader_election, Phase, PhaseArgs, SState0), - Events = [{demonitor, process, P} || P <- Pids], - Monitors1 = lists:foldl(fun(P, M) -> - maps:remove(P, M) - end, Monitors, Pids), - rabbit_log:debug("rabbit_stream_coordinator: ~p entering ~p", [StreamId, Phase]), - {State#?MODULE{monitors = Monitors1, - streams = Streams#{StreamId => SState}}, - ok, Events ++ [{aux, {phase, StreamId, Phase, PhaseArgs}}]}; - follower -> - case rabbit_misc:is_process_alive(maps:get(leader_pid, Conf0)) of - true -> - Phase = phase_start_replica, - PhaseArgs = [node(Pid), Conf0, 1], - SState = update_stream_state(undefined, - replica_restart, - Phase, PhaseArgs, - SState0), - rabbit_log:debug("rabbit_stream_coordinator: ~p replica on node ~p is down, entering ~p", [StreamId, node(Pid), Phase]), - {State#?MODULE{monitors = Monitors, - streams = Streams#{StreamId => SState}}, - ok, [{aux, {phase, StreamId, Phase, PhaseArgs}}]}; - false -> - SState = SState0#{pending_cmds => Pending0 ++ [Cmd]}, - reply_and_run_pending(undefined, StreamId, ok, ok, [], State#?MODULE{streams = Streams#{StreamId => SState}}) - end - end; - #{pending_cmds := Pending0} = SState0 -> - SState = SState0#{pending_cmds => Pending0 ++ [Cmd]}, - {State#?MODULE{streams = Streams#{StreamId => SState}}, ok, []} - end; - undefined -> - {State, ok, []} - end; -apply(_Meta, {start_leader_election, StreamId, NewEpoch, Offsets}, - #?MODULE{streams = Streams} = State) -> - #{conf := Conf0} = SState0 = maps:get(StreamId, Streams), - #{leader_node := Leader, - replica_nodes := Replicas, - replica_pids := ReplicaPids0} = Conf0, - NewLeader = find_max_offset(Offsets), - rabbit_log:info("rabbit_stream_coordinator: ~p starting new leader on node ~p", - [StreamId, NewLeader]), - {ReplicaPids, _} = delete_replica_pid(NewLeader, ReplicaPids0), - Conf = rabbit_stream_queue:update_stream_conf( - Conf0#{epoch => NewEpoch, - leader_node => NewLeader, - replica_nodes => lists:delete(NewLeader, Replicas ++ [Leader]), - replica_pids => ReplicaPids}), - Phase = phase_start_new_leader, - PhaseArgs = [Conf], - SState = SState0#{conf => Conf, - phase => Phase, - phase_args => PhaseArgs}, - rabbit_log:debug("rabbit_stream_coordinator: ~p entering phase_start_new_leader", - [StreamId]), - {State#?MODULE{streams = Streams#{StreamId => SState}}, ok, - [{aux, {phase, StreamId, Phase, PhaseArgs}}]}; -apply(_Meta, {leader_elected, StreamId, NewLeaderPid}, - #?MODULE{streams = Streams, monitors = Monitors0} = State) -> - rabbit_log:info("rabbit_stream_coordinator: ~p leader elected", [StreamId]), - #{conf := Conf0, - pending_cmds := Pending0} = SState0 = maps:get(StreamId, Streams), - #{leader_pid := LeaderPid, - replica_nodes := Replicas} = Conf0, - Conf = Conf0#{leader_pid => NewLeaderPid}, - Phase = phase_repair_mnesia, - PhaseArgs = [update, Conf], - Pending = Pending0 ++ [{start_replica, #{stream_id => StreamId, node => R, - retries => 1, from => undefined}} - || R <- Replicas], - SState = SState0#{conf => Conf, - phase => Phase, - phase_args => PhaseArgs, - pending_replicas => Replicas, - pending_cmds => Pending}, - Monitors = maps:put(NewLeaderPid, {StreamId, leader}, maps:remove(LeaderPid, Monitors0)), - rabbit_log:debug("rabbit_stream_coordinator: ~p entering ~p after " - "leader election", [StreamId, Phase]), - {State#?MODULE{streams = Streams#{StreamId => SState}, - monitors = Monitors}, ok, - [{monitor, process, NewLeaderPid}, - {aux, {phase, StreamId, Phase, PhaseArgs}}]}; -apply(_Meta, {replicas_stopped, StreamId}, #?MODULE{streams = Streams} = State) -> - case maps:get(StreamId, Streams, undefined) of - undefined -> - {State, {error, not_found}, []}; - #{conf := Conf0} = SState0 -> - Phase = phase_check_quorum, - Conf = Conf0#{replica_pids => []}, - PhaseArgs = [Conf], - SState = SState0#{conf => Conf, - phase => Phase, - phase_args => PhaseArgs}, - rabbit_log:info("rabbit_stream_coordinator: ~p all replicas have been stopped, " - "checking quorum available", [StreamId]), - {State#?MODULE{streams = Streams#{StreamId => SState}}, ok, - [{aux, {phase, StreamId, Phase, PhaseArgs}}]} - end; -apply(_Meta, {stream_updated, #{name := StreamId} = Conf}, #?MODULE{streams = Streams} = State) -> - SState0 = maps:get(StreamId, Streams), - Phase = phase_repair_mnesia, - PhaseArgs = [update, Conf], - SState = SState0#{conf => Conf, - phase => Phase, - phase_args => PhaseArgs}, - rabbit_log:debug("rabbit_stream_coordinator: ~p entering ~p after" - " stream_updated", [StreamId, Phase]), - {State#?MODULE{streams = Streams#{StreamId => SState}}, ok, - [{aux, {phase, StreamId, Phase, PhaseArgs}}]}; -apply(_, {timeout, {pipeline, Cmds}}, State) -> - Actions = [{mod_call, ra, pipeline_command, [{?MODULE, node()}, Cmd]} || Cmd <- Cmds], - {State, ok, Actions}; -apply(_, {timeout, {aux, Cmd}}, State) -> - {State, ok, [{aux, Cmd}]}; -apply(Meta, {_, #{from := From}} = Cmd, State) -> - ?MODULE:apply(Meta#{from => From}, Cmd, State). - -state_enter(leader, #?MODULE{streams = Streams, monitors = Monitors}) -> - maps:fold(fun(_, #{conf := #{name := StreamId}, - pending_replicas := Pending, - state := State, - phase := Phase, - phase_args := PhaseArgs}, Acc) -> - restart_aux_phase(State, Phase, PhaseArgs, StreamId) ++ - pipeline_restart_replica_cmds(StreamId, Pending) ++ - Acc - end, [{monitor, process, P} || P <- maps:keys(Monitors)], Streams); -state_enter(follower, #?MODULE{monitors = Monitors}) -> - [{monitor, process, P} || P <- maps:keys(Monitors)]; -state_enter(recover, _) -> - put('$rabbit_vm_category', ?MODULE), - []; -state_enter(_, _) -> - []. - -restart_aux_phase(running, _, _, _) -> - []; -restart_aux_phase(_State, Phase, PhaseArgs, StreamId) -> - [{aux, {phase, StreamId, Phase, PhaseArgs}}]. - -pipeline_restart_replica_cmds(StreamId, Pending) -> - [{timer, {pipeline, [{start_replica, #{stream_id => StreamId, - node => Node, - from => undefined, - retries => 1}} - || Node <- Pending]}, ?RESTART_TIMEOUT}]. - -tick(_Ts, _State) -> - [{aux, maybe_resize_coordinator_cluster}]. - -maybe_resize_coordinator_cluster() -> - spawn(fun() -> - case ra:members({?MODULE, node()}) of - {_, Members, _} -> - MemberNodes = [Node || {_, Node} <- Members], - Running = rabbit_mnesia:cluster_nodes(running), - All = rabbit_mnesia:cluster_nodes(all), - case Running -- MemberNodes of - [] -> - ok; - New -> - rabbit_log:warning("New rabbit node(s) detected, " - "adding stream coordinator in: ~p", [New]), - add_members(Members, New) - end, - case MemberNodes -- All of - [] -> - ok; - Old -> - rabbit_log:warning("Rabbit node(s) removed from the cluster, " - "deleting stream coordinator in: ~p", [Old]), - remove_members(Members, Old) - end; - _ -> - ok - end - end). - -add_members(_, []) -> - ok; -add_members(Members, [Node | Nodes]) -> - Conf = make_ra_conf(Node, [N || {_, N} <- Members]), - case ra:start_server(Conf) of - ok -> - case ra:add_member(Members, {?MODULE, Node}) of - {ok, NewMembers, _} -> - add_members(NewMembers, Nodes); - _ -> - add_members(Members, Nodes) - end; - Error -> - rabbit_log:warning("Stream coordinator failed to start on node ~p : ~p", - [Node, Error]), - add_members(Members, Nodes) - end. - -remove_members(_, []) -> - ok; -remove_members(Members, [Node | Nodes]) -> - case ra:remove_member(Members, {?MODULE, Node}) of - {ok, NewMembers, _} -> - remove_members(NewMembers, Nodes); - _ -> - remove_members(Members, Nodes) - end. - -init_aux(_Name) -> - {#{}, undefined}. - -%% TODO ensure the dead writer is restarted as a replica at some point in time, increasing timeout? -handle_aux(leader, _, maybe_resize_coordinator_cluster, {Monitors, undefined}, LogState, _) -> - Pid = maybe_resize_coordinator_cluster(), - {no_reply, {Monitors, Pid}, LogState, [{monitor, process, aux, Pid}]}; -handle_aux(leader, _, maybe_resize_coordinator_cluster, AuxState, LogState, _) -> - %% Coordinator resizing is still happening, let's ignore this tick event - {no_reply, AuxState, LogState}; -handle_aux(leader, _, {down, Pid, _}, {Monitors, Pid}, LogState, _) -> - %% Coordinator resizing has finished - {no_reply, {Monitors, undefined}, LogState}; -handle_aux(leader, _, {phase, _, Fun, Args} = Cmd, {Monitors, Coordinator}, LogState, _) -> - Pid = erlang:apply(?MODULE, Fun, Args), - Actions = [{monitor, process, aux, Pid}], - {no_reply, {maps:put(Pid, Cmd, Monitors), Coordinator}, LogState, Actions}; -handle_aux(leader, _, {down, Pid, normal}, {Monitors, Coordinator}, LogState, _) -> - {no_reply, {maps:remove(Pid, Monitors), Coordinator}, LogState}; -handle_aux(leader, _, {down, Pid, Reason}, {Monitors0, Coordinator}, LogState, _) -> - %% The phase has failed, let's retry it - case maps:get(Pid, Monitors0) of - {phase, StreamId, phase_start_new_leader, Args} -> - rabbit_log:warning("Error while starting new leader for stream queue ~p, " - "restarting election: ~p", [StreamId, Reason]), - Monitors = maps:remove(Pid, Monitors0), - Cmd = {phase, StreamId, phase_check_quorum, Args}, - {no_reply, {Monitors, Coordinator}, LogState, [{timer, {aux, Cmd}, ?PHASE_RETRY_TIMEOUT}]}; - {phase, StreamId, Fun, _} = Cmd -> - rabbit_log:warning("Error while executing coordinator phase ~p for stream queue ~p ~p", - [Fun, StreamId, Reason]), - Monitors = maps:remove(Pid, Monitors0), - {no_reply, {Monitors, Coordinator}, LogState, [{timer, {aux, Cmd}, ?PHASE_RETRY_TIMEOUT}]} - end; -handle_aux(_, _, _, AuxState, LogState, _) -> - {no_reply, AuxState, LogState}. - -reply_and_run_pending(From, StreamId, Reply, WrapReply, Actions0, #?MODULE{streams = Streams} = State) -> - #{pending_cmds := Pending} = SState0 = maps:get(StreamId, Streams), - AuxActions = [{mod_call, ra, pipeline_command, [{?MODULE, node()}, Cmd]} - || Cmd <- Pending], - SState = maps:put(pending_cmds, [], SState0), - Actions = case From of - undefined -> - AuxActions ++ Actions0; - _ -> - wrap_reply(From, WrapReply) ++ AuxActions ++ Actions0 - end, - {State#?MODULE{streams = Streams#{StreamId => SState}}, Reply, Actions}. - -wrap_reply(From, Reply) -> - [{reply, From, {wrap_reply, Reply}}]. - -add_pending_cmd(From, {CmdName, CmdMap}, #{pending_cmds := Pending0} = StreamState) -> - %% Remove from pending the leader election and automatic replica restart when - %% the command is delete_cluster - Pending = case CmdName of - delete_cluster -> - lists:filter(fun({down, _, _}) -> - false; - (_) -> - true - end, Pending0); - _ -> - Pending0 - end, - maps:put(pending_cmds, Pending ++ [{CmdName, maps:put(from, From, CmdMap)}], - StreamState). - -clear_stream_state(StreamState) -> - StreamState#{reply_to => undefined, - state => running, - phase => undefined, - phase_args => undefined}. - -update_stream_state(From, State, Phase, PhaseArgs, StreamState) -> - StreamState#{reply_to => From, - state => State, - phase => Phase, - phase_args => PhaseArgs}. - -phase_start_replica(Node, #{name := StreamId} = Conf0, - Retries) -> - spawn( - fun() -> - %% If a new leader hasn't yet been elected, this will fail with a badmatch - %% as get_reader_context returns a no proc. An unhandled failure will - %% crash this monitored process and restart it later. - %% TODO However, do we want that crash in the log? We might need to try/catch - %% to provide a log message instead as it's 'expected'. We could try to - %% verify first that the leader is alive, but there would still be potential - %% for a race condition in here. - try - case osiris_replica:start(Node, Conf0) of - {ok, Pid} -> - ra:pipeline_command({?MODULE, node()}, - {start_replica_reply, StreamId, Pid}); - {error, already_present} -> - ra:pipeline_command({?MODULE, node()}, {phase_finished, StreamId, ok}); - {error, {already_started, _}} -> - ra:pipeline_command({?MODULE, node()}, {phase_finished, StreamId, ok}); - {error, Reason} = Error -> - rabbit_log:warning("Error while starting replica for ~p : ~p", - [maps:get(name, Conf0), Reason]), - ra:pipeline_command({?MODULE, node()}, - {start_replica_failed, StreamId, Node, Retries, Error}) - end - catch _:E-> - rabbit_log:warning("Error while starting replica for ~p : ~p", - [maps:get(name, Conf0), E]), - ra:pipeline_command({?MODULE, node()}, - {start_replica_failed, StreamId, Node, Retries, {error, E}}) - end - end). - -phase_delete_replica(Node, Conf) -> - spawn( - fun() -> - ok = osiris_replica:delete(Node, Conf), - ra:pipeline_command({?MODULE, node()}, {stream_updated, Conf}) - end). - -phase_stop_replicas(#{replica_nodes := Replicas, - name := StreamId} = Conf) -> - spawn( - fun() -> - [try - osiris_replica:stop(Node, Conf) - catch _:{{nodedown, _}, _} -> - %% It could be the old leader that is still down, it's normal. - ok - end || Node <- Replicas], - ra:pipeline_command({?MODULE, node()}, {replicas_stopped, StreamId}) - end). - -phase_start_new_leader(#{name := StreamId, leader_node := Node, leader_pid := LPid} = Conf) -> - spawn(fun() -> - osiris_replica:stop(Node, Conf), - %% If the start fails, the monitor will capture the crash and restart it - case osiris_writer:start(Conf) of - {ok, Pid} -> - ra:pipeline_command({?MODULE, node()}, - {leader_elected, StreamId, Pid}); - {error, already_present} -> - ra:pipeline_command({?MODULE, node()}, - {leader_elected, StreamId, LPid}); - {error, {already_started, Pid}} -> - ra:pipeline_command({?MODULE, node()}, - {leader_elected, StreamId, Pid}) - end - end). - -phase_check_quorum(#{name := StreamId, - epoch := Epoch, - replica_nodes := Nodes} = Conf) -> - spawn(fun() -> - Offsets = find_replica_offsets(Conf), - case is_quorum(length(Nodes) + 1, length(Offsets)) of - true -> - ra:pipeline_command({?MODULE, node()}, - {start_leader_election, StreamId, Epoch + 1, Offsets}); - false -> - %% Let's crash this process so the monitor will restart it - exit({not_enough_quorum, StreamId}) - end - end). - -find_replica_offsets(#{replica_nodes := Nodes, - leader_node := Leader} = Conf) -> - lists:foldl( - fun(Node, Acc) -> - try - %% osiris_log:overview/1 needs the directory - last item of the list - case rpc:call(Node, rabbit, is_running, []) of - false -> - Acc; - true -> - case rpc:call(Node, ?MODULE, log_overview, [Conf]) of - {badrpc, nodedown} -> - Acc; - {_Range, Offsets} -> - [{Node, select_highest_offset(Offsets)} | Acc] - end - end - catch - _:_ -> - Acc - end - end, [], Nodes ++ [Leader]). - -select_highest_offset([]) -> - empty; -select_highest_offset(Offsets) -> - lists:last(Offsets). - -log_overview(Config) -> - Dir = osiris_log:directory(Config), - osiris_log:overview(Dir). - -find_max_offset(Offsets) -> - [{Node, _} | _] = lists:sort(fun({_, {Ao, E}}, {_, {Bo, E}}) -> - Ao >= Bo; - ({_, {_, Ae}}, {_, {_, Be}}) -> - Ae >= Be; - ({_, empty}, _) -> - false; - (_, {_, empty}) -> - true - end, Offsets), - Node. - -is_quorum(1, 1) -> - true; -is_quorum(NumReplicas, NumAlive) -> - NumAlive >= ((NumReplicas div 2) + 1). - -phase_repair_mnesia(new, Q) -> - spawn(fun() -> - Reply = rabbit_amqqueue:internal_declare(Q, false), - #{name := StreamId} = amqqueue:get_type_state(Q), - ra:pipeline_command({?MODULE, node()}, {phase_finished, StreamId, Reply}) - end); - -phase_repair_mnesia(update, #{reference := QName, - leader_pid := LeaderPid, - name := StreamId} = Conf) -> - Fun = fun (Q) -> - amqqueue:set_type_state(amqqueue:set_pid(Q, LeaderPid), Conf) - end, - spawn(fun() -> - case rabbit_misc:execute_mnesia_transaction( - fun() -> - rabbit_amqqueue:update(QName, Fun) - end) of - not_found -> - %% This can happen during recovery - [Q] = mnesia:dirty_read(rabbit_durable_queue, QName), - rabbit_amqqueue:ensure_rabbit_queue_record_is_initialized(Fun(Q)); - _ -> - ok - end, - ra:pipeline_command({?MODULE, node()}, {phase_finished, StreamId, ok}) - end). - -phase_start_cluster(Q0) -> - spawn( - fun() -> - case osiris:start_cluster(amqqueue:get_type_state(Q0)) of - {ok, #{leader_pid := Pid} = Conf} -> - Q = amqqueue:set_type_state(amqqueue:set_pid(Q0, Pid), Conf), - ra:pipeline_command({?MODULE, node()}, {start_cluster_reply, Q}); - {error, {already_started, _}} -> - ra:pipeline_command({?MODULE, node()}, {start_cluster_finished, {error, already_started}}) - end - end). - -phase_delete_cluster(#{name := StreamId, - reference := QName} = Conf, ActingUser) -> - spawn( - fun() -> - ok = osiris:delete_cluster(Conf), - _ = rabbit_amqqueue:internal_delete(QName, ActingUser), - ra:pipeline_command({?MODULE, node()}, {delete_cluster_reply, StreamId}) - end). - -format_ra_event(ServerId, Evt) -> - {stream_coordinator_event, ServerId, Evt}. - -make_ra_conf(Node, Nodes) -> - UId = ra:new_uid(ra_lib:to_binary(?MODULE)), - Formatter = {?MODULE, format_ra_event, []}, - Members = [{?MODULE, N} || N <- Nodes], - TickTimeout = application:get_env(rabbit, stream_tick_interval, - ?TICK_TIMEOUT), - #{cluster_name => ?MODULE, - id => {?MODULE, Node}, - uid => UId, - friendly_name => atom_to_list(?MODULE), - metrics_key => ?MODULE, - initial_members => Members, - log_init_args => #{uid => UId}, - tick_timeout => TickTimeout, - machine => {module, ?MODULE, #{}}, - ra_event_formatter => Formatter}. - -add_unique(Node, Nodes) -> - case lists:member(Node, Nodes) of - true -> - Nodes; - _ -> - [Node | Nodes] - end. - -delete_replica_pid(Node, ReplicaPids) -> - lists:partition(fun(P) -> node(P) =/= Node end, ReplicaPids). - -apply_leader_locator_strategy(#{leader_locator_strategy := <<"client-local">>} = Conf, _) -> - Conf; -apply_leader_locator_strategy(#{leader_node := Leader, - replica_nodes := Replicas0, - leader_locator_strategy := <<"random">>, - name := StreamId} = Conf, _) -> - Replicas = [Leader | Replicas0], - ClusterSize = length(Replicas), - Hash = erlang:phash2(StreamId), - Pos = (Hash rem ClusterSize) + 1, - NewLeader = lists:nth(Pos, Replicas), - NewReplicas = lists:delete(NewLeader, Replicas), - Conf#{leader_node => NewLeader, - replica_nodes => NewReplicas}; -apply_leader_locator_strategy(#{leader_node := Leader, - replica_nodes := Replicas0, - leader_locator_strategy := <<"least-leaders">>} = Conf, - Streams) -> - Replicas = [Leader | Replicas0], - Counters0 = maps:from_list([{R, 0} || R <- Replicas]), - Counters = maps:to_list(maps:fold(fun(_Key, #{conf := #{leader_node := L}}, Acc) -> - maps:update_with(L, fun(V) -> V + 1 end, 0, Acc) - end, Counters0, Streams)), - Ordered = lists:sort(fun({_, V1}, {_, V2}) -> - V1 =< V2 - end, Counters), - %% We could have potentially introduced nodes that are not in the list of replicas if - %% initial cluster size is smaller than the cluster size. Let's select the first one - %% that is on the list of replicas - NewLeader = select_first_matching_node(Ordered, Replicas), - NewReplicas = lists:delete(NewLeader, Replicas), - Conf#{leader_node => NewLeader, - replica_nodes => NewReplicas}. - -select_first_matching_node([{N, _} | Rest], Replicas) -> - case lists:member(N, Replicas) of - true -> N; - false -> select_first_matching_node(Rest, Replicas) - end. diff --git a/src/rabbit_stream_queue.erl b/src/rabbit_stream_queue.erl deleted file mode 100644 index 4e428495b0..0000000000 --- a/src/rabbit_stream_queue.erl +++ /dev/null @@ -1,734 +0,0 @@ -%% The contents of this file are subject to the Mozilla Public License -%% Version 1.1 (the "License"); you may not use this file except in -%% compliance with the License. You may obtain a copy of the License -%% at https://www.mozilla.org/MPL/ -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and -%% limitations under the License. -%% -%% The Original Code is RabbitMQ. -%% -%% Copyright (c) 2012-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_stream_queue). - --behaviour(rabbit_queue_type). - --export([is_enabled/0, - declare/2, - delete/4, - purge/1, - policy_changed/1, - recover/2, - is_recoverable/1, - consume/3, - cancel/5, - handle_event/2, - deliver/2, - settle/4, - credit/4, - dequeue/4, - info/2, - init/1, - close/1, - update/2, - state_info/1, - stat/1, - capabilities/0]). - --export([set_retention_policy/3]). --export([add_replica/3, - delete_replica/3]). --export([format_osiris_event/2]). --export([update_stream_conf/1]). - --include("rabbit.hrl"). --include("amqqueue.hrl"). - --define(INFO_KEYS, [name, durable, auto_delete, arguments, leader, members, online, state, - messages, messages_ready, messages_unacknowledged, committed_offset, - policy, operator_policy, effective_policy_definition, type]). - --type appender_seq() :: non_neg_integer(). - --record(stream, {name :: rabbit_types:r('queue'), - credit :: integer(), - max :: non_neg_integer(), - start_offset = 0 :: non_neg_integer(), - listening_offset = 0 :: non_neg_integer(), - log :: undefined | osiris_log:state()}). - --record(stream_client, {name :: term(), - leader :: pid(), - next_seq = 1 :: non_neg_integer(), - correlation = #{} :: #{appender_seq() => term()}, - soft_limit :: non_neg_integer(), - slow = false :: boolean(), - readers = #{} :: #{term() => #stream{}} - }). - --import(rabbit_queue_type_util, [args_policy_lookup/3]). - --type client() :: #stream_client{}. - --spec is_enabled() -> boolean(). -is_enabled() -> - rabbit_feature_flags:is_enabled(stream_queue). - --spec declare(amqqueue:amqqueue(), node()) -> - {'new' | 'existing', amqqueue:amqqueue()} | - {protocol_error, Type :: atom(), Reason :: string(), Args :: term()}. -declare(Q0, Node) when ?amqqueue_is_stream(Q0) -> - case rabbit_queue_type_util:run_checks( - [fun rabbit_queue_type_util:check_auto_delete/1, - fun rabbit_queue_type_util:check_exclusive/1, - fun rabbit_queue_type_util:check_non_durable/1], - Q0) of - ok -> - start_cluster(Q0, Node); - Err -> - Err - end. - -start_cluster(Q0, Node) -> - Arguments = amqqueue:get_arguments(Q0), - QName = amqqueue:get_name(Q0), - Opts = amqqueue:get_options(Q0), - ActingUser = maps:get(user, Opts, ?UNKNOWN_USER), - Conf0 = make_stream_conf(Node, Q0), - case rabbit_stream_coordinator:start_cluster( - amqqueue:set_type_state(Q0, Conf0)) of - {ok, {error, already_started}, _} -> - {protocol_error, precondition_failed, "safe queue name already in use '~s'", - [Node]}; - {ok, {created, Q}, _} -> - rabbit_event:notify(queue_created, - [{name, QName}, - {durable, true}, - {auto_delete, false}, - {arguments, Arguments}, - {user_who_performed_action, - ActingUser}]), - {new, Q}; - {ok, {error, Error}, _} -> - _ = rabbit_amqqueue:internal_delete(QName, ActingUser), - {protocol_error, internal_error, "Cannot declare a queue '~s' on node '~s': ~255p", - [rabbit_misc:rs(QName), node(), Error]}; - {ok, {existing, Q}, _} -> - {existing, Q}; - {error, coordinator_unavailable} -> - _ = rabbit_amqqueue:internal_delete(QName, ActingUser), - {protocol_error, internal_error, - "Cannot declare a queue '~s' on node '~s': coordinator unavailable", - [rabbit_misc:rs(QName), node()]} - end. - --spec delete(amqqueue:amqqueue(), boolean(), - boolean(), rabbit_types:username()) -> - rabbit_types:ok(non_neg_integer()) | - rabbit_types:error(in_use | not_empty). -delete(Q, _IfUnused, _IfEmpty, ActingUser) -> - Name = maps:get(name, amqqueue:get_type_state(Q)), - {ok, Reply, _} = rabbit_stream_coordinator:delete_cluster(Name, ActingUser), - Reply. - --spec purge(amqqueue:amqqueue()) -> - {ok, non_neg_integer()} | {error, term()}. -purge(_) -> - {error, not_supported}. - --spec policy_changed(amqqueue:amqqueue()) -> 'ok'. -policy_changed(Q) -> - Name = maps:get(name, amqqueue:get_type_state(Q)), - _ = rabbit_stream_coordinator:policy_changed(Name), - ok. - -stat(_) -> - {ok, 0, 0}. - -consume(Q, #{prefetch_count := 0}, _) - when ?amqqueue_is_stream(Q) -> - {protocol_error, precondition_failed, "consumer prefetch count is not set for '~s'", - [rabbit_misc:rs(amqqueue:get_name(Q))]}; -consume(Q, #{no_ack := true}, _) - when ?amqqueue_is_stream(Q) -> - {protocol_error, not_implemented, - "automatic acknowledgement not supported by stream queues ~s", - [rabbit_misc:rs(amqqueue:get_name(Q))]}; -consume(Q, #{limiter_active := true}, _State) - when ?amqqueue_is_stream(Q) -> - {error, global_qos_not_supported_for_queue_type}; -consume(Q, Spec, QState0) when ?amqqueue_is_stream(Q) -> - %% Messages should include the offset as a custom header. - case check_queue_exists_in_local_node(Q) of - ok -> - #{no_ack := NoAck, - channel_pid := ChPid, - prefetch_count := ConsumerPrefetchCount, - consumer_tag := ConsumerTag, - exclusive_consume := ExclusiveConsume, - args := Args, - ok_msg := OkMsg} = Spec, - QName = amqqueue:get_name(Q), - Offset = case rabbit_misc:table_lookup(Args, <<"x-stream-offset">>) of - undefined -> - next; - {_, <<"first">>} -> - first; - {_, <<"last">>} -> - last; - {_, <<"next">>} -> - next; - {timestamp, V} -> - {timestamp, V}; - {_, V} -> - V - end, - rabbit_core_metrics:consumer_created(ChPid, ConsumerTag, ExclusiveConsume, - not NoAck, QName, - ConsumerPrefetchCount, false, - up, Args), - %% FIXME: reply needs to be sent before the stream begins sending - %% really it should be sent by the stream queue process like classic queues - %% do - maybe_send_reply(ChPid, OkMsg), - QState = begin_stream(QState0, Q, ConsumerTag, Offset, - ConsumerPrefetchCount), - {ok, QState, []}; - Err -> - Err - end. - -get_local_pid(#{leader_pid := Pid}) when node(Pid) == node() -> - Pid; -get_local_pid(#{replica_pids := ReplicaPids}) -> - [Local | _] = lists:filter(fun(Pid) -> - node(Pid) == node() - end, ReplicaPids), - Local. - -begin_stream(#stream_client{readers = Readers0} = State, - Q, Tag, Offset, Max) -> - LocalPid = get_local_pid(amqqueue:get_type_state(Q)), - {ok, Seg0} = osiris:init_reader(LocalPid, Offset), - NextOffset = osiris_log:next_offset(Seg0) - 1, - osiris:register_offset_listener(LocalPid, NextOffset), - %% TODO: avoid double calls to the same process - StartOffset = case Offset of - first -> NextOffset; - last -> NextOffset; - next -> NextOffset; - {timestamp, _} -> NextOffset; - _ -> Offset - end, - Str0 = #stream{name = amqqueue:get_name(Q), - credit = Max, - start_offset = StartOffset, - listening_offset = NextOffset, - log = Seg0, - max = Max}, - State#stream_client{readers = Readers0#{Tag => Str0}}. - -cancel(_Q, ConsumerTag, OkMsg, ActingUser, #stream_client{readers = Readers0, - name = QName} = State) -> - Readers = maps:remove(ConsumerTag, Readers0), - rabbit_core_metrics:consumer_deleted(self(), ConsumerTag, QName), - rabbit_event:notify(consumer_deleted, [{consumer_tag, ConsumerTag}, - {channel, self()}, - {queue, QName}, - {user_who_performed_action, ActingUser}]), - maybe_send_reply(self(), OkMsg), - {ok, State#stream_client{readers = Readers}}. - -credit(CTag, Credit, Drain, #stream_client{readers = Readers0, - name = Name, - leader = Leader} = State) -> - {Readers1, Msgs} = case Readers0 of - #{CTag := #stream{credit = Credit0} = Str0} -> - Str1 = Str0#stream{credit = Credit0 + Credit}, - {Str, Msgs0} = stream_entries(Name, Leader, Str1), - {Readers0#{CTag => Str}, Msgs0}; - _ -> - {Readers0, []} - end, - {Readers, Actions} = - case Drain of - true -> - case Readers1 of - #{CTag := #stream{credit = Credit1} = Str2} -> - {Readers0#{CTag => Str2#stream{credit = 0}}, [{send_drained, {CTag, Credit1}}]}; - _ -> - {Readers1, []} - end; - false -> - {Readers1, []} - end, - {State#stream_client{readers = Readers}, [{send_credit_reply, length(Msgs)}, - {deliver, CTag, true, Msgs}] ++ Actions}. - -deliver(QSs, #delivery{confirm = Confirm} = Delivery) -> - lists:foldl( - fun({_Q, stateless}, {Qs, Actions}) -> - %% TODO what do we do with stateless? - %% QRef = amqqueue:get_pid(Q), - %% ok = rabbit_fifo_client:untracked_enqueue( - %% [QRef], Delivery#delivery.message), - {Qs, Actions}; - ({Q, S0}, {Qs, Actions}) -> - S = deliver(Confirm, Delivery, S0), - {[{Q, S} | Qs], Actions} - end, {[], []}, QSs). - -deliver(_Confirm, #delivery{message = Msg, msg_seq_no = MsgId}, - #stream_client{name = Name, - leader = LeaderPid, - next_seq = Seq, - correlation = Correlation0, - soft_limit = SftLmt, - slow = Slow0} = State) -> - ok = osiris:write(LeaderPid, Seq, msg_to_iodata(Msg)), - Correlation = case MsgId of - undefined -> - Correlation0; - _ when is_number(MsgId) -> - Correlation0#{Seq => MsgId} - end, - Slow = case maps:size(Correlation) >= SftLmt of - true when not Slow0 -> - credit_flow:block(Name), - true; - Bool -> - Bool - end, - State#stream_client{next_seq = Seq + 1, - correlation = Correlation, - slow = Slow}. --spec dequeue(_, _, _, client()) -> no_return(). -dequeue(_, _, _, #stream_client{name = Name}) -> - {protocol_error, not_implemented, "basic.get not supported by stream queues ~s", - [rabbit_misc:rs(Name)]}. - -handle_event({osiris_written, From, Corrs}, State = #stream_client{correlation = Correlation0, - soft_limit = SftLmt, - slow = Slow0, - name = Name}) -> - MsgIds = maps:values(maps:with(Corrs, Correlation0)), - Correlation = maps:without(Corrs, Correlation0), - Slow = case maps:size(Correlation) < SftLmt of - true when Slow0 -> - credit_flow:unblock(Name), - false; - _ -> - Slow0 - end, - {ok, State#stream_client{correlation = Correlation, - slow = Slow}, [{settled, From, MsgIds}]}; -handle_event({osiris_offset, _From, _Offs}, State = #stream_client{leader = Leader, - readers = Readers0, - name = Name}) -> - %% offset isn't actually needed as we use the atomic to read the - %% current committed - {Readers, TagMsgs} = maps:fold( - fun (Tag, Str0, {Acc, TM}) -> - {Str, Msgs} = stream_entries(Name, Leader, Str0), - %% HACK for now, better to just return but - %% tricky with acks credits - %% that also evaluate the stream - % gen_server:cast(self(), {stream_delivery, Tag, Msgs}), - {Acc#{Tag => Str}, [{Tag, Leader, Msgs} | TM]} - end, {#{}, []}, Readers0), - Ack = true, - Deliveries = [{deliver, Tag, Ack, OffsetMsg} - || {Tag, _LeaderPid, OffsetMsg} <- TagMsgs], - {ok, State#stream_client{readers = Readers}, Deliveries}. - -is_recoverable(Q) -> - Node = node(), - #{replica_nodes := Nodes, - leader_node := Leader} = amqqueue:get_type_state(Q), - lists:member(Node, Nodes ++ [Leader]). - -recover(_VHost, Queues) -> - lists:foldl( - fun (Q0, {R0, F0}) -> - {ok, Q} = recover(Q0), - {[Q | R0], F0} - end, {[], []}, Queues). - -settle(complete, CTag, MsgIds, #stream_client{readers = Readers0, - name = Name, - leader = Leader} = State) -> - Credit = length(MsgIds), - {Readers, Msgs} = case Readers0 of - #{CTag := #stream{credit = Credit0} = Str0} -> - Str1 = Str0#stream{credit = Credit0 + Credit}, - {Str, Msgs0} = stream_entries(Name, Leader, Str1), - {Readers0#{CTag => Str}, Msgs0}; - _ -> - {Readers0, []} - end, - {State#stream_client{readers = Readers}, [{deliver, CTag, true, Msgs}]}; -settle(_, _, _, #stream_client{name = Name}) -> - {protocol_error, not_implemented, - "basic.nack and basic.reject not supported by stream queues ~s", - [rabbit_misc:rs(Name)]}. - -info(Q, all_items) -> - info(Q, ?INFO_KEYS); -info(Q, Items) -> - lists:foldr(fun(Item, Acc) -> - [{Item, i(Item, Q)} | Acc] - end, [], Items). - -i(name, Q) when ?is_amqqueue(Q) -> amqqueue:get_name(Q); -i(durable, Q) when ?is_amqqueue(Q) -> amqqueue:is_durable(Q); -i(auto_delete, Q) when ?is_amqqueue(Q) -> amqqueue:is_auto_delete(Q); -i(arguments, Q) when ?is_amqqueue(Q) -> amqqueue:get_arguments(Q); -i(leader, Q) when ?is_amqqueue(Q) -> - #{leader_node := Leader} = amqqueue:get_type_state(Q), - Leader; -i(members, Q) when ?is_amqqueue(Q) -> - #{replica_nodes := Nodes} = amqqueue:get_type_state(Q), - Nodes; -i(online, Q) -> - #{replica_pids := ReplicaPids, - leader_pid := LeaderPid} = amqqueue:get_type_state(Q), - [node(P) || P <- ReplicaPids ++ [LeaderPid], rabbit_misc:is_process_alive(P)]; -i(state, Q) when ?is_amqqueue(Q) -> - %% TODO the coordinator should answer this, I guess?? - running; -i(messages, Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - case ets:lookup(queue_coarse_metrics, QName) of - [{_, _, _, M, _}] -> - M; - [] -> - 0 - end; -i(messages_ready, Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - case ets:lookup(queue_coarse_metrics, QName) of - [{_, MR, _, _, _}] -> - MR; - [] -> - 0 - end; -i(messages_unacknowledged, Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - case ets:lookup(queue_coarse_metrics, QName) of - [{_, _, MU, _, _}] -> - MU; - [] -> - 0 - end; -i(committed_offset, Q) -> - %% TODO should it be on a metrics table? - Data = osiris_counters:overview(), - maps:get(committed_offset, - maps:get({osiris_writer, amqqueue:get_name(Q)}, Data)); -i(policy, Q) -> - case rabbit_policy:name(Q) of - none -> ''; - Policy -> Policy - end; -i(operator_policy, Q) -> - case rabbit_policy:name_op(Q) of - none -> ''; - Policy -> Policy - end; -i(effective_policy_definition, Q) -> - case rabbit_policy:effective_definition(Q) of - undefined -> []; - Def -> Def - end; -i(type, _) -> - stream; -i(_, _) -> - ''. - -init(Q) when ?is_amqqueue(Q) -> - Leader = amqqueue:get_pid(Q), - {ok, SoftLimit} = application:get_env(rabbit, stream_messages_soft_limit), - #stream_client{name = amqqueue:get_name(Q), - leader = Leader, - soft_limit = SoftLimit}. - -close(#stream_client{readers = Readers}) -> - _ = maps:map(fun (_, #stream{log = Log}) -> - osiris_log:close(Log) - end, Readers), - ok. - -update(_, State) -> - State. - -state_info(_) -> - #{}. - -set_retention_policy(Name, VHost, Policy) -> - case rabbit_amqqueue:check_max_age(Policy) of - {error, _} = E -> - E; - MaxAge -> - QName = rabbit_misc:r(VHost, queue, Name), - Fun = fun(Q) -> - Conf = amqqueue:get_type_state(Q), - amqqueue:set_type_state(Q, Conf#{max_age => MaxAge}) - end, - case rabbit_misc:execute_mnesia_transaction( - fun() -> rabbit_amqqueue:update(QName, Fun) end) of - not_found -> - {error, not_found}; - _ -> - ok - end - end. - -add_replica(VHost, Name, Node) -> - QName = rabbit_misc:r(VHost, queue, Name), - case rabbit_amqqueue:lookup(QName) of - {ok, Q} when ?amqqueue_is_classic(Q) -> - {error, classic_queue_not_supported}; - {ok, Q} when ?amqqueue_is_quorum(Q) -> - {error, quorum_queue_not_supported}; - {ok, Q} when ?amqqueue_is_stream(Q) -> - case lists:member(Node, rabbit_mnesia:cluster_nodes(running)) of - false -> - {error, node_not_running}; - true -> - #{name := StreamId} = amqqueue:get_type_state(Q), - {ok, Reply, _} = rabbit_stream_coordinator:add_replica(StreamId, Node), - Reply - end; - E -> - E - end. - -delete_replica(VHost, Name, Node) -> - QName = rabbit_misc:r(VHost, queue, Name), - case rabbit_amqqueue:lookup(QName) of - {ok, Q} when ?amqqueue_is_classic(Q) -> - {error, classic_queue_not_supported}; - {ok, Q} when ?amqqueue_is_quorum(Q) -> - {error, quorum_queue_not_supported}; - {ok, Q} when ?amqqueue_is_stream(Q) -> - case lists:member(Node, rabbit_mnesia:cluster_nodes(running)) of - false -> - {error, node_not_running}; - true -> - #{name := StreamId} = amqqueue:get_type_state(Q), - {ok, Reply, _} = rabbit_stream_coordinator:delete_replica(StreamId, Node), - Reply - end; - E -> - E - end. - -make_stream_conf(Node, Q) -> - QName = amqqueue:get_name(Q), - Name = queue_name(QName), - %% MaxLength = args_policy_lookup(<<"max-length">>, fun min/2, Q), - MaxBytes = args_policy_lookup(<<"max-length-bytes">>, fun min/2, Q), - MaxAge = max_age(args_policy_lookup(<<"max-age">>, fun max_age/2, Q)), - MaxSegmentSize = args_policy_lookup(<<"max-segment-size">>, fun min/2, Q), - LeaderLocator = queue_leader_locator(args_policy_lookup(<<"queue-leader-locator">>, - fun res_arg/2, Q)), - InitialClusterSize = initial_cluster_size(args_policy_lookup(<<"initial-cluster-size">>, - fun res_arg/2, Q)), - Replicas0 = rabbit_mnesia:cluster_nodes(all) -- [Node], - Replicas = select_stream_nodes(InitialClusterSize - 1, Replicas0), - Formatter = {?MODULE, format_osiris_event, [QName]}, - Retention = lists:filter(fun({_, R}) -> - R =/= undefined - end, [{max_bytes, MaxBytes}, - {max_age, MaxAge}]), - add_if_defined(max_segment_size, MaxSegmentSize, #{reference => QName, - name => Name, - retention => Retention, - leader_locator_strategy => LeaderLocator, - leader_node => Node, - replica_nodes => Replicas, - event_formatter => Formatter, - epoch => 1}). - -select_stream_nodes(Size, All) when length(All) =< Size -> - All; -select_stream_nodes(Size, All) -> - Node = node(), - case lists:member(Node, All) of - true -> - select_stream_nodes(Size - 1, lists:delete(Node, All), [Node]); - false -> - select_stream_nodes(Size, All, []) - end. - -select_stream_nodes(0, _, Selected) -> - Selected; -select_stream_nodes(Size, Rest, Selected) -> - S = lists:nth(rand:uniform(length(Rest)), Rest), - select_stream_nodes(Size - 1, lists:delete(S, Rest), [S | Selected]). - -update_stream_conf(#{reference := QName} = Conf) -> - case rabbit_amqqueue:lookup(QName) of - {ok, Q} -> - MaxBytes = args_policy_lookup(<<"max-length-bytes">>, fun min/2, Q), - MaxAge = max_age(args_policy_lookup(<<"max-age">>, fun max_age/2, Q)), - MaxSegmentSize = args_policy_lookup(<<"max-segment-size">>, fun min/2, Q), - Retention = lists:filter(fun({_, R}) -> - R =/= undefined - end, [{max_bytes, MaxBytes}, - {max_age, MaxAge}]), - add_if_defined(max_segment_size, MaxSegmentSize, Conf#{retention => Retention}); - _ -> - Conf - end. - -add_if_defined(_, undefined, Map) -> - Map; -add_if_defined(Key, Value, Map) -> - maps:put(Key, Value, Map). - -format_osiris_event(Evt, QRef) -> - {'$gen_cast', {queue_event, QRef, Evt}}. - -max_age(undefined) -> - undefined; -max_age(Bin) when is_binary(Bin) -> - rabbit_amqqueue:check_max_age(Bin); -max_age(Age) -> - Age. - -max_age(Age1, Age2) -> - min(rabbit_amqqueue:check_max_age(Age1), rabbit_amqqueue:check_max_age(Age2)). - -queue_leader_locator(undefined) -> <<"client-local">>; -queue_leader_locator(Val) -> Val. - -initial_cluster_size(undefined) -> - length(rabbit_mnesia:cluster_nodes(running)); -initial_cluster_size(Val) -> - Val. - -res_arg(PolVal, undefined) -> PolVal; -res_arg(_, ArgVal) -> ArgVal. - -queue_name(#resource{virtual_host = VHost, name = Name}) -> - Timestamp = erlang:integer_to_binary(erlang:system_time()), - osiris_util:to_base64uri(erlang:binary_to_list(<<VHost/binary, "_", Name/binary, "_", - Timestamp/binary>>)). - -recover(Q) -> - rabbit_stream_coordinator:recover(), - {ok, Q}. - -check_queue_exists_in_local_node(Q) -> - Conf = amqqueue:get_type_state(Q), - AllNodes = [maps:get(leader_node, Conf) | maps:get(replica_nodes, Conf)], - case lists:member(node(), AllNodes) of - true -> - ok; - false -> - {protocol_error, precondition_failed, - "queue '~s' does not a have a replica on the local node", - [rabbit_misc:rs(amqqueue:get_name(Q))]} - end. - -maybe_send_reply(_ChPid, undefined) -> ok; -maybe_send_reply(ChPid, Msg) -> ok = rabbit_channel:send_command(ChPid, Msg). - -stream_entries(Name, Id, Str) -> - stream_entries(Name, Id, Str, []). - -stream_entries(Name, LeaderPid, - #stream{name = QName, - credit = Credit, - start_offset = StartOffs, - listening_offset = LOffs, - log = Seg0} = Str0, MsgIn) - when Credit > 0 -> - case osiris_log:read_chunk_parsed(Seg0) of - {end_of_stream, Seg} -> - NextOffset = osiris_log:next_offset(Seg), - case NextOffset > LOffs of - true -> - osiris:register_offset_listener(LeaderPid, NextOffset), - {Str0#stream{log = Seg, - listening_offset = NextOffset}, MsgIn}; - false -> - {Str0#stream{log = Seg}, MsgIn} - end; - {Records, Seg} -> - Msgs = [begin - Msg0 = binary_to_msg(QName, B), - Msg = rabbit_basic:add_header(<<"x-stream-offset">>, - long, O, Msg0), - {Name, LeaderPid, O, false, Msg} - end || {O, B} <- Records, - O >= StartOffs], - - NumMsgs = length(Msgs), - - Str = Str0#stream{credit = Credit - NumMsgs, - log = Seg}, - case Str#stream.credit < 1 of - true -> - %% we are done here - {Str, MsgIn ++ Msgs}; - false -> - %% if there are fewer Msgs than Entries0 it means there were non-events - %% in the log and we should recurse and try again - stream_entries(Name, LeaderPid, Str, MsgIn ++ Msgs) - end - end; -stream_entries(_Name, _Id, Str, Msgs) -> - {Str, Msgs}. - -binary_to_msg(#resource{virtual_host = VHost, - kind = queue, - name = QName}, Data) -> - R0 = rabbit_msg_record:init(Data), - %% if the message annotation isn't present the data most likely came from - %% the rabbitmq-stream plugin so we'll choose defaults that simulate use - %% of the direct exchange - {utf8, Exchange} = rabbit_msg_record:message_annotation(<<"x-exchange">>, - R0, {utf8, <<>>}), - {utf8, RoutingKey} = rabbit_msg_record:message_annotation(<<"x-routing-key">>, - R0, {utf8, QName}), - {Props, Payload} = rabbit_msg_record:to_amqp091(R0), - XName = #resource{kind = exchange, - virtual_host = VHost, - name = Exchange}, - Content = #content{class_id = 60, - properties = Props, - properties_bin = none, - payload_fragments_rev = [Payload]}, - {ok, Msg} = rabbit_basic:message(XName, RoutingKey, Content), - Msg. - - -msg_to_iodata(#basic_message{exchange_name = #resource{name = Exchange}, - routing_keys = [RKey | _], - content = Content}) -> - #content{properties = Props, - payload_fragments_rev = Payload} = - rabbit_binary_parser:ensure_content_decoded(Content), - R0 = rabbit_msg_record:from_amqp091(Props, lists:reverse(Payload)), - %% TODO durable? - R = rabbit_msg_record:add_message_annotations( - #{<<"x-exchange">> => {utf8, Exchange}, - <<"x-routing-key">> => {utf8, RKey}}, R0), - rabbit_msg_record:to_iodata(R). - -capabilities() -> - #{policies => [<<"max-length-bytes">>, <<"max-age">>, <<"max-segment-size">>, - <<"queue-leader-locator">>, <<"initial-cluster-size">>], - queue_arguments => [<<"x-dead-letter-exchange">>, <<"x-dead-letter-routing-key">>, - <<"x-max-length">>, <<"x-max-length-bytes">>, - <<"x-single-active-consumer">>, <<"x-queue-type">>, - <<"x-max-age">>, <<"x-max-segment-size">>, - <<"x-initial-cluster-size">>, <<"x-queue-leader-locator">>], - consumer_arguments => [<<"x-stream-offset">>], - server_named => false}. diff --git a/src/rabbit_sup.erl b/src/rabbit_sup.erl deleted file mode 100644 index 06643b155d..0000000000 --- a/src/rabbit_sup.erl +++ /dev/null @@ -1,109 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_sup). - --behaviour(supervisor). - --export([start_link/0, start_child/1, start_child/2, start_child/3, start_child/4, - start_supervisor_child/1, start_supervisor_child/2, - start_supervisor_child/3, - start_restartable_child/1, start_restartable_child/2, - start_delayed_restartable_child/1, start_delayed_restartable_child/2, - stop_child/1]). - --export([init/1]). - --include("rabbit.hrl"). - --define(SERVER, ?MODULE). - -%%---------------------------------------------------------------------------- - --spec start_link() -> rabbit_types:ok_pid_or_error(). - -start_link() -> supervisor:start_link({local, ?SERVER}, ?MODULE, []). - --spec start_child(atom()) -> 'ok'. - -start_child(Mod) -> start_child(Mod, []). - --spec start_child(atom(), [any()]) -> 'ok'. - -start_child(Mod, Args) -> start_child(Mod, Mod, Args). - --spec start_child(atom(), atom(), [any()]) -> 'ok'. - -start_child(ChildId, Mod, Args) -> - child_reply(supervisor:start_child( - ?SERVER, - {ChildId, {Mod, start_link, Args}, - transient, ?WORKER_WAIT, worker, [Mod]})). - --spec start_child(atom(), atom(), atom(), [any()]) -> 'ok'. - -start_child(ChildId, Mod, Fun, Args) -> - child_reply(supervisor:start_child( - ?SERVER, - {ChildId, {Mod, Fun, Args}, - transient, ?WORKER_WAIT, worker, [Mod]})). - --spec start_supervisor_child(atom()) -> 'ok'. - -start_supervisor_child(Mod) -> start_supervisor_child(Mod, []). - --spec start_supervisor_child(atom(), [any()]) -> 'ok'. - -start_supervisor_child(Mod, Args) -> start_supervisor_child(Mod, Mod, Args). - --spec start_supervisor_child(atom(), atom(), [any()]) -> 'ok'. - -start_supervisor_child(ChildId, Mod, Args) -> - child_reply(supervisor:start_child( - ?SERVER, - {ChildId, {Mod, start_link, Args}, - transient, infinity, supervisor, [Mod]})). - --spec start_restartable_child(atom()) -> 'ok'. - -start_restartable_child(M) -> start_restartable_child(M, [], false). - --spec start_restartable_child(atom(), [any()]) -> 'ok'. - -start_restartable_child(M, A) -> start_restartable_child(M, A, false). - --spec start_delayed_restartable_child(atom()) -> 'ok'. - -start_delayed_restartable_child(M) -> start_restartable_child(M, [], true). - --spec start_delayed_restartable_child(atom(), [any()]) -> 'ok'. - -start_delayed_restartable_child(M, A) -> start_restartable_child(M, A, true). - -start_restartable_child(Mod, Args, Delay) -> - Name = list_to_atom(atom_to_list(Mod) ++ "_sup"), - child_reply(supervisor:start_child( - ?SERVER, - {Name, {rabbit_restartable_sup, start_link, - [Name, {Mod, start_link, Args}, Delay]}, - transient, infinity, supervisor, [rabbit_restartable_sup]})). - --spec stop_child(atom()) -> rabbit_types:ok_or_error(any()). - -stop_child(ChildId) -> - case supervisor:terminate_child(?SERVER, ChildId) of - ok -> supervisor:delete_child(?SERVER, ChildId); - E -> E - end. - -init([]) -> {ok, {{one_for_all, 0, 1}, []}}. - - -%%---------------------------------------------------------------------------- - -child_reply({ok, _}) -> ok; -child_reply(X) -> X. diff --git a/src/rabbit_sysmon_handler.erl b/src/rabbit_sysmon_handler.erl deleted file mode 100644 index 8f7298ed6e..0000000000 --- a/src/rabbit_sysmon_handler.erl +++ /dev/null @@ -1,235 +0,0 @@ -%% Copyright (c) 2011 Basho Technologies, Inc. All Rights Reserved. -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% https://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. - -%% @doc A custom event handler to the `sysmon_handler' application's -%% `system_monitor' event manager. -%% -%% This module attempts to discover more information about a process -%% that generates a system_monitor event. - --module(rabbit_sysmon_handler). - --behaviour(gen_event). - -%% API --export([add_handler/0]). - -%% gen_event callbacks --export([init/1, handle_event/2, handle_call/2, - handle_info/2, terminate/2, code_change/3]). - --record(state, {timer_ref :: reference() | undefined}). - --define(INACTIVITY_TIMEOUT, 5000). - -%%%=================================================================== -%%% gen_event callbacks -%%%=================================================================== - -add_handler() -> - %% Vulnerable to race conditions (installing handler multiple - %% times), but risk is zero in the common OTP app startup case. - case lists:member(?MODULE, gen_event:which_handlers(sysmon_handler)) of - true -> - ok; - false -> - sysmon_handler_filter:add_custom_handler(?MODULE, []) - end. - -%%%=================================================================== -%%% gen_event callbacks -%%%=================================================================== - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Whenever a new event handler is added to an event manager, -%% this function is called to initialize the event handler. -%% -%% @spec init(Args) -> {ok, State} -%% @end -%%-------------------------------------------------------------------- -init([]) -> - {ok, #state{}, hibernate}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Whenever an event manager receives an event sent using -%% gen_event:notify/2 or gen_event:sync_notify/2, this function is -%% called for each installed event handler to handle the event. -%% -%% @spec handle_event(Event, State) -> -%% {ok, State} | -%% {swap_handler, Args1, State1, Mod2, Args2} | -%% remove_handler -%% @end -%%-------------------------------------------------------------------- -handle_event({monitor, Pid, Type, _Info}, - State=#state{timer_ref=TimerRef}) when Pid == self() -> - %% Reset the inactivity timeout - NewTimerRef = reset_timer(TimerRef), - maybe_collect_garbage(Type), - {ok, State#state{timer_ref=NewTimerRef}}; -handle_event({monitor, PidOrPort, Type, Info}, State=#state{timer_ref=TimerRef}) -> - %% Reset the inactivity timeout - NewTimerRef = reset_timer(TimerRef), - {Fmt, Args} = format_pretty_proc_or_port_info(PidOrPort), - rabbit_log:warning("~p ~w ~w " ++ Fmt ++ " ~w", [?MODULE, Type, PidOrPort] ++ Args ++ [Info]), - {ok, State#state{timer_ref=NewTimerRef}}; -handle_event({suppressed, Type, Info}, State=#state{timer_ref=TimerRef}) -> - %% Reset the inactivity timeout - NewTimerRef = reset_timer(TimerRef), - rabbit_log:debug("~p encountered a suppressed event of type ~w: ~w", [?MODULE, Type, Info]), - {ok, State#state{timer_ref=NewTimerRef}}; -handle_event(Event, State=#state{timer_ref=TimerRef}) -> - NewTimerRef = reset_timer(TimerRef), - rabbit_log:warning("~p unhandled event: ~p", [?MODULE, Event]), - {ok, State#state{timer_ref=NewTimerRef}}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Whenever an event manager receives a request sent using -%% gen_event:call/3,4, this function is called for the specified -%% event handler to handle the request. -%% -%% @spec handle_call(Request, State) -> -%% {ok, Reply, State} | -%% {swap_handler, Reply, Args1, State1, Mod2, Args2} | -%% {remove_handler, Reply} -%% @end -%%-------------------------------------------------------------------- -handle_call(_Call, State) -> - Reply = not_supported, - {ok, Reply, State}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% This function is called for each installed event handler when -%% an event manager receives any other message than an event or a -%% synchronous request (or a system message). -%% -%% @spec handle_info(Info, State) -> -%% {ok, State} | -%% {swap_handler, Args1, State1, Mod2, Args2} | -%% remove_handler -%% @end -%%-------------------------------------------------------------------- -handle_info(inactivity_timeout, State) -> - %% No events have arrived for the timeout period - %% so hibernate to free up resources. - {ok, State, hibernate}; -handle_info(Info, State) -> - rabbit_log:info("handle_info got ~p", [Info]), - {ok, State}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Whenever an event handler is deleted from an event manager, this -%% function is called. It should be the opposite of Module:init/1 and -%% do any necessary cleaning up. -%% -%% @spec terminate(Reason, State) -> void() -%% @end -%%-------------------------------------------------------------------- -terminate(_Reason, _State) -> - ok. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Convert process state when code is changed -%% -%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} -%% @end -%%-------------------------------------------------------------------- -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%%=================================================================== -%%% Internal functions -%%%=================================================================== - -format_pretty_proc_or_port_info(PidOrPort) -> - try - case get_pretty_proc_or_port_info(PidOrPort) of - undefined -> - {"", []}; - Res -> - Res - end - catch C:E:S -> - {"Pid ~w, ~W ~W at ~w\n", - [PidOrPort, C, 20, E, 20, S]} - end. - -get_pretty_proc_or_port_info(Pid) when is_pid(Pid) -> - Infos = [registered_name, initial_call, current_function, message_queue_len], - case process_info(Pid, Infos) of - undefined -> - undefined; - [] -> - undefined; - [{registered_name, RN0}, ICT1, {_, CF}, {_, MQL}] -> - ICT = case proc_lib:translate_initial_call(Pid) of - {proc_lib, init_p, 5} -> % not by proc_lib, see docs - ICT1; - ICT2 -> - {initial_call, ICT2} - end, - RNL = if RN0 == [] -> []; - true -> [{name, RN0}] - end, - {"~w", [RNL ++ [ICT, CF, {message_queue_len, MQL}]]} - end; -get_pretty_proc_or_port_info(Port) when is_port(Port) -> - PortInfo = erlang:port_info(Port), - {value, {name, Name}, PortInfo2} = lists:keytake(name, 1, PortInfo), - QueueSize = [erlang:port_info(Port, queue_size)], - Connected = case proplists:get_value(connected, PortInfo2) of - undefined -> - []; - ConnectedPid -> - case proc_lib:translate_initial_call(ConnectedPid) of - {proc_lib, init_p, 5} -> % not by proc_lib, see docs - []; - ICT -> - [{initial_call, ICT}] - end - end, - {"name ~s ~w", [Name, lists:append([PortInfo2, QueueSize, Connected])]}. - - -%% @doc If the message type is due to a large heap warning -%% and the source is ourself, go ahead and collect garbage -%% to avoid the death spiral. --spec maybe_collect_garbage(atom()) -> ok. -maybe_collect_garbage(large_heap) -> - erlang:garbage_collect(), - ok; -maybe_collect_garbage(_) -> - ok. - --spec reset_timer(undefined | reference()) -> reference(). -reset_timer(undefined) -> - erlang:send_after(?INACTIVITY_TIMEOUT, self(), inactivity_timeout); -reset_timer(TimerRef) -> - _ = erlang:cancel_timer(TimerRef), - reset_timer(undefined). diff --git a/src/rabbit_sysmon_minder.erl b/src/rabbit_sysmon_minder.erl deleted file mode 100644 index a0402e5ebe..0000000000 --- a/src/rabbit_sysmon_minder.erl +++ /dev/null @@ -1,156 +0,0 @@ -%% ------------------------------------------------------------------- -%% Copyright (c) 2007-2010 Basho Technologies, Inc. All Rights Reserved. -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% https://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - --module(rabbit_sysmon_minder). - --behaviour(gen_server). - -%% API --export([start_link/0]). - -%% gen_server callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {}). - -%%%=================================================================== -%%% API -%%%=================================================================== - -%%-------------------------------------------------------------------- -%% @doc -%% Starts the server -%% -%% @spec start_link() -> {ok, Pid} | ignore | {error, Error} -%% @end -%%-------------------------------------------------------------------- -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -%%%=================================================================== -%%% gen_server callbacks -%%%=================================================================== - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Initializes the server -%% -%% @spec init(Args) -> {ok, State} | -%% {ok, State, Timeout} | -%% ignore | -%% {stop, Reason} -%% @end -%%-------------------------------------------------------------------- -init([]) -> - %% Add our system_monitor event handler. We do that here because - %% we have a process at our disposal (i.e. ourself) to receive the - %% notification in the very unlikely event that the - %% sysmon_handler has crashed and been removed from the - %% sysmon_handler gen_event server. (If we had a supervisor - %% or app-starting process add the handler, then if the handler - %% crashes, nobody will act on the crash notification.) - rabbit_sysmon_handler:add_handler(), - {ok, #state{}}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Handling call messages -%% -%% @spec handle_call(Request, From, State) -> -%% {reply, Reply, State} | -%% {reply, Reply, State, Timeout} | -%% {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, Reply, State} | -%% {stop, Reason, State} -%% @end -%%-------------------------------------------------------------------- -handle_call(_Request, _From, State) -> - Reply = ok, - {reply, Reply, State}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Handling cast messages -%% -%% @spec handle_cast(Msg, State) -> {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, State} -%% @end -%%-------------------------------------------------------------------- -handle_cast(_Msg, State) -> - {noreply, State}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Handling all non call/cast messages -%% -%% @spec handle_info(Info, State) -> {noreply, State} | -%% {noreply, State, Timeout} | -%% {stop, Reason, State} -%% @end -%%-------------------------------------------------------------------- -handle_info({gen_event_EXIT, rabbit_sysmon_handler, _}, State) -> - %% SASL will create an error message, no need for us to duplicate it. - %% - %% Our handler should never crash, but it did indeed crash. If - %% there's a pathological condition somewhere that's generating - %% lots of unforseen things that crash core's custom handler, we - %% could make things worse by jumping back into the exploding - %% volcano. Wait a little bit before jumping back. Besides, the - %% system_monitor data is nice but is not critical: there is no - %% need to make things worse if things are indeed bad, and if we - %% miss a few seconds of system_monitor events, the world will not - %% end. - timer:sleep(2*1000), - rabbit_sysmon_handler:add_handler(), - {noreply, State}; -handle_info(_Info, State) -> - {noreply, State}. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% This function is called by a gen_server when it is about to -%% terminate. It should be the opposite of Module:init/1 and do any -%% necessary cleaning up. When it returns, the gen_server terminates -%% with Reason. The return value is ignored. -%% -%% @spec terminate(Reason, State) -> void() -%% @end -%%-------------------------------------------------------------------- -terminate(_Reason, _State) -> - ok. - -%%-------------------------------------------------------------------- -%% @private -%% @doc -%% Convert process state when code is changed -%% -%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState} -%% @end -%%-------------------------------------------------------------------- -code_change(_OldVsn, State, _Extra) -> - {ok, State}. diff --git a/src/rabbit_table.erl b/src/rabbit_table.erl deleted file mode 100644 index 77534763d0..0000000000 --- a/src/rabbit_table.erl +++ /dev/null @@ -1,416 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_table). - --export([ - create/0, create/2, ensure_local_copies/1, ensure_table_copy/2, - wait_for_replicated/1, wait/1, wait/2, - force_load/0, is_present/0, is_empty/0, needs_default_data/0, - check_schema_integrity/1, clear_ram_only_tables/0, retry_timeout/0, - wait_for_replicated/0, exists/1]). - -%% for testing purposes --export([definitions/0]). - --include_lib("rabbit_common/include/rabbit.hrl"). - -%%---------------------------------------------------------------------------- - --type retry() :: boolean(). --type mnesia_table() :: atom(). - -%%---------------------------------------------------------------------------- -%% Main interface -%%---------------------------------------------------------------------------- - --spec create() -> 'ok'. - -create() -> - lists:foreach( - fun ({Table, Def}) -> create(Table, Def) end, - definitions()), - ensure_secondary_indexes(), - ok. - --spec create(mnesia_table(), list()) -> rabbit_types:ok_or_error(any()). - -create(TableName, TableDefinition) -> - TableDefinition1 = proplists:delete(match, TableDefinition), - rabbit_log:debug("Will create a schema database table '~s'", [TableName]), - case mnesia:create_table(TableName, TableDefinition1) of - {atomic, ok} -> ok; - {aborted,{already_exists, TableName}} -> ok; - {aborted, {already_exists, TableName, _}} -> ok; - {aborted, Reason} -> - throw({error, {table_creation_failed, TableName, TableDefinition1, Reason}}) - end. - --spec exists(mnesia_table()) -> boolean(). -exists(Table) -> - lists:member(Table, mnesia:system_info(tables)). - -%% Sets up secondary indexes in a blank node database. -ensure_secondary_indexes() -> - ensure_secondary_index(rabbit_queue, vhost), - ok. - -ensure_secondary_index(Table, Field) -> - case mnesia:add_table_index(Table, Field) of - {atomic, ok} -> ok; - {aborted, {already_exists, Table, _}} -> ok - end. - --spec ensure_table_copy(mnesia_table(), node()) -> ok | {error, any()}. -ensure_table_copy(TableName, Node) -> - rabbit_log:debug("Will add a local schema database copy for table '~s'", [TableName]), - case mnesia:add_table_copy(TableName, Node, disc_copies) of - {atomic, ok} -> ok; - {aborted,{already_exists, TableName}} -> ok; - {aborted, {already_exists, TableName, _}} -> ok; - {aborted, Reason} -> {error, Reason} - end. - -%% This arity only exists for backwards compatibility with certain -%% plugins. See https://github.com/rabbitmq/rabbitmq-clusterer/issues/19. - --spec wait_for_replicated() -> 'ok'. - -wait_for_replicated() -> - wait_for_replicated(false). - --spec wait_for_replicated(retry()) -> 'ok'. - -wait_for_replicated(Retry) -> - wait([Tab || {Tab, TabDef} <- definitions(), - not lists:member({local_content, true}, TabDef)], Retry). - --spec wait([atom()]) -> 'ok'. - -wait(TableNames) -> - wait(TableNames, _Retry = false). - -wait(TableNames, Retry) -> - {Timeout, Retries} = retry_timeout(Retry), - wait(TableNames, Timeout, Retries). - -wait(TableNames, Timeout, Retries) -> - %% We might be in ctl here for offline ops, in which case we can't - %% get_env() for the rabbit app. - rabbit_log:info("Waiting for Mnesia tables for ~p ms, ~p retries left~n", - [Timeout, Retries - 1]), - Result = case mnesia:wait_for_tables(TableNames, Timeout) of - ok -> - ok; - {timeout, BadTabs} -> - AllNodes = rabbit_mnesia:cluster_nodes(all), - {error, {timeout_waiting_for_tables, AllNodes, BadTabs}}; - {error, Reason} -> - AllNodes = rabbit_mnesia:cluster_nodes(all), - {error, {failed_waiting_for_tables, AllNodes, Reason}} - end, - case {Retries, Result} of - {_, ok} -> - rabbit_log:info("Successfully synced tables from a peer"), - ok; - {1, {error, _} = Error} -> - throw(Error); - {_, {error, Error}} -> - rabbit_log:warning("Error while waiting for Mnesia tables: ~p~n", [Error]), - wait(TableNames, Timeout, Retries - 1) - end. - -retry_timeout(_Retry = false) -> - {retry_timeout(), 1}; -retry_timeout(_Retry = true) -> - Retries = case application:get_env(rabbit, mnesia_table_loading_retry_limit) of - {ok, T} -> T; - undefined -> 10 - end, - {retry_timeout(), Retries}. - --spec retry_timeout() -> non_neg_integer() | infinity. - -retry_timeout() -> - case application:get_env(rabbit, mnesia_table_loading_retry_timeout) of - {ok, T} -> T; - undefined -> 30000 - end. - --spec force_load() -> 'ok'. - -force_load() -> [mnesia:force_load_table(T) || T <- names()], ok. - --spec is_present() -> boolean(). - -is_present() -> names() -- mnesia:system_info(tables) =:= []. - --spec is_empty() -> boolean(). - -is_empty() -> is_empty(names()). - --spec needs_default_data() -> boolean(). - -needs_default_data() -> is_empty([rabbit_user, rabbit_user_permission, - rabbit_vhost]). - -is_empty(Names) -> - lists:all(fun (Tab) -> mnesia:dirty_first(Tab) == '$end_of_table' end, - Names). - --spec check_schema_integrity(retry()) -> rabbit_types:ok_or_error(any()). - -check_schema_integrity(Retry) -> - Tables = mnesia:system_info(tables), - case check(fun (Tab, TabDef) -> - case lists:member(Tab, Tables) of - false -> {error, {table_missing, Tab}}; - true -> check_attributes(Tab, TabDef) - end - end) of - ok -> wait(names(), Retry), - check(fun check_content/2); - Other -> Other - end. - --spec clear_ram_only_tables() -> 'ok'. - -clear_ram_only_tables() -> - Node = node(), - lists:foreach( - fun (TabName) -> - case lists:member(Node, mnesia:table_info(TabName, ram_copies)) of - true -> {atomic, ok} = mnesia:clear_table(TabName); - false -> ok - end - end, names()), - ok. - -%% The sequence in which we delete the schema and then the other -%% tables is important: if we delete the schema first when moving to -%% RAM mnesia will loudly complain since it doesn't make much sense to -%% do that. But when moving to disc, we need to move the schema first. - --spec ensure_local_copies('disc' | 'ram') -> 'ok'. - -ensure_local_copies(disc) -> - create_local_copy(schema, disc_copies), - create_local_copies(disc); -ensure_local_copies(ram) -> - create_local_copies(ram), - create_local_copy(schema, ram_copies). - -%%-------------------------------------------------------------------- -%% Internal helpers -%%-------------------------------------------------------------------- - -create_local_copies(Type) -> - lists:foreach( - fun ({Tab, TabDef}) -> - HasDiscCopies = has_copy_type(TabDef, disc_copies), - HasDiscOnlyCopies = has_copy_type(TabDef, disc_only_copies), - LocalTab = proplists:get_bool(local_content, TabDef), - StorageType = - if - Type =:= disc orelse LocalTab -> - if - HasDiscCopies -> disc_copies; - HasDiscOnlyCopies -> disc_only_copies; - true -> ram_copies - end; - Type =:= ram -> - ram_copies - end, - ok = create_local_copy(Tab, StorageType) - end, definitions(Type)), - ok. - -create_local_copy(Tab, Type) -> - StorageType = mnesia:table_info(Tab, storage_type), - {atomic, ok} = - if - StorageType == unknown -> - mnesia:add_table_copy(Tab, node(), Type); - StorageType /= Type -> - mnesia:change_table_copy_type(Tab, node(), Type); - true -> {atomic, ok} - end, - ok. - -has_copy_type(TabDef, DiscType) -> - lists:member(node(), proplists:get_value(DiscType, TabDef, [])). - -check_attributes(Tab, TabDef) -> - {_, ExpAttrs} = proplists:lookup(attributes, TabDef), - case mnesia:table_info(Tab, attributes) of - ExpAttrs -> ok; - Attrs -> {error, {table_attributes_mismatch, Tab, ExpAttrs, Attrs}} - end. - -check_content(Tab, TabDef) -> - {_, Match} = proplists:lookup(match, TabDef), - case mnesia:dirty_first(Tab) of - '$end_of_table' -> - ok; - Key -> - ObjList = mnesia:dirty_read(Tab, Key), - MatchComp = ets:match_spec_compile([{Match, [], ['$_']}]), - case ets:match_spec_run(ObjList, MatchComp) of - ObjList -> ok; - _ -> {error, {table_content_invalid, Tab, Match, ObjList}} - end - end. - -check(Fun) -> - case [Error || {Tab, TabDef} <- definitions(), - begin - {Ret, Error} = case Fun(Tab, TabDef) of - ok -> {false, none}; - {error, E} -> {true, E} - end, - Ret - end] of - [] -> ok; - Errors -> {error, Errors} - end. - -%%-------------------------------------------------------------------- -%% Table definitions -%%-------------------------------------------------------------------- - -names() -> [Tab || {Tab, _} <- definitions()]. - -%% The tables aren't supposed to be on disk on a ram node -definitions(disc) -> - definitions(); -definitions(ram) -> - [{Tab, [{disc_copies, []}, {ram_copies, [node()]} | - proplists:delete( - ram_copies, proplists:delete(disc_copies, TabDef))]} || - {Tab, TabDef} <- definitions()]. - -definitions() -> - [{rabbit_user, - [{record_name, internal_user}, - {attributes, internal_user:fields()}, - {disc_copies, [node()]}, - {match, internal_user:pattern_match_all()}]}, - {rabbit_user_permission, - [{record_name, user_permission}, - {attributes, record_info(fields, user_permission)}, - {disc_copies, [node()]}, - {match, #user_permission{user_vhost = #user_vhost{_='_'}, - permission = #permission{_='_'}, - _='_'}}]}, - {rabbit_topic_permission, - [{record_name, topic_permission}, - {attributes, record_info(fields, topic_permission)}, - {disc_copies, [node()]}, - {match, #topic_permission{topic_permission_key = #topic_permission_key{_='_'}, - permission = #permission{_='_'}, - _='_'}}]}, - {rabbit_vhost, - [ - {record_name, vhost}, - {attributes, vhost:fields()}, - {disc_copies, [node()]}, - {match, vhost:pattern_match_all()}]}, - {rabbit_listener, - [{record_name, listener}, - {attributes, record_info(fields, listener)}, - {type, bag}, - {match, #listener{_='_'}}]}, - {rabbit_durable_route, - [{record_name, route}, - {attributes, record_info(fields, route)}, - {disc_copies, [node()]}, - {match, #route{binding = binding_match(), _='_'}}]}, - {rabbit_semi_durable_route, - [{record_name, route}, - {attributes, record_info(fields, route)}, - {type, ordered_set}, - {match, #route{binding = binding_match(), _='_'}}]}, - {rabbit_route, - [{record_name, route}, - {attributes, record_info(fields, route)}, - {type, ordered_set}, - {match, #route{binding = binding_match(), _='_'}}]}, - {rabbit_reverse_route, - [{record_name, reverse_route}, - {attributes, record_info(fields, reverse_route)}, - {type, ordered_set}, - {match, #reverse_route{reverse_binding = reverse_binding_match(), - _='_'}}]}, - {rabbit_topic_trie_node, - [{record_name, topic_trie_node}, - {attributes, record_info(fields, topic_trie_node)}, - {type, ordered_set}, - {match, #topic_trie_node{trie_node = trie_node_match(), _='_'}}]}, - {rabbit_topic_trie_edge, - [{record_name, topic_trie_edge}, - {attributes, record_info(fields, topic_trie_edge)}, - {type, ordered_set}, - {match, #topic_trie_edge{trie_edge = trie_edge_match(), _='_'}}]}, - {rabbit_topic_trie_binding, - [{record_name, topic_trie_binding}, - {attributes, record_info(fields, topic_trie_binding)}, - {type, ordered_set}, - {match, #topic_trie_binding{trie_binding = trie_binding_match(), - _='_'}}]}, - {rabbit_durable_exchange, - [{record_name, exchange}, - {attributes, record_info(fields, exchange)}, - {disc_copies, [node()]}, - {match, #exchange{name = exchange_name_match(), _='_'}}]}, - {rabbit_exchange, - [{record_name, exchange}, - {attributes, record_info(fields, exchange)}, - {match, #exchange{name = exchange_name_match(), _='_'}}]}, - {rabbit_exchange_serial, - [{record_name, exchange_serial}, - {attributes, record_info(fields, exchange_serial)}, - {match, #exchange_serial{name = exchange_name_match(), _='_'}}]}, - {rabbit_runtime_parameters, - [{record_name, runtime_parameters}, - {attributes, record_info(fields, runtime_parameters)}, - {disc_copies, [node()]}, - {match, #runtime_parameters{_='_'}}]}, - {rabbit_durable_queue, - [{record_name, amqqueue}, - {attributes, amqqueue:fields()}, - {disc_copies, [node()]}, - {match, amqqueue:pattern_match_on_name(queue_name_match())}]}, - {rabbit_queue, - [{record_name, amqqueue}, - {attributes, amqqueue:fields()}, - {match, amqqueue:pattern_match_on_name(queue_name_match())}]} - ] - ++ gm:table_definitions() - ++ mirrored_supervisor:table_definitions(). - -binding_match() -> - #binding{source = exchange_name_match(), - destination = binding_destination_match(), - _='_'}. -reverse_binding_match() -> - #reverse_binding{destination = binding_destination_match(), - source = exchange_name_match(), - _='_'}. -binding_destination_match() -> - resource_match('_'). -trie_node_match() -> - #trie_node{exchange_name = exchange_name_match(), _='_'}. -trie_edge_match() -> - #trie_edge{exchange_name = exchange_name_match(), _='_'}. -trie_binding_match() -> - #trie_binding{exchange_name = exchange_name_match(), _='_'}. -exchange_name_match() -> - resource_match(exchange). -queue_name_match() -> - resource_match(queue). -resource_match(Kind) -> - #resource{kind = Kind, _='_'}. diff --git a/src/rabbit_trace.erl b/src/rabbit_trace.erl deleted file mode 100644 index 74b892330e..0000000000 --- a/src/rabbit_trace.erl +++ /dev/null @@ -1,128 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_trace). - --export([init/1, enabled/1, tap_in/6, tap_out/5, start/1, stop/1]). - --include("rabbit.hrl"). --include("rabbit_framing.hrl"). - --define(TRACE_VHOSTS, trace_vhosts). --define(XNAME, <<"amq.rabbitmq.trace">>). - -%%---------------------------------------------------------------------------- - --type state() :: rabbit_types:exchange() | 'none'. - -%%---------------------------------------------------------------------------- - --spec init(rabbit_types:vhost()) -> state(). - -init(VHost) -> - case enabled(VHost) of - false -> none; - true -> {ok, X} = rabbit_exchange:lookup( - rabbit_misc:r(VHost, exchange, ?XNAME)), - X - end. - --spec enabled(rabbit_types:vhost()) -> boolean(). - -enabled(VHost) -> - {ok, VHosts} = application:get_env(rabbit, ?TRACE_VHOSTS), - lists:member(VHost, VHosts). - --spec tap_in(rabbit_types:basic_message(), [rabbit_amqqueue:name()], - binary(), rabbit_channel:channel_number(), - rabbit_types:username(), state()) -> 'ok'. - -tap_in(_Msg, _QNames, _ConnName, _ChannelNum, _Username, none) -> ok; -tap_in(Msg = #basic_message{exchange_name = #resource{name = XName, - virtual_host = VHost}}, - QNames, ConnName, ChannelNum, Username, TraceX) -> - trace(TraceX, Msg, <<"publish">>, XName, - [{<<"vhost">>, longstr, VHost}, - {<<"connection">>, longstr, ConnName}, - {<<"channel">>, signedint, ChannelNum}, - {<<"user">>, longstr, Username}, - {<<"routed_queues">>, array, - [{longstr, QName#resource.name} || QName <- QNames]}]). - --spec tap_out(rabbit_amqqueue:qmsg(), binary(), - rabbit_channel:channel_number(), - rabbit_types:username(), state()) -> 'ok'. - -tap_out(_Msg, _ConnName, _ChannelNum, _Username, none) -> ok; -tap_out({#resource{name = QName, virtual_host = VHost}, - _QPid, _QMsgId, Redelivered, Msg}, - ConnName, ChannelNum, Username, TraceX) -> - RedeliveredNum = case Redelivered of true -> 1; false -> 0 end, - trace(TraceX, Msg, <<"deliver">>, QName, - [{<<"redelivered">>, signedint, RedeliveredNum}, - {<<"vhost">>, longstr, VHost}, - {<<"connection">>, longstr, ConnName}, - {<<"channel">>, signedint, ChannelNum}, - {<<"user">>, longstr, Username}]). - -%%---------------------------------------------------------------------------- - --spec start(rabbit_types:vhost()) -> 'ok'. - -start(VHost) -> - rabbit_log:info("Enabling tracing for vhost '~s'~n", [VHost]), - update_config(fun (VHosts) -> [VHost | VHosts -- [VHost]] end). - --spec stop(rabbit_types:vhost()) -> 'ok'. - -stop(VHost) -> - rabbit_log:info("Disabling tracing for vhost '~s'~n", [VHost]), - update_config(fun (VHosts) -> VHosts -- [VHost] end). - -update_config(Fun) -> - {ok, VHosts0} = application:get_env(rabbit, ?TRACE_VHOSTS), - VHosts = Fun(VHosts0), - application:set_env(rabbit, ?TRACE_VHOSTS, VHosts), - rabbit_channel:refresh_config_local(), - ok. - -%%---------------------------------------------------------------------------- - -trace(#exchange{name = Name}, #basic_message{exchange_name = Name}, - _RKPrefix, _RKSuffix, _Extra) -> - ok; -trace(X, Msg = #basic_message{content = #content{payload_fragments_rev = PFR}}, - RKPrefix, RKSuffix, Extra) -> - ok = rabbit_basic:publish( - X, <<RKPrefix/binary, ".", RKSuffix/binary>>, - #'P_basic'{headers = msg_to_table(Msg) ++ Extra}, PFR), - ok. - -msg_to_table(#basic_message{exchange_name = #resource{name = XName}, - routing_keys = RoutingKeys, - content = Content}) -> - #content{properties = Props} = - rabbit_binary_parser:ensure_content_decoded(Content), - {PropsTable, _Ix} = - lists:foldl(fun (K, {L, Ix}) -> - V = element(Ix, Props), - NewL = case V of - undefined -> L; - _ -> [{a2b(K), type(V), V} | L] - end, - {NewL, Ix + 1} - end, {[], 2}, record_info(fields, 'P_basic')), - [{<<"exchange_name">>, longstr, XName}, - {<<"routing_keys">>, array, [{longstr, K} || K <- RoutingKeys]}, - {<<"properties">>, table, PropsTable}, - {<<"node">>, longstr, a2b(node())}]. - -a2b(A) -> list_to_binary(atom_to_list(A)). - -type(V) when is_list(V) -> table; -type(V) when is_integer(V) -> signedint; -type(_V) -> longstr. diff --git a/src/rabbit_tracking.erl b/src/rabbit_tracking.erl deleted file mode 100644 index a124d20226..0000000000 --- a/src/rabbit_tracking.erl +++ /dev/null @@ -1,103 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_tracking). - -%% Common behaviour and processing functions for tracking components -%% -%% See in use: -%% * rabbit_connection_tracking -%% * rabbit_channel_tracking - --callback boot() -> ok. --callback update_tracked(term()) -> ok. --callback handle_cast(term()) -> ok. --callback register_tracked( - rabbit_types:tracked_connection() | - rabbit_types:tracked_channel()) -> 'ok'. --callback unregister_tracked( - rabbit_types:tracked_connection_id() | - rabbit_types:tracked_channel_id()) -> 'ok'. --callback count_tracked_items_in(term()) -> non_neg_integer(). --callback clear_tracking_tables() -> 'ok'. --callback shutdown_tracked_items(list(), term()) -> ok. - --export([id/2, count_tracked_items/4, match_tracked_items/2, - clear_tracking_table/1, delete_tracking_table/3, - delete_tracked_entry/3]). - -%%---------------------------------------------------------------------------- - --spec id(atom(), term()) -> - rabbit_types:tracked_connection_id() | rabbit_types:tracked_channel_id(). - -id(Node, Name) -> {Node, Name}. - --spec count_tracked_items(function(), integer(), term(), string()) -> - non_neg_integer(). - -count_tracked_items(TableNameFun, CountRecPosition, Key, ContextMsg) -> - lists:foldl(fun (Node, Acc) -> - Tab = TableNameFun(Node), - try - N = case mnesia:dirty_read(Tab, Key) of - [] -> 0; - [Val] -> - element(CountRecPosition, Val) - end, - Acc + N - catch _:Err -> - rabbit_log:error( - "Failed to fetch number of ~p ~p on node ~p:~n~p~n", - [ContextMsg, Key, Node, Err]), - Acc - end - end, 0, rabbit_nodes:all_running()). - --spec match_tracked_items(function(), tuple()) -> term(). - -match_tracked_items(TableNameFun, MatchSpec) -> - lists:foldl( - fun (Node, Acc) -> - Tab = TableNameFun(Node), - Acc ++ mnesia:dirty_match_object( - Tab, - MatchSpec) - end, [], rabbit_nodes:all_running()). - --spec clear_tracking_table(atom()) -> ok. - -clear_tracking_table(TableName) -> - case mnesia:clear_table(TableName) of - {atomic, ok} -> ok; - {aborted, _} -> ok - end. - --spec delete_tracking_table(atom(), node(), string()) -> ok. - -delete_tracking_table(TableName, Node, ContextMsg) -> - case mnesia:delete_table(TableName) of - {atomic, ok} -> ok; - {aborted, {no_exists, _}} -> ok; - {aborted, Error} -> - rabbit_log:error("Failed to delete a ~p table for node ~p: ~p", - [ContextMsg, Node, Error]), - ok - end. - --spec delete_tracked_entry({atom(), atom(), list()}, function(), term()) -> ok. - -delete_tracked_entry(_ExistsCheckSpec = {M, F, A}, TableNameFun, Key) -> - ClusterNodes = rabbit_nodes:all_running(), - ExistsInCluster = - lists:any(fun(Node) -> rpc:call(Node, M, F, A) end, ClusterNodes), - case ExistsInCluster of - false -> - [mnesia:dirty_delete(TableNameFun(Node), Key) || Node <- ClusterNodes]; - true -> - ok - end. diff --git a/src/rabbit_upgrade.erl b/src/rabbit_upgrade.erl deleted file mode 100644 index b1b128fecc..0000000000 --- a/src/rabbit_upgrade.erl +++ /dev/null @@ -1,314 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_upgrade). - --export([maybe_upgrade_mnesia/0, maybe_upgrade_local/0, - maybe_migrate_queues_to_per_vhost_storage/0, - nodes_running/1, secondary_upgrade/1]). - --include("rabbit.hrl"). - --define(VERSION_FILENAME, "schema_version"). --define(LOCK_FILENAME, "schema_upgrade_lock"). - -%% ------------------------------------------------------------------- - -%% The upgrade logic is quite involved, due to the existence of -%% clusters. -%% -%% Firstly, we have two different types of upgrades to do: Mnesia and -%% everything else. Mnesia upgrades must only be done by one node in -%% the cluster (we treat a non-clustered node as a single-node -%% cluster). This is the primary upgrader. The other upgrades need to -%% be done by all nodes. -%% -%% The primary upgrader has to start first (and do its Mnesia -%% upgrades). Secondary upgraders need to reset their Mnesia database -%% and then rejoin the cluster. They can't do the Mnesia upgrades as -%% well and then merge databases since the cookie for each table will -%% end up different and the merge will fail. -%% -%% This in turn means that we need to determine whether we are the -%% primary or secondary upgrader *before* Mnesia comes up. If we -%% didn't then the secondary upgrader would try to start Mnesia, and -%% either hang waiting for a node which is not yet up, or fail since -%% its schema differs from the other nodes in the cluster. -%% -%% Also, the primary upgrader needs to start Mnesia to do its -%% upgrades, but needs to forcibly load tables rather than wait for -%% them (in case it was not the last node to shut down, in which case -%% it would wait forever). -%% -%% This in turn means that maybe_upgrade_mnesia/0 has to be patched -%% into the boot process by prelaunch before the mnesia application is -%% started. By the time Mnesia is started the upgrades have happened -%% (on the primary), or Mnesia has been reset (on the secondary) and -%% rabbit_mnesia:init_db_unchecked/2 can then make the node rejoin the cluster -%% in the normal way. -%% -%% The non-mnesia upgrades are then triggered by -%% rabbit_mnesia:init_db_unchecked/2. Of course, it's possible for a given -%% upgrade process to only require Mnesia upgrades, or only require -%% non-Mnesia upgrades. In the latter case no Mnesia resets and -%% reclusterings occur. -%% -%% The primary upgrader needs to be a disc node. Ideally we would like -%% it to be the last disc node to shut down (since otherwise there's a -%% risk of data loss). On each node we therefore record the disc nodes -%% that were still running when we shut down. A disc node that knows -%% other nodes were up when it shut down, or a ram node, will refuse -%% to be the primary upgrader, and will thus not start when upgrades -%% are needed. -%% -%% However, this is racy if several nodes are shut down at once. Since -%% rabbit records the running nodes, and shuts down before mnesia, the -%% race manifests as all disc nodes thinking they are not the primary -%% upgrader. Therefore the user can remove the record of the last disc -%% node to shut down to get things going again. This may lose any -%% mnesia changes that happened after the node chosen as the primary -%% upgrader was shut down. - -%% ------------------------------------------------------------------- - -ensure_backup_taken() -> - case filelib:is_file(lock_filename()) of - false -> case filelib:is_dir(backup_dir()) of - false -> ok = take_backup(); - _ -> ok - end; - true -> - rabbit_log:error("Found lock file at ~s. - Either previous upgrade is in progress or has failed. - Database backup path: ~s", - [lock_filename(), backup_dir()]), - throw({error, previous_upgrade_failed}) - end. - -take_backup() -> - BackupDir = backup_dir(), - info("upgrades: Backing up mnesia dir to ~p~n", [BackupDir]), - case rabbit_mnesia:copy_db(BackupDir) of - ok -> info("upgrades: Mnesia dir backed up to ~p~n", - [BackupDir]); - {error, E} -> throw({could_not_back_up_mnesia_dir, E, BackupDir}) - end. - -ensure_backup_removed() -> - case filelib:is_dir(backup_dir()) of - true -> ok = remove_backup(); - _ -> ok - end. - -remove_backup() -> - ok = rabbit_file:recursive_delete([backup_dir()]), - info("upgrades: Mnesia backup removed~n", []). - --spec maybe_upgrade_mnesia() -> 'ok'. - -maybe_upgrade_mnesia() -> - AllNodes = rabbit_mnesia:cluster_nodes(all), - ok = rabbit_mnesia_rename:maybe_finish(AllNodes), - %% Mnesia upgrade is the first upgrade scope, - %% so we should create a backup here if there are any upgrades - case rabbit_version:all_upgrades_required([mnesia, local, message_store]) of - {error, starting_from_scratch} -> - ok; - {error, version_not_available} -> - case AllNodes of - [] -> die("Cluster upgrade needed but upgrading from " - "< 2.1.1.~nUnfortunately you will need to " - "rebuild the cluster.", []); - _ -> ok - end; - {error, _} = Err -> - throw(Err); - {ok, []} -> - ok; - {ok, Upgrades} -> - ensure_backup_taken(), - run_mnesia_upgrades(proplists:get_value(mnesia, Upgrades, []), - AllNodes) - end. - -run_mnesia_upgrades([], _) -> ok; -run_mnesia_upgrades(Upgrades, AllNodes) -> - case upgrade_mode(AllNodes) of - primary -> primary_upgrade(Upgrades, AllNodes); - secondary -> secondary_upgrade(AllNodes) - end. - -upgrade_mode(AllNodes) -> - case nodes_running(AllNodes) of - [] -> - AfterUs = rabbit_nodes:all_running() -- [node()], - case {node_type_legacy(), AfterUs} of - {disc, []} -> - primary; - {disc, _} -> - Filename = rabbit_node_monitor:running_nodes_filename(), - die("Cluster upgrade needed but other disc nodes shut " - "down after this one.~nPlease first start the last " - "disc node to shut down.~n~nNote: if several disc " - "nodes were shut down simultaneously they may " - "all~nshow this message. In which case, remove " - "the lock file on one of them and~nstart that node. " - "The lock file on this node is:~n~n ~s ", [Filename]); - {ram, _} -> - die("Cluster upgrade needed but this is a ram node.~n" - "Please first start the last disc node to shut down.", - []) - end; - [Another|_] -> - MyVersion = rabbit_version:desired_for_scope(mnesia), - case rpc:call(Another, rabbit_version, desired_for_scope, - [mnesia]) of - {badrpc, {'EXIT', {undef, _}}} -> - die_because_cluster_upgrade_needed(unknown_old_version, - MyVersion); - {badrpc, Reason} -> - die_because_cluster_upgrade_needed({unknown, Reason}, - MyVersion); - CV -> case rabbit_version:matches( - MyVersion, CV) of - true -> secondary; - false -> die_because_cluster_upgrade_needed( - CV, MyVersion) - end - end - end. - --spec die_because_cluster_upgrade_needed(any(), any()) -> no_return(). - -die_because_cluster_upgrade_needed(ClusterVersion, MyVersion) -> - %% The other node(s) are running an - %% unexpected version. - die("Cluster upgrade needed but other nodes are " - "running ~p~nand I want ~p", - [ClusterVersion, MyVersion]). - --spec die(string(), list()) -> no_return(). - -die(Msg, Args) -> - %% We don't throw or exit here since that gets thrown - %% straight out into do_boot, generating an erl_crash.dump - %% and displaying any error message in a confusing way. - rabbit_log:error(Msg, Args), - Str = rabbit_misc:format( - "~n~n****~n~n" ++ Msg ++ "~n~n****~n~n~n", Args), - io:format(Str), - error_logger:logfile(close), - case application:get_env(rabbit, halt_on_upgrade_failure) of - {ok, false} -> throw({upgrade_error, Str}); - _ -> halt(1) %% i.e. true or undefined - end. - -primary_upgrade(Upgrades, Nodes) -> - Others = Nodes -- [node()], - ok = apply_upgrades( - mnesia, - Upgrades, - fun () -> - rabbit_table:force_load(), - case Others of - [] -> ok; - _ -> info("mnesia upgrades: Breaking cluster~n", []), - [{atomic, ok} = mnesia:del_table_copy(schema, Node) - || Node <- Others] - end - end), - ok. - -secondary_upgrade(AllNodes) -> - %% must do this before we wipe out schema - NodeType = node_type_legacy(), - rabbit_misc:ensure_ok(mnesia:delete_schema([node()]), - cannot_delete_schema), - rabbit_misc:ensure_ok(mnesia:start(), cannot_start_mnesia), - ok = rabbit_mnesia:init_db_unchecked(AllNodes, NodeType), - ok = rabbit_version:record_desired_for_scope(mnesia), - ok. - -nodes_running(Nodes) -> - [N || N <- Nodes, rabbit:is_running(N)]. - -%% ------------------------------------------------------------------- - --spec maybe_upgrade_local() -> - 'ok' | - 'version_not_available' | - 'starting_from_scratch'. - -maybe_upgrade_local() -> - case rabbit_version:upgrades_required(local) of - {error, version_not_available} -> version_not_available; - {error, starting_from_scratch} -> starting_from_scratch; - {error, _} = Err -> throw(Err); - {ok, []} -> ensure_backup_removed(), - ok; - {ok, Upgrades} -> mnesia:stop(), - ok = apply_upgrades(local, Upgrades, - fun () -> ok end), - ok - end. - -%% ------------------------------------------------------------------- - -maybe_migrate_queues_to_per_vhost_storage() -> - Result = case rabbit_version:upgrades_required(message_store) of - {error, version_not_available} -> version_not_available; - {error, starting_from_scratch} -> - starting_from_scratch; - {error, _} = Err -> throw(Err); - {ok, []} -> ok; - {ok, Upgrades} -> apply_upgrades(message_store, - Upgrades, - fun() -> ok end), - ok - end, - %% Message store upgrades should be - %% the last group. - %% Backup can be deleted here. - ensure_backup_removed(), - Result. - -%% ------------------------------------------------------------------- - -apply_upgrades(Scope, Upgrades, Fun) -> - ok = rabbit_file:lock_file(lock_filename()), - info("~s upgrades: ~w to apply~n", [Scope, length(Upgrades)]), - rabbit_misc:ensure_ok(mnesia:start(), cannot_start_mnesia), - Fun(), - [apply_upgrade(Scope, Upgrade) || Upgrade <- Upgrades], - info("~s upgrades: All upgrades applied successfully~n", [Scope]), - ok = rabbit_version:record_desired_for_scope(Scope), - ok = file:delete(lock_filename()). - -apply_upgrade(Scope, {M, F}) -> - info("~s upgrades: Applying ~w:~w~n", [Scope, M, F]), - ok = apply(M, F, []). - -%% ------------------------------------------------------------------- - -dir() -> rabbit_mnesia:dir(). - -lock_filename() -> lock_filename(dir()). -lock_filename(Dir) -> filename:join(Dir, ?LOCK_FILENAME). -backup_dir() -> dir() ++ "-upgrade-backup". - -node_type_legacy() -> - %% This is pretty ugly but we can't start Mnesia and ask it (will - %% hang), we can't look at the config file (may not include us - %% even if we're a disc node). We also can't use - %% rabbit_mnesia:node_type/0 because that will give false - %% positives on Rabbit up to 2.5.1. - case filelib:is_regular(filename:join(dir(), "rabbit_durable_exchange.DCD")) of - true -> disc; - false -> ram - end. - -info(Msg, Args) -> rabbit_log:info(Msg, Args). diff --git a/src/rabbit_upgrade_functions.erl b/src/rabbit_upgrade_functions.erl deleted file mode 100644 index 59417c72bb..0000000000 --- a/src/rabbit_upgrade_functions.erl +++ /dev/null @@ -1,662 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_upgrade_functions). - -%% If you are tempted to add include("rabbit.hrl"). here, don't. Using record -%% defs here leads to pain later. - --compile([nowarn_export_all, export_all]). - --rabbit_upgrade({remove_user_scope, mnesia, []}). --rabbit_upgrade({hash_passwords, mnesia, []}). --rabbit_upgrade({add_ip_to_listener, mnesia, []}). --rabbit_upgrade({add_opts_to_listener, mnesia, [add_ip_to_listener]}). --rabbit_upgrade({internal_exchanges, mnesia, []}). --rabbit_upgrade({user_to_internal_user, mnesia, [hash_passwords]}). --rabbit_upgrade({topic_trie, mnesia, []}). --rabbit_upgrade({semi_durable_route, mnesia, []}). --rabbit_upgrade({exchange_event_serial, mnesia, []}). --rabbit_upgrade({trace_exchanges, mnesia, [internal_exchanges]}). --rabbit_upgrade({user_admin_to_tags, mnesia, [user_to_internal_user]}). --rabbit_upgrade({ha_mirrors, mnesia, []}). --rabbit_upgrade({gm, mnesia, []}). --rabbit_upgrade({exchange_scratch, mnesia, [trace_exchanges]}). --rabbit_upgrade({mirrored_supervisor, mnesia, []}). --rabbit_upgrade({topic_trie_node, mnesia, []}). --rabbit_upgrade({runtime_parameters, mnesia, []}). --rabbit_upgrade({exchange_scratches, mnesia, [exchange_scratch]}). --rabbit_upgrade({policy, mnesia, - [exchange_scratches, ha_mirrors]}). --rabbit_upgrade({sync_slave_pids, mnesia, [policy]}). --rabbit_upgrade({no_mirror_nodes, mnesia, [sync_slave_pids]}). --rabbit_upgrade({gm_pids, mnesia, [no_mirror_nodes]}). --rabbit_upgrade({exchange_decorators, mnesia, [policy]}). --rabbit_upgrade({policy_apply_to, mnesia, [runtime_parameters]}). --rabbit_upgrade({queue_decorators, mnesia, [gm_pids]}). --rabbit_upgrade({internal_system_x, mnesia, [exchange_decorators]}). --rabbit_upgrade({cluster_name, mnesia, [runtime_parameters]}). --rabbit_upgrade({down_slave_nodes, mnesia, [queue_decorators]}). --rabbit_upgrade({queue_state, mnesia, [down_slave_nodes]}). --rabbit_upgrade({recoverable_slaves, mnesia, [queue_state]}). --rabbit_upgrade({policy_version, mnesia, [recoverable_slaves]}). --rabbit_upgrade({slave_pids_pending_shutdown, mnesia, [policy_version]}). --rabbit_upgrade({user_password_hashing, mnesia, [hash_passwords]}). --rabbit_upgrade({operator_policies, mnesia, [slave_pids_pending_shutdown, internal_system_x]}). --rabbit_upgrade({vhost_limits, mnesia, []}). --rabbit_upgrade({queue_vhost_field, mnesia, [operator_policies]}). --rabbit_upgrade({topic_permission, mnesia, []}). --rabbit_upgrade({queue_options, mnesia, [queue_vhost_field]}). --rabbit_upgrade({exchange_options, mnesia, [operator_policies]}). - -%% ------------------------------------------------------------------- - -%% replaces vhost.dummy (used to avoid having a single-field record -%% which Mnesia doesn't like) with vhost.limits (which is actually -%% used) - --spec vhost_limits() -> 'ok'. - -vhost_limits() -> - transform( - rabbit_vhost, - fun ({vhost, VHost, _Dummy}) -> - {vhost, VHost, undefined} - end, - [virtual_host, limits]). - -%% It's a bad idea to use records or record_info here, even for the -%% destination form. Because in the future, the destination form of -%% your current transform may not match the record any more, and it -%% would be messy to have to go back and fix old transforms at that -%% point. - --spec remove_user_scope() -> 'ok'. - -remove_user_scope() -> - transform( - rabbit_user_permission, - fun ({user_permission, UV, {permission, _Scope, Conf, Write, Read}}) -> - {user_permission, UV, {permission, Conf, Write, Read}} - end, - [user_vhost, permission]). - -%% this is an early migration that hashes passwords using MD5, -%% only relevant to those migrating from 2.1.1. -%% all users created after in 3.6.0 or later will use SHA-256 (unless configured -%% otherwise) - --spec hash_passwords() -> 'ok'. - -hash_passwords() -> - transform( - rabbit_user, - fun ({user, Username, Password, IsAdmin}) -> - Hash = rabbit_auth_backend_internal:hash_password(rabbit_password_hashing_md5, Password), - {user, Username, Hash, IsAdmin} - end, - [username, password_hash, is_admin]). - --spec add_ip_to_listener() -> 'ok'. - -add_ip_to_listener() -> - transform( - rabbit_listener, - fun ({listener, Node, Protocol, Host, Port}) -> - {listener, Node, Protocol, Host, {0,0,0,0}, Port} - end, - [node, protocol, host, ip_address, port]). - --spec add_opts_to_listener() -> 'ok'. - -add_opts_to_listener() -> - transform( - rabbit_listener, - fun ({listener, Node, Protocol, Host, IP, Port}) -> - {listener, Node, Protocol, Host, IP, Port, []} - end, - [node, protocol, host, ip_address, port, opts]). - --spec internal_exchanges() -> 'ok'. - -internal_exchanges() -> - Tables = [rabbit_exchange, rabbit_durable_exchange], - AddInternalFun = - fun ({exchange, Name, Type, Durable, AutoDelete, Args}) -> - {exchange, Name, Type, Durable, AutoDelete, false, Args} - end, - [ ok = transform(T, - AddInternalFun, - [name, type, durable, auto_delete, internal, arguments]) - || T <- Tables ], - ok. - --spec user_to_internal_user() -> 'ok'. - -user_to_internal_user() -> - transform( - rabbit_user, - fun({user, Username, PasswordHash, IsAdmin}) -> - {internal_user, Username, PasswordHash, IsAdmin} - end, - [username, password_hash, is_admin], internal_user). - --spec topic_trie() -> 'ok'. - -topic_trie() -> - create(rabbit_topic_trie_edge, [{record_name, topic_trie_edge}, - {attributes, [trie_edge, node_id]}, - {type, ordered_set}]), - create(rabbit_topic_trie_binding, [{record_name, topic_trie_binding}, - {attributes, [trie_binding, value]}, - {type, ordered_set}]). - --spec semi_durable_route() -> 'ok'. - -semi_durable_route() -> - create(rabbit_semi_durable_route, [{record_name, route}, - {attributes, [binding, value]}]). - --spec exchange_event_serial() -> 'ok'. - -exchange_event_serial() -> - create(rabbit_exchange_serial, [{record_name, exchange_serial}, - {attributes, [name, next]}]). - --spec trace_exchanges() -> 'ok'. - -trace_exchanges() -> - [declare_exchange( - rabbit_misc:r(VHost, exchange, <<"amq.rabbitmq.trace">>), topic) || - VHost <- rabbit_vhost:list_names()], - ok. - --spec user_admin_to_tags() -> 'ok'. - -user_admin_to_tags() -> - transform( - rabbit_user, - fun({internal_user, Username, PasswordHash, true}) -> - {internal_user, Username, PasswordHash, [administrator]}; - ({internal_user, Username, PasswordHash, false}) -> - {internal_user, Username, PasswordHash, [management]} - end, - [username, password_hash, tags], internal_user). - --spec ha_mirrors() -> 'ok'. - -ha_mirrors() -> - Tables = [rabbit_queue, rabbit_durable_queue], - AddMirrorPidsFun = - fun ({amqqueue, Name, Durable, AutoDelete, Owner, Arguments, Pid}) -> - {amqqueue, Name, Durable, AutoDelete, Owner, Arguments, Pid, - [], undefined} - end, - [ ok = transform(T, - AddMirrorPidsFun, - [name, durable, auto_delete, exclusive_owner, arguments, - pid, slave_pids, mirror_nodes]) - || T <- Tables ], - ok. - --spec gm() -> 'ok'. - -gm() -> - create(gm_group, [{record_name, gm_group}, - {attributes, [name, version, members]}]). - --spec exchange_scratch() -> 'ok'. - -exchange_scratch() -> - ok = exchange_scratch(rabbit_exchange), - ok = exchange_scratch(rabbit_durable_exchange). - -exchange_scratch(Table) -> - transform( - Table, - fun ({exchange, Name, Type, Dur, AutoDel, Int, Args}) -> - {exchange, Name, Type, Dur, AutoDel, Int, Args, undefined} - end, - [name, type, durable, auto_delete, internal, arguments, scratch]). - --spec mirrored_supervisor() -> 'ok'. - -mirrored_supervisor() -> - create(mirrored_sup_childspec, - [{record_name, mirrored_sup_childspec}, - {attributes, [key, mirroring_pid, childspec]}]). - --spec topic_trie_node() -> 'ok'. - -topic_trie_node() -> - create(rabbit_topic_trie_node, - [{record_name, topic_trie_node}, - {attributes, [trie_node, edge_count, binding_count]}, - {type, ordered_set}]). - --spec runtime_parameters() -> 'ok'. - -runtime_parameters() -> - create(rabbit_runtime_parameters, - [{record_name, runtime_parameters}, - {attributes, [key, value]}, - {disc_copies, [node()]}]). - -exchange_scratches() -> - ok = exchange_scratches(rabbit_exchange), - ok = exchange_scratches(rabbit_durable_exchange). - -exchange_scratches(Table) -> - transform( - Table, - fun ({exchange, Name, Type = <<"x-federation">>, Dur, AutoDel, Int, Args, - Scratch}) -> - Scratches = orddict:store(federation, Scratch, orddict:new()), - {exchange, Name, Type, Dur, AutoDel, Int, Args, Scratches}; - %% We assert here that nothing else uses the scratch mechanism ATM - ({exchange, Name, Type, Dur, AutoDel, Int, Args, undefined}) -> - {exchange, Name, Type, Dur, AutoDel, Int, Args, undefined} - end, - [name, type, durable, auto_delete, internal, arguments, scratches]). - --spec policy() -> 'ok'. - -policy() -> - ok = exchange_policy(rabbit_exchange), - ok = exchange_policy(rabbit_durable_exchange), - ok = queue_policy(rabbit_queue), - ok = queue_policy(rabbit_durable_queue). - -exchange_policy(Table) -> - transform( - Table, - fun ({exchange, Name, Type, Dur, AutoDel, Int, Args, Scratches}) -> - {exchange, Name, Type, Dur, AutoDel, Int, Args, Scratches, - undefined} - end, - [name, type, durable, auto_delete, internal, arguments, scratches, - policy]). - -queue_policy(Table) -> - transform( - Table, - fun ({amqqueue, Name, Dur, AutoDel, Excl, Args, Pid, SPids, MNodes}) -> - {amqqueue, Name, Dur, AutoDel, Excl, Args, Pid, SPids, MNodes, - undefined} - end, - [name, durable, auto_delete, exclusive_owner, arguments, pid, - slave_pids, mirror_nodes, policy]). - --spec sync_slave_pids() -> 'ok'. - -sync_slave_pids() -> - Tables = [rabbit_queue, rabbit_durable_queue], - AddSyncSlavesFun = - fun ({amqqueue, N, D, AD, Excl, Args, Pid, SPids, MNodes, Pol}) -> - {amqqueue, N, D, AD, Excl, Args, Pid, SPids, [], MNodes, Pol} - end, - [ok = transform(T, AddSyncSlavesFun, - [name, durable, auto_delete, exclusive_owner, arguments, - pid, slave_pids, sync_slave_pids, mirror_nodes, policy]) - || T <- Tables], - ok. - --spec no_mirror_nodes() -> 'ok'. - -no_mirror_nodes() -> - Tables = [rabbit_queue, rabbit_durable_queue], - RemoveMirrorNodesFun = - fun ({amqqueue, N, D, AD, O, A, Pid, SPids, SSPids, _MNodes, Pol}) -> - {amqqueue, N, D, AD, O, A, Pid, SPids, SSPids, Pol} - end, - [ok = transform(T, RemoveMirrorNodesFun, - [name, durable, auto_delete, exclusive_owner, arguments, - pid, slave_pids, sync_slave_pids, policy]) - || T <- Tables], - ok. - --spec gm_pids() -> 'ok'. - -gm_pids() -> - Tables = [rabbit_queue, rabbit_durable_queue], - AddGMPidsFun = - fun ({amqqueue, N, D, AD, O, A, Pid, SPids, SSPids, Pol}) -> - {amqqueue, N, D, AD, O, A, Pid, SPids, SSPids, Pol, []} - end, - [ok = transform(T, AddGMPidsFun, - [name, durable, auto_delete, exclusive_owner, arguments, - pid, slave_pids, sync_slave_pids, policy, gm_pids]) - || T <- Tables], - ok. - --spec exchange_decorators() -> 'ok'. - -exchange_decorators() -> - ok = exchange_decorators(rabbit_exchange), - ok = exchange_decorators(rabbit_durable_exchange). - -exchange_decorators(Table) -> - transform( - Table, - fun ({exchange, Name, Type, Dur, AutoDel, Int, Args, Scratches, - Policy}) -> - {exchange, Name, Type, Dur, AutoDel, Int, Args, Scratches, Policy, - {[], []}} - end, - [name, type, durable, auto_delete, internal, arguments, scratches, policy, - decorators]). - --spec policy_apply_to() -> 'ok'. - -policy_apply_to() -> - transform( - rabbit_runtime_parameters, - fun ({runtime_parameters, Key = {_VHost, <<"policy">>, _Name}, Value}) -> - ApplyTo = apply_to(proplists:get_value(<<"definition">>, Value)), - {runtime_parameters, Key, [{<<"apply-to">>, ApplyTo} | Value]}; - ({runtime_parameters, Key, Value}) -> - {runtime_parameters, Key, Value} - end, - [key, value]), - rabbit_policy:invalidate(), - ok. - -apply_to(Def) -> - case [proplists:get_value(K, Def) || - K <- [<<"federation-upstream-set">>, <<"ha-mode">>]] of - [undefined, undefined] -> <<"all">>; - [_, undefined] -> <<"exchanges">>; - [undefined, _] -> <<"queues">>; - [_, _] -> <<"all">> - end. - --spec queue_decorators() -> 'ok'. - -queue_decorators() -> - ok = queue_decorators(rabbit_queue), - ok = queue_decorators(rabbit_durable_queue). - -queue_decorators(Table) -> - transform( - Table, - fun ({amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, Policy, GmPids}) -> - {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, Policy, GmPids, []} - end, - [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, - sync_slave_pids, policy, gm_pids, decorators]). - --spec internal_system_x() -> 'ok'. - -internal_system_x() -> - transform( - rabbit_durable_exchange, - fun ({exchange, Name = {resource, _, _, <<"amq.rabbitmq.", _/binary>>}, - Type, Dur, AutoDel, _Int, Args, Scratches, Policy, Decorators}) -> - {exchange, Name, Type, Dur, AutoDel, true, Args, Scratches, - Policy, Decorators}; - (X) -> - X - end, - [name, type, durable, auto_delete, internal, arguments, scratches, policy, - decorators]). - --spec cluster_name() -> 'ok'. - -cluster_name() -> - {atomic, ok} = mnesia:transaction(fun cluster_name_tx/0), - ok. - -cluster_name_tx() -> - %% mnesia:transform_table/4 does not let us delete records - T = rabbit_runtime_parameters, - mnesia:write_lock_table(T), - Ks = [K || {_VHost, <<"federation">>, <<"local-nodename">>} = K - <- mnesia:all_keys(T)], - case Ks of - [] -> ok; - [K|Tl] -> [{runtime_parameters, _K, Name}] = mnesia:read(T, K, write), - R = {runtime_parameters, cluster_name, Name}, - mnesia:write(T, R, write), - case Tl of - [] -> ok; - _ -> {VHost, _, _} = K, - error_logger:warning_msg( - "Multiple local-nodenames found, picking '~s' " - "from '~s' for cluster name~n", [Name, VHost]) - end - end, - [mnesia:delete(T, K, write) || K <- Ks], - ok. - --spec down_slave_nodes() -> 'ok'. - -down_slave_nodes() -> - ok = down_slave_nodes(rabbit_queue), - ok = down_slave_nodes(rabbit_durable_queue). - -down_slave_nodes(Table) -> - transform( - Table, - fun ({amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, Policy, GmPids, Decorators}) -> - {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, [], Policy, GmPids, Decorators} - end, - [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, - sync_slave_pids, down_slave_nodes, policy, gm_pids, decorators]). - --spec queue_state() -> 'ok'. - -queue_state() -> - ok = queue_state(rabbit_queue), - ok = queue_state(rabbit_durable_queue). - -queue_state(Table) -> - transform( - Table, - fun ({amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, GmPids, Decorators}) -> - {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, GmPids, Decorators, - live} - end, - [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, - sync_slave_pids, down_slave_nodes, policy, gm_pids, decorators, state]). - --spec recoverable_slaves() -> 'ok'. - -recoverable_slaves() -> - ok = recoverable_slaves(rabbit_queue), - ok = recoverable_slaves(rabbit_durable_queue). - -recoverable_slaves(Table) -> - transform( - Table, fun (Q) -> Q end, %% Don't change shape of record - [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, - sync_slave_pids, recoverable_slaves, policy, gm_pids, decorators, - state]). - -policy_version() -> - ok = policy_version(rabbit_queue), - ok = policy_version(rabbit_durable_queue). - -policy_version(Table) -> - transform( - Table, - fun ({amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, GmPids, Decorators, - State}) -> - {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, GmPids, Decorators, - State, 0} - end, - [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, - sync_slave_pids, recoverable_slaves, policy, gm_pids, decorators, state, - policy_version]). - -slave_pids_pending_shutdown() -> - ok = slave_pids_pending_shutdown(rabbit_queue), - ok = slave_pids_pending_shutdown(rabbit_durable_queue). - -slave_pids_pending_shutdown(Table) -> - transform( - Table, - fun ({amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, GmPids, Decorators, - State, PolicyVersion}) -> - {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, GmPids, Decorators, - State, PolicyVersion, []} - end, - [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, - sync_slave_pids, recoverable_slaves, policy, gm_pids, decorators, state, - policy_version, slave_pids_pending_shutdown]). - --spec operator_policies() -> 'ok'. - -operator_policies() -> - ok = exchange_operator_policies(rabbit_exchange), - ok = exchange_operator_policies(rabbit_durable_exchange), - ok = queue_operator_policies(rabbit_queue), - ok = queue_operator_policies(rabbit_durable_queue). - -exchange_operator_policies(Table) -> - transform( - Table, - fun ({exchange, Name, Type, Dur, AutoDel, Internal, - Args, Scratches, Policy, Decorators}) -> - {exchange, Name, Type, Dur, AutoDel, Internal, - Args, Scratches, Policy, undefined, Decorators} - end, - [name, type, durable, auto_delete, internal, arguments, scratches, policy, - operator_policy, decorators]). - -queue_operator_policies(Table) -> - transform( - Table, - fun ({amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, GmPids, Decorators, - State, PolicyVersion, SlavePidsPendingShutdown}) -> - {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, undefined, GmPids, - Decorators, State, PolicyVersion, SlavePidsPendingShutdown} - end, - [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, - sync_slave_pids, recoverable_slaves, policy, operator_policy, - gm_pids, decorators, state, policy_version, slave_pids_pending_shutdown]). - --spec queue_vhost_field() -> 'ok'. - -queue_vhost_field() -> - ok = queue_vhost_field(rabbit_queue), - ok = queue_vhost_field(rabbit_durable_queue), - {atomic, ok} = mnesia:add_table_index(rabbit_queue, vhost), - {atomic, ok} = mnesia:add_table_index(rabbit_durable_queue, vhost), - ok. - -queue_vhost_field(Table) -> - transform( - Table, - fun ({amqqueue, Name = {resource, VHost, queue, _QName}, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, OperatorPolicy, GmPids, Decorators, - State, PolicyVersion, SlavePidsPendingShutdown}) -> - {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, OperatorPolicy, GmPids, Decorators, - State, PolicyVersion, SlavePidsPendingShutdown, VHost} - end, - [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, - sync_slave_pids, recoverable_slaves, policy, operator_policy, - gm_pids, decorators, state, policy_version, slave_pids_pending_shutdown, vhost]). - --spec queue_options() -> 'ok'. - -queue_options() -> - ok = queue_options(rabbit_queue), - ok = queue_options(rabbit_durable_queue), - ok. - -queue_options(Table) -> - transform( - Table, - fun ({amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, OperatorPolicy, GmPids, Decorators, - State, PolicyVersion, SlavePidsPendingShutdown, VHost}) -> - {amqqueue, Name, Durable, AutoDelete, ExclusiveOwner, Arguments, - Pid, SlavePids, SyncSlavePids, DSN, Policy, OperatorPolicy, GmPids, Decorators, - State, PolicyVersion, SlavePidsPendingShutdown, VHost, #{}} - end, - [name, durable, auto_delete, exclusive_owner, arguments, pid, slave_pids, - sync_slave_pids, recoverable_slaves, policy, operator_policy, - gm_pids, decorators, state, policy_version, slave_pids_pending_shutdown, vhost, options]). - -%% Prior to 3.6.0, passwords were hashed using MD5, this populates -%% existing records with said default. Users created with 3.6.0+ will -%% have internal_user.hashing_algorithm populated by the internal -%% authn backend. - --spec user_password_hashing() -> 'ok'. - -user_password_hashing() -> - transform( - rabbit_user, - fun ({internal_user, Username, Hash, Tags}) -> - {internal_user, Username, Hash, Tags, rabbit_password_hashing_md5} - end, - [username, password_hash, tags, hashing_algorithm]). - --spec topic_permission() -> 'ok'. -topic_permission() -> - create(rabbit_topic_permission, - [{record_name, topic_permission}, - {attributes, [topic_permission_key, permission]}, - {disc_copies, [node()]}]). - --spec exchange_options() -> 'ok'. - -exchange_options() -> - ok = exchange_options(rabbit_exchange), - ok = exchange_options(rabbit_durable_exchange). - -exchange_options(Table) -> - transform( - Table, - fun ({exchange, Name, Type, Dur, AutoDel, Internal, - Args, Scratches, Policy, OperatorPolicy, Decorators}) -> - {exchange, Name, Type, Dur, AutoDel, Internal, - Args, Scratches, Policy, OperatorPolicy, Decorators, #{}} - end, - [name, type, durable, auto_delete, internal, arguments, scratches, policy, - operator_policy, decorators, options]). - -%%-------------------------------------------------------------------- - -transform(TableName, Fun, FieldList) -> - rabbit_table:wait([TableName]), - {atomic, ok} = mnesia:transform_table(TableName, Fun, FieldList), - ok. - -transform(TableName, Fun, FieldList, NewRecordName) -> - rabbit_table:wait([TableName]), - {atomic, ok} = mnesia:transform_table(TableName, Fun, FieldList, - NewRecordName), - ok. - -create(Tab, TabDef) -> - rabbit_log:debug("Will create a schema table named '~s'", [Tab]), - {atomic, ok} = mnesia:create_table(Tab, TabDef), - ok. - -%% Dumb replacement for rabbit_exchange:declare that does not require -%% the exchange type registry or worker pool to be running by dint of -%% not validating anything and assuming the exchange type does not -%% require serialisation. NB: this assumes the -%% pre-exchange-scratch-space format -declare_exchange(XName, Type) -> - X = {exchange, XName, Type, true, false, false, []}, - ok = mnesia:dirty_write(rabbit_durable_exchange, X). diff --git a/src/rabbit_upgrade_preparation.erl b/src/rabbit_upgrade_preparation.erl deleted file mode 100644 index fc1de24610..0000000000 --- a/src/rabbit_upgrade_preparation.erl +++ /dev/null @@ -1,51 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_upgrade_preparation). - --export([await_online_quorum_plus_one/1, await_online_synchronised_mirrors/1]). - -%% -%% API -%% - --define(SAMPLING_INTERVAL, 200). - -await_online_quorum_plus_one(Timeout) -> - Iterations = ceil(Timeout / ?SAMPLING_INTERVAL), - do_await_safe_online_quorum(Iterations). - - -await_online_synchronised_mirrors(Timeout) -> - Iterations = ceil(Timeout / ?SAMPLING_INTERVAL), - do_await_online_synchronised_mirrors(Iterations). - - -%% -%% Implementation -%% - -do_await_safe_online_quorum(0) -> - false; -do_await_safe_online_quorum(IterationsLeft) -> - case rabbit_quorum_queue:list_with_minimum_quorum() of - [] -> true; - List when is_list(List) -> - timer:sleep(?SAMPLING_INTERVAL), - do_await_safe_online_quorum(IterationsLeft - 1) - end. - - -do_await_online_synchronised_mirrors(0) -> - false; -do_await_online_synchronised_mirrors(IterationsLeft) -> - case rabbit_amqqueue:list_local_mirrored_classic_without_synchronised_mirrors() of - [] -> true; - List when is_list(List) -> - timer:sleep(?SAMPLING_INTERVAL), - do_await_online_synchronised_mirrors(IterationsLeft - 1) - end. diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl deleted file mode 100644 index cf6fa4a189..0000000000 --- a/src/rabbit_variable_queue.erl +++ /dev/null @@ -1,3015 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_variable_queue). - --export([init/3, terminate/2, delete_and_terminate/2, delete_crashed/1, - purge/1, purge_acks/1, - publish/6, publish_delivered/5, - batch_publish/4, batch_publish_delivered/4, - discard/4, drain_confirmed/1, - dropwhile/2, fetchwhile/4, fetch/2, drop/2, ack/2, requeue/2, - ackfold/4, fold/3, len/1, is_empty/1, depth/1, - set_ram_duration_target/2, ram_duration/1, needs_timeout/1, timeout/1, - handle_pre_hibernate/1, resume/1, msg_rates/1, - info/2, invoke/3, is_duplicate/2, set_queue_mode/2, - zip_msgs_and_acks/4, multiple_routing_keys/0, handle_info/2]). - --export([start/2, stop/1]). - -%% exported for testing only --export([start_msg_store/3, stop_msg_store/1, init/6]). - --export([move_messages_to_vhost_store/0]). - --export([migrate_queue/3, migrate_message/3, get_per_vhost_store_client/2, - get_global_store_client/1, log_upgrade_verbose/1, - log_upgrade_verbose/2]). - --include_lib("stdlib/include/qlc.hrl"). - --define(QUEUE_MIGRATION_BATCH_SIZE, 100). --define(EMPTY_START_FUN_STATE, {fun (ok) -> finished end, ok}). - -%%---------------------------------------------------------------------------- -%% Messages, and their position in the queue, can be in memory or on -%% disk, or both. Persistent messages will have both message and -%% position pushed to disk as soon as they arrive; transient messages -%% can be written to disk (and thus both types can be evicted from -%% memory) under memory pressure. The question of whether a message is -%% in RAM and whether it is persistent are orthogonal. -%% -%% Messages are persisted using the queue index and the message -%% store. Normally the queue index holds the position of the message -%% *within this queue* along with a couple of small bits of metadata, -%% while the message store holds the message itself (including headers -%% and other properties). -%% -%% However, as an optimisation, small messages can be embedded -%% directly in the queue index and bypass the message store -%% altogether. -%% -%% Definitions: -%% -%% alpha: this is a message where both the message itself, and its -%% position within the queue are held in RAM -%% -%% beta: this is a message where the message itself is only held on -%% disk (if persisted to the message store) but its position -%% within the queue is held in RAM. -%% -%% gamma: this is a message where the message itself is only held on -%% disk, but its position is both in RAM and on disk. -%% -%% delta: this is a collection of messages, represented by a single -%% term, where the messages and their position are only held on -%% disk. -%% -%% Note that for persistent messages, the message and its position -%% within the queue are always held on disk, *in addition* to being in -%% one of the above classifications. -%% -%% Also note that within this code, the term gamma seldom -%% appears. It's frequently the case that gammas are defined by betas -%% who have had their queue position recorded on disk. -%% -%% In general, messages move q1 -> q2 -> delta -> q3 -> q4, though -%% many of these steps are frequently skipped. q1 and q4 only hold -%% alphas, q2 and q3 hold both betas and gammas. When a message -%% arrives, its classification is determined. It is then added to the -%% rightmost appropriate queue. -%% -%% If a new message is determined to be a beta or gamma, q1 is -%% empty. If a new message is determined to be a delta, q1 and q2 are -%% empty (and actually q4 too). -%% -%% When removing messages from a queue, if q4 is empty then q3 is read -%% directly. If q3 becomes empty then the next segment's worth of -%% messages from delta are read into q3, reducing the size of -%% delta. If the queue is non empty, either q4 or q3 contain -%% entries. It is never permitted for delta to hold all the messages -%% in the queue. -%% -%% The duration indicated to us by the memory_monitor is used to -%% calculate, given our current ingress and egress rates, how many -%% messages we should hold in RAM (i.e. as alphas). We track the -%% ingress and egress rates for both messages and pending acks and -%% rates for both are considered when calculating the number of -%% messages to hold in RAM. When we need to push alphas to betas or -%% betas to gammas, we favour writing out messages that are further -%% from the head of the queue. This minimises writes to disk, as the -%% messages closer to the tail of the queue stay in the queue for -%% longer, thus do not need to be replaced as quickly by sending other -%% messages to disk. -%% -%% Whilst messages are pushed to disk and forgotten from RAM as soon -%% as requested by a new setting of the queue RAM duration, the -%% inverse is not true: we only load messages back into RAM as -%% demanded as the queue is read from. Thus only publishes to the -%% queue will take up available spare capacity. -%% -%% When we report our duration to the memory monitor, we calculate -%% average ingress and egress rates over the last two samples, and -%% then calculate our duration based on the sum of the ingress and -%% egress rates. More than two samples could be used, but it's a -%% balance between responding quickly enough to changes in -%% producers/consumers versus ignoring temporary blips. The problem -%% with temporary blips is that with just a few queues, they can have -%% substantial impact on the calculation of the average duration and -%% hence cause unnecessary I/O. Another alternative is to increase the -%% amqqueue_process:RAM_DURATION_UPDATE_PERIOD to beyond 5 -%% seconds. However, that then runs the risk of being too slow to -%% inform the memory monitor of changes. Thus a 5 second interval, -%% plus a rolling average over the last two samples seems to work -%% well in practice. -%% -%% The sum of the ingress and egress rates is used because the egress -%% rate alone is not sufficient. Adding in the ingress rate means that -%% queues which are being flooded by messages are given more memory, -%% resulting in them being able to process the messages faster (by -%% doing less I/O, or at least deferring it) and thus helping keep -%% their mailboxes empty and thus the queue as a whole is more -%% responsive. If such a queue also has fast but previously idle -%% consumers, the consumer can then start to be driven as fast as it -%% can go, whereas if only egress rate was being used, the incoming -%% messages may have to be written to disk and then read back in, -%% resulting in the hard disk being a bottleneck in driving the -%% consumers. Generally, we want to give Rabbit every chance of -%% getting rid of messages as fast as possible and remaining -%% responsive, and using only the egress rate impacts that goal. -%% -%% Once the queue has more alphas than the target_ram_count, the -%% surplus must be converted to betas, if not gammas, if not rolled -%% into delta. The conditions under which these transitions occur -%% reflect the conflicting goals of minimising RAM cost per msg, and -%% minimising CPU cost per msg. Once the msg has become a beta, its -%% payload is no longer in RAM, thus a read from the msg_store must -%% occur before the msg can be delivered, but the RAM cost of a beta -%% is the same as a gamma, so converting a beta to gamma will not free -%% up any further RAM. To reduce the RAM cost further, the gamma must -%% be rolled into delta. Whilst recovering a beta or a gamma to an -%% alpha requires only one disk read (from the msg_store), recovering -%% a msg from within delta will require two reads (queue_index and -%% then msg_store). But delta has a near-0 per-msg RAM cost. So the -%% conflict is between using delta more, which will free up more -%% memory, but require additional CPU and disk ops, versus using delta -%% less and gammas and betas more, which will cost more memory, but -%% require fewer disk ops and less CPU overhead. -%% -%% In the case of a persistent msg published to a durable queue, the -%% msg is immediately written to the msg_store and queue_index. If -%% then additionally converted from an alpha, it'll immediately go to -%% a gamma (as it's already in queue_index), and cannot exist as a -%% beta. Thus a durable queue with a mixture of persistent and -%% transient msgs in it which has more messages than permitted by the -%% target_ram_count may contain an interspersed mixture of betas and -%% gammas in q2 and q3. -%% -%% There is then a ratio that controls how many betas and gammas there -%% can be. This is based on the target_ram_count and thus expresses -%% the fact that as the number of permitted alphas in the queue falls, -%% so should the number of betas and gammas fall (i.e. delta -%% grows). If q2 and q3 contain more than the permitted number of -%% betas and gammas, then the surplus are forcibly converted to gammas -%% (as necessary) and then rolled into delta. The ratio is that -%% delta/(betas+gammas+delta) equals -%% (betas+gammas+delta)/(target_ram_count+betas+gammas+delta). I.e. as -%% the target_ram_count shrinks to 0, so must betas and gammas. -%% -%% The conversion of betas to deltas is done if there are at least -%% ?IO_BATCH_SIZE betas in q2 & q3. This value should not be too small, -%% otherwise the frequent operations on the queues of q2 and q3 will not be -%% effectively amortised (switching the direction of queue access defeats -%% amortisation). Note that there is a natural upper bound due to credit_flow -%% limits on the alpha to beta conversion. -%% -%% The conversion from alphas to betas is chunked due to the -%% credit_flow limits of the msg_store. This further smooths the -%% effects of changes to the target_ram_count and ensures the queue -%% remains responsive even when there is a large amount of IO work to -%% do. The 'resume' callback is utilised to ensure that conversions -%% are done as promptly as possible whilst ensuring the queue remains -%% responsive. -%% -%% In the queue we keep track of both messages that are pending -%% delivery and messages that are pending acks. In the event of a -%% queue purge, we only need to load qi segments if the queue has -%% elements in deltas (i.e. it came under significant memory -%% pressure). In the event of a queue deletion, in addition to the -%% preceding, by keeping track of pending acks in RAM, we do not need -%% to search through qi segments looking for messages that are yet to -%% be acknowledged. -%% -%% Pending acks are recorded in memory by storing the message itself. -%% If the message has been sent to disk, we do not store the message -%% content. During memory reduction, pending acks containing message -%% content have that content removed and the corresponding messages -%% are pushed out to disk. -%% -%% Messages from pending acks are returned to q4, q3 and delta during -%% requeue, based on the limits of seq_id contained in each. Requeued -%% messages retain their original seq_id, maintaining order -%% when requeued. -%% -%% The order in which alphas are pushed to betas and pending acks -%% are pushed to disk is determined dynamically. We always prefer to -%% push messages for the source (alphas or acks) that is growing the -%% fastest (with growth measured as avg. ingress - avg. egress). -%% -%% Notes on Clean Shutdown -%% (This documents behaviour in variable_queue, queue_index and -%% msg_store.) -%% -%% In order to try to achieve as fast a start-up as possible, if a -%% clean shutdown occurs, we try to save out state to disk to reduce -%% work on startup. In the msg_store this takes the form of the -%% index_module's state, plus the file_summary ets table, and client -%% refs. In the VQ, this takes the form of the count of persistent -%% messages in the queue and references into the msg_stores. The -%% queue_index adds to these terms the details of its segments and -%% stores the terms in the queue directory. -%% -%% Two message stores are used. One is created for persistent messages -%% to durable queues that must survive restarts, and the other is used -%% for all other messages that just happen to need to be written to -%% disk. On start up we can therefore nuke the transient message -%% store, and be sure that the messages in the persistent store are -%% all that we need. -%% -%% The references to the msg_stores are there so that the msg_store -%% knows to only trust its saved state if all of the queues it was -%% previously talking to come up cleanly. Likewise, the queues -%% themselves (esp queue_index) skips work in init if all the queues -%% and msg_store were shutdown cleanly. This gives both good speed -%% improvements and also robustness so that if anything possibly went -%% wrong in shutdown (or there was subsequent manual tampering), all -%% messages and queues that can be recovered are recovered, safely. -%% -%% To delete transient messages lazily, the variable_queue, on -%% startup, stores the next_seq_id reported by the queue_index as the -%% transient_threshold. From that point on, whenever it's reading a -%% message off disk via the queue_index, if the seq_id is below this -%% threshold and the message is transient then it drops the message -%% (the message itself won't exist on disk because it would have been -%% stored in the transient msg_store which would have had its saved -%% state nuked on startup). This avoids the expensive operation of -%% scanning the entire queue on startup in order to delete transient -%% messages that were only pushed to disk to save memory. -%% -%%---------------------------------------------------------------------------- - --behaviour(rabbit_backing_queue). - --record(vqstate, - { q1, - q2, - delta, - q3, - q4, - next_seq_id, - ram_pending_ack, %% msgs using store, still in RAM - disk_pending_ack, %% msgs in store, paged out - qi_pending_ack, %% msgs using qi, *can't* be paged out - index_state, - msg_store_clients, - durable, - transient_threshold, - qi_embed_msgs_below, - - len, %% w/o unacked - bytes, %% w/o unacked - unacked_bytes, - persistent_count, %% w unacked - persistent_bytes, %% w unacked - delta_transient_bytes, %% - - target_ram_count, - ram_msg_count, %% w/o unacked - ram_msg_count_prev, - ram_ack_count_prev, - ram_bytes, %% w unacked - out_counter, - in_counter, - rates, - msgs_on_disk, - msg_indices_on_disk, - unconfirmed, - confirmed, - ack_out_counter, - ack_in_counter, - %% Unlike the other counters these two do not feed into - %% #rates{} and get reset - disk_read_count, - disk_write_count, - - io_batch_size, - - %% default queue or lazy queue - mode, - %% number of reduce_memory_usage executions, once it - %% reaches a threshold the queue will manually trigger a runtime GC - %% see: maybe_execute_gc/1 - memory_reduction_run_count, - %% Queue data is grouped by VHost. We need to store it - %% to work with queue index. - virtual_host, - waiting_bump = false - }). - --record(rates, { in, out, ack_in, ack_out, timestamp }). - --record(msg_status, - { seq_id, - msg_id, - msg, - is_persistent, - is_delivered, - msg_in_store, - index_on_disk, - persist_to, - msg_props - }). - --record(delta, - { start_seq_id, %% start_seq_id is inclusive - count, - transient, - end_seq_id %% end_seq_id is exclusive - }). - --define(HEADER_GUESS_SIZE, 100). %% see determine_persist_to/2 --define(PERSISTENT_MSG_STORE, msg_store_persistent). --define(TRANSIENT_MSG_STORE, msg_store_transient). - --define(QUEUE, lqueue). - --include_lib("rabbit_common/include/rabbit.hrl"). --include_lib("rabbit_common/include/rabbit_framing.hrl"). --include("amqqueue.hrl"). - -%%---------------------------------------------------------------------------- - --rabbit_upgrade({multiple_routing_keys, local, []}). --rabbit_upgrade({move_messages_to_vhost_store, message_store, []}). - --type seq_id() :: non_neg_integer(). - --type rates() :: #rates { in :: float(), - out :: float(), - ack_in :: float(), - ack_out :: float(), - timestamp :: rabbit_types:timestamp()}. - --type delta() :: #delta { start_seq_id :: non_neg_integer(), - count :: non_neg_integer(), - end_seq_id :: non_neg_integer() }. - -%% The compiler (rightfully) complains that ack() and state() are -%% unused. For this reason we duplicate a -spec from -%% rabbit_backing_queue with the only intent being to remove -%% warnings. The problem here is that we can't parameterise the BQ -%% behaviour by these two types as we would like to. We still leave -%% these here for documentation purposes. --type ack() :: seq_id(). --type state() :: #vqstate { - q1 :: ?QUEUE:?QUEUE(), - q2 :: ?QUEUE:?QUEUE(), - delta :: delta(), - q3 :: ?QUEUE:?QUEUE(), - q4 :: ?QUEUE:?QUEUE(), - next_seq_id :: seq_id(), - ram_pending_ack :: gb_trees:tree(), - disk_pending_ack :: gb_trees:tree(), - qi_pending_ack :: gb_trees:tree(), - index_state :: any(), - msg_store_clients :: 'undefined' | {{any(), binary()}, - {any(), binary()}}, - durable :: boolean(), - transient_threshold :: non_neg_integer(), - qi_embed_msgs_below :: non_neg_integer(), - - len :: non_neg_integer(), - bytes :: non_neg_integer(), - unacked_bytes :: non_neg_integer(), - - persistent_count :: non_neg_integer(), - persistent_bytes :: non_neg_integer(), - - target_ram_count :: non_neg_integer() | 'infinity', - ram_msg_count :: non_neg_integer(), - ram_msg_count_prev :: non_neg_integer(), - ram_ack_count_prev :: non_neg_integer(), - ram_bytes :: non_neg_integer(), - out_counter :: non_neg_integer(), - in_counter :: non_neg_integer(), - rates :: rates(), - msgs_on_disk :: gb_sets:set(), - msg_indices_on_disk :: gb_sets:set(), - unconfirmed :: gb_sets:set(), - confirmed :: gb_sets:set(), - ack_out_counter :: non_neg_integer(), - ack_in_counter :: non_neg_integer(), - disk_read_count :: non_neg_integer(), - disk_write_count :: non_neg_integer(), - - io_batch_size :: pos_integer(), - mode :: 'default' | 'lazy', - memory_reduction_run_count :: non_neg_integer()}. - --define(BLANK_DELTA, #delta { start_seq_id = undefined, - count = 0, - transient = 0, - end_seq_id = undefined }). --define(BLANK_DELTA_PATTERN(Z), #delta { start_seq_id = Z, - count = 0, - transient = 0, - end_seq_id = Z }). - --define(MICROS_PER_SECOND, 1000000.0). - -%% We're sampling every 5s for RAM duration; a half life that is of -%% the same order of magnitude is probably about right. --define(RATE_AVG_HALF_LIFE, 5.0). - -%% We will recalculate the #rates{} every time we get asked for our -%% RAM duration, or every N messages published, whichever is -%% sooner. We do this since the priority calculations in -%% rabbit_amqqueue_process need fairly fresh rates. --define(MSGS_PER_RATE_CALC, 100). - -%% we define the garbage collector threshold -%% it needs to tune the `reduce_memory_use` calls. Thus, the garbage collection. -%% see: rabbitmq-server-973 and rabbitmq-server-964 --define(DEFAULT_EXPLICIT_GC_RUN_OP_THRESHOLD, 1000). --define(EXPLICIT_GC_RUN_OP_THRESHOLD(Mode), - case get(explicit_gc_run_operation_threshold) of - undefined -> - Val = explicit_gc_run_operation_threshold_for_mode(Mode), - put(explicit_gc_run_operation_threshold, Val), - Val; - Val -> Val - end). - -explicit_gc_run_operation_threshold_for_mode(Mode) -> - {Key, Fallback} = case Mode of - lazy -> {lazy_queue_explicit_gc_run_operation_threshold, - ?DEFAULT_EXPLICIT_GC_RUN_OP_THRESHOLD}; - _ -> {queue_explicit_gc_run_operation_threshold, - ?DEFAULT_EXPLICIT_GC_RUN_OP_THRESHOLD} - end, - rabbit_misc:get_env(rabbit, Key, Fallback). - -%%---------------------------------------------------------------------------- -%% Public API -%%---------------------------------------------------------------------------- - -start(VHost, DurableQueues) -> - {AllTerms, StartFunState} = rabbit_queue_index:start(VHost, DurableQueues), - %% Group recovery terms by vhost. - ClientRefs = [Ref || Terms <- AllTerms, - Terms /= non_clean_shutdown, - begin - Ref = proplists:get_value(persistent_ref, Terms), - Ref =/= undefined - end], - start_msg_store(VHost, ClientRefs, StartFunState), - {ok, AllTerms}. - -stop(VHost) -> - ok = stop_msg_store(VHost), - ok = rabbit_queue_index:stop(VHost). - -start_msg_store(VHost, Refs, StartFunState) when is_list(Refs); Refs == undefined -> - rabbit_log:info("Starting message stores for vhost '~s'~n", [VHost]), - do_start_msg_store(VHost, ?TRANSIENT_MSG_STORE, undefined, ?EMPTY_START_FUN_STATE), - do_start_msg_store(VHost, ?PERSISTENT_MSG_STORE, Refs, StartFunState), - ok. - -do_start_msg_store(VHost, Type, Refs, StartFunState) -> - case rabbit_vhost_msg_store:start(VHost, Type, Refs, StartFunState) of - {ok, _} -> - rabbit_log:info("Started message store of type ~s for vhost '~s'~n", [abbreviated_type(Type), VHost]); - {error, {no_such_vhost, VHost}} = Err -> - rabbit_log:error("Failed to start message store of type ~s for vhost '~s': the vhost no longer exists!~n", - [Type, VHost]), - exit(Err); - {error, Error} -> - rabbit_log:error("Failed to start message store of type ~s for vhost '~s': ~p~n", - [Type, VHost, Error]), - exit({error, Error}) - end. - -abbreviated_type(?TRANSIENT_MSG_STORE) -> transient; -abbreviated_type(?PERSISTENT_MSG_STORE) -> persistent. - -stop_msg_store(VHost) -> - rabbit_vhost_msg_store:stop(VHost, ?TRANSIENT_MSG_STORE), - rabbit_vhost_msg_store:stop(VHost, ?PERSISTENT_MSG_STORE), - ok. - -init(Queue, Recover, Callback) -> - init( - Queue, Recover, Callback, - fun (MsgIds, ActionTaken) -> - msgs_written_to_disk(Callback, MsgIds, ActionTaken) - end, - fun (MsgIds) -> msg_indices_written_to_disk(Callback, MsgIds) end, - fun (MsgIds) -> msgs_and_indices_written_to_disk(Callback, MsgIds) end). - -init(Q, new, AsyncCallback, MsgOnDiskFun, MsgIdxOnDiskFun, MsgAndIdxOnDiskFun) when ?is_amqqueue(Q) -> - QueueName = amqqueue:get_name(Q), - IsDurable = amqqueue:is_durable(Q), - IndexState = rabbit_queue_index:init(QueueName, - MsgIdxOnDiskFun, MsgAndIdxOnDiskFun), - VHost = QueueName#resource.virtual_host, - init(IsDurable, IndexState, 0, 0, [], - case IsDurable of - true -> msg_store_client_init(?PERSISTENT_MSG_STORE, - MsgOnDiskFun, AsyncCallback, VHost); - false -> undefined - end, - msg_store_client_init(?TRANSIENT_MSG_STORE, undefined, - AsyncCallback, VHost), VHost); - -%% We can be recovering a transient queue if it crashed -init(Q, Terms, AsyncCallback, MsgOnDiskFun, MsgIdxOnDiskFun, MsgAndIdxOnDiskFun) when ?is_amqqueue(Q) -> - QueueName = amqqueue:get_name(Q), - IsDurable = amqqueue:is_durable(Q), - {PRef, RecoveryTerms} = process_recovery_terms(Terms), - VHost = QueueName#resource.virtual_host, - {PersistentClient, ContainsCheckFun} = - case IsDurable of - true -> C = msg_store_client_init(?PERSISTENT_MSG_STORE, PRef, - MsgOnDiskFun, AsyncCallback, - VHost), - {C, fun (MsgId) when is_binary(MsgId) -> - rabbit_msg_store:contains(MsgId, C); - (#basic_message{is_persistent = Persistent}) -> - Persistent - end}; - false -> {undefined, fun(_MsgId) -> false end} - end, - TransientClient = msg_store_client_init(?TRANSIENT_MSG_STORE, - undefined, AsyncCallback, - VHost), - {DeltaCount, DeltaBytes, IndexState} = - rabbit_queue_index:recover( - QueueName, RecoveryTerms, - rabbit_vhost_msg_store:successfully_recovered_state( - VHost, - ?PERSISTENT_MSG_STORE), - ContainsCheckFun, MsgIdxOnDiskFun, MsgAndIdxOnDiskFun), - init(IsDurable, IndexState, DeltaCount, DeltaBytes, RecoveryTerms, - PersistentClient, TransientClient, VHost). - -process_recovery_terms(Terms=non_clean_shutdown) -> - {rabbit_guid:gen(), Terms}; -process_recovery_terms(Terms) -> - case proplists:get_value(persistent_ref, Terms) of - undefined -> {rabbit_guid:gen(), []}; - PRef -> {PRef, Terms} - end. - -terminate(_Reason, State) -> - State1 = #vqstate { virtual_host = VHost, - persistent_count = PCount, - persistent_bytes = PBytes, - index_state = IndexState, - msg_store_clients = {MSCStateP, MSCStateT} } = - purge_pending_ack(true, State), - PRef = case MSCStateP of - undefined -> undefined; - _ -> ok = maybe_client_terminate(MSCStateP), - rabbit_msg_store:client_ref(MSCStateP) - end, - ok = rabbit_msg_store:client_delete_and_terminate(MSCStateT), - Terms = [{persistent_ref, PRef}, - {persistent_count, PCount}, - {persistent_bytes, PBytes}], - a(State1#vqstate { - index_state = rabbit_queue_index:terminate(VHost, Terms, IndexState), - msg_store_clients = undefined }). - -%% the only difference between purge and delete is that delete also -%% needs to delete everything that's been delivered and not ack'd. -delete_and_terminate(_Reason, State) -> - %% Normally when we purge messages we interact with the qi by - %% issues delivers and acks for every purged message. In this case - %% we don't need to do that, so we just delete the qi. - State1 = purge_and_index_reset(State), - State2 = #vqstate { msg_store_clients = {MSCStateP, MSCStateT} } = - purge_pending_ack_delete_and_terminate(State1), - case MSCStateP of - undefined -> ok; - _ -> rabbit_msg_store:client_delete_and_terminate(MSCStateP) - end, - rabbit_msg_store:client_delete_and_terminate(MSCStateT), - a(State2 #vqstate { msg_store_clients = undefined }). - -delete_crashed(Q) when ?is_amqqueue(Q) -> - QName = amqqueue:get_name(Q), - ok = rabbit_queue_index:erase(QName). - -purge(State = #vqstate { len = Len }) -> - case is_pending_ack_empty(State) and is_unconfirmed_empty(State) of - true -> - {Len, purge_and_index_reset(State)}; - false -> - {Len, purge_when_pending_acks(State)} - end. - -purge_acks(State) -> a(purge_pending_ack(false, State)). - -publish(Msg, MsgProps, IsDelivered, ChPid, Flow, State) -> - State1 = - publish1(Msg, MsgProps, IsDelivered, ChPid, Flow, - fun maybe_write_to_disk/4, - State), - a(maybe_reduce_memory_use(maybe_update_rates(State1))). - -batch_publish(Publishes, ChPid, Flow, State) -> - {ChPid, Flow, State1} = - lists:foldl(fun batch_publish1/2, {ChPid, Flow, State}, Publishes), - State2 = ui(State1), - a(maybe_reduce_memory_use(maybe_update_rates(State2))). - -publish_delivered(Msg, MsgProps, ChPid, Flow, State) -> - {SeqId, State1} = - publish_delivered1(Msg, MsgProps, ChPid, Flow, - fun maybe_write_to_disk/4, - State), - {SeqId, a(maybe_reduce_memory_use(maybe_update_rates(State1)))}. - -batch_publish_delivered(Publishes, ChPid, Flow, State) -> - {ChPid, Flow, SeqIds, State1} = - lists:foldl(fun batch_publish_delivered1/2, - {ChPid, Flow, [], State}, Publishes), - State2 = ui(State1), - {lists:reverse(SeqIds), a(maybe_reduce_memory_use(maybe_update_rates(State2)))}. - -discard(_MsgId, _ChPid, _Flow, State) -> State. - -drain_confirmed(State = #vqstate { confirmed = C }) -> - case gb_sets:is_empty(C) of - true -> {[], State}; %% common case - false -> {gb_sets:to_list(C), State #vqstate { - confirmed = gb_sets:new() }} - end. - -dropwhile(Pred, State) -> - {MsgProps, State1} = - remove_by_predicate(Pred, State), - {MsgProps, a(State1)}. - -fetchwhile(Pred, Fun, Acc, State) -> - {MsgProps, Acc1, State1} = - fetch_by_predicate(Pred, Fun, Acc, State), - {MsgProps, Acc1, a(State1)}. - -fetch(AckRequired, State) -> - case queue_out(State) of - {empty, State1} -> - {empty, a(State1)}; - {{value, MsgStatus}, State1} -> - %% it is possible that the message wasn't read from disk - %% at this point, so read it in. - {Msg, State2} = read_msg(MsgStatus, State1), - {AckTag, State3} = remove(AckRequired, MsgStatus, State2), - {{Msg, MsgStatus#msg_status.is_delivered, AckTag}, a(State3)} - end. - -drop(AckRequired, State) -> - case queue_out(State) of - {empty, State1} -> - {empty, a(State1)}; - {{value, MsgStatus}, State1} -> - {AckTag, State2} = remove(AckRequired, MsgStatus, State1), - {{MsgStatus#msg_status.msg_id, AckTag}, a(State2)} - end. - -%% Duplicated from rabbit_backing_queue --spec ack([ack()], state()) -> {[rabbit_guid:guid()], state()}. - -ack([], State) -> - {[], State}; -%% optimisation: this head is essentially a partial evaluation of the -%% general case below, for the single-ack case. -ack([SeqId], State) -> - case remove_pending_ack(true, SeqId, State) of - {none, _} -> - {[], State}; - {#msg_status { msg_id = MsgId, - is_persistent = IsPersistent, - msg_in_store = MsgInStore, - index_on_disk = IndexOnDisk }, - State1 = #vqstate { index_state = IndexState, - msg_store_clients = MSCState, - ack_out_counter = AckOutCount }} -> - IndexState1 = case IndexOnDisk of - true -> rabbit_queue_index:ack([SeqId], IndexState); - false -> IndexState - end, - case MsgInStore of - true -> ok = msg_store_remove(MSCState, IsPersistent, [MsgId]); - false -> ok - end, - {[MsgId], - a(State1 #vqstate { index_state = IndexState1, - ack_out_counter = AckOutCount + 1 })} - end; -ack(AckTags, State) -> - {{IndexOnDiskSeqIds, MsgIdsByStore, AllMsgIds}, - State1 = #vqstate { index_state = IndexState, - msg_store_clients = MSCState, - ack_out_counter = AckOutCount }} = - lists:foldl( - fun (SeqId, {Acc, State2}) -> - case remove_pending_ack(true, SeqId, State2) of - {none, _} -> - {Acc, State2}; - {MsgStatus, State3} -> - {accumulate_ack(MsgStatus, Acc), State3} - end - end, {accumulate_ack_init(), State}, AckTags), - IndexState1 = rabbit_queue_index:ack(IndexOnDiskSeqIds, IndexState), - remove_msgs_by_id(MsgIdsByStore, MSCState), - {lists:reverse(AllMsgIds), - a(State1 #vqstate { index_state = IndexState1, - ack_out_counter = AckOutCount + length(AckTags) })}. - -requeue(AckTags, #vqstate { mode = default, - delta = Delta, - q3 = Q3, - q4 = Q4, - in_counter = InCounter, - len = Len } = State) -> - {SeqIds, Q4a, MsgIds, State1} = queue_merge(lists:sort(AckTags), Q4, [], - beta_limit(Q3), - fun publish_alpha/2, State), - {SeqIds1, Q3a, MsgIds1, State2} = queue_merge(SeqIds, Q3, MsgIds, - delta_limit(Delta), - fun publish_beta/2, State1), - {Delta1, MsgIds2, State3} = delta_merge(SeqIds1, Delta, MsgIds1, - State2), - MsgCount = length(MsgIds2), - {MsgIds2, a(maybe_reduce_memory_use( - maybe_update_rates(ui( - State3 #vqstate { delta = Delta1, - q3 = Q3a, - q4 = Q4a, - in_counter = InCounter + MsgCount, - len = Len + MsgCount }))))}; -requeue(AckTags, #vqstate { mode = lazy, - delta = Delta, - q3 = Q3, - in_counter = InCounter, - len = Len } = State) -> - {SeqIds, Q3a, MsgIds, State1} = queue_merge(lists:sort(AckTags), Q3, [], - delta_limit(Delta), - fun publish_beta/2, State), - {Delta1, MsgIds1, State2} = delta_merge(SeqIds, Delta, MsgIds, - State1), - MsgCount = length(MsgIds1), - {MsgIds1, a(maybe_reduce_memory_use( - maybe_update_rates(ui( - State2 #vqstate { delta = Delta1, - q3 = Q3a, - in_counter = InCounter + MsgCount, - len = Len + MsgCount }))))}. - -ackfold(MsgFun, Acc, State, AckTags) -> - {AccN, StateN} = - lists:foldl(fun(SeqId, {Acc0, State0}) -> - MsgStatus = lookup_pending_ack(SeqId, State0), - {Msg, State1} = read_msg(MsgStatus, State0), - {MsgFun(Msg, SeqId, Acc0), State1} - end, {Acc, State}, AckTags), - {AccN, a(StateN)}. - -fold(Fun, Acc, State = #vqstate{index_state = IndexState}) -> - {Its, IndexState1} = lists:foldl(fun inext/2, {[], IndexState}, - [msg_iterator(State), - disk_ack_iterator(State), - ram_ack_iterator(State), - qi_ack_iterator(State)]), - ifold(Fun, Acc, Its, State#vqstate{index_state = IndexState1}). - -len(#vqstate { len = Len }) -> Len. - -is_empty(State) -> 0 == len(State). - -depth(State) -> - len(State) + count_pending_acks(State). - -set_ram_duration_target( - DurationTarget, State = #vqstate { - rates = #rates { in = AvgIngressRate, - out = AvgEgressRate, - ack_in = AvgAckIngressRate, - ack_out = AvgAckEgressRate }, - target_ram_count = TargetRamCount }) -> - Rate = - AvgEgressRate + AvgIngressRate + AvgAckEgressRate + AvgAckIngressRate, - TargetRamCount1 = - case DurationTarget of - infinity -> infinity; - _ -> trunc(DurationTarget * Rate) %% msgs = sec * msgs/sec - end, - State1 = State #vqstate { target_ram_count = TargetRamCount1 }, - a(case TargetRamCount1 == infinity orelse - (TargetRamCount =/= infinity andalso - TargetRamCount1 >= TargetRamCount) of - true -> State1; - false -> reduce_memory_use(State1) - end). - -maybe_update_rates(State = #vqstate{ in_counter = InCount, - out_counter = OutCount }) - when InCount + OutCount > ?MSGS_PER_RATE_CALC -> - update_rates(State); -maybe_update_rates(State) -> - State. - -update_rates(State = #vqstate{ in_counter = InCount, - out_counter = OutCount, - ack_in_counter = AckInCount, - ack_out_counter = AckOutCount, - rates = #rates{ in = InRate, - out = OutRate, - ack_in = AckInRate, - ack_out = AckOutRate, - timestamp = TS }}) -> - Now = erlang:monotonic_time(), - - Rates = #rates { in = update_rate(Now, TS, InCount, InRate), - out = update_rate(Now, TS, OutCount, OutRate), - ack_in = update_rate(Now, TS, AckInCount, AckInRate), - ack_out = update_rate(Now, TS, AckOutCount, AckOutRate), - timestamp = Now }, - - State#vqstate{ in_counter = 0, - out_counter = 0, - ack_in_counter = 0, - ack_out_counter = 0, - rates = Rates }. - -update_rate(Now, TS, Count, Rate) -> - Time = erlang:convert_time_unit(Now - TS, native, micro_seconds) / - ?MICROS_PER_SECOND, - if - Time == 0 -> Rate; - true -> rabbit_misc:moving_average(Time, ?RATE_AVG_HALF_LIFE, - Count / Time, Rate) - end. - -ram_duration(State) -> - State1 = #vqstate { rates = #rates { in = AvgIngressRate, - out = AvgEgressRate, - ack_in = AvgAckIngressRate, - ack_out = AvgAckEgressRate }, - ram_msg_count = RamMsgCount, - ram_msg_count_prev = RamMsgCountPrev, - ram_pending_ack = RPA, - qi_pending_ack = QPA, - ram_ack_count_prev = RamAckCountPrev } = - update_rates(State), - - RamAckCount = gb_trees:size(RPA) + gb_trees:size(QPA), - - Duration = %% msgs+acks / (msgs+acks/sec) == sec - case lists:all(fun (X) -> X < 0.01 end, - [AvgEgressRate, AvgIngressRate, - AvgAckEgressRate, AvgAckIngressRate]) of - true -> infinity; - false -> (RamMsgCountPrev + RamMsgCount + - RamAckCount + RamAckCountPrev) / - (4 * (AvgEgressRate + AvgIngressRate + - AvgAckEgressRate + AvgAckIngressRate)) - end, - - {Duration, State1}. - -needs_timeout(#vqstate { index_state = IndexState }) -> - case rabbit_queue_index:needs_sync(IndexState) of - confirms -> timed; - other -> idle; - false -> false - end. - -timeout(State = #vqstate { index_state = IndexState }) -> - State #vqstate { index_state = rabbit_queue_index:sync(IndexState) }. - -handle_pre_hibernate(State = #vqstate { index_state = IndexState }) -> - State #vqstate { index_state = rabbit_queue_index:flush(IndexState) }. - -handle_info(bump_reduce_memory_use, State = #vqstate{ waiting_bump = true }) -> - State#vqstate{ waiting_bump = false }; -handle_info(bump_reduce_memory_use, State) -> - State. - -resume(State) -> a(reduce_memory_use(State)). - -msg_rates(#vqstate { rates = #rates { in = AvgIngressRate, - out = AvgEgressRate } }) -> - {AvgIngressRate, AvgEgressRate}. - -info(messages_ready_ram, #vqstate{ram_msg_count = RamMsgCount}) -> - RamMsgCount; -info(messages_unacknowledged_ram, #vqstate{ram_pending_ack = RPA, - qi_pending_ack = QPA}) -> - gb_trees:size(RPA) + gb_trees:size(QPA); -info(messages_ram, State) -> - info(messages_ready_ram, State) + info(messages_unacknowledged_ram, State); -info(messages_persistent, #vqstate{persistent_count = PersistentCount}) -> - PersistentCount; -info(messages_paged_out, #vqstate{delta = #delta{transient = Count}}) -> - Count; -info(message_bytes, #vqstate{bytes = Bytes, - unacked_bytes = UBytes}) -> - Bytes + UBytes; -info(message_bytes_ready, #vqstate{bytes = Bytes}) -> - Bytes; -info(message_bytes_unacknowledged, #vqstate{unacked_bytes = UBytes}) -> - UBytes; -info(message_bytes_ram, #vqstate{ram_bytes = RamBytes}) -> - RamBytes; -info(message_bytes_persistent, #vqstate{persistent_bytes = PersistentBytes}) -> - PersistentBytes; -info(message_bytes_paged_out, #vqstate{delta_transient_bytes = PagedOutBytes}) -> - PagedOutBytes; -info(head_message_timestamp, #vqstate{ - q3 = Q3, - q4 = Q4, - ram_pending_ack = RPA, - qi_pending_ack = QPA}) -> - head_message_timestamp(Q3, Q4, RPA, QPA); -info(disk_reads, #vqstate{disk_read_count = Count}) -> - Count; -info(disk_writes, #vqstate{disk_write_count = Count}) -> - Count; -info(backing_queue_status, #vqstate { - q1 = Q1, q2 = Q2, delta = Delta, q3 = Q3, q4 = Q4, - mode = Mode, - len = Len, - target_ram_count = TargetRamCount, - next_seq_id = NextSeqId, - rates = #rates { in = AvgIngressRate, - out = AvgEgressRate, - ack_in = AvgAckIngressRate, - ack_out = AvgAckEgressRate }}) -> - - [ {mode , Mode}, - {q1 , ?QUEUE:len(Q1)}, - {q2 , ?QUEUE:len(Q2)}, - {delta , Delta}, - {q3 , ?QUEUE:len(Q3)}, - {q4 , ?QUEUE:len(Q4)}, - {len , Len}, - {target_ram_count , TargetRamCount}, - {next_seq_id , NextSeqId}, - {avg_ingress_rate , AvgIngressRate}, - {avg_egress_rate , AvgEgressRate}, - {avg_ack_ingress_rate, AvgAckIngressRate}, - {avg_ack_egress_rate , AvgAckEgressRate} ]; -info(_, _) -> - ''. - -invoke(?MODULE, Fun, State) -> Fun(?MODULE, State); -invoke( _, _, State) -> State. - -is_duplicate(_Msg, State) -> {false, State}. - -set_queue_mode(Mode, State = #vqstate { mode = Mode }) -> - State; -set_queue_mode(lazy, State = #vqstate { - target_ram_count = TargetRamCount }) -> - %% To become a lazy queue we need to page everything to disk first. - State1 = convert_to_lazy(State), - %% restore the original target_ram_count - a(State1 #vqstate { mode = lazy, target_ram_count = TargetRamCount }); -set_queue_mode(default, State) -> - %% becoming a default queue means loading messages from disk like - %% when a queue is recovered. - a(maybe_deltas_to_betas(State #vqstate { mode = default })); -set_queue_mode(_, State) -> - State. - -zip_msgs_and_acks(Msgs, AckTags, Accumulator, _State) -> - lists:foldl(fun ({{#basic_message{ id = Id }, _Props}, AckTag}, Acc) -> - [{Id, AckTag} | Acc] - end, Accumulator, lists:zip(Msgs, AckTags)). - -convert_to_lazy(State) -> - State1 = #vqstate { delta = Delta, q3 = Q3, len = Len } = - set_ram_duration_target(0, State), - case Delta#delta.count + ?QUEUE:len(Q3) == Len of - true -> - State1; - false -> - %% When pushing messages to disk, we might have been - %% blocked by the msg_store, so we need to see if we have - %% to wait for more credit, and then keep paging messages. - %% - %% The amqqueue_process could have taken care of this, but - %% between the time it receives the bump_credit msg and - %% calls BQ:resume to keep paging messages to disk, some - %% other request may arrive to the BQ which at this moment - %% is not in a proper state for a lazy BQ (unless all - %% messages have been paged to disk already). - wait_for_msg_store_credit(), - convert_to_lazy(resume(State1)) - end. - -wait_for_msg_store_credit() -> - case credit_flow:blocked() of - true -> receive - {bump_credit, Msg} -> - credit_flow:handle_bump_msg(Msg) - end; - false -> ok - end. - -%% Get the Timestamp property of the first msg, if present. This is -%% the one with the oldest timestamp among the heads of the pending -%% acks and unread queues. We can't check disk_pending_acks as these -%% are paged out - we assume some will soon be paged in rather than -%% forcing it to happen. Pending ack msgs are included as they are -%% regarded as unprocessed until acked, this also prevents the result -%% apparently oscillating during repeated rejects. Q3 is only checked -%% when Q4 is empty as any Q4 msg will be earlier. -head_message_timestamp(Q3, Q4, RPA, QPA) -> - HeadMsgs = [ HeadMsgStatus#msg_status.msg || - HeadMsgStatus <- - [ get_qs_head([Q4, Q3]), - get_pa_head(RPA), - get_pa_head(QPA) ], - HeadMsgStatus /= undefined, - HeadMsgStatus#msg_status.msg /= undefined ], - - Timestamps = - [Timestamp || HeadMsg <- HeadMsgs, - Timestamp <- [rabbit_basic:extract_timestamp( - HeadMsg#basic_message.content)], - Timestamp /= undefined - ], - - case Timestamps == [] of - true -> ''; - false -> lists:min(Timestamps) - end. - -get_qs_head(Qs) -> - catch lists:foldl( - fun (Q, Acc) -> - case get_q_head(Q) of - undefined -> Acc; - Val -> throw(Val) - end - end, undefined, Qs). - -get_q_head(Q) -> - get_collection_head(Q, fun ?QUEUE:is_empty/1, fun ?QUEUE:peek/1). - -get_pa_head(PA) -> - get_collection_head(PA, fun gb_trees:is_empty/1, fun gb_trees:smallest/1). - -get_collection_head(Col, IsEmpty, GetVal) -> - case IsEmpty(Col) of - false -> - {_, MsgStatus} = GetVal(Col), - MsgStatus; - true -> undefined - end. - -%%---------------------------------------------------------------------------- -%% Minor helpers -%%---------------------------------------------------------------------------- -a(State = #vqstate { q1 = Q1, q2 = Q2, delta = Delta, q3 = Q3, q4 = Q4, - mode = default, - len = Len, - bytes = Bytes, - unacked_bytes = UnackedBytes, - persistent_count = PersistentCount, - persistent_bytes = PersistentBytes, - ram_msg_count = RamMsgCount, - ram_bytes = RamBytes}) -> - E1 = ?QUEUE:is_empty(Q1), - E2 = ?QUEUE:is_empty(Q2), - ED = Delta#delta.count == 0, - E3 = ?QUEUE:is_empty(Q3), - E4 = ?QUEUE:is_empty(Q4), - LZ = Len == 0, - - %% if q1 has messages then q3 cannot be empty. See publish/6. - true = E1 or not E3, - %% if q2 has messages then we have messages in delta (paged to - %% disk). See push_alphas_to_betas/2. - true = E2 or not ED, - %% if delta has messages then q3 cannot be empty. This is enforced - %% by paging, where min([?SEGMENT_ENTRY_COUNT, len(q3)]) messages - %% are always kept on RAM. - true = ED or not E3, - %% if the queue length is 0, then q3 and q4 must be empty. - true = LZ == (E3 and E4), - - true = Len >= 0, - true = Bytes >= 0, - true = UnackedBytes >= 0, - true = PersistentCount >= 0, - true = PersistentBytes >= 0, - true = RamMsgCount >= 0, - true = RamMsgCount =< Len, - true = RamBytes >= 0, - true = RamBytes =< Bytes + UnackedBytes, - - State; -a(State = #vqstate { q1 = Q1, q2 = Q2, delta = Delta, q3 = Q3, q4 = Q4, - mode = lazy, - len = Len, - bytes = Bytes, - unacked_bytes = UnackedBytes, - persistent_count = PersistentCount, - persistent_bytes = PersistentBytes, - ram_msg_count = RamMsgCount, - ram_bytes = RamBytes}) -> - E1 = ?QUEUE:is_empty(Q1), - E2 = ?QUEUE:is_empty(Q2), - ED = Delta#delta.count == 0, - E3 = ?QUEUE:is_empty(Q3), - E4 = ?QUEUE:is_empty(Q4), - LZ = Len == 0, - L3 = ?QUEUE:len(Q3), - - %% q1 must always be empty, since q1 only gets messages during - %% publish, but for lazy queues messages go straight to delta. - true = E1, - - %% q2 only gets messages from q1 when push_alphas_to_betas is - %% called for a non empty delta, which won't be the case for a - %% lazy queue. This means q2 must always be empty. - true = E2, - - %% q4 must always be empty, since q1 only gets messages during - %% publish, but for lazy queues messages go straight to delta. - true = E4, - - %% if the queue is empty, then delta is empty and q3 is empty. - true = LZ == (ED and E3), - - %% There should be no messages in q1, q2, and q4 - true = Delta#delta.count + L3 == Len, - - true = Len >= 0, - true = Bytes >= 0, - true = UnackedBytes >= 0, - true = PersistentCount >= 0, - true = PersistentBytes >= 0, - true = RamMsgCount >= 0, - true = RamMsgCount =< Len, - true = RamBytes >= 0, - true = RamBytes =< Bytes + UnackedBytes, - - State. - -d(Delta = #delta { start_seq_id = Start, count = Count, end_seq_id = End }) - when Start + Count =< End -> - Delta. - -m(MsgStatus = #msg_status { is_persistent = IsPersistent, - msg_in_store = MsgInStore, - index_on_disk = IndexOnDisk }) -> - true = (not IsPersistent) or IndexOnDisk, - true = msg_in_ram(MsgStatus) or MsgInStore, - MsgStatus. - -one_if(true ) -> 1; -one_if(false) -> 0. - -cons_if(true, E, L) -> [E | L]; -cons_if(false, _E, L) -> L. - -gb_sets_maybe_insert(false, _Val, Set) -> Set; -gb_sets_maybe_insert(true, Val, Set) -> gb_sets:add(Val, Set). - -msg_status(IsPersistent, IsDelivered, SeqId, - Msg = #basic_message {id = MsgId}, MsgProps, IndexMaxSize) -> - #msg_status{seq_id = SeqId, - msg_id = MsgId, - msg = Msg, - is_persistent = IsPersistent, - is_delivered = IsDelivered, - msg_in_store = false, - index_on_disk = false, - persist_to = determine_persist_to(Msg, MsgProps, IndexMaxSize), - msg_props = MsgProps}. - -beta_msg_status({Msg = #basic_message{id = MsgId}, - SeqId, MsgProps, IsPersistent, IsDelivered}) -> - MS0 = beta_msg_status0(SeqId, MsgProps, IsPersistent, IsDelivered), - MS0#msg_status{msg_id = MsgId, - msg = Msg, - persist_to = queue_index, - msg_in_store = false}; - -beta_msg_status({MsgId, SeqId, MsgProps, IsPersistent, IsDelivered}) -> - MS0 = beta_msg_status0(SeqId, MsgProps, IsPersistent, IsDelivered), - MS0#msg_status{msg_id = MsgId, - msg = undefined, - persist_to = msg_store, - msg_in_store = true}. - -beta_msg_status0(SeqId, MsgProps, IsPersistent, IsDelivered) -> - #msg_status{seq_id = SeqId, - msg = undefined, - is_persistent = IsPersistent, - is_delivered = IsDelivered, - index_on_disk = true, - msg_props = MsgProps}. - -trim_msg_status(MsgStatus) -> - case persist_to(MsgStatus) of - msg_store -> MsgStatus#msg_status{msg = undefined}; - queue_index -> MsgStatus - end. - -with_msg_store_state({MSCStateP, MSCStateT}, true, Fun) -> - {Result, MSCStateP1} = Fun(MSCStateP), - {Result, {MSCStateP1, MSCStateT}}; -with_msg_store_state({MSCStateP, MSCStateT}, false, Fun) -> - {Result, MSCStateT1} = Fun(MSCStateT), - {Result, {MSCStateP, MSCStateT1}}. - -with_immutable_msg_store_state(MSCState, IsPersistent, Fun) -> - {Res, MSCState} = with_msg_store_state(MSCState, IsPersistent, - fun (MSCState1) -> - {Fun(MSCState1), MSCState1} - end), - Res. - -msg_store_client_init(MsgStore, MsgOnDiskFun, Callback, VHost) -> - msg_store_client_init(MsgStore, rabbit_guid:gen(), MsgOnDiskFun, - Callback, VHost). - -msg_store_client_init(MsgStore, Ref, MsgOnDiskFun, Callback, VHost) -> - CloseFDsFun = msg_store_close_fds_fun(MsgStore =:= ?PERSISTENT_MSG_STORE), - rabbit_vhost_msg_store:client_init(VHost, MsgStore, - Ref, MsgOnDiskFun, - fun () -> - Callback(?MODULE, CloseFDsFun) - end). - -msg_store_write(MSCState, IsPersistent, MsgId, Msg) -> - with_immutable_msg_store_state( - MSCState, IsPersistent, - fun (MSCState1) -> - rabbit_msg_store:write_flow(MsgId, Msg, MSCState1) - end). - -msg_store_read(MSCState, IsPersistent, MsgId) -> - with_msg_store_state( - MSCState, IsPersistent, - fun (MSCState1) -> - rabbit_msg_store:read(MsgId, MSCState1) - end). - -msg_store_remove(MSCState, IsPersistent, MsgIds) -> - with_immutable_msg_store_state( - MSCState, IsPersistent, - fun (MCSState1) -> - rabbit_msg_store:remove(MsgIds, MCSState1) - end). - -msg_store_close_fds(MSCState, IsPersistent) -> - with_msg_store_state( - MSCState, IsPersistent, - fun (MSCState1) -> rabbit_msg_store:close_all_indicated(MSCState1) end). - -msg_store_close_fds_fun(IsPersistent) -> - fun (?MODULE, State = #vqstate { msg_store_clients = MSCState }) -> - {ok, MSCState1} = msg_store_close_fds(MSCState, IsPersistent), - State #vqstate { msg_store_clients = MSCState1 } - end. - -maybe_write_delivered(false, _SeqId, IndexState) -> - IndexState; -maybe_write_delivered(true, SeqId, IndexState) -> - rabbit_queue_index:deliver([SeqId], IndexState). - -betas_from_index_entries(List, TransientThreshold, DelsAndAcksFun, State) -> - {Filtered, Delivers, Acks, RamReadyCount, RamBytes, TransientCount, TransientBytes} = - lists:foldr( - fun ({_MsgOrId, SeqId, _MsgProps, IsPersistent, IsDelivered} = M, - {Filtered1, Delivers1, Acks1, RRC, RB, TC, TB} = Acc) -> - case SeqId < TransientThreshold andalso not IsPersistent of - true -> {Filtered1, - cons_if(not IsDelivered, SeqId, Delivers1), - [SeqId | Acks1], RRC, RB, TC, TB}; - false -> MsgStatus = m(beta_msg_status(M)), - HaveMsg = msg_in_ram(MsgStatus), - Size = msg_size(MsgStatus), - case is_msg_in_pending_acks(SeqId, State) of - false -> {?QUEUE:in_r(MsgStatus, Filtered1), - Delivers1, Acks1, - RRC + one_if(HaveMsg), - RB + one_if(HaveMsg) * Size, - TC + one_if(not IsPersistent), - TB + one_if(not IsPersistent) * Size}; - true -> Acc %% [0] - end - end - end, {?QUEUE:new(), [], [], 0, 0, 0, 0}, List), - {Filtered, RamReadyCount, RamBytes, DelsAndAcksFun(Delivers, Acks, State), - TransientCount, TransientBytes}. -%% [0] We don't increase RamBytes here, even though it pertains to -%% unacked messages too, since if HaveMsg then the message must have -%% been stored in the QI, thus the message must have been in -%% qi_pending_ack, thus it must already have been in RAM. - -is_msg_in_pending_acks(SeqId, #vqstate { ram_pending_ack = RPA, - disk_pending_ack = DPA, - qi_pending_ack = QPA }) -> - (gb_trees:is_defined(SeqId, RPA) orelse - gb_trees:is_defined(SeqId, DPA) orelse - gb_trees:is_defined(SeqId, QPA)). - -expand_delta(SeqId, ?BLANK_DELTA_PATTERN(X), IsPersistent) -> - d(#delta { start_seq_id = SeqId, count = 1, end_seq_id = SeqId + 1, - transient = one_if(not IsPersistent)}); -expand_delta(SeqId, #delta { start_seq_id = StartSeqId, - count = Count, - transient = Transient } = Delta, - IsPersistent ) - when SeqId < StartSeqId -> - d(Delta #delta { start_seq_id = SeqId, count = Count + 1, - transient = Transient + one_if(not IsPersistent)}); -expand_delta(SeqId, #delta { count = Count, - end_seq_id = EndSeqId, - transient = Transient } = Delta, - IsPersistent) - when SeqId >= EndSeqId -> - d(Delta #delta { count = Count + 1, end_seq_id = SeqId + 1, - transient = Transient + one_if(not IsPersistent)}); -expand_delta(_SeqId, #delta { count = Count, - transient = Transient } = Delta, - IsPersistent ) -> - d(Delta #delta { count = Count + 1, - transient = Transient + one_if(not IsPersistent) }). - -%%---------------------------------------------------------------------------- -%% Internal major helpers for Public API -%%---------------------------------------------------------------------------- - -init(IsDurable, IndexState, DeltaCount, DeltaBytes, Terms, - PersistentClient, TransientClient, VHost) -> - {LowSeqId, NextSeqId, IndexState1} = rabbit_queue_index:bounds(IndexState), - - {DeltaCount1, DeltaBytes1} = - case Terms of - non_clean_shutdown -> {DeltaCount, DeltaBytes}; - _ -> {proplists:get_value(persistent_count, - Terms, DeltaCount), - proplists:get_value(persistent_bytes, - Terms, DeltaBytes)} - end, - Delta = case DeltaCount1 == 0 andalso DeltaCount /= undefined of - true -> ?BLANK_DELTA; - false -> d(#delta { start_seq_id = LowSeqId, - count = DeltaCount1, - transient = 0, - end_seq_id = NextSeqId }) - end, - Now = erlang:monotonic_time(), - IoBatchSize = rabbit_misc:get_env(rabbit, msg_store_io_batch_size, - ?IO_BATCH_SIZE), - - {ok, IndexMaxSize} = application:get_env( - rabbit, queue_index_embed_msgs_below), - State = #vqstate { - q1 = ?QUEUE:new(), - q2 = ?QUEUE:new(), - delta = Delta, - q3 = ?QUEUE:new(), - q4 = ?QUEUE:new(), - next_seq_id = NextSeqId, - ram_pending_ack = gb_trees:empty(), - disk_pending_ack = gb_trees:empty(), - qi_pending_ack = gb_trees:empty(), - index_state = IndexState1, - msg_store_clients = {PersistentClient, TransientClient}, - durable = IsDurable, - transient_threshold = NextSeqId, - qi_embed_msgs_below = IndexMaxSize, - - len = DeltaCount1, - persistent_count = DeltaCount1, - bytes = DeltaBytes1, - persistent_bytes = DeltaBytes1, - delta_transient_bytes = 0, - - target_ram_count = infinity, - ram_msg_count = 0, - ram_msg_count_prev = 0, - ram_ack_count_prev = 0, - ram_bytes = 0, - unacked_bytes = 0, - out_counter = 0, - in_counter = 0, - rates = blank_rates(Now), - msgs_on_disk = gb_sets:new(), - msg_indices_on_disk = gb_sets:new(), - unconfirmed = gb_sets:new(), - confirmed = gb_sets:new(), - ack_out_counter = 0, - ack_in_counter = 0, - disk_read_count = 0, - disk_write_count = 0, - - io_batch_size = IoBatchSize, - - mode = default, - memory_reduction_run_count = 0, - virtual_host = VHost}, - a(maybe_deltas_to_betas(State)). - -blank_rates(Now) -> - #rates { in = 0.0, - out = 0.0, - ack_in = 0.0, - ack_out = 0.0, - timestamp = Now}. - -in_r(MsgStatus = #msg_status { msg = undefined }, - State = #vqstate { mode = default, q3 = Q3, q4 = Q4 }) -> - case ?QUEUE:is_empty(Q4) of - true -> State #vqstate { q3 = ?QUEUE:in_r(MsgStatus, Q3) }; - false -> {Msg, State1 = #vqstate { q4 = Q4a }} = - read_msg(MsgStatus, State), - MsgStatus1 = MsgStatus#msg_status{msg = Msg}, - stats(ready0, {MsgStatus, MsgStatus1}, 0, - State1 #vqstate { q4 = ?QUEUE:in_r(MsgStatus1, Q4a) }) - end; -in_r(MsgStatus, - State = #vqstate { mode = default, q4 = Q4 }) -> - State #vqstate { q4 = ?QUEUE:in_r(MsgStatus, Q4) }; -%% lazy queues -in_r(MsgStatus = #msg_status { seq_id = SeqId, is_persistent = IsPersistent }, - State = #vqstate { mode = lazy, q3 = Q3, delta = Delta}) -> - case ?QUEUE:is_empty(Q3) of - true -> - {_MsgStatus1, State1} = - maybe_write_to_disk(true, true, MsgStatus, State), - State2 = stats(ready0, {MsgStatus, none}, 1, State1), - Delta1 = expand_delta(SeqId, Delta, IsPersistent), - State2 #vqstate{ delta = Delta1}; - false -> - State #vqstate { q3 = ?QUEUE:in_r(MsgStatus, Q3) } - end. - -queue_out(State = #vqstate { mode = default, q4 = Q4 }) -> - case ?QUEUE:out(Q4) of - {empty, _Q4} -> - case fetch_from_q3(State) of - {empty, _State1} = Result -> Result; - {loaded, {MsgStatus, State1}} -> {{value, MsgStatus}, State1} - end; - {{value, MsgStatus}, Q4a} -> - {{value, MsgStatus}, State #vqstate { q4 = Q4a }} - end; -%% lazy queues -queue_out(State = #vqstate { mode = lazy }) -> - case fetch_from_q3(State) of - {empty, _State1} = Result -> Result; - {loaded, {MsgStatus, State1}} -> {{value, MsgStatus}, State1} - end. - -read_msg(#msg_status{msg = undefined, - msg_id = MsgId, - is_persistent = IsPersistent}, State) -> - read_msg(MsgId, IsPersistent, State); -read_msg(#msg_status{msg = Msg}, State) -> - {Msg, State}. - -read_msg(MsgId, IsPersistent, State = #vqstate{msg_store_clients = MSCState, - disk_read_count = Count}) -> - {{ok, Msg = #basic_message {}}, MSCState1} = - msg_store_read(MSCState, IsPersistent, MsgId), - {Msg, State #vqstate {msg_store_clients = MSCState1, - disk_read_count = Count + 1}}. - -stats(Signs, Statuses, DeltaPaged, State) -> - stats0(expand_signs(Signs), expand_statuses(Statuses), DeltaPaged, State). - -expand_signs(ready0) -> {0, 0, true}; -expand_signs(lazy_pub) -> {1, 0, true}; -expand_signs({A, B}) -> {A, B, false}. - -expand_statuses({none, A}) -> {false, msg_in_ram(A), A}; -expand_statuses({B, none}) -> {msg_in_ram(B), false, B}; -expand_statuses({lazy, A}) -> {false , false, A}; -expand_statuses({B, A}) -> {msg_in_ram(B), msg_in_ram(A), B}. - -%% In this function at least, we are religious: the variable name -%% contains "Ready" or "Unacked" iff that is what it counts. If -%% neither is present it counts both. -stats0({DeltaReady, DeltaUnacked, ReadyMsgPaged}, - {InRamBefore, InRamAfter, MsgStatus}, DeltaPaged, - State = #vqstate{len = ReadyCount, - bytes = ReadyBytes, - ram_msg_count = RamReadyCount, - persistent_count = PersistentCount, - unacked_bytes = UnackedBytes, - ram_bytes = RamBytes, - delta_transient_bytes = DeltaBytes, - persistent_bytes = PersistentBytes}) -> - S = msg_size(MsgStatus), - DeltaTotal = DeltaReady + DeltaUnacked, - DeltaRam = case {InRamBefore, InRamAfter} of - {false, false} -> 0; - {false, true} -> 1; - {true, false} -> -1; - {true, true} -> 0 - end, - DeltaRamReady = case DeltaReady of - 1 -> one_if(InRamAfter); - -1 -> -one_if(InRamBefore); - 0 when ReadyMsgPaged -> DeltaRam; - 0 -> 0 - end, - DeltaPersistent = DeltaTotal * one_if(MsgStatus#msg_status.is_persistent), - State#vqstate{len = ReadyCount + DeltaReady, - ram_msg_count = RamReadyCount + DeltaRamReady, - persistent_count = PersistentCount + DeltaPersistent, - bytes = ReadyBytes + DeltaReady * S, - unacked_bytes = UnackedBytes + DeltaUnacked * S, - ram_bytes = RamBytes + DeltaRam * S, - persistent_bytes = PersistentBytes + DeltaPersistent * S, - delta_transient_bytes = DeltaBytes + DeltaPaged * one_if(not MsgStatus#msg_status.is_persistent) * S}. - -msg_size(#msg_status{msg_props = #message_properties{size = Size}}) -> Size. - -msg_in_ram(#msg_status{msg = Msg}) -> Msg =/= undefined. - -%% first param: AckRequired -remove(true, MsgStatus = #msg_status { - seq_id = SeqId, - is_delivered = IsDelivered, - index_on_disk = IndexOnDisk }, - State = #vqstate {out_counter = OutCount, - index_state = IndexState}) -> - %% Mark it delivered if necessary - IndexState1 = maybe_write_delivered( - IndexOnDisk andalso not IsDelivered, - SeqId, IndexState), - - State1 = record_pending_ack( - MsgStatus #msg_status { - is_delivered = true }, State), - - State2 = stats({-1, 1}, {MsgStatus, MsgStatus}, 0, State1), - - {SeqId, maybe_update_rates( - State2 #vqstate {out_counter = OutCount + 1, - index_state = IndexState1})}; - -%% This function body has the same behaviour as remove_queue_entries/3 -%% but instead of removing messages based on a ?QUEUE, this removes -%% just one message, the one referenced by the MsgStatus provided. -remove(false, MsgStatus = #msg_status { - seq_id = SeqId, - msg_id = MsgId, - is_persistent = IsPersistent, - is_delivered = IsDelivered, - msg_in_store = MsgInStore, - index_on_disk = IndexOnDisk }, - State = #vqstate {out_counter = OutCount, - index_state = IndexState, - msg_store_clients = MSCState}) -> - %% Mark it delivered if necessary - IndexState1 = maybe_write_delivered( - IndexOnDisk andalso not IsDelivered, - SeqId, IndexState), - - %% Remove from msg_store and queue index, if necessary - case MsgInStore of - true -> ok = msg_store_remove(MSCState, IsPersistent, [MsgId]); - false -> ok - end, - - IndexState2 = - case IndexOnDisk of - true -> rabbit_queue_index:ack([SeqId], IndexState1); - false -> IndexState1 - end, - - State1 = stats({-1, 0}, {MsgStatus, none}, 0, State), - - {undefined, maybe_update_rates( - State1 #vqstate {out_counter = OutCount + 1, - index_state = IndexState2})}. - -%% This function exists as a way to improve dropwhile/2 -%% performance. The idea of having this function is to optimise calls -%% to rabbit_queue_index by batching delivers and acks, instead of -%% sending them one by one. -%% -%% Instead of removing every message as their are popped from the -%% queue, it first accumulates them and then removes them by calling -%% remove_queue_entries/3, since the behaviour of -%% remove_queue_entries/3 when used with -%% process_delivers_and_acks_fun(deliver_and_ack) is the same as -%% calling remove(false, MsgStatus, State). -%% -%% remove/3 also updates the out_counter in every call, but here we do -%% it just once at the end. -remove_by_predicate(Pred, State = #vqstate {out_counter = OutCount}) -> - {MsgProps, QAcc, State1} = - collect_by_predicate(Pred, ?QUEUE:new(), State), - State2 = - remove_queue_entries( - QAcc, process_delivers_and_acks_fun(deliver_and_ack), State1), - %% maybe_update_rates/1 is called in remove/2 for every - %% message. Since we update out_counter only once, we call it just - %% there. - {MsgProps, maybe_update_rates( - State2 #vqstate { - out_counter = OutCount + ?QUEUE:len(QAcc)})}. - -%% This function exists as a way to improve fetchwhile/4 -%% performance. The idea of having this function is to optimise calls -%% to rabbit_queue_index by batching delivers, instead of sending them -%% one by one. -%% -%% Fun is the function passed to fetchwhile/4 that's -%% applied to every fetched message and used to build the fetchwhile/4 -%% result accumulator FetchAcc. -fetch_by_predicate(Pred, Fun, FetchAcc, - State = #vqstate { - index_state = IndexState, - out_counter = OutCount}) -> - {MsgProps, QAcc, State1} = - collect_by_predicate(Pred, ?QUEUE:new(), State), - - {Delivers, FetchAcc1, State2} = - process_queue_entries(QAcc, Fun, FetchAcc, State1), - - IndexState1 = rabbit_queue_index:deliver(Delivers, IndexState), - - {MsgProps, FetchAcc1, maybe_update_rates( - State2 #vqstate { - index_state = IndexState1, - out_counter = OutCount + ?QUEUE:len(QAcc)})}. - -%% We try to do here the same as what remove(true, State) does but -%% processing several messages at the same time. The idea is to -%% optimize rabbit_queue_index:deliver/2 calls by sending a list of -%% SeqIds instead of one by one, thus process_queue_entries1 will -%% accumulate the required deliveries, will record_pending_ack for -%% each message, and will update stats, like remove/2 does. -%% -%% For the meaning of Fun and FetchAcc arguments see -%% fetch_by_predicate/4 above. -process_queue_entries(Q, Fun, FetchAcc, State) -> - ?QUEUE:foldl(fun (MsgStatus, Acc) -> - process_queue_entries1(MsgStatus, Fun, Acc) - end, - {[], FetchAcc, State}, Q). - -process_queue_entries1( - #msg_status { seq_id = SeqId, is_delivered = IsDelivered, - index_on_disk = IndexOnDisk} = MsgStatus, - Fun, - {Delivers, FetchAcc, State}) -> - {Msg, State1} = read_msg(MsgStatus, State), - State2 = record_pending_ack( - MsgStatus #msg_status { - is_delivered = true }, State1), - {cons_if(IndexOnDisk andalso not IsDelivered, SeqId, Delivers), - Fun(Msg, SeqId, FetchAcc), - stats({-1, 1}, {MsgStatus, MsgStatus}, 0, State2)}. - -collect_by_predicate(Pred, QAcc, State) -> - case queue_out(State) of - {empty, State1} -> - {undefined, QAcc, State1}; - {{value, MsgStatus = #msg_status { msg_props = MsgProps }}, State1} -> - case Pred(MsgProps) of - true -> collect_by_predicate(Pred, ?QUEUE:in(MsgStatus, QAcc), - State1); - false -> {MsgProps, QAcc, in_r(MsgStatus, State1)} - end - end. - -%%---------------------------------------------------------------------------- -%% Helpers for Public API purge/1 function -%%---------------------------------------------------------------------------- - -%% The difference between purge_when_pending_acks/1 -%% vs. purge_and_index_reset/1 is that the first one issues a deliver -%% and an ack to the queue index for every message that's being -%% removed, while the later just resets the queue index state. -purge_when_pending_acks(State) -> - State1 = purge1(process_delivers_and_acks_fun(deliver_and_ack), State), - a(State1). - -purge_and_index_reset(State) -> - State1 = purge1(process_delivers_and_acks_fun(none), State), - a(reset_qi_state(State1)). - -%% This function removes messages from each of {q1, q2, q3, q4}. -%% -%% With remove_queue_entries/3 q1 and q4 are emptied, while q2 and q3 -%% are specially handled by purge_betas_and_deltas/2. -%% -%% purge_betas_and_deltas/2 loads messages from the queue index, -%% filling up q3 and in some cases moving messages form q2 to q3 while -%% resetting q2 to an empty queue (see maybe_deltas_to_betas/2). The -%% messages loaded into q3 are removed by calling -%% remove_queue_entries/3 until there are no more messages to be read -%% from the queue index. Messages are read in batches from the queue -%% index. -purge1(AfterFun, State = #vqstate { q4 = Q4}) -> - State1 = remove_queue_entries(Q4, AfterFun, State), - - State2 = #vqstate {q1 = Q1} = - purge_betas_and_deltas(AfterFun, State1#vqstate{q4 = ?QUEUE:new()}), - - State3 = remove_queue_entries(Q1, AfterFun, State2), - - a(State3#vqstate{q1 = ?QUEUE:new()}). - -reset_qi_state(State = #vqstate{index_state = IndexState}) -> - State#vqstate{index_state = - rabbit_queue_index:reset_state(IndexState)}. - -is_pending_ack_empty(State) -> - count_pending_acks(State) =:= 0. - -is_unconfirmed_empty(#vqstate { unconfirmed = UC }) -> - gb_sets:is_empty(UC). - -count_pending_acks(#vqstate { ram_pending_ack = RPA, - disk_pending_ack = DPA, - qi_pending_ack = QPA }) -> - gb_trees:size(RPA) + gb_trees:size(DPA) + gb_trees:size(QPA). - -purge_betas_and_deltas(DelsAndAcksFun, State = #vqstate { mode = Mode }) -> - State0 = #vqstate { q3 = Q3 } = - case Mode of - lazy -> maybe_deltas_to_betas(DelsAndAcksFun, State); - _ -> State - end, - - case ?QUEUE:is_empty(Q3) of - true -> State0; - false -> State1 = remove_queue_entries(Q3, DelsAndAcksFun, State0), - purge_betas_and_deltas(DelsAndAcksFun, - maybe_deltas_to_betas( - DelsAndAcksFun, - State1#vqstate{q3 = ?QUEUE:new()})) - end. - -remove_queue_entries(Q, DelsAndAcksFun, - State = #vqstate{msg_store_clients = MSCState}) -> - {MsgIdsByStore, Delivers, Acks, State1} = - ?QUEUE:foldl(fun remove_queue_entries1/2, - {maps:new(), [], [], State}, Q), - remove_msgs_by_id(MsgIdsByStore, MSCState), - DelsAndAcksFun(Delivers, Acks, State1). - -remove_queue_entries1( - #msg_status { msg_id = MsgId, seq_id = SeqId, is_delivered = IsDelivered, - msg_in_store = MsgInStore, index_on_disk = IndexOnDisk, - is_persistent = IsPersistent} = MsgStatus, - {MsgIdsByStore, Delivers, Acks, State}) -> - {case MsgInStore of - true -> rabbit_misc:maps_cons(IsPersistent, MsgId, MsgIdsByStore); - false -> MsgIdsByStore - end, - cons_if(IndexOnDisk andalso not IsDelivered, SeqId, Delivers), - cons_if(IndexOnDisk, SeqId, Acks), - stats({-1, 0}, {MsgStatus, none}, 0, State)}. - -process_delivers_and_acks_fun(deliver_and_ack) -> - fun (Delivers, Acks, State = #vqstate { index_state = IndexState }) -> - IndexState1 = - rabbit_queue_index:ack( - Acks, rabbit_queue_index:deliver(Delivers, IndexState)), - State #vqstate { index_state = IndexState1 } - end; -process_delivers_and_acks_fun(_) -> - fun (_, _, State) -> - State - end. - -%%---------------------------------------------------------------------------- -%% Internal gubbins for publishing -%%---------------------------------------------------------------------------- - -publish1(Msg = #basic_message { is_persistent = IsPersistent, id = MsgId }, - MsgProps = #message_properties { needs_confirming = NeedsConfirming }, - IsDelivered, _ChPid, _Flow, PersistFun, - State = #vqstate { q1 = Q1, q3 = Q3, q4 = Q4, - mode = default, - qi_embed_msgs_below = IndexMaxSize, - next_seq_id = SeqId, - in_counter = InCount, - durable = IsDurable, - unconfirmed = UC }) -> - IsPersistent1 = IsDurable andalso IsPersistent, - MsgStatus = msg_status(IsPersistent1, IsDelivered, SeqId, Msg, MsgProps, IndexMaxSize), - {MsgStatus1, State1} = PersistFun(false, false, MsgStatus, State), - State2 = case ?QUEUE:is_empty(Q3) of - false -> State1 #vqstate { q1 = ?QUEUE:in(m(MsgStatus1), Q1) }; - true -> State1 #vqstate { q4 = ?QUEUE:in(m(MsgStatus1), Q4) } - end, - InCount1 = InCount + 1, - UC1 = gb_sets_maybe_insert(NeedsConfirming, MsgId, UC), - stats({1, 0}, {none, MsgStatus1}, 0, - State2#vqstate{ next_seq_id = SeqId + 1, - in_counter = InCount1, - unconfirmed = UC1 }); -publish1(Msg = #basic_message { is_persistent = IsPersistent, id = MsgId }, - MsgProps = #message_properties { needs_confirming = NeedsConfirming }, - IsDelivered, _ChPid, _Flow, PersistFun, - State = #vqstate { mode = lazy, - qi_embed_msgs_below = IndexMaxSize, - next_seq_id = SeqId, - in_counter = InCount, - durable = IsDurable, - unconfirmed = UC, - delta = Delta}) -> - IsPersistent1 = IsDurable andalso IsPersistent, - MsgStatus = msg_status(IsPersistent1, IsDelivered, SeqId, Msg, MsgProps, IndexMaxSize), - {MsgStatus1, State1} = PersistFun(true, true, MsgStatus, State), - Delta1 = expand_delta(SeqId, Delta, IsPersistent), - UC1 = gb_sets_maybe_insert(NeedsConfirming, MsgId, UC), - stats(lazy_pub, {lazy, m(MsgStatus1)}, 1, - State1#vqstate{ delta = Delta1, - next_seq_id = SeqId + 1, - in_counter = InCount + 1, - unconfirmed = UC1}). - -batch_publish1({Msg, MsgProps, IsDelivered}, {ChPid, Flow, State}) -> - {ChPid, Flow, publish1(Msg, MsgProps, IsDelivered, ChPid, Flow, - fun maybe_prepare_write_to_disk/4, State)}. - -publish_delivered1(Msg = #basic_message { is_persistent = IsPersistent, - id = MsgId }, - MsgProps = #message_properties { - needs_confirming = NeedsConfirming }, - _ChPid, _Flow, PersistFun, - State = #vqstate { mode = default, - qi_embed_msgs_below = IndexMaxSize, - next_seq_id = SeqId, - out_counter = OutCount, - in_counter = InCount, - durable = IsDurable, - unconfirmed = UC }) -> - IsPersistent1 = IsDurable andalso IsPersistent, - MsgStatus = msg_status(IsPersistent1, true, SeqId, Msg, MsgProps, IndexMaxSize), - {MsgStatus1, State1} = PersistFun(false, false, MsgStatus, State), - State2 = record_pending_ack(m(MsgStatus1), State1), - UC1 = gb_sets_maybe_insert(NeedsConfirming, MsgId, UC), - State3 = stats({0, 1}, {none, MsgStatus1}, 0, - State2 #vqstate { next_seq_id = SeqId + 1, - out_counter = OutCount + 1, - in_counter = InCount + 1, - unconfirmed = UC1 }), - {SeqId, State3}; -publish_delivered1(Msg = #basic_message { is_persistent = IsPersistent, - id = MsgId }, - MsgProps = #message_properties { - needs_confirming = NeedsConfirming }, - _ChPid, _Flow, PersistFun, - State = #vqstate { mode = lazy, - qi_embed_msgs_below = IndexMaxSize, - next_seq_id = SeqId, - out_counter = OutCount, - in_counter = InCount, - durable = IsDurable, - unconfirmed = UC }) -> - IsPersistent1 = IsDurable andalso IsPersistent, - MsgStatus = msg_status(IsPersistent1, true, SeqId, Msg, MsgProps, IndexMaxSize), - {MsgStatus1, State1} = PersistFun(true, true, MsgStatus, State), - State2 = record_pending_ack(m(MsgStatus1), State1), - UC1 = gb_sets_maybe_insert(NeedsConfirming, MsgId, UC), - State3 = stats({0, 1}, {none, MsgStatus1}, 0, - State2 #vqstate { next_seq_id = SeqId + 1, - out_counter = OutCount + 1, - in_counter = InCount + 1, - unconfirmed = UC1 }), - {SeqId, State3}. - -batch_publish_delivered1({Msg, MsgProps}, {ChPid, Flow, SeqIds, State}) -> - {SeqId, State1} = - publish_delivered1(Msg, MsgProps, ChPid, Flow, - fun maybe_prepare_write_to_disk/4, - State), - {ChPid, Flow, [SeqId | SeqIds], State1}. - -maybe_write_msg_to_disk(_Force, MsgStatus = #msg_status { - msg_in_store = true }, State) -> - {MsgStatus, State}; -maybe_write_msg_to_disk(Force, MsgStatus = #msg_status { - msg = Msg, msg_id = MsgId, - is_persistent = IsPersistent }, - State = #vqstate{ msg_store_clients = MSCState, - disk_write_count = Count}) - when Force orelse IsPersistent -> - case persist_to(MsgStatus) of - msg_store -> ok = msg_store_write(MSCState, IsPersistent, MsgId, - prepare_to_store(Msg)), - {MsgStatus#msg_status{msg_in_store = true}, - State#vqstate{disk_write_count = Count + 1}}; - queue_index -> {MsgStatus, State} - end; -maybe_write_msg_to_disk(_Force, MsgStatus, State) -> - {MsgStatus, State}. - -%% Due to certain optimisations made inside -%% rabbit_queue_index:pre_publish/7 we need to have two separate -%% functions for index persistence. This one is only used when paging -%% during memory pressure. We didn't want to modify -%% maybe_write_index_to_disk/3 because that function is used in other -%% places. -maybe_batch_write_index_to_disk(_Force, - MsgStatus = #msg_status { - index_on_disk = true }, State) -> - {MsgStatus, State}; -maybe_batch_write_index_to_disk(Force, - MsgStatus = #msg_status { - msg = Msg, - msg_id = MsgId, - seq_id = SeqId, - is_persistent = IsPersistent, - is_delivered = IsDelivered, - msg_props = MsgProps}, - State = #vqstate { - target_ram_count = TargetRamCount, - disk_write_count = DiskWriteCount, - index_state = IndexState}) - when Force orelse IsPersistent -> - {MsgOrId, DiskWriteCount1} = - case persist_to(MsgStatus) of - msg_store -> {MsgId, DiskWriteCount}; - queue_index -> {prepare_to_store(Msg), DiskWriteCount + 1} - end, - IndexState1 = rabbit_queue_index:pre_publish( - MsgOrId, SeqId, MsgProps, IsPersistent, IsDelivered, - TargetRamCount, IndexState), - {MsgStatus#msg_status{index_on_disk = true}, - State#vqstate{index_state = IndexState1, - disk_write_count = DiskWriteCount1}}; -maybe_batch_write_index_to_disk(_Force, MsgStatus, State) -> - {MsgStatus, State}. - -maybe_write_index_to_disk(_Force, MsgStatus = #msg_status { - index_on_disk = true }, State) -> - {MsgStatus, State}; -maybe_write_index_to_disk(Force, MsgStatus = #msg_status { - msg = Msg, - msg_id = MsgId, - seq_id = SeqId, - is_persistent = IsPersistent, - is_delivered = IsDelivered, - msg_props = MsgProps}, - State = #vqstate{target_ram_count = TargetRamCount, - disk_write_count = DiskWriteCount, - index_state = IndexState}) - when Force orelse IsPersistent -> - {MsgOrId, DiskWriteCount1} = - case persist_to(MsgStatus) of - msg_store -> {MsgId, DiskWriteCount}; - queue_index -> {prepare_to_store(Msg), DiskWriteCount + 1} - end, - IndexState1 = rabbit_queue_index:publish( - MsgOrId, SeqId, MsgProps, IsPersistent, TargetRamCount, - IndexState), - IndexState2 = maybe_write_delivered(IsDelivered, SeqId, IndexState1), - {MsgStatus#msg_status{index_on_disk = true}, - State#vqstate{index_state = IndexState2, - disk_write_count = DiskWriteCount1}}; - -maybe_write_index_to_disk(_Force, MsgStatus, State) -> - {MsgStatus, State}. - -maybe_write_to_disk(ForceMsg, ForceIndex, MsgStatus, State) -> - {MsgStatus1, State1} = maybe_write_msg_to_disk(ForceMsg, MsgStatus, State), - maybe_write_index_to_disk(ForceIndex, MsgStatus1, State1). - -maybe_prepare_write_to_disk(ForceMsg, ForceIndex, MsgStatus, State) -> - {MsgStatus1, State1} = maybe_write_msg_to_disk(ForceMsg, MsgStatus, State), - maybe_batch_write_index_to_disk(ForceIndex, MsgStatus1, State1). - -determine_persist_to(#basic_message{ - content = #content{properties = Props, - properties_bin = PropsBin}}, - #message_properties{size = BodySize}, - IndexMaxSize) -> - %% The >= is so that you can set the env to 0 and never persist - %% to the index. - %% - %% We want this to be fast, so we avoid size(term_to_binary()) - %% here, or using the term size estimation from truncate.erl, both - %% of which are too slow. So instead, if the message body size - %% goes over the limit then we avoid any other checks. - %% - %% If it doesn't we need to decide if the properties will push - %% it past the limit. If we have the encoded properties (usual - %% case) we can just check their size. If we don't (message came - %% via the direct client), we make a guess based on the number of - %% headers. - case BodySize >= IndexMaxSize of - true -> msg_store; - false -> Est = case is_binary(PropsBin) of - true -> BodySize + size(PropsBin); - false -> #'P_basic'{headers = Hs} = Props, - case Hs of - undefined -> 0; - _ -> length(Hs) - end * ?HEADER_GUESS_SIZE + BodySize - end, - case Est >= IndexMaxSize of - true -> msg_store; - false -> queue_index - end - end. - -persist_to(#msg_status{persist_to = To}) -> To. - -prepare_to_store(Msg) -> - Msg#basic_message{ - %% don't persist any recoverable decoded properties - content = rabbit_binary_parser:clear_decoded_content( - Msg #basic_message.content)}. - -%%---------------------------------------------------------------------------- -%% Internal gubbins for acks -%%---------------------------------------------------------------------------- - -record_pending_ack(#msg_status { seq_id = SeqId } = MsgStatus, - State = #vqstate { ram_pending_ack = RPA, - disk_pending_ack = DPA, - qi_pending_ack = QPA, - ack_in_counter = AckInCount}) -> - Insert = fun (Tree) -> gb_trees:insert(SeqId, MsgStatus, Tree) end, - {RPA1, DPA1, QPA1} = - case {msg_in_ram(MsgStatus), persist_to(MsgStatus)} of - {false, _} -> {RPA, Insert(DPA), QPA}; - {_, queue_index} -> {RPA, DPA, Insert(QPA)}; - {_, msg_store} -> {Insert(RPA), DPA, QPA} - end, - State #vqstate { ram_pending_ack = RPA1, - disk_pending_ack = DPA1, - qi_pending_ack = QPA1, - ack_in_counter = AckInCount + 1}. - -lookup_pending_ack(SeqId, #vqstate { ram_pending_ack = RPA, - disk_pending_ack = DPA, - qi_pending_ack = QPA}) -> - case gb_trees:lookup(SeqId, RPA) of - {value, V} -> V; - none -> case gb_trees:lookup(SeqId, DPA) of - {value, V} -> V; - none -> gb_trees:get(SeqId, QPA) - end - end. - -%% First parameter = UpdateStats -remove_pending_ack(true, SeqId, State) -> - case remove_pending_ack(false, SeqId, State) of - {none, _} -> - {none, State}; - {MsgStatus, State1} -> - {MsgStatus, stats({0, -1}, {MsgStatus, none}, 0, State1)} - end; -remove_pending_ack(false, SeqId, State = #vqstate{ram_pending_ack = RPA, - disk_pending_ack = DPA, - qi_pending_ack = QPA}) -> - case gb_trees:lookup(SeqId, RPA) of - {value, V} -> RPA1 = gb_trees:delete(SeqId, RPA), - {V, State #vqstate { ram_pending_ack = RPA1 }}; - none -> case gb_trees:lookup(SeqId, DPA) of - {value, V} -> - DPA1 = gb_trees:delete(SeqId, DPA), - {V, State#vqstate{disk_pending_ack = DPA1}}; - none -> - case gb_trees:lookup(SeqId, QPA) of - {value, V} -> - QPA1 = gb_trees:delete(SeqId, QPA), - {V, State#vqstate{qi_pending_ack = QPA1}}; - none -> - {none, State} - end - end - end. - -purge_pending_ack(KeepPersistent, - State = #vqstate { index_state = IndexState, - msg_store_clients = MSCState }) -> - {IndexOnDiskSeqIds, MsgIdsByStore, State1} = purge_pending_ack1(State), - case KeepPersistent of - true -> remove_transient_msgs_by_id(MsgIdsByStore, MSCState), - State1; - false -> IndexState1 = - rabbit_queue_index:ack(IndexOnDiskSeqIds, IndexState), - remove_msgs_by_id(MsgIdsByStore, MSCState), - State1 #vqstate { index_state = IndexState1 } - end. - -purge_pending_ack_delete_and_terminate( - State = #vqstate { index_state = IndexState, - msg_store_clients = MSCState }) -> - {_, MsgIdsByStore, State1} = purge_pending_ack1(State), - IndexState1 = rabbit_queue_index:delete_and_terminate(IndexState), - remove_msgs_by_id(MsgIdsByStore, MSCState), - State1 #vqstate { index_state = IndexState1 }. - -purge_pending_ack1(State = #vqstate { ram_pending_ack = RPA, - disk_pending_ack = DPA, - qi_pending_ack = QPA }) -> - F = fun (_SeqId, MsgStatus, Acc) -> accumulate_ack(MsgStatus, Acc) end, - {IndexOnDiskSeqIds, MsgIdsByStore, _AllMsgIds} = - rabbit_misc:gb_trees_fold( - F, rabbit_misc:gb_trees_fold( - F, rabbit_misc:gb_trees_fold( - F, accumulate_ack_init(), RPA), DPA), QPA), - State1 = State #vqstate { ram_pending_ack = gb_trees:empty(), - disk_pending_ack = gb_trees:empty(), - qi_pending_ack = gb_trees:empty()}, - {IndexOnDiskSeqIds, MsgIdsByStore, State1}. - -%% MsgIdsByStore is an map with two keys: -%% -%% true: holds a list of Persistent Message Ids. -%% false: holds a list of Transient Message Ids. -%% -%% When we call maps:to_list/1 we get two sets of msg ids, where -%% IsPersistent is either true for persistent messages or false for -%% transient ones. The msg_store_remove/3 function takes this boolean -%% flag to determine from which store the messages should be removed -%% from. -remove_msgs_by_id(MsgIdsByStore, MSCState) -> - [ok = msg_store_remove(MSCState, IsPersistent, MsgIds) - || {IsPersistent, MsgIds} <- maps:to_list(MsgIdsByStore)]. - -remove_transient_msgs_by_id(MsgIdsByStore, MSCState) -> - case maps:find(false, MsgIdsByStore) of - error -> ok; - {ok, MsgIds} -> ok = msg_store_remove(MSCState, false, MsgIds) - end. - -accumulate_ack_init() -> {[], maps:new(), []}. - -accumulate_ack(#msg_status { seq_id = SeqId, - msg_id = MsgId, - is_persistent = IsPersistent, - msg_in_store = MsgInStore, - index_on_disk = IndexOnDisk }, - {IndexOnDiskSeqIdsAcc, MsgIdsByStore, AllMsgIds}) -> - {cons_if(IndexOnDisk, SeqId, IndexOnDiskSeqIdsAcc), - case MsgInStore of - true -> rabbit_misc:maps_cons(IsPersistent, MsgId, MsgIdsByStore); - false -> MsgIdsByStore - end, - [MsgId | AllMsgIds]}. - -%%---------------------------------------------------------------------------- -%% Internal plumbing for confirms (aka publisher acks) -%%---------------------------------------------------------------------------- - -record_confirms(MsgIdSet, State = #vqstate { msgs_on_disk = MOD, - msg_indices_on_disk = MIOD, - unconfirmed = UC, - confirmed = C }) -> - State #vqstate { - msgs_on_disk = rabbit_misc:gb_sets_difference(MOD, MsgIdSet), - msg_indices_on_disk = rabbit_misc:gb_sets_difference(MIOD, MsgIdSet), - unconfirmed = rabbit_misc:gb_sets_difference(UC, MsgIdSet), - confirmed = gb_sets:union(C, MsgIdSet) }. - -msgs_written_to_disk(Callback, MsgIdSet, ignored) -> - Callback(?MODULE, - fun (?MODULE, State) -> record_confirms(MsgIdSet, State) end); -msgs_written_to_disk(Callback, MsgIdSet, written) -> - Callback(?MODULE, - fun (?MODULE, State = #vqstate { msgs_on_disk = MOD, - msg_indices_on_disk = MIOD, - unconfirmed = UC }) -> - Confirmed = gb_sets:intersection(UC, MsgIdSet), - record_confirms(gb_sets:intersection(MsgIdSet, MIOD), - State #vqstate { - msgs_on_disk = - gb_sets:union(MOD, Confirmed) }) - end). - -msg_indices_written_to_disk(Callback, MsgIdSet) -> - Callback(?MODULE, - fun (?MODULE, State = #vqstate { msgs_on_disk = MOD, - msg_indices_on_disk = MIOD, - unconfirmed = UC }) -> - Confirmed = gb_sets:intersection(UC, MsgIdSet), - record_confirms(gb_sets:intersection(MsgIdSet, MOD), - State #vqstate { - msg_indices_on_disk = - gb_sets:union(MIOD, Confirmed) }) - end). - -msgs_and_indices_written_to_disk(Callback, MsgIdSet) -> - Callback(?MODULE, - fun (?MODULE, State) -> record_confirms(MsgIdSet, State) end). - -%%---------------------------------------------------------------------------- -%% Internal plumbing for requeue -%%---------------------------------------------------------------------------- - -publish_alpha(#msg_status { msg = undefined } = MsgStatus, State) -> - {Msg, State1} = read_msg(MsgStatus, State), - MsgStatus1 = MsgStatus#msg_status { msg = Msg }, - {MsgStatus1, stats({1, -1}, {MsgStatus, MsgStatus1}, 0, State1)}; -publish_alpha(MsgStatus, State) -> - {MsgStatus, stats({1, -1}, {MsgStatus, MsgStatus}, 0, State)}. - -publish_beta(MsgStatus, State) -> - {MsgStatus1, State1} = maybe_prepare_write_to_disk(true, false, MsgStatus, State), - MsgStatus2 = m(trim_msg_status(MsgStatus1)), - {MsgStatus2, stats({1, -1}, {MsgStatus, MsgStatus2}, 0, State1)}. - -%% Rebuild queue, inserting sequence ids to maintain ordering -queue_merge(SeqIds, Q, MsgIds, Limit, PubFun, State) -> - queue_merge(SeqIds, Q, ?QUEUE:new(), MsgIds, - Limit, PubFun, State). - -queue_merge([SeqId | Rest] = SeqIds, Q, Front, MsgIds, - Limit, PubFun, State) - when Limit == undefined orelse SeqId < Limit -> - case ?QUEUE:out(Q) of - {{value, #msg_status { seq_id = SeqIdQ } = MsgStatus}, Q1} - when SeqIdQ < SeqId -> - %% enqueue from the remaining queue - queue_merge(SeqIds, Q1, ?QUEUE:in(MsgStatus, Front), MsgIds, - Limit, PubFun, State); - {_, _Q1} -> - %% enqueue from the remaining list of sequence ids - case msg_from_pending_ack(SeqId, State) of - {none, _} -> - queue_merge(Rest, Q, Front, MsgIds, Limit, PubFun, State); - {MsgStatus, State1} -> - {#msg_status { msg_id = MsgId } = MsgStatus1, State2} = - PubFun(MsgStatus, State1), - queue_merge(Rest, Q, ?QUEUE:in(MsgStatus1, Front), [MsgId | MsgIds], - Limit, PubFun, State2) - end - end; -queue_merge(SeqIds, Q, Front, MsgIds, - _Limit, _PubFun, State) -> - {SeqIds, ?QUEUE:join(Front, Q), MsgIds, State}. - -delta_merge([], Delta, MsgIds, State) -> - {Delta, MsgIds, State}; -delta_merge(SeqIds, Delta, MsgIds, State) -> - lists:foldl(fun (SeqId, {Delta0, MsgIds0, State0} = Acc) -> - case msg_from_pending_ack(SeqId, State0) of - {none, _} -> - Acc; - {#msg_status { msg_id = MsgId, - is_persistent = IsPersistent } = MsgStatus, State1} -> - {_MsgStatus, State2} = - maybe_prepare_write_to_disk(true, true, MsgStatus, State1), - {expand_delta(SeqId, Delta0, IsPersistent), [MsgId | MsgIds0], - stats({1, -1}, {MsgStatus, none}, 1, State2)} - end - end, {Delta, MsgIds, State}, SeqIds). - -%% Mostly opposite of record_pending_ack/2 -msg_from_pending_ack(SeqId, State) -> - case remove_pending_ack(false, SeqId, State) of - {none, _} -> - {none, State}; - {#msg_status { msg_props = MsgProps } = MsgStatus, State1} -> - {MsgStatus #msg_status { - msg_props = MsgProps #message_properties { needs_confirming = false } }, - State1} - end. - -beta_limit(Q) -> - case ?QUEUE:peek(Q) of - {value, #msg_status { seq_id = SeqId }} -> SeqId; - empty -> undefined - end. - -delta_limit(?BLANK_DELTA_PATTERN(_X)) -> undefined; -delta_limit(#delta { start_seq_id = StartSeqId }) -> StartSeqId. - -%%---------------------------------------------------------------------------- -%% Iterator -%%---------------------------------------------------------------------------- - -ram_ack_iterator(State) -> - {ack, gb_trees:iterator(State#vqstate.ram_pending_ack)}. - -disk_ack_iterator(State) -> - {ack, gb_trees:iterator(State#vqstate.disk_pending_ack)}. - -qi_ack_iterator(State) -> - {ack, gb_trees:iterator(State#vqstate.qi_pending_ack)}. - -msg_iterator(State) -> istate(start, State). - -istate(start, State) -> {q4, State#vqstate.q4, State}; -istate(q4, State) -> {q3, State#vqstate.q3, State}; -istate(q3, State) -> {delta, State#vqstate.delta, State}; -istate(delta, State) -> {q2, State#vqstate.q2, State}; -istate(q2, State) -> {q1, State#vqstate.q1, State}; -istate(q1, _State) -> done. - -next({ack, It}, IndexState) -> - case gb_trees:next(It) of - none -> {empty, IndexState}; - {_SeqId, MsgStatus, It1} -> Next = {ack, It1}, - {value, MsgStatus, true, Next, IndexState} - end; -next(done, IndexState) -> {empty, IndexState}; -next({delta, #delta{start_seq_id = SeqId, - end_seq_id = SeqId}, State}, IndexState) -> - next(istate(delta, State), IndexState); -next({delta, #delta{start_seq_id = SeqId, - end_seq_id = SeqIdEnd} = Delta, State}, IndexState) -> - SeqIdB = rabbit_queue_index:next_segment_boundary(SeqId), - SeqId1 = lists:min([SeqIdB, SeqIdEnd]), - {List, IndexState1} = rabbit_queue_index:read(SeqId, SeqId1, IndexState), - next({delta, Delta#delta{start_seq_id = SeqId1}, List, State}, IndexState1); -next({delta, Delta, [], State}, IndexState) -> - next({delta, Delta, State}, IndexState); -next({delta, Delta, [{_, SeqId, _, _, _} = M | Rest], State}, IndexState) -> - case is_msg_in_pending_acks(SeqId, State) of - false -> Next = {delta, Delta, Rest, State}, - {value, beta_msg_status(M), false, Next, IndexState}; - true -> next({delta, Delta, Rest, State}, IndexState) - end; -next({Key, Q, State}, IndexState) -> - case ?QUEUE:out(Q) of - {empty, _Q} -> next(istate(Key, State), IndexState); - {{value, MsgStatus}, QN} -> Next = {Key, QN, State}, - {value, MsgStatus, false, Next, IndexState} - end. - -inext(It, {Its, IndexState}) -> - case next(It, IndexState) of - {empty, IndexState1} -> - {Its, IndexState1}; - {value, MsgStatus1, Unacked, It1, IndexState1} -> - {[{MsgStatus1, Unacked, It1} | Its], IndexState1} - end. - -ifold(_Fun, Acc, [], State0) -> - {Acc, State0}; -ifold(Fun, Acc, Its0, State0) -> - [{MsgStatus, Unacked, It} | Rest] = - lists:sort(fun ({#msg_status{seq_id = SeqId1}, _, _}, - {#msg_status{seq_id = SeqId2}, _, _}) -> - SeqId1 =< SeqId2 - end, Its0), - {Msg, State1} = read_msg(MsgStatus, State0), - case Fun(Msg, MsgStatus#msg_status.msg_props, Unacked, Acc) of - {stop, Acc1} -> - {Acc1, State1}; - {cont, Acc1} -> - IndexState0 = State1#vqstate.index_state, - {Its1, IndexState1} = inext(It, {Rest, IndexState0}), - State2 = State1#vqstate{index_state = IndexState1}, - ifold(Fun, Acc1, Its1, State2) - end. - -%%---------------------------------------------------------------------------- -%% Phase changes -%%---------------------------------------------------------------------------- - -maybe_reduce_memory_use(State = #vqstate {memory_reduction_run_count = MRedRunCount, - mode = Mode}) -> - case MRedRunCount >= ?EXPLICIT_GC_RUN_OP_THRESHOLD(Mode) of - true -> State1 = reduce_memory_use(State), - State1#vqstate{memory_reduction_run_count = 0}; - false -> State#vqstate{memory_reduction_run_count = MRedRunCount + 1} - end. - -reduce_memory_use(State = #vqstate { target_ram_count = infinity }) -> - State; -reduce_memory_use(State = #vqstate { - mode = default, - ram_pending_ack = RPA, - ram_msg_count = RamMsgCount, - target_ram_count = TargetRamCount, - io_batch_size = IoBatchSize, - rates = #rates { in = AvgIngress, - out = AvgEgress, - ack_in = AvgAckIngress, - ack_out = AvgAckEgress } }) -> - {CreditDiscBound, _} =rabbit_misc:get_env(rabbit, - msg_store_credit_disc_bound, - ?CREDIT_DISC_BOUND), - {NeedResumeA2B, State1} = {_, #vqstate { q2 = Q2, q3 = Q3 }} = - case chunk_size(RamMsgCount + gb_trees:size(RPA), TargetRamCount) of - 0 -> {false, State}; - %% Reduce memory of pending acks and alphas. The order is - %% determined based on which is growing faster. Whichever - %% comes second may very well get a quota of 0 if the - %% first manages to push out the max number of messages. - A2BChunk -> - %% In case there are few messages to be sent to a message store - %% and many messages to be embedded to the queue index, - %% we should limit the number of messages to be flushed - %% to avoid blocking the process. - A2BChunkActual = case A2BChunk > CreditDiscBound * 2 of - true -> CreditDiscBound * 2; - false -> A2BChunk - end, - Funs = case ((AvgAckIngress - AvgAckEgress) > - (AvgIngress - AvgEgress)) of - true -> [fun limit_ram_acks/2, - fun push_alphas_to_betas/2]; - false -> [fun push_alphas_to_betas/2, - fun limit_ram_acks/2] - end, - {Quota, State2} = lists:foldl(fun (ReduceFun, {QuotaN, StateN}) -> - ReduceFun(QuotaN, StateN) - end, {A2BChunkActual, State}, Funs), - {(Quota == 0) andalso (A2BChunk > A2BChunkActual), State2} - end, - Permitted = permitted_beta_count(State1), - {NeedResumeB2D, State3} = - %% If there are more messages with their queue position held in RAM, - %% a.k.a. betas, in Q2 & Q3 than IoBatchSize, - %% write their queue position to disk, a.k.a. push_betas_to_deltas - case chunk_size(?QUEUE:len(Q2) + ?QUEUE:len(Q3), - Permitted) of - B2DChunk when B2DChunk >= IoBatchSize -> - %% Same as for alphas to betas. Limit a number of messages - %% to be flushed to disk at once to avoid blocking the process. - B2DChunkActual = case B2DChunk > CreditDiscBound * 2 of - true -> CreditDiscBound * 2; - false -> B2DChunk - end, - StateBD = push_betas_to_deltas(B2DChunkActual, State1), - {B2DChunk > B2DChunkActual, StateBD}; - _ -> - {false, State1} - end, - %% We can be blocked by the credit flow, or limited by a batch size, - %% or finished with flushing. - %% If blocked by the credit flow - the credit grant will resume processing, - %% if limited by a batch - the batch continuation message should be sent. - %% The continuation message will be prioritised over publishes, - %% but not consumptions, so the queue can make progess. - Blocked = credit_flow:blocked(), - case {Blocked, NeedResumeA2B orelse NeedResumeB2D} of - %% Credit bump will continue paging - {true, _} -> State3; - %% Finished with paging - {false, false} -> State3; - %% Planning next batch - {false, true} -> - %% We don't want to use self-credit-flow, because it's harder to - %% reason about. So the process sends a (prioritised) message to - %% itself and sets a waiting_bump value to keep the message box clean - maybe_bump_reduce_memory_use(State3) - end; -%% When using lazy queues, there are no alphas, so we don't need to -%% call push_alphas_to_betas/2. -reduce_memory_use(State = #vqstate { - mode = lazy, - ram_pending_ack = RPA, - ram_msg_count = RamMsgCount, - target_ram_count = TargetRamCount }) -> - State1 = #vqstate { q3 = Q3 } = - case chunk_size(RamMsgCount + gb_trees:size(RPA), TargetRamCount) of - 0 -> State; - S1 -> {_, State2} = limit_ram_acks(S1, State), - State2 - end, - - State3 = - case chunk_size(?QUEUE:len(Q3), - permitted_beta_count(State1)) of - 0 -> - State1; - S2 -> - push_betas_to_deltas(S2, State1) - end, - garbage_collect(), - State3. - -maybe_bump_reduce_memory_use(State = #vqstate{ waiting_bump = true }) -> - State; -maybe_bump_reduce_memory_use(State) -> - self() ! bump_reduce_memory_use, - State#vqstate{ waiting_bump = true }. - -limit_ram_acks(0, State) -> - {0, ui(State)}; -limit_ram_acks(Quota, State = #vqstate { ram_pending_ack = RPA, - disk_pending_ack = DPA }) -> - case gb_trees:is_empty(RPA) of - true -> - {Quota, ui(State)}; - false -> - {SeqId, MsgStatus, RPA1} = gb_trees:take_largest(RPA), - {MsgStatus1, State1} = - maybe_prepare_write_to_disk(true, false, MsgStatus, State), - MsgStatus2 = m(trim_msg_status(MsgStatus1)), - DPA1 = gb_trees:insert(SeqId, MsgStatus2, DPA), - limit_ram_acks(Quota - 1, - stats({0, 0}, {MsgStatus, MsgStatus2}, 0, - State1 #vqstate { ram_pending_ack = RPA1, - disk_pending_ack = DPA1 })) - end. - -permitted_beta_count(#vqstate { len = 0 }) -> - infinity; -permitted_beta_count(#vqstate { mode = lazy, - target_ram_count = TargetRamCount}) -> - TargetRamCount; -permitted_beta_count(#vqstate { target_ram_count = 0, q3 = Q3 }) -> - lists:min([?QUEUE:len(Q3), rabbit_queue_index:next_segment_boundary(0)]); -permitted_beta_count(#vqstate { q1 = Q1, - q4 = Q4, - target_ram_count = TargetRamCount, - len = Len }) -> - BetaDelta = Len - ?QUEUE:len(Q1) - ?QUEUE:len(Q4), - lists:max([rabbit_queue_index:next_segment_boundary(0), - BetaDelta - ((BetaDelta * BetaDelta) div - (BetaDelta + TargetRamCount))]). - -chunk_size(Current, Permitted) - when Permitted =:= infinity orelse Permitted >= Current -> - 0; -chunk_size(Current, Permitted) -> - Current - Permitted. - -fetch_from_q3(State = #vqstate { mode = default, - q1 = Q1, - q2 = Q2, - delta = #delta { count = DeltaCount }, - q3 = Q3, - q4 = Q4 }) -> - case ?QUEUE:out(Q3) of - {empty, _Q3} -> - {empty, State}; - {{value, MsgStatus}, Q3a} -> - State1 = State #vqstate { q3 = Q3a }, - State2 = case {?QUEUE:is_empty(Q3a), 0 == DeltaCount} of - {true, true} -> - %% q3 is now empty, it wasn't before; - %% delta is still empty. So q2 must be - %% empty, and we know q4 is empty - %% otherwise we wouldn't be loading from - %% q3. As such, we can just set q4 to Q1. - true = ?QUEUE:is_empty(Q2), %% ASSERTION - true = ?QUEUE:is_empty(Q4), %% ASSERTION - State1 #vqstate { q1 = ?QUEUE:new(), q4 = Q1 }; - {true, false} -> - maybe_deltas_to_betas(State1); - {false, _} -> - %% q3 still isn't empty, we've not - %% touched delta, so the invariants - %% between q1, q2, delta and q3 are - %% maintained - State1 - end, - {loaded, {MsgStatus, State2}} - end; -%% lazy queues -fetch_from_q3(State = #vqstate { mode = lazy, - delta = #delta { count = DeltaCount }, - q3 = Q3 }) -> - case ?QUEUE:out(Q3) of - {empty, _Q3} when DeltaCount =:= 0 -> - {empty, State}; - {empty, _Q3} -> - fetch_from_q3(maybe_deltas_to_betas(State)); - {{value, MsgStatus}, Q3a} -> - State1 = State #vqstate { q3 = Q3a }, - {loaded, {MsgStatus, State1}} - end. - -maybe_deltas_to_betas(State) -> - AfterFun = process_delivers_and_acks_fun(deliver_and_ack), - maybe_deltas_to_betas(AfterFun, State). - -maybe_deltas_to_betas(_DelsAndAcksFun, - State = #vqstate {delta = ?BLANK_DELTA_PATTERN(X) }) -> - State; -maybe_deltas_to_betas(DelsAndAcksFun, - State = #vqstate { - q2 = Q2, - delta = Delta, - q3 = Q3, - index_state = IndexState, - ram_msg_count = RamMsgCount, - ram_bytes = RamBytes, - disk_read_count = DiskReadCount, - delta_transient_bytes = DeltaTransientBytes, - transient_threshold = TransientThreshold }) -> - #delta { start_seq_id = DeltaSeqId, - count = DeltaCount, - transient = Transient, - end_seq_id = DeltaSeqIdEnd } = Delta, - DeltaSeqId1 = - lists:min([rabbit_queue_index:next_segment_boundary(DeltaSeqId), - DeltaSeqIdEnd]), - {List, IndexState1} = rabbit_queue_index:read(DeltaSeqId, DeltaSeqId1, - IndexState), - {Q3a, RamCountsInc, RamBytesInc, State1, TransientCount, TransientBytes} = - betas_from_index_entries(List, TransientThreshold, - DelsAndAcksFun, - State #vqstate { index_state = IndexState1 }), - State2 = State1 #vqstate { ram_msg_count = RamMsgCount + RamCountsInc, - ram_bytes = RamBytes + RamBytesInc, - disk_read_count = DiskReadCount + RamCountsInc }, - case ?QUEUE:len(Q3a) of - 0 -> - %% we ignored every message in the segment due to it being - %% transient and below the threshold - maybe_deltas_to_betas( - DelsAndAcksFun, - State2 #vqstate { - delta = d(Delta #delta { start_seq_id = DeltaSeqId1 })}); - Q3aLen -> - Q3b = ?QUEUE:join(Q3, Q3a), - case DeltaCount - Q3aLen of - 0 -> - %% delta is now empty, but it wasn't before, so - %% can now join q2 onto q3 - State2 #vqstate { q2 = ?QUEUE:new(), - delta = ?BLANK_DELTA, - q3 = ?QUEUE:join(Q3b, Q2), - delta_transient_bytes = 0}; - N when N > 0 -> - Delta1 = d(#delta { start_seq_id = DeltaSeqId1, - count = N, - transient = Transient - TransientCount, - end_seq_id = DeltaSeqIdEnd }), - State2 #vqstate { delta = Delta1, - q3 = Q3b, - delta_transient_bytes = DeltaTransientBytes - TransientBytes } - end - end. - -push_alphas_to_betas(Quota, State) -> - {Quota1, State1} = - push_alphas_to_betas( - fun ?QUEUE:out/1, - fun (MsgStatus, Q1a, - State0 = #vqstate { q3 = Q3, delta = #delta { count = 0, - transient = 0 } }) -> - State0 #vqstate { q1 = Q1a, q3 = ?QUEUE:in(MsgStatus, Q3) }; - (MsgStatus, Q1a, State0 = #vqstate { q2 = Q2 }) -> - State0 #vqstate { q1 = Q1a, q2 = ?QUEUE:in(MsgStatus, Q2) } - end, Quota, State #vqstate.q1, State), - {Quota2, State2} = - push_alphas_to_betas( - fun ?QUEUE:out_r/1, - fun (MsgStatus, Q4a, State0 = #vqstate { q3 = Q3 }) -> - State0 #vqstate { q3 = ?QUEUE:in_r(MsgStatus, Q3), q4 = Q4a } - end, Quota1, State1 #vqstate.q4, State1), - {Quota2, State2}. - -push_alphas_to_betas(_Generator, _Consumer, Quota, _Q, - State = #vqstate { ram_msg_count = RamMsgCount, - target_ram_count = TargetRamCount }) - when Quota =:= 0 orelse - TargetRamCount =:= infinity orelse - TargetRamCount >= RamMsgCount -> - {Quota, ui(State)}; -push_alphas_to_betas(Generator, Consumer, Quota, Q, State) -> - %% We consume credits from the message_store whenever we need to - %% persist a message to disk. See: - %% rabbit_variable_queue:msg_store_write/4. So perhaps the - %% msg_store is trying to throttle down our queue. - case credit_flow:blocked() of - true -> {Quota, ui(State)}; - false -> case Generator(Q) of - {empty, _Q} -> - {Quota, ui(State)}; - {{value, MsgStatus}, Qa} -> - {MsgStatus1, State1} = - maybe_prepare_write_to_disk(true, false, MsgStatus, - State), - MsgStatus2 = m(trim_msg_status(MsgStatus1)), - State2 = stats( - ready0, {MsgStatus, MsgStatus2}, 0, State1), - State3 = Consumer(MsgStatus2, Qa, State2), - push_alphas_to_betas(Generator, Consumer, Quota - 1, - Qa, State3) - end - end. - -push_betas_to_deltas(Quota, State = #vqstate { mode = default, - q2 = Q2, - delta = Delta, - q3 = Q3}) -> - PushState = {Quota, Delta, State}, - {Q3a, PushState1} = push_betas_to_deltas( - fun ?QUEUE:out_r/1, - fun rabbit_queue_index:next_segment_boundary/1, - Q3, PushState), - {Q2a, PushState2} = push_betas_to_deltas( - fun ?QUEUE:out/1, - fun (Q2MinSeqId) -> Q2MinSeqId end, - Q2, PushState1), - {_, Delta1, State1} = PushState2, - State1 #vqstate { q2 = Q2a, - delta = Delta1, - q3 = Q3a }; -%% In the case of lazy queues we want to page as many messages as -%% possible from q3. -push_betas_to_deltas(Quota, State = #vqstate { mode = lazy, - delta = Delta, - q3 = Q3}) -> - PushState = {Quota, Delta, State}, - {Q3a, PushState1} = push_betas_to_deltas( - fun ?QUEUE:out_r/1, - fun (Q2MinSeqId) -> Q2MinSeqId end, - Q3, PushState), - {_, Delta1, State1} = PushState1, - State1 #vqstate { delta = Delta1, - q3 = Q3a }. - - -push_betas_to_deltas(Generator, LimitFun, Q, PushState) -> - case ?QUEUE:is_empty(Q) of - true -> - {Q, PushState}; - false -> - {value, #msg_status { seq_id = MinSeqId }} = ?QUEUE:peek(Q), - {value, #msg_status { seq_id = MaxSeqId }} = ?QUEUE:peek_r(Q), - Limit = LimitFun(MinSeqId), - case MaxSeqId < Limit of - true -> {Q, PushState}; - false -> push_betas_to_deltas1(Generator, Limit, Q, PushState) - end - end. - -push_betas_to_deltas1(_Generator, _Limit, Q, {0, Delta, State}) -> - {Q, {0, Delta, ui(State)}}; -push_betas_to_deltas1(Generator, Limit, Q, {Quota, Delta, State}) -> - case Generator(Q) of - {empty, _Q} -> - {Q, {Quota, Delta, ui(State)}}; - {{value, #msg_status { seq_id = SeqId }}, _Qa} - when SeqId < Limit -> - {Q, {Quota, Delta, ui(State)}}; - {{value, MsgStatus = #msg_status { seq_id = SeqId }}, Qa} -> - {#msg_status { index_on_disk = true, - is_persistent = IsPersistent }, State1} = - maybe_batch_write_index_to_disk(true, MsgStatus, State), - State2 = stats(ready0, {MsgStatus, none}, 1, State1), - Delta1 = expand_delta(SeqId, Delta, IsPersistent), - push_betas_to_deltas1(Generator, Limit, Qa, - {Quota - 1, Delta1, State2}) - end. - -%% Flushes queue index batch caches and updates queue index state. -ui(#vqstate{index_state = IndexState, - target_ram_count = TargetRamCount} = State) -> - IndexState1 = rabbit_queue_index:flush_pre_publish_cache( - TargetRamCount, IndexState), - State#vqstate{index_state = IndexState1}. - -%%---------------------------------------------------------------------------- -%% Upgrading -%%---------------------------------------------------------------------------- - --spec multiple_routing_keys() -> 'ok'. - -multiple_routing_keys() -> - transform_storage( - fun ({basic_message, ExchangeName, Routing_Key, Content, - MsgId, Persistent}) -> - {ok, {basic_message, ExchangeName, [Routing_Key], Content, - MsgId, Persistent}}; - (_) -> {error, corrupt_message} - end), - ok. - - -%% Assumes message store is not running -transform_storage(TransformFun) -> - transform_store(?PERSISTENT_MSG_STORE, TransformFun), - transform_store(?TRANSIENT_MSG_STORE, TransformFun). - -transform_store(Store, TransformFun) -> - rabbit_msg_store:force_recovery(rabbit_mnesia:dir(), Store), - rabbit_msg_store:transform_dir(rabbit_mnesia:dir(), Store, TransformFun). - -move_messages_to_vhost_store() -> - case list_persistent_queues() of - [] -> - log_upgrade("No durable queues found." - " Skipping message store migration"), - ok; - Queues -> - move_messages_to_vhost_store(Queues) - end, - ok = delete_old_store(), - ok = rabbit_queue_index:cleanup_global_recovery_terms(). - -move_messages_to_vhost_store(Queues) -> - log_upgrade("Moving messages to per-vhost message store"), - %% Move the queue index for each persistent queue to the new store - lists:foreach( - fun(Queue) -> - QueueName = amqqueue:get_name(Queue), - rabbit_queue_index:move_to_per_vhost_stores(QueueName) - end, - Queues), - %% Legacy (global) msg_store may require recovery. - %% This upgrade step should only be started - %% if we are upgrading from a pre-3.7.0 version. - {QueuesWithTerms, RecoveryRefs, StartFunState} = read_old_recovery_terms(Queues), - - OldStore = run_old_persistent_store(RecoveryRefs, StartFunState), - - VHosts = rabbit_vhost:list_names(), - - %% New store should not be recovered. - NewMsgStore = start_new_store(VHosts), - %% Recovery terms should be started for all vhosts for new store. - [ok = rabbit_recovery_terms:open_table(VHost) || VHost <- VHosts], - - MigrationBatchSize = application:get_env(rabbit, queue_migration_batch_size, - ?QUEUE_MIGRATION_BATCH_SIZE), - in_batches(MigrationBatchSize, - {rabbit_variable_queue, migrate_queue, [OldStore, NewMsgStore]}, - QueuesWithTerms, - "message_store upgrades: Migrating batch ~p of ~p queues. Out of total ~p ~n", - "message_store upgrades: Batch ~p of ~p queues migrated ~n. ~p total left"), - - log_upgrade("Message store migration finished"), - ok = rabbit_sup:stop_child(OldStore), - [ok= rabbit_recovery_terms:close_table(VHost) || VHost <- VHosts], - ok = stop_new_store(NewMsgStore). - -in_batches(Size, MFA, List, MessageStart, MessageEnd) -> - in_batches(Size, 1, MFA, List, MessageStart, MessageEnd). - -in_batches(_, _, _, [], _, _) -> ok; -in_batches(Size, BatchNum, MFA, List, MessageStart, MessageEnd) -> - Length = length(List), - {Batch, Tail} = case Size > Length of - true -> {List, []}; - false -> lists:split(Size, List) - end, - ProcessedLength = (BatchNum - 1) * Size, - rabbit_log:info(MessageStart, [BatchNum, Size, ProcessedLength + Length]), - {M, F, A} = MFA, - Keys = [ rpc:async_call(node(), M, F, [El | A]) || El <- Batch ], - lists:foreach(fun(Key) -> - case rpc:yield(Key) of - {badrpc, Err} -> throw(Err); - _ -> ok - end - end, - Keys), - rabbit_log:info(MessageEnd, [BatchNum, Size, length(Tail)]), - in_batches(Size, BatchNum + 1, MFA, Tail, MessageStart, MessageEnd). - -migrate_queue({QueueName = #resource{virtual_host = VHost, name = Name}, - RecoveryTerm}, - OldStore, NewStore) -> - log_upgrade_verbose( - "Migrating messages in queue ~s in vhost ~s to per-vhost message store~n", - [Name, VHost]), - OldStoreClient = get_global_store_client(OldStore), - NewStoreClient = get_per_vhost_store_client(QueueName, NewStore), - %% WARNING: During scan_queue_segments queue index state is being recovered - %% and terminated. This can cause side effects! - rabbit_queue_index:scan_queue_segments( - %% We migrate only persistent messages which are found in message store - %% and are not acked yet - fun (_SeqId, MsgId, _MsgProps, true, _IsDelivered, no_ack, OldC) - when is_binary(MsgId) -> - migrate_message(MsgId, OldC, NewStoreClient); - (_SeqId, _MsgId, _MsgProps, - _IsPersistent, _IsDelivered, _IsAcked, OldC) -> - OldC - end, - OldStoreClient, - QueueName), - rabbit_msg_store:client_terminate(OldStoreClient), - rabbit_msg_store:client_terminate(NewStoreClient), - NewClientRef = rabbit_msg_store:client_ref(NewStoreClient), - case RecoveryTerm of - non_clean_shutdown -> ok; - Term when is_list(Term) -> - NewRecoveryTerm = lists:keyreplace(persistent_ref, 1, RecoveryTerm, - {persistent_ref, NewClientRef}), - rabbit_queue_index:update_recovery_term(QueueName, NewRecoveryTerm) - end, - log_upgrade_verbose("Finished migrating queue ~s in vhost ~s", [Name, VHost]), - {QueueName, NewClientRef}. - -migrate_message(MsgId, OldC, NewC) -> - case rabbit_msg_store:read(MsgId, OldC) of - {{ok, Msg}, OldC1} -> - ok = rabbit_msg_store:write(MsgId, Msg, NewC), - OldC1; - _ -> OldC - end. - -get_per_vhost_store_client(#resource{virtual_host = VHost}, NewStore) -> - {VHost, StorePid} = lists:keyfind(VHost, 1, NewStore), - rabbit_msg_store:client_init(StorePid, rabbit_guid:gen(), - fun(_,_) -> ok end, fun() -> ok end). - -get_global_store_client(OldStore) -> - rabbit_msg_store:client_init(OldStore, - rabbit_guid:gen(), - fun(_,_) -> ok end, - fun() -> ok end). - -list_persistent_queues() -> - Node = node(), - mnesia:async_dirty( - fun () -> - qlc:e(qlc:q([Q || Q <- mnesia:table(rabbit_durable_queue), - ?amqqueue_is_classic(Q), - amqqueue:qnode(Q) == Node, - mnesia:read(rabbit_queue, amqqueue:get_name(Q), read) =:= []])) - end). - -read_old_recovery_terms([]) -> - {[], [], ?EMPTY_START_FUN_STATE}; -read_old_recovery_terms(Queues) -> - QueueNames = [amqqueue:get_name(Q) || Q <- Queues], - {AllTerms, StartFunState} = rabbit_queue_index:read_global_recovery_terms(QueueNames), - Refs = [Ref || Terms <- AllTerms, - Terms /= non_clean_shutdown, - begin - Ref = proplists:get_value(persistent_ref, Terms), - Ref =/= undefined - end], - {lists:zip(QueueNames, AllTerms), Refs, StartFunState}. - -run_old_persistent_store(Refs, StartFunState) -> - OldStoreName = ?PERSISTENT_MSG_STORE, - ok = rabbit_sup:start_child(OldStoreName, rabbit_msg_store, start_global_store_link, - [OldStoreName, rabbit_mnesia:dir(), - Refs, StartFunState]), - OldStoreName. - -start_new_store(VHosts) -> - %% Ensure vhost supervisor is started, so we can add vhosts to it. - lists:map(fun(VHost) -> - VHostDir = rabbit_vhost:msg_store_dir_path(VHost), - {ok, Pid} = rabbit_msg_store:start_link(?PERSISTENT_MSG_STORE, - VHostDir, - undefined, - ?EMPTY_START_FUN_STATE), - {VHost, Pid} - end, - VHosts). - -stop_new_store(NewStore) -> - lists:foreach(fun({_VHost, StorePid}) -> - unlink(StorePid), - exit(StorePid, shutdown) - end, - NewStore), - ok. - -delete_old_store() -> - log_upgrade("Removing the old message store data"), - rabbit_file:recursive_delete( - [filename:join([rabbit_mnesia:dir(), ?PERSISTENT_MSG_STORE])]), - %% Delete old transient store as well - rabbit_file:recursive_delete( - [filename:join([rabbit_mnesia:dir(), ?TRANSIENT_MSG_STORE])]), - ok. - -log_upgrade(Msg) -> - log_upgrade(Msg, []). - -log_upgrade(Msg, Args) -> - rabbit_log:info("message_store upgrades: " ++ Msg, Args). - -log_upgrade_verbose(Msg) -> - log_upgrade_verbose(Msg, []). - -log_upgrade_verbose(Msg, Args) -> - rabbit_log_upgrade:info(Msg, Args). - -maybe_client_terminate(MSCStateP) -> - %% Queue might have been asked to stop by the supervisor, it needs a clean - %% shutdown in order for the supervising strategy to work - if it reaches max - %% restarts might bring the vhost down. - try - rabbit_msg_store:client_terminate(MSCStateP) - catch - _:_ -> - ok - end. diff --git a/src/rabbit_version.erl b/src/rabbit_version.erl deleted file mode 100644 index 3f5462c7b4..0000000000 --- a/src/rabbit_version.erl +++ /dev/null @@ -1,227 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_version). - --export([recorded/0, matches/2, desired/0, desired_for_scope/1, - record_desired/0, record_desired_for_scope/1, - upgrades_required/1, all_upgrades_required/1, - check_version_consistency/3, - check_version_consistency/4, check_otp_consistency/1, - version_error/3]). - -%% ------------------------------------------------------------------- - --export_type([scope/0, step/0]). - --type scope() :: atom(). --type scope_version() :: [atom()]. --type step() :: {atom(), atom()}. - --type version() :: [atom()]. - -%% ------------------------------------------------------------------- - --define(VERSION_FILENAME, "schema_version"). --define(SCOPES, [mnesia, local]). - -%% ------------------------------------------------------------------- - --spec recorded() -> rabbit_types:ok_or_error2(version(), any()). - -recorded() -> case rabbit_file:read_term_file(schema_filename()) of - {ok, [V]} -> {ok, V}; - {error, _} = Err -> Err - end. - -record(V) -> ok = rabbit_file:write_term_file(schema_filename(), [V]). - -recorded_for_scope(Scope) -> - case recorded() of - {error, _} = Err -> - Err; - {ok, Version} -> - {ok, case lists:keysearch(Scope, 1, categorise_by_scope(Version)) of - false -> []; - {value, {Scope, SV1}} -> SV1 - end} - end. - -record_for_scope(Scope, ScopeVersion) -> - case recorded() of - {error, _} = Err -> - Err; - {ok, Version} -> - Version1 = lists:keystore(Scope, 1, categorise_by_scope(Version), - {Scope, ScopeVersion}), - ok = record([Name || {_Scope, Names} <- Version1, Name <- Names]) - end. - -%% ------------------------------------------------------------------- - --spec matches([A], [A]) -> boolean(). - -matches(VerA, VerB) -> - lists:usort(VerA) =:= lists:usort(VerB). - -%% ------------------------------------------------------------------- - --spec desired() -> version(). - -desired() -> [Name || Scope <- ?SCOPES, Name <- desired_for_scope(Scope)]. - --spec desired_for_scope(scope()) -> scope_version(). - -desired_for_scope(Scope) -> with_upgrade_graph(fun heads/1, Scope). - --spec record_desired() -> 'ok'. - -record_desired() -> record(desired()). - --spec record_desired_for_scope - (scope()) -> rabbit_types:ok_or_error(any()). - -record_desired_for_scope(Scope) -> - record_for_scope(Scope, desired_for_scope(Scope)). - --spec upgrades_required - (scope()) -> rabbit_types:ok_or_error2([step()], any()). - -upgrades_required(Scope) -> - case recorded_for_scope(Scope) of - {error, enoent} -> - case filelib:is_file(rabbit_guid:filename()) of - false -> {error, starting_from_scratch}; - true -> {error, version_not_available} - end; - {ok, CurrentHeads} -> - with_upgrade_graph( - fun (G) -> - case unknown_heads(CurrentHeads, G) of - [] -> {ok, upgrades_to_apply(CurrentHeads, G)}; - Unknown -> {error, {future_upgrades_found, Unknown}} - end - end, Scope) - end. - -all_upgrades_required(Scopes) -> - case recorded() of - {error, enoent} -> - case filelib:is_file(rabbit_guid:filename()) of - false -> {error, starting_from_scratch}; - true -> {error, version_not_available} - end; - {ok, _} -> - lists:foldl( - fun - (_, {error, Err}) -> {error, Err}; - (Scope, {ok, Acc}) -> - case upgrades_required(Scope) of - %% Lift errors from any scope. - {error, Err} -> {error, Err}; - %% Filter non-upgradable scopes - {ok, []} -> {ok, Acc}; - {ok, Upgrades} -> {ok, [{Scope, Upgrades} | Acc]} - end - end, - {ok, []}, - Scopes) - end. - -%% ------------------------------------------------------------------- - -with_upgrade_graph(Fun, Scope) -> - case rabbit_misc:build_acyclic_graph( - fun ({_App, Module, Steps}) -> vertices(Module, Steps, Scope) end, - fun ({_App, Module, Steps}) -> edges(Module, Steps, Scope) end, - rabbit_misc:all_module_attributes(rabbit_upgrade)) of - {ok, G} -> try - Fun(G) - after - true = digraph:delete(G) - end; - {error, {vertex, duplicate, StepName}} -> - throw({error, {duplicate_upgrade_step, StepName}}); - {error, {edge, {bad_vertex, StepName}, _From, _To}} -> - throw({error, {dependency_on_unknown_upgrade_step, StepName}}); - {error, {edge, {bad_edge, StepNames}, _From, _To}} -> - throw({error, {cycle_in_upgrade_steps, StepNames}}) - end. - -vertices(Module, Steps, Scope0) -> - [{StepName, {Module, StepName}} || {StepName, Scope1, _Reqs} <- Steps, - Scope0 == Scope1]. - -edges(_Module, Steps, Scope0) -> - [{Require, StepName} || {StepName, Scope1, Requires} <- Steps, - Require <- Requires, - Scope0 == Scope1]. -unknown_heads(Heads, G) -> - [H || H <- Heads, digraph:vertex(G, H) =:= false]. - -upgrades_to_apply(Heads, G) -> - %% Take all the vertices which can reach the known heads. That's - %% everything we've already applied. Subtract that from all - %% vertices: that's what we have to apply. - Unsorted = sets:to_list( - sets:subtract( - sets:from_list(digraph:vertices(G)), - sets:from_list(digraph_utils:reaching(Heads, G)))), - %% Form a subgraph from that list and find a topological ordering - %% so we can invoke them in order. - [element(2, digraph:vertex(G, StepName)) || - StepName <- digraph_utils:topsort(digraph_utils:subgraph(G, Unsorted))]. - -heads(G) -> - lists:sort([V || V <- digraph:vertices(G), digraph:out_degree(G, V) =:= 0]). - -%% ------------------------------------------------------------------- - -categorise_by_scope(Version) when is_list(Version) -> - Categorised = - [{Scope, Name} || {_App, _Module, Attributes} <- - rabbit_misc:all_module_attributes(rabbit_upgrade), - {Name, Scope, _Requires} <- Attributes, - lists:member(Name, Version)], - maps:to_list( - lists:foldl(fun ({Scope, Name}, CatVersion) -> - rabbit_misc:maps_cons(Scope, Name, CatVersion) - end, maps:new(), Categorised)). - -dir() -> rabbit_mnesia:dir(). - -schema_filename() -> filename:join(dir(), ?VERSION_FILENAME). - -%% -------------------------------------------------------------------- - --spec check_version_consistency - (string(), string(), string()) -> rabbit_types:ok_or_error(any()). - -check_version_consistency(This, Remote, Name) -> - check_version_consistency(This, Remote, Name, fun (A, B) -> A =:= B end). - --spec check_version_consistency - (string(), string(), string(), - fun((string(), string()) -> boolean())) -> - rabbit_types:ok_or_error(any()). - -check_version_consistency(This, Remote, Name, Comp) -> - case Comp(This, Remote) of - true -> ok; - false -> version_error(Name, This, Remote) - end. - -version_error(Name, This, Remote) -> - {error, {inconsistent_cluster, - rabbit_misc:format("~s version mismatch: local node is ~s, " - "remote node ~s", [Name, This, Remote])}}. - --spec check_otp_consistency - (string()) -> rabbit_types:ok_or_error(any()). - -check_otp_consistency(Remote) -> - check_version_consistency(rabbit_misc:otp_release(), Remote, "OTP"). diff --git a/src/rabbit_vhost.erl b/src/rabbit_vhost.erl deleted file mode 100644 index c8c5fc961a..0000000000 --- a/src/rabbit_vhost.erl +++ /dev/null @@ -1,422 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_vhost). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("vhost.hrl"). - --export([recover/0, recover/1]). --export([add/2, add/4, delete/2, exists/1, with/2, with_user_and_vhost/3, assert/1, update/2, - set_limits/2, vhost_cluster_state/1, is_running_on_all_nodes/1, await_running_on_all_nodes/2, - list/0, count/0, list_names/0, all/0, parse_tags/1]). --export([info/1, info/2, info_all/0, info_all/1, info_all/2, info_all/3]). --export([dir/1, msg_store_dir_path/1, msg_store_dir_wildcard/0]). --export([delete_storage/1]). --export([vhost_down/1]). --export([put_vhost/5]). - -%% -%% API -%% - -recover() -> - %% Clear out remnants of old incarnation, in case we restarted - %% faster than other nodes handled DOWN messages from us. - rabbit_amqqueue:on_node_down(node()), - - rabbit_amqqueue:warn_file_limit(), - - %% Prepare rabbit_semi_durable_route table - rabbit_binding:recover(), - - %% rabbit_vhost_sup_sup will start the actual recovery. - %% So recovery will be run every time a vhost supervisor is restarted. - ok = rabbit_vhost_sup_sup:start(), - - [ok = rabbit_vhost_sup_sup:init_vhost(VHost) || VHost <- list_names()], - ok. - -recover(VHost) -> - VHostDir = msg_store_dir_path(VHost), - rabbit_log:info("Making sure data directory '~ts' for vhost '~s' exists~n", - [VHostDir, VHost]), - VHostStubFile = filename:join(VHostDir, ".vhost"), - ok = rabbit_file:ensure_dir(VHostStubFile), - ok = file:write_file(VHostStubFile, VHost), - {Recovered, Failed} = rabbit_amqqueue:recover(VHost), - AllQs = Recovered ++ Failed, - QNames = [amqqueue:get_name(Q) || Q <- AllQs], - ok = rabbit_binding:recover(rabbit_exchange:recover(VHost), QNames), - ok = rabbit_amqqueue:start(Recovered), - %% Start queue mirrors. - ok = rabbit_mirror_queue_misc:on_vhost_up(VHost), - ok. - --define(INFO_KEYS, vhost:info_keys()). - --spec parse_tags(binary() | string() | atom()) -> [atom()]. -parse_tags(undefined) -> - []; -parse_tags("") -> - []; -parse_tags(<<"">>) -> - []; -parse_tags(Val) when is_binary(Val) -> - parse_tags(rabbit_data_coercion:to_list(Val)); -parse_tags(Val) when is_list(Val) -> - [trim_tag(Tag) || Tag <- re:split(Val, ",", [{return, list}])]. - --spec add(vhost:name(), rabbit_types:username()) -> rabbit_types:ok_or_error(any()). - -add(VHost, ActingUser) -> - case exists(VHost) of - true -> ok; - false -> do_add(VHost, <<"">>, [], ActingUser) - end. - --spec add(vhost:name(), binary(), [atom()], rabbit_types:username()) -> rabbit_types:ok_or_error(any()). - -add(Name, Description, Tags, ActingUser) -> - case exists(Name) of - true -> ok; - false -> do_add(Name, Description, Tags, ActingUser) - end. - -do_add(Name, Description, Tags, ActingUser) -> - case Description of - undefined -> - rabbit_log:info("Adding vhost '~s' without a description", [Name]); - Value -> - rabbit_log:info("Adding vhost '~s' (description: '~s')", [Name, Value]) - end, - VHost = rabbit_misc:execute_mnesia_transaction( - fun () -> - case mnesia:wread({rabbit_vhost, Name}) of - [] -> - Row = vhost:new(Name, [], #{description => Description, tags => Tags}), - rabbit_log:debug("Inserting a virtual host record ~p", [Row]), - ok = mnesia:write(rabbit_vhost, Row, write), - Row; - %% the vhost already exists - [Row] -> - Row - end - end, - fun (VHost1, true) -> - VHost1; - (VHost1, false) -> - [begin - Resource = rabbit_misc:r(Name, exchange, ExchangeName), - rabbit_log:debug("Will declare an exchange ~p", [Resource]), - _ = rabbit_exchange:declare(Resource, Type, true, false, Internal, [], ActingUser) - end || {ExchangeName, Type, Internal} <- - [{<<"">>, direct, false}, - {<<"amq.direct">>, direct, false}, - {<<"amq.topic">>, topic, false}, - %% per 0-9-1 pdf - {<<"amq.match">>, headers, false}, - %% per 0-9-1 xml - {<<"amq.headers">>, headers, false}, - {<<"amq.fanout">>, fanout, false}, - {<<"amq.rabbitmq.trace">>, topic, true}]], - VHost1 - end), - case rabbit_vhost_sup_sup:start_on_all_nodes(Name) of - ok -> - rabbit_event:notify(vhost_created, info(VHost) - ++ [{user_who_performed_action, ActingUser}, - {description, Description}, - {tags, Tags}]), - ok; - {error, Reason} -> - Msg = rabbit_misc:format("failed to set up vhost '~s': ~p", - [Name, Reason]), - {error, Msg} - end. - --spec delete(vhost:name(), rabbit_types:username()) -> rabbit_types:ok_or_error(any()). - -delete(VHost, ActingUser) -> - %% FIXME: We are forced to delete the queues and exchanges outside - %% the TX below. Queue deletion involves sending messages to the queue - %% process, which in turn results in further mnesia actions and - %% eventually the termination of that process. Exchange deletion causes - %% notifications which must be sent outside the TX - rabbit_log:info("Deleting vhost '~s'~n", [VHost]), - QDelFun = fun (Q) -> rabbit_amqqueue:delete(Q, false, false, ActingUser) end, - [begin - Name = amqqueue:get_name(Q), - assert_benign(rabbit_amqqueue:with(Name, QDelFun), ActingUser) - end || Q <- rabbit_amqqueue:list(VHost)], - [assert_benign(rabbit_exchange:delete(Name, false, ActingUser), ActingUser) || - #exchange{name = Name} <- rabbit_exchange:list(VHost)], - Funs = rabbit_misc:execute_mnesia_transaction( - with(VHost, fun () -> internal_delete(VHost, ActingUser) end)), - ok = rabbit_event:notify(vhost_deleted, [{name, VHost}, - {user_who_performed_action, ActingUser}]), - [case Fun() of - ok -> ok; - {error, {no_such_vhost, VHost}} -> ok - end || Fun <- Funs], - %% After vhost was deleted from mnesia DB, we try to stop vhost supervisors - %% on all the nodes. - rabbit_vhost_sup_sup:delete_on_all_nodes(VHost), - ok. - -put_vhost(Name, Description, Tags0, Trace, Username) -> - Tags = case Tags0 of - undefined -> <<"">>; - null -> <<"">>; - "undefined" -> <<"">>; - "null" -> <<"">>; - Other -> Other - end, - Result = case exists(Name) of - true -> ok; - false -> add(Name, Description, parse_tags(Tags), Username), - %% wait for up to 45 seconds for the vhost to initialise - %% on all nodes - case await_running_on_all_nodes(Name, 45000) of - ok -> - maybe_grant_full_permissions(Name, Username); - {error, timeout} -> - {error, timeout} - end - end, - case Trace of - true -> rabbit_trace:start(Name); - false -> rabbit_trace:stop(Name); - undefined -> ok - end, - Result. - -%% when definitions are loaded on boot, Username here will be ?INTERNAL_USER, -%% which does not actually exist -maybe_grant_full_permissions(_Name, ?INTERNAL_USER) -> - ok; -maybe_grant_full_permissions(Name, Username) -> - U = rabbit_auth_backend_internal:lookup_user(Username), - maybe_grant_full_permissions(U, Name, Username). - -maybe_grant_full_permissions({ok, _}, Name, Username) -> - rabbit_auth_backend_internal:set_permissions( - Username, Name, <<".*">>, <<".*">>, <<".*">>, Username); -maybe_grant_full_permissions(_, _Name, _Username) -> - ok. - - -%% 50 ms --define(AWAIT_SAMPLE_INTERVAL, 50). - --spec await_running_on_all_nodes(vhost:name(), integer()) -> ok | {error, timeout}. -await_running_on_all_nodes(VHost, Timeout) -> - Attempts = round(Timeout / ?AWAIT_SAMPLE_INTERVAL), - await_running_on_all_nodes0(VHost, Attempts). - -await_running_on_all_nodes0(_VHost, 0) -> - {error, timeout}; -await_running_on_all_nodes0(VHost, Attempts) -> - case is_running_on_all_nodes(VHost) of - true -> ok; - _ -> - timer:sleep(?AWAIT_SAMPLE_INTERVAL), - await_running_on_all_nodes0(VHost, Attempts - 1) - end. - --spec is_running_on_all_nodes(vhost:name()) -> boolean(). -is_running_on_all_nodes(VHost) -> - States = vhost_cluster_state(VHost), - lists:all(fun ({_Node, State}) -> State =:= running end, - States). - --spec vhost_cluster_state(vhost:name()) -> [{atom(), atom()}]. -vhost_cluster_state(VHost) -> - Nodes = rabbit_nodes:all_running(), - lists:map(fun(Node) -> - State = case rabbit_misc:rpc_call(Node, - rabbit_vhost_sup_sup, is_vhost_alive, - [VHost]) of - {badrpc, nodedown} -> nodedown; - true -> running; - false -> stopped - end, - {Node, State} - end, - Nodes). - -vhost_down(VHost) -> - ok = rabbit_event:notify(vhost_down, - [{name, VHost}, - {node, node()}, - {user_who_performed_action, ?INTERNAL_USER}]). - -delete_storage(VHost) -> - VhostDir = msg_store_dir_path(VHost), - rabbit_log:info("Deleting message store directory for vhost '~s' at '~s'~n", [VHost, VhostDir]), - %% Message store should be closed when vhost supervisor is closed. - case rabbit_file:recursive_delete([VhostDir]) of - ok -> ok; - {error, {_, enoent}} -> - %% a concurrent delete did the job for us - rabbit_log:warning("Tried to delete storage directories for vhost '~s', it failed with an ENOENT", [VHost]), - ok; - Other -> - rabbit_log:warning("Tried to delete storage directories for vhost '~s': ~p", [VHost, Other]), - Other - end. - -assert_benign(ok, _) -> ok; -assert_benign({ok, _}, _) -> ok; -assert_benign({ok, _, _}, _) -> ok; -assert_benign({error, not_found}, _) -> ok; -assert_benign({error, {absent, Q, _}}, ActingUser) -> - %% Removing the mnesia entries here is safe. If/when the down node - %% restarts, it will clear out the on-disk storage of the queue. - QName = amqqueue:get_name(Q), - rabbit_amqqueue:internal_delete(QName, ActingUser). - -internal_delete(VHost, ActingUser) -> - [ok = rabbit_auth_backend_internal:clear_permissions( - proplists:get_value(user, Info), VHost, ActingUser) - || Info <- rabbit_auth_backend_internal:list_vhost_permissions(VHost)], - TopicPermissions = rabbit_auth_backend_internal:list_vhost_topic_permissions(VHost), - [ok = rabbit_auth_backend_internal:clear_topic_permissions( - proplists:get_value(user, TopicPermission), VHost, ActingUser) - || TopicPermission <- TopicPermissions], - Fs1 = [rabbit_runtime_parameters:clear(VHost, - proplists:get_value(component, Info), - proplists:get_value(name, Info), - ActingUser) - || Info <- rabbit_runtime_parameters:list(VHost)], - Fs2 = [rabbit_policy:delete(VHost, proplists:get_value(name, Info), ActingUser) - || Info <- rabbit_policy:list(VHost)], - ok = mnesia:delete({rabbit_vhost, VHost}), - Fs1 ++ Fs2. - --spec exists(vhost:name()) -> boolean(). - -exists(VHost) -> - mnesia:dirty_read({rabbit_vhost, VHost}) /= []. - --spec list_names() -> [vhost:name()]. -list_names() -> mnesia:dirty_all_keys(rabbit_vhost). - -%% Exists for backwards compatibility, prefer list_names/0. --spec list() -> [vhost:name()]. -list() -> list_names(). - --spec all() -> [vhost:vhost()]. -all() -> mnesia:dirty_match_object(rabbit_vhost, vhost:pattern_match_all()). - --spec count() -> non_neg_integer(). -count() -> - length(list()). - --spec with(vhost:name(), rabbit_misc:thunk(A)) -> A. - -with(VHost, Thunk) -> - fun () -> - case mnesia:read({rabbit_vhost, VHost}) of - [] -> - mnesia:abort({no_such_vhost, VHost}); - [_V] -> - Thunk() - end - end. - --spec with_user_and_vhost - (rabbit_types:username(), vhost:name(), rabbit_misc:thunk(A)) -> A. - -with_user_and_vhost(Username, VHost, Thunk) -> - rabbit_misc:with_user(Username, with(VHost, Thunk)). - -%% Like with/2 but outside an Mnesia tx - --spec assert(vhost:name()) -> 'ok'. - -assert(VHost) -> case exists(VHost) of - true -> ok; - false -> throw({error, {no_such_vhost, VHost}}) - end. - --spec update(vhost:name(), fun((vhost:vhost()) -> vhost:vhost())) -> vhost:vhost(). - -update(VHost, Fun) -> - case mnesia:read({rabbit_vhost, VHost}) of - [] -> - mnesia:abort({no_such_vhost, VHost}); - [V] -> - V1 = Fun(V), - ok = mnesia:write(rabbit_vhost, V1, write), - V1 - end. - -set_limits(VHost, undefined) -> - vhost:set_limits(VHost, []); -set_limits(VHost, Limits) -> - vhost:set_limits(VHost, Limits). - - -dir(Vhost) -> - <<Num:128>> = erlang:md5(Vhost), - rabbit_misc:format("~.36B", [Num]). - -msg_store_dir_path(VHost) -> - EncodedName = dir(VHost), - rabbit_data_coercion:to_list(filename:join([msg_store_dir_base(), EncodedName])). - -msg_store_dir_wildcard() -> - rabbit_data_coercion:to_list(filename:join([msg_store_dir_base(), "*"])). - -msg_store_dir_base() -> - Dir = rabbit_mnesia:dir(), - filename:join([Dir, "msg_stores", "vhosts"]). - --spec trim_tag(list() | binary() | atom()) -> atom(). -trim_tag(Val) -> - rabbit_data_coercion:to_atom(string:trim(rabbit_data_coercion:to_list(Val))). - -%%---------------------------------------------------------------------------- - -infos(Items, X) -> [{Item, i(Item, X)} || Item <- Items]. - -i(name, VHost) -> vhost:get_name(VHost); -i(tracing, VHost) -> rabbit_trace:enabled(vhost:get_name(VHost)); -i(cluster_state, VHost) -> vhost_cluster_state(vhost:get_name(VHost)); -i(description, VHost) -> vhost:get_description(VHost); -i(tags, VHost) -> vhost:get_tags(VHost); -i(metadata, VHost) -> vhost:get_metadata(VHost); -i(Item, VHost) -> - rabbit_log:error("Don't know how to compute a virtual host info item '~s' for virtual host '~p'", [Item, VHost]), - throw({bad_argument, Item}). - --spec info(vhost:vhost() | vhost:name()) -> rabbit_types:infos(). - -info(VHost) when ?is_vhost(VHost) -> - infos(?INFO_KEYS, VHost); -info(Key) -> - case mnesia:dirty_read({rabbit_vhost, Key}) of - [] -> []; - [VHost] -> infos(?INFO_KEYS, VHost) - end. - --spec info(vhost:vhost(), rabbit_types:info_keys()) -> rabbit_types:infos(). -info(VHost, Items) -> infos(Items, VHost). - --spec info_all() -> [rabbit_types:infos()]. -info_all() -> info_all(?INFO_KEYS). - --spec info_all(rabbit_types:info_keys()) -> [rabbit_types:infos()]. -info_all(Items) -> [info(VHost, Items) || VHost <- all()]. - -info_all(Ref, AggregatorPid) -> info_all(?INFO_KEYS, Ref, AggregatorPid). - --spec info_all(rabbit_types:info_keys(), reference(), pid()) -> 'ok'. -info_all(Items, Ref, AggregatorPid) -> - rabbit_control_misc:emitting_map( - AggregatorPid, Ref, fun(VHost) -> info(VHost, Items) end, all()). diff --git a/src/rabbit_vhost_limit.erl b/src/rabbit_vhost_limit.erl deleted file mode 100644 index bee01f3054..0000000000 --- a/src/rabbit_vhost_limit.erl +++ /dev/null @@ -1,205 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_vhost_limit). - --behaviour(rabbit_runtime_parameter). - --include("rabbit.hrl"). - --export([register/0]). --export([parse_set/3, set/3, clear/2]). --export([list/0, list/1]). --export([update_limit/4, clear_limit/3, get_limit/2]). --export([validate/5, notify/5, notify_clear/4]). --export([connection_limit/1, queue_limit/1, - is_over_queue_limit/1, would_exceed_queue_limit/2, - is_over_connection_limit/1]). - --import(rabbit_misc, [pget/2, pget/3]). - --rabbit_boot_step({?MODULE, - [{description, "vhost limit parameters"}, - {mfa, {rabbit_vhost_limit, register, []}}, - {requires, rabbit_registry}, - {enables, recovery}]}). - -%%---------------------------------------------------------------------------- - -register() -> - rabbit_registry:register(runtime_parameter, <<"vhost-limits">>, ?MODULE). - -validate(_VHost, <<"vhost-limits">>, Name, Term, _User) -> - rabbit_parameter_validation:proplist( - Name, vhost_limit_validation(), Term). - -notify(VHost, <<"vhost-limits">>, <<"limits">>, Limits, ActingUser) -> - rabbit_event:notify(vhost_limits_set, [{name, <<"limits">>}, - {user_who_performed_action, ActingUser} - | Limits]), - update_vhost(VHost, Limits). - -notify_clear(VHost, <<"vhost-limits">>, <<"limits">>, ActingUser) -> - rabbit_event:notify(vhost_limits_cleared, [{name, <<"limits">>}, - {user_who_performed_action, ActingUser}]), - %% If the function is called as a part of vhost deletion, the vhost can - %% be already deleted. - case rabbit_vhost:exists(VHost) of - true -> update_vhost(VHost, undefined); - false -> ok - end. - -connection_limit(VirtualHost) -> - get_limit(VirtualHost, <<"max-connections">>). - -queue_limit(VirtualHost) -> - get_limit(VirtualHost, <<"max-queues">>). - - -query_limits(VHost) -> - case rabbit_runtime_parameters:list(VHost, <<"vhost-limits">>) of - [] -> []; - Params -> [ {pget(vhost, Param), pget(value, Param)} - || Param <- Params, - pget(value, Param) =/= undefined, - pget(name, Param) == <<"limits">> ] - end. - - --spec list() -> [{vhost:name(), rabbit_types:infos()}]. -list() -> - query_limits('_'). - --spec list(vhost:name()) -> rabbit_types:infos(). -list(VHost) -> - case query_limits(VHost) of - [] -> []; - [{VHost, Value}] -> Value - end. - --spec is_over_connection_limit(vhost:name()) -> {true, non_neg_integer()} | false. - -is_over_connection_limit(VirtualHost) -> - case rabbit_vhost_limit:connection_limit(VirtualHost) of - %% no limit configured - undefined -> false; - %% with limit = 0, no connections are allowed - {ok, 0} -> {true, 0}; - {ok, Limit} when is_integer(Limit) andalso Limit > 0 -> - ConnectionCount = - rabbit_connection_tracking:count_tracked_items_in({vhost, VirtualHost}), - case ConnectionCount >= Limit of - false -> false; - true -> {true, Limit} - end; - %% any negative value means "no limit". Note that parameter validation - %% will replace negative integers with 'undefined', so this is to be - %% explicit and extra defensive - {ok, Limit} when is_integer(Limit) andalso Limit < 0 -> false; - %% ignore non-integer limits - {ok, _Limit} -> false - end. - --spec would_exceed_queue_limit(non_neg_integer(), vhost:name()) -> - {true, non_neg_integer(), non_neg_integer()} | false. - -would_exceed_queue_limit(AdditionalCount, VirtualHost) -> - case queue_limit(VirtualHost) of - undefined -> - %% no limit configured - false; - {ok, 0} -> - %% with limit = 0, no queues can be declared (perhaps not very - %% useful but consistent with the connection limit) - {true, 0, 0}; - {ok, Limit} when is_integer(Limit) andalso Limit > 0 -> - QueueCount = rabbit_amqqueue:count(VirtualHost), - case (AdditionalCount + QueueCount) > Limit of - false -> false; - true -> {true, Limit, QueueCount} - end; - {ok, Limit} when is_integer(Limit) andalso Limit < 0 -> - %% any negative value means "no limit". Note that parameter validation - %% will replace negative integers with 'undefined', so this is to be - %% explicit and extra defensive - false; - {ok, _Limit} -> - %% ignore non-integer limits - false - end. - --spec is_over_queue_limit(vhost:name()) -> {true, non_neg_integer()} | false. - -is_over_queue_limit(VirtualHost) -> - case would_exceed_queue_limit(1, VirtualHost) of - {true, Limit, _QueueCount} -> {true, Limit}; - false -> false - end. - -%%---------------------------------------------------------------------------- - -parse_set(VHost, Defn, ActingUser) -> - Definition = rabbit_data_coercion:to_binary(Defn), - case rabbit_json:try_decode(Definition) of - {ok, Term} -> - set(VHost, maps:to_list(Term), ActingUser); - {error, Reason} -> - {error_string, - rabbit_misc:format("JSON decoding error. Reason: ~ts", [Reason])} - end. - -set(VHost, Defn, ActingUser) -> - rabbit_runtime_parameters:set_any(VHost, <<"vhost-limits">>, - <<"limits">>, Defn, ActingUser). - -clear(VHost, ActingUser) -> - rabbit_runtime_parameters:clear_any(VHost, <<"vhost-limits">>, - <<"limits">>, ActingUser). - -update_limit(VHost, Name, Value, ActingUser) -> - OldDef = case rabbit_runtime_parameters:list(VHost, <<"vhost-limits">>) of - [] -> []; - [Param] -> pget(value, Param, []) - end, - NewDef = [{Name, Value} | lists:keydelete(Name, 1, OldDef)], - set(VHost, NewDef, ActingUser). - -clear_limit(VHost, Name, ActingUser) -> - OldDef = case rabbit_runtime_parameters:list(VHost, <<"vhost-limits">>) of - [] -> []; - [Param] -> pget(value, Param, []) - end, - NewDef = lists:keydelete(Name, 1, OldDef), - set(VHost, NewDef, ActingUser). - -vhost_limit_validation() -> - [{<<"max-connections">>, fun rabbit_parameter_validation:integer/2, optional}, - {<<"max-queues">>, fun rabbit_parameter_validation:integer/2, optional}]. - -update_vhost(VHostName, Limits) -> - rabbit_misc:execute_mnesia_transaction( - fun() -> - rabbit_vhost:update(VHostName, - fun(VHost) -> - rabbit_vhost:set_limits(VHost, Limits) - end) - end), - ok. - -get_limit(VirtualHost, Limit) -> - case rabbit_runtime_parameters:list(VirtualHost, <<"vhost-limits">>) of - [] -> undefined; - [Param] -> case pget(value, Param) of - undefined -> undefined; - Val -> case pget(Limit, Val) of - undefined -> undefined; - %% no limit - N when N < 0 -> undefined; - N when N >= 0 -> {ok, N} - end - end - end. diff --git a/src/rabbit_vhost_msg_store.erl b/src/rabbit_vhost_msg_store.erl deleted file mode 100644 index 8667b4d143..0000000000 --- a/src/rabbit_vhost_msg_store.erl +++ /dev/null @@ -1,68 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_vhost_msg_store). - --include("rabbit.hrl"). - --export([start/4, stop/2, client_init/5, successfully_recovered_state/2]). --export([vhost_store_pid/2]). - -start(VHost, Type, ClientRefs, StartupFunState) when is_list(ClientRefs); - ClientRefs == undefined -> - case rabbit_vhost_sup_sup:get_vhost_sup(VHost) of - {ok, VHostSup} -> - VHostDir = rabbit_vhost:msg_store_dir_path(VHost), - supervisor2:start_child(VHostSup, - {Type, {rabbit_msg_store, start_link, - [Type, VHostDir, ClientRefs, StartupFunState]}, - transient, ?MSG_STORE_WORKER_WAIT, worker, [rabbit_msg_store]}); - %% we can get here if a vhost is added and removed concurrently - %% e.g. some integration tests do it - {error, {no_such_vhost, VHost}} = E -> - rabbit_log:error("Failed to start a message store for vhost ~s: vhost no longer exists!", - [VHost]), - E - end. - -stop(VHost, Type) -> - case rabbit_vhost_sup_sup:get_vhost_sup(VHost) of - {ok, VHostSup} -> - ok = supervisor2:terminate_child(VHostSup, Type), - ok = supervisor2:delete_child(VHostSup, Type); - %% see start/4 - {error, {no_such_vhost, VHost}} -> - rabbit_log:error("Failed to stop a message store for vhost ~s: vhost no longer exists!", - [VHost]), - - ok - end. - -client_init(VHost, Type, Ref, MsgOnDiskFun, CloseFDsFun) -> - with_vhost_store(VHost, Type, fun(StorePid) -> - rabbit_msg_store:client_init(StorePid, Ref, MsgOnDiskFun, CloseFDsFun) - end). - -with_vhost_store(VHost, Type, Fun) -> - case vhost_store_pid(VHost, Type) of - no_pid -> - throw({message_store_not_started, Type, VHost}); - Pid when is_pid(Pid) -> - Fun(Pid) - end. - -vhost_store_pid(VHost, Type) -> - {ok, VHostSup} = rabbit_vhost_sup_sup:get_vhost_sup(VHost), - case supervisor2:find_child(VHostSup, Type) of - [Pid] -> Pid; - [] -> no_pid - end. - -successfully_recovered_state(VHost, Type) -> - with_vhost_store(VHost, Type, fun(StorePid) -> - rabbit_msg_store:successfully_recovered_state(StorePid) - end). diff --git a/src/rabbit_vhost_process.erl b/src/rabbit_vhost_process.erl deleted file mode 100644 index cf70d49010..0000000000 --- a/src/rabbit_vhost_process.erl +++ /dev/null @@ -1,96 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2017-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% This module implements a vhost identity process. - -%% On start this process will try to recover the vhost data and -%% processes structure (queues and message stores). -%% If recovered successfully, the process will save it's PID -%% to vhost process registry. If vhost process PID is in the registry and the -%% process is alive - the vhost is considered running. - -%% On termination, the ptocess will notify of vhost going down. - -%% The process will also check periodically if the vhost still -%% present in mnesia DB and stop the vhost supervision tree when it -%% disappears. - --module(rabbit_vhost_process). - -%% Transitional step until we can require Erlang/OTP 21 and -%% use the now recommended try/catch syntax for obtaining the stack trace. --compile(nowarn_deprecated_function). - --include("rabbit.hrl"). - --define(TICKTIME_RATIO, 4). - --behaviour(gen_server2). --export([start_link/1]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - -start_link(VHost) -> - gen_server2:start_link(?MODULE, [VHost], []). - - -init([VHost]) -> - process_flag(trap_exit, true), - rabbit_log:debug("Recovering data for VHost ~p~n", [VHost]), - try - %% Recover the vhost data and save it to vhost registry. - ok = rabbit_vhost:recover(VHost), - rabbit_vhost_sup_sup:save_vhost_process(VHost, self()), - Interval = interval(), - timer:send_interval(Interval, check_vhost), - true = erlang:garbage_collect(), - {ok, VHost} - catch _:Reason:Stacktrace -> - rabbit_amqqueue:mark_local_durable_queues_stopped(VHost), - rabbit_log:error("Unable to recover vhost ~p data. Reason ~p~n" - " Stacktrace ~p", - [VHost, Reason, Stacktrace]), - {stop, Reason} - end. - -handle_call(_,_,VHost) -> - {reply, ok, VHost}. - -handle_cast(_, VHost) -> - {noreply, VHost}. - -handle_info(check_vhost, VHost) -> - case rabbit_vhost:exists(VHost) of - true -> {noreply, VHost}; - false -> - rabbit_log:warning("Virtual host '~s' is gone. " - "Stopping its top level supervisor.", - [VHost]), - %% Stop vhost's top supervisor in a one-off process to avoid a deadlock: - %% us (a child process) waiting for supervisor shutdown and our supervisor(s) - %% waiting for us to shutdown. - spawn( - fun() -> - rabbit_vhost_sup_sup:stop_and_delete_vhost(VHost) - end), - {noreply, VHost} - end; -handle_info(_, VHost) -> - {noreply, VHost}. - -terminate(shutdown, VHost) -> - %% Notify that vhost is stopped. - rabbit_vhost:vhost_down(VHost); -terminate(_, _VHost) -> - ok. - -code_change(_OldVsn, VHost, _Extra) -> - {ok, VHost}. - -interval() -> - application:get_env(kernel, net_ticktime, 60000) * ?TICKTIME_RATIO. diff --git a/src/rabbit_vhost_sup.erl b/src/rabbit_vhost_sup.erl deleted file mode 100644 index d82d827ecf..0000000000 --- a/src/rabbit_vhost_sup.erl +++ /dev/null @@ -1,22 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2017-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_vhost_sup). - --include("rabbit.hrl"). - -%% Each vhost gets an instance of this supervisor that supervises -%% message stores and queues (via rabbit_amqqueue_sup_sup). --behaviour(supervisor2). --export([init/1]). --export([start_link/1]). - -start_link(VHost) -> - supervisor2:start_link(?MODULE, [VHost]). - -init([_VHost]) -> - {ok, {{one_for_all, 0, 1}, []}}. diff --git a/src/rabbit_vhost_sup_sup.erl b/src/rabbit_vhost_sup_sup.erl deleted file mode 100644 index c201237daa..0000000000 --- a/src/rabbit_vhost_sup_sup.erl +++ /dev/null @@ -1,271 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_vhost_sup_sup). - --include("rabbit.hrl"). - --behaviour(supervisor2). - --export([init/1]). - --export([start_link/0, start/0]). --export([init_vhost/1, - start_vhost/1, start_vhost/2, - get_vhost_sup/1, get_vhost_sup/2, - save_vhost_sup/3, - save_vhost_process/2]). --export([delete_on_all_nodes/1, start_on_all_nodes/1]). --export([is_vhost_alive/1]). --export([check/0]). - -%% Internal --export([stop_and_delete_vhost/1]). - --record(vhost_sup, {vhost, vhost_sup_pid, wrapper_pid, vhost_process_pid}). - -start() -> - case supervisor:start_child(rabbit_sup, {?MODULE, - {?MODULE, start_link, []}, - permanent, infinity, supervisor, - [?MODULE]}) of - {ok, _} -> ok; - {error, Err} -> {error, Err} - end. - -start_link() -> - supervisor2:start_link({local, ?MODULE}, ?MODULE, []). - -init([]) -> - %% This assumes that a single vhost termination should not shut down nodes - %% unless the operator opts in. - RestartStrategy = vhost_restart_strategy(), - ets:new(?MODULE, [named_table, public, {keypos, #vhost_sup.vhost}]), - {ok, {{simple_one_for_one, 0, 5}, - [{rabbit_vhost, {rabbit_vhost_sup_wrapper, start_link, []}, - RestartStrategy, ?SUPERVISOR_WAIT, supervisor, - [rabbit_vhost_sup_wrapper, rabbit_vhost_sup]}]}}. - -start_on_all_nodes(VHost) -> - %% Do not try to start a vhost on booting peer nodes - AllBooted = [Node || Node <- rabbit_nodes:all_running(), rabbit:is_booted(Node)], - Nodes = [node() | AllBooted], - Results = [{Node, start_vhost(VHost, Node)} || Node <- Nodes], - Failures = lists:filter(fun - ({_, {ok, _}}) -> false; - ({_, {error, {already_started, _}}}) -> false; - (_) -> true - end, - Results), - case Failures of - [] -> ok; - Errors -> {error, {failed_to_start_vhost_on_nodes, Errors}} - end. - -delete_on_all_nodes(VHost) -> - [ stop_and_delete_vhost(VHost, Node) || Node <- rabbit_nodes:all_running() ], - ok. - -stop_and_delete_vhost(VHost) -> - StopResult = case lookup_vhost_sup_record(VHost) of - not_found -> ok; - #vhost_sup{wrapper_pid = WrapperPid, - vhost_sup_pid = VHostSupPid} -> - case is_process_alive(WrapperPid) of - false -> ok; - true -> - rabbit_log:info("Stopping vhost supervisor ~p" - " for vhost '~s'~n", - [VHostSupPid, VHost]), - case supervisor2:terminate_child(?MODULE, WrapperPid) of - ok -> - true = ets:delete(?MODULE, VHost), - ok; - Other -> - Other - end - end - end, - ok = rabbit_vhost:delete_storage(VHost), - StopResult. - -%% We take an optimistic approach whan stopping a remote VHost supervisor. -stop_and_delete_vhost(VHost, Node) when Node == node(self()) -> - stop_and_delete_vhost(VHost); -stop_and_delete_vhost(VHost, Node) -> - case rabbit_misc:rpc_call(Node, rabbit_vhost_sup_sup, stop_and_delete_vhost, [VHost]) of - ok -> ok; - {badrpc, RpcErr} -> - rabbit_log:error("Failed to stop and delete a vhost ~p" - " on node ~p." - " Reason: ~p", - [VHost, Node, RpcErr]), - {error, RpcErr} - end. - --spec init_vhost(rabbit_types:vhost()) -> ok | {error, {no_such_vhost, rabbit_types:vhost()}}. -init_vhost(VHost) -> - case start_vhost(VHost) of - {ok, _} -> ok; - {error, {already_started, _}} -> - rabbit_log:warning( - "Attempting to start an already started vhost '~s'.", - [VHost]), - ok; - {error, {no_such_vhost, VHost}} -> - {error, {no_such_vhost, VHost}}; - {error, Reason} -> - case vhost_restart_strategy() of - permanent -> - rabbit_log:error( - "Unable to initialize vhost data store for vhost '~s'." - " Reason: ~p", - [VHost, Reason]), - throw({error, Reason}); - transient -> - rabbit_log:warning( - "Unable to initialize vhost data store for vhost '~s'." - " The vhost will be stopped for this node. " - " Reason: ~p", - [VHost, Reason]), - ok - end - end. - --type vhost_error() :: {no_such_vhost, rabbit_types:vhost()} | - {vhost_supervisor_not_running, rabbit_types:vhost()}. - --spec get_vhost_sup(rabbit_types:vhost(), node()) -> {ok, pid()} | {error, vhost_error() | term()}. -get_vhost_sup(VHost, Node) -> - case rabbit_misc:rpc_call(Node, rabbit_vhost_sup_sup, get_vhost_sup, [VHost]) of - {ok, Pid} when is_pid(Pid) -> - {ok, Pid}; - {error, Err} -> - {error, Err}; - {badrpc, RpcErr} -> - {error, RpcErr} - end. - --spec get_vhost_sup(rabbit_types:vhost()) -> {ok, pid()} | {error, vhost_error()}. -get_vhost_sup(VHost) -> - case rabbit_vhost:exists(VHost) of - false -> - {error, {no_such_vhost, VHost}}; - true -> - case vhost_sup_pid(VHost) of - no_pid -> - {error, {vhost_supervisor_not_running, VHost}}; - {ok, Pid} when is_pid(Pid) -> - {ok, Pid} - end - end. - --spec start_vhost(rabbit_types:vhost(), node()) -> {ok, pid()} | {error, term()}. -start_vhost(VHost, Node) -> - case rabbit_misc:rpc_call(Node, rabbit_vhost_sup_sup, start_vhost, [VHost]) of - {ok, Pid} -> {ok, Pid}; - {error, Err} -> {error, Err}; - {badrpc, RpcErr} -> {error, RpcErr} - end. - --spec start_vhost(rabbit_types:vhost()) -> {ok, pid()} | {error, term()}. -start_vhost(VHost) -> - case rabbit_vhost:exists(VHost) of - false -> {error, {no_such_vhost, VHost}}; - true -> - case whereis(?MODULE) of - Pid when is_pid(Pid) -> - supervisor2:start_child(?MODULE, [VHost]); - undefined -> - {error, rabbit_vhost_sup_sup_not_running} - end - end. - --spec is_vhost_alive(rabbit_types:vhost()) -> boolean(). -is_vhost_alive(VHost) -> -%% A vhost is considered alive if it's supervision tree is alive and -%% saved in the ETS table - case lookup_vhost_sup_record(VHost) of - #vhost_sup{wrapper_pid = WrapperPid, - vhost_sup_pid = VHostSupPid, - vhost_process_pid = VHostProcessPid} - when is_pid(WrapperPid), - is_pid(VHostSupPid), - is_pid(VHostProcessPid) -> - is_process_alive(WrapperPid) - andalso - is_process_alive(VHostSupPid) - andalso - is_process_alive(VHostProcessPid); - _ -> false - end. - - --spec save_vhost_sup(rabbit_types:vhost(), pid(), pid()) -> ok. -save_vhost_sup(VHost, WrapperPid, VHostPid) -> - true = ets:insert(?MODULE, #vhost_sup{vhost = VHost, - vhost_sup_pid = VHostPid, - wrapper_pid = WrapperPid}), - ok. - --spec save_vhost_process(rabbit_types:vhost(), pid()) -> ok. -save_vhost_process(VHost, VHostProcessPid) -> - true = ets:update_element(?MODULE, VHost, - {#vhost_sup.vhost_process_pid, VHostProcessPid}), - ok. - --spec lookup_vhost_sup_record(rabbit_types:vhost()) -> #vhost_sup{} | not_found. -lookup_vhost_sup_record(VHost) -> - case ets:info(?MODULE, name) of - ?MODULE -> - case ets:lookup(?MODULE, VHost) of - [] -> not_found; - [#vhost_sup{} = VHostSup] -> VHostSup - end; - undefined -> not_found - end. - --spec vhost_sup_pid(rabbit_types:vhost()) -> no_pid | {ok, pid()}. -vhost_sup_pid(VHost) -> - case lookup_vhost_sup_record(VHost) of - not_found -> - no_pid; - #vhost_sup{vhost_sup_pid = Pid} = VHostSup -> - case erlang:is_process_alive(Pid) of - true -> {ok, Pid}; - false -> - ets:delete_object(?MODULE, VHostSup), - no_pid - end - end. - -vhost_restart_strategy() -> - %% This assumes that a single vhost termination should not shut down nodes - %% unless the operator opts in. - case application:get_env(rabbit, vhost_restart_strategy, continue) of - continue -> transient; - stop_node -> permanent; - transient -> transient; - permanent -> permanent - end. - -check() -> - VHosts = rabbit_vhost:list_names(), - lists:filter( - fun(V) -> - case rabbit_vhost_sup_sup:get_vhost_sup(V) of - {ok, Sup} -> - MsgStores = [Pid || {Name, Pid, _, _} <- supervisor:which_children(Sup), - lists:member(Name, [msg_store_persistent, - msg_store_transient])], - not is_vhost_alive(V) orelse (not lists:all(fun(P) -> - erlang:is_process_alive(P) - end, MsgStores)); - {error, _} -> - true - end - end, VHosts). diff --git a/src/rabbit_vhost_sup_wrapper.erl b/src/rabbit_vhost_sup_wrapper.erl deleted file mode 100644 index ed239ade69..0000000000 --- a/src/rabbit_vhost_sup_wrapper.erl +++ /dev/null @@ -1,57 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2017-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% This module is a wrapper around vhost supervisor to -%% provide exactly once restart semantics. - --module(rabbit_vhost_sup_wrapper). - --include("rabbit.hrl"). - --behaviour(supervisor2). --export([init/1]). --export([start_link/1]). --export([start_vhost_sup/1]). - -start_link(VHost) -> - %% Using supervisor, because supervisor2 does not stop a started child when - %% another one fails to start. Bug? - case rabbit_vhost_sup_sup:get_vhost_sup(VHost) of - {ok, Pid} -> - {error, {already_started, Pid}}; - {error, _} -> - supervisor:start_link(?MODULE, [VHost]) - end. - -init([VHost]) -> - %% 2 restarts in 5 minutes. One per message store. - {ok, {{one_for_all, 2, 300}, - [ - %% rabbit_vhost_sup is an empty supervisor container for - %% all data processes. - {rabbit_vhost_sup, - {rabbit_vhost_sup_wrapper, start_vhost_sup, [VHost]}, - permanent, infinity, supervisor, - [rabbit_vhost_sup]}, - %% rabbit_vhost_process is a vhost identity process, which - %% is responsible for data recovery and vhost aliveness status. - %% See the module comments for more info. - {rabbit_vhost_process, - {rabbit_vhost_process, start_link, [VHost]}, - permanent, ?WORKER_WAIT, worker, - [rabbit_vhost_process]}]}}. - - -start_vhost_sup(VHost) -> - case rabbit_vhost_sup:start_link(VHost) of - {ok, Pid} -> - %% Save vhost sup record with wrapper pid and vhost sup pid. - ok = rabbit_vhost_sup_sup:save_vhost_sup(VHost, self(), Pid), - {ok, Pid}; - Other -> - Other - end. diff --git a/src/rabbit_vm.erl b/src/rabbit_vm.erl deleted file mode 100644 index b014e090c5..0000000000 --- a/src/rabbit_vm.erl +++ /dev/null @@ -1,427 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(rabbit_vm). - --export([memory/0, binary/0, ets_tables_memory/1]). - --define(MAGIC_PLUGINS, ["cowboy", "ranch", "sockjs"]). - -%%---------------------------------------------------------------------------- - --spec memory() -> rabbit_types:infos(). - -memory() -> - All = interesting_sups(), - {Sums, _Other} = sum_processes( - lists:append(All), distinguishers(), [memory]), - - [Qs, QsSlave, Qqs, Ssqs, Srqs, SCoor, ConnsReader, ConnsWriter, ConnsChannel, - ConnsOther, MsgIndexProc, MgmtDbProc, Plugins] = - [aggregate(Names, Sums, memory, fun (X) -> X end) - || Names <- distinguished_interesting_sups()], - - MnesiaETS = mnesia_memory(), - MsgIndexETS = ets_memory(msg_stores()), - MetricsETS = ets_memory([rabbit_metrics]), - QuorumETS = ets_memory([ra_log_ets]), - MetricsProc = try - [{_, M}] = process_info(whereis(rabbit_metrics), [memory]), - M - catch - error:badarg -> - 0 - end, - MgmtDbETS = ets_memory([rabbit_mgmt_storage]), - [{total, ErlangTotal}, - {processes, Processes}, - {ets, ETS}, - {atom, Atom}, - {binary, Bin}, - {code, Code}, - {system, System}] = - erlang:memory([total, processes, ets, atom, binary, code, system]), - - Strategy = vm_memory_monitor:get_memory_calculation_strategy(), - Allocated = recon_alloc:memory(allocated), - Rss = vm_memory_monitor:get_rss_memory(), - - AllocatedUnused = max(Allocated - ErlangTotal, 0), - OSReserved = max(Rss - Allocated, 0), - - OtherProc = Processes - - ConnsReader - ConnsWriter - ConnsChannel - ConnsOther - - Qs - QsSlave - Qqs - Ssqs - Srqs - SCoor - MsgIndexProc - Plugins - - MgmtDbProc - MetricsProc, - - [ - %% Connections - {connection_readers, ConnsReader}, - {connection_writers, ConnsWriter}, - {connection_channels, ConnsChannel}, - {connection_other, ConnsOther}, - - %% Queues - {queue_procs, Qs}, - {queue_slave_procs, QsSlave}, - {quorum_queue_procs, Qqs}, - {stream_queue_procs, Ssqs}, - {stream_queue_replica_reader_procs, Srqs}, - {stream_queue_coordinator_procs, SCoor}, - - %% Processes - {plugins, Plugins}, - {other_proc, lists:max([0, OtherProc])}, %% [1] - - %% Metrics - {metrics, MetricsETS + MetricsProc}, - {mgmt_db, MgmtDbETS + MgmtDbProc}, - - %% ETS - {mnesia, MnesiaETS}, - {quorum_ets, QuorumETS}, - {other_ets, ETS - MnesiaETS - MetricsETS - MgmtDbETS - MsgIndexETS - QuorumETS}, - - %% Messages (mostly, some binaries are not messages) - {binary, Bin}, - {msg_index, MsgIndexETS + MsgIndexProc}, - - %% System - {code, Code}, - {atom, Atom}, - {other_system, System - ETS - Bin - Code - Atom}, - {allocated_unused, AllocatedUnused}, - {reserved_unallocated, OSReserved}, - {strategy, Strategy}, - {total, [{erlang, ErlangTotal}, - {rss, Rss}, - {allocated, Allocated}]} - ]. -%% [1] - erlang:memory(processes) can be less than the sum of its -%% parts. Rather than display something nonsensical, just silence any -%% claims about negative memory. See -%% http://erlang.org/pipermail/erlang-questions/2012-September/069320.html - --spec binary() -> rabbit_types:infos(). - -binary() -> - All = interesting_sups(), - {Sums, Rest} = - sum_processes( - lists:append(All), - fun (binary, Info, Acc) -> - lists:foldl(fun ({Ptr, Sz, _RefCnt}, Acc0) -> - sets:add_element({Ptr, Sz}, Acc0) - end, Acc, Info) - end, distinguishers(), [{binary, sets:new()}]), - [Other, Qs, QsSlave, Qqs, Ssqs, Srqs, Scoor, ConnsReader, ConnsWriter, - ConnsChannel, ConnsOther, MsgIndexProc, MgmtDbProc, Plugins] = - [aggregate(Names, [{other, Rest} | Sums], binary, fun sum_binary/1) - || Names <- [[other] | distinguished_interesting_sups()]], - [{connection_readers, ConnsReader}, - {connection_writers, ConnsWriter}, - {connection_channels, ConnsChannel}, - {connection_other, ConnsOther}, - {queue_procs, Qs}, - {queue_slave_procs, QsSlave}, - {quorum_queue_procs, Qqs}, - {stream_queue_procs, Ssqs}, - {stream_queue_replica_reader_procs, Srqs}, - {stream_queue_coordinator_procs, Scoor}, - {plugins, Plugins}, - {mgmt_db, MgmtDbProc}, - {msg_index, MsgIndexProc}, - {other, Other}]. - -%%---------------------------------------------------------------------------- - -mnesia_memory() -> - case mnesia:system_info(is_running) of - yes -> lists:sum([bytes(mnesia:table_info(Tab, memory)) || - Tab <- mnesia:system_info(tables)]); - _ -> 0 - end. - -ets_memory(Owners) -> - lists:sum([V || {_K, V} <- ets_tables_memory(Owners)]). - --spec ets_tables_memory(Owners) -> rabbit_types:infos() - when Owners :: all | OwnerProcessName | [OwnerProcessName], - OwnerProcessName :: atom(). - -ets_tables_memory(all) -> - [{ets:info(T, name), bytes(ets:info(T, memory))} - || T <- ets:all(), - is_atom(T)]; -ets_tables_memory(OwnerName) when is_atom(OwnerName) -> - ets_tables_memory([OwnerName]); -ets_tables_memory(Owners) when is_list(Owners) -> - OwnerPids = lists:map(fun(O) when is_pid(O) -> O; - (O) when is_atom(O) -> whereis(O) - end, - Owners), - [{ets:info(T, name), bytes(ets:info(T, memory))} - || T <- ets:all(), - lists:member(ets:info(T, owner), OwnerPids)]. - -bytes(Words) -> try - Words * erlang:system_info(wordsize) - catch - _:_ -> 0 - end. - -interesting_sups() -> - [queue_sups(), quorum_sups(), stream_server_sups(), stream_reader_sups(), - conn_sups() | interesting_sups0()]. - -queue_sups() -> - all_vhosts_children(rabbit_amqqueue_sup_sup). - -quorum_sups() -> - %% TODO: in the future not all ra servers may be queues and we needs - %% some way to filter this - case whereis(ra_server_sup_sup) of - undefined -> - []; - _ -> - [Pid || {_, Pid, _, _} <- - supervisor:which_children(ra_server_sup_sup)] - end. - -stream_server_sups() -> [osiris_server_sup]. -stream_reader_sups() -> [osiris_replica_reader_sup]. - -msg_stores() -> - all_vhosts_children(msg_store_transient) - ++ - all_vhosts_children(msg_store_persistent). - -all_vhosts_children(Name) -> - case whereis(rabbit_vhost_sup_sup) of - undefined -> []; - Pid when is_pid(Pid) -> - lists:filtermap( - fun({_, VHostSupWrapper, _, _}) -> - case supervisor2:find_child(VHostSupWrapper, - rabbit_vhost_sup) of - [] -> false; - [VHostSup] -> - case supervisor2:find_child(VHostSup, Name) of - [QSup] -> {true, QSup}; - [] -> false - end - end - end, - supervisor:which_children(rabbit_vhost_sup_sup)) - end. - -interesting_sups0() -> - MsgIndexProcs = msg_stores(), - MgmtDbProcs = [rabbit_mgmt_sup_sup], - PluginProcs = plugin_sups(), - [MsgIndexProcs, MgmtDbProcs, PluginProcs]. - -conn_sups() -> - Ranches = lists:flatten(ranch_server_sups()), - [amqp_sup|Ranches]. - -ranch_server_sups() -> - try - ets:match(ranch_server, {{conns_sup, '_'}, '$1'}) - catch - %% Ranch ETS table doesn't exist yet - error:badarg -> [] - end. - -with(Sups, With) -> [{Sup, With} || Sup <- Sups]. - -distinguishers() -> with(queue_sups(), fun queue_type/1) ++ - with(conn_sups(), fun conn_type/1) ++ - with(quorum_sups(), fun ra_type/1). - -distinguished_interesting_sups() -> - [ - with(queue_sups(), master), - with(queue_sups(), slave), - with(quorum_sups(), quorum), - stream_server_sups(), - stream_reader_sups(), - with(quorum_sups(), stream), - with(conn_sups(), reader), - with(conn_sups(), writer), - with(conn_sups(), channel), - with(conn_sups(), other)] - ++ interesting_sups0(). - -plugin_sups() -> - lists:append([plugin_sup(App) || - {App, _, _} <- rabbit_misc:which_applications(), - is_plugin(atom_to_list(App))]). - -plugin_sup(App) -> - case application_controller:get_master(App) of - undefined -> []; - Master -> case application_master:get_child(Master) of - {Pid, _} when is_pid(Pid) -> [process_name(Pid)]; - Pid when is_pid(Pid) -> [process_name(Pid)]; - _ -> [] - end - end. - -process_name(Pid) -> - case process_info(Pid, registered_name) of - {registered_name, Name} -> Name; - _ -> Pid - end. - -is_plugin("rabbitmq_" ++ _) -> true; -is_plugin(App) -> lists:member(App, ?MAGIC_PLUGINS). - -aggregate(Names, Sums, Key, Fun) -> - lists:sum([extract(Name, Sums, Key, Fun) || Name <- Names]). - -extract(Name, Sums, Key, Fun) -> - case keyfind(Name, Sums) of - {value, Accs} -> Fun(keyfetch(Key, Accs)); - false -> 0 - end. - -sum_binary(Set) -> - sets:fold(fun({_Pt, Sz}, Acc) -> Acc + Sz end, 0, Set). - -queue_type(PDict) -> - case keyfind(process_name, PDict) of - {value, {rabbit_mirror_queue_slave, _}} -> slave; - _ -> master - end. - -conn_type(PDict) -> - case keyfind(process_name, PDict) of - {value, {rabbit_reader, _}} -> reader; - {value, {rabbit_writer, _}} -> writer; - {value, {rabbit_channel, _}} -> channel; - _ -> other - end. - -ra_type(PDict) -> - case keyfind('$rabbit_vm_category', PDict) of - {value, rabbit_stream_coordinator} -> stream; - _ -> quorum - end. - -%%---------------------------------------------------------------------------- - -%% NB: this code is non-rabbit specific. - --type process() :: pid() | atom(). --type info_key() :: atom(). --type info_value() :: any(). --type info_item() :: {info_key(), info_value()}. --type accumulate() :: fun ((info_key(), info_value(), info_value()) -> - info_value()). --type distinguisher() :: fun (([{term(), term()}]) -> atom()). --type distinguishers() :: [{info_key(), distinguisher()}]. --spec sum_processes([process()], distinguishers(), [info_key()]) -> - {[{process(), [info_item()]}], [info_item()]}. --spec sum_processes([process()], accumulate(), distinguishers(), - [info_item()]) -> - {[{process(), [info_item()]}], [info_item()]}. - -sum_processes(Names, Distinguishers, Items) -> - sum_processes(Names, fun (_, X, Y) -> X + Y end, Distinguishers, - [{Item, 0} || Item <- Items]). - -%% summarize the process_info of all processes based on their -%% '$ancestor' hierarchy, recorded in their process dictionary. -%% -%% The function takes -%% -%% 1) a list of names/pids of processes that are accumulation points -%% in the hierarchy. -%% -%% 2) a function that aggregates individual info items -taking the -%% info item key, value and accumulated value as the input and -%% producing a new accumulated value. -%% -%% 3) a list of info item key / initial accumulator value pairs. -%% -%% The process_info of a process is accumulated at the nearest of its -%% ancestors that is mentioned in the first argument, or, if no such -%% ancestor exists or the ancestor information is absent, in a special -%% 'other' bucket. -%% -%% The result is a pair consisting of -%% -%% 1) a k/v list, containing for each of the accumulation names/pids a -%% list of info items, containing the accumulated data, and -%% -%% 2) the 'other' bucket - a list of info items containing the -%% accumulated data of all processes with no matching ancestors -%% -%% Note that this function operates on names as well as pids, but -%% these must match whatever is contained in the '$ancestor' process -%% dictionary entry. Generally that means for all registered processes -%% the name should be used. -sum_processes(Names, Fun, Distinguishers, Acc0) -> - Items = [Item || {Item, _Blank0} <- Acc0], - {NameAccs, OtherAcc} = - lists:foldl( - fun (Pid, Acc) -> - InfoItems = [registered_name, dictionary | Items], - case process_info(Pid, InfoItems) of - undefined -> - Acc; - [{registered_name, RegName}, {dictionary, D} | Vals] -> - %% see docs for process_info/2 for the - %% special handling of 'registered_name' - %% info items - Extra = case RegName of - [] -> []; - N -> [N] - end, - Name0 = find_ancestor(Extra, D, Names), - Name = case keyfind(Name0, Distinguishers) of - {value, DistFun} -> {Name0, DistFun(D)}; - false -> Name0 - end, - accumulate( - Name, Fun, orddict:from_list(Vals), Acc, Acc0) - end - end, {orddict:new(), Acc0}, processes()), - %% these conversions aren't strictly necessary; we do them simply - %% for the sake of encapsulating the representation. - {[{Name, orddict:to_list(Accs)} || - {Name, Accs} <- orddict:to_list(NameAccs)], - orddict:to_list(OtherAcc)}. - -find_ancestor(Extra, D, Names) -> - Ancestors = case keyfind('$ancestors', D) of - {value, Ancs} -> Ancs; - false -> [] - end, - case lists:splitwith(fun (A) -> not lists:member(A, Names) end, - Extra ++ Ancestors) of - {_, []} -> undefined; - {_, [Name | _]} -> Name - end. - -accumulate(undefined, Fun, ValsDict, {NameAccs, OtherAcc}, _Acc0) -> - {NameAccs, orddict:merge(Fun, ValsDict, OtherAcc)}; -accumulate(Name, Fun, ValsDict, {NameAccs, OtherAcc}, Acc0) -> - F = fun (NameAcc) -> orddict:merge(Fun, ValsDict, NameAcc) end, - {case orddict:is_key(Name, NameAccs) of - true -> orddict:update(Name, F, NameAccs); - false -> orddict:store( Name, F(Acc0), NameAccs) - end, OtherAcc}. - -keyfetch(K, L) -> {value, {_, V}} = lists:keysearch(K, 1, L), - V. - -keyfind(K, L) -> case lists:keysearch(K, 1, L) of - {value, {_, V}} -> {value, V}; - false -> false - end. diff --git a/src/supervised_lifecycle.erl b/src/supervised_lifecycle.erl deleted file mode 100644 index 0e1bb9b5c8..0000000000 --- a/src/supervised_lifecycle.erl +++ /dev/null @@ -1,53 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - -%% Invoke callbacks on startup and termination. -%% -%% Simply hook this process into a supervision hierarchy, to have the -%% callbacks invoked at a precise point during the establishment and -%% teardown of that hierarchy, respectively. -%% -%% Or launch the process independently, and link to it, to have the -%% callbacks invoked on startup and when the linked process -%% terminates, respectively. - --module(supervised_lifecycle). - --behavior(gen_server). - --export([start_link/3]). - -%% gen_server callbacks --export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, - code_change/3]). - -%%---------------------------------------------------------------------------- - --spec start_link(atom(), rabbit_types:mfargs(), rabbit_types:mfargs()) -> - rabbit_types:ok_pid_or_error(). - -start_link(Name, StartMFA, StopMFA) -> - gen_server:start_link({local, Name}, ?MODULE, [StartMFA, StopMFA], []). - -%%---------------------------------------------------------------------------- - -init([{M, F, A}, StopMFA]) -> - process_flag(trap_exit, true), - apply(M, F, A), - {ok, StopMFA}. - -handle_call(_Request, _From, State) -> {noreply, State}. - -handle_cast(_Msg, State) -> {noreply, State}. - -handle_info(_Info, State) -> {noreply, State}. - -terminate(_Reason, {M, F, A}) -> - apply(M, F, A), - ok. - -code_change(_OldVsn, State, _Extra) -> {ok, State}. diff --git a/src/tcp_listener.erl b/src/tcp_listener.erl deleted file mode 100644 index 93c24ab397..0000000000 --- a/src/tcp_listener.erl +++ /dev/null @@ -1,90 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(tcp_listener). - -%% Represents a running TCP listener (a process that listens for inbound -%% TCP or TLS connections). Every protocol supported typically has one -%% or two listeners, plain TCP and (optionally) TLS, but there can -%% be more, e.g. when multiple network interfaces are involved. -%% -%% A listener has 6 properties (is a tuple of 6): -%% -%% * IP address -%% * Port -%% * Node -%% * Label (human-friendly name, e.g. AMQP 0-9-1) -%% * Startup callback -%% * Shutdown callback -%% -%% Listeners use Ranch in embedded mode to accept and "bridge" client -%% connections with protocol entry points such as rabbit_reader. -%% -%% Listeners are tracked in a Mnesia table so that they can be -%% -%% * Shut down -%% * Listed (e.g. in the management UI) -%% -%% Every tcp_listener process has callbacks that are executed on start -%% and termination. Those must take care of listener registration -%% among other things. -%% -%% Listeners are supervised by tcp_listener_sup (one supervisor per protocol). -%% -%% See also rabbit_networking and tcp_listener_sup. - --behaviour(gen_server). - --export([start_link/5]). - --export([init/1, handle_call/3, handle_cast/2, handle_info/2, - terminate/2, code_change/3]). - --record(state, {on_startup, on_shutdown, label, ip, port}). - -%%---------------------------------------------------------------------------- - --type mfargs() :: {atom(), atom(), [any()]}. - --spec start_link - (inet:ip_address(), inet:port_number(), - mfargs(), mfargs(), string()) -> - rabbit_types:ok_pid_or_error(). - -start_link(IPAddress, Port, - OnStartup, OnShutdown, Label) -> - gen_server:start_link( - ?MODULE, {IPAddress, Port, - OnStartup, OnShutdown, Label}, []). - -%%-------------------------------------------------------------------- - -init({IPAddress, Port, {M,F,A} = OnStartup, OnShutdown, Label}) -> - process_flag(trap_exit, true), - error_logger:info_msg( - "started ~s on ~s:~p~n", - [Label, rabbit_misc:ntoab(IPAddress), Port]), - apply(M, F, A ++ [IPAddress, Port]), - {ok, #state{on_startup = OnStartup, on_shutdown = OnShutdown, - label = Label, ip=IPAddress, port=Port}}. - -handle_call(_Request, _From, State) -> - {noreply, State}. - -handle_cast(_Msg, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, #state{on_shutdown = {M,F,A}, label=Label, ip=IPAddress, port=Port}) -> - error_logger:info_msg("stopped ~s on ~s:~p~n", - [Label, rabbit_misc:ntoab(IPAddress), Port]), - apply(M, F, A ++ [IPAddress, Port]). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. diff --git a/src/tcp_listener_sup.erl b/src/tcp_listener_sup.erl deleted file mode 100644 index 82128bb2af..0000000000 --- a/src/tcp_listener_sup.erl +++ /dev/null @@ -1,54 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(tcp_listener_sup). - -%% Supervises TCP listeners. There is a separate supervisor for every -%% protocol. In case of AMQP 0-9-1, it resides under rabbit_sup. Plugins -%% that provide protocol support (e.g. STOMP) have an instance of this supervisor in their -%% app supervision tree. -%% -%% See also rabbit_networking and tcp_listener. - --behaviour(supervisor). - --export([start_link/10]). --export([init/1]). - --type mfargs() :: {atom(), atom(), [any()]}. - --spec start_link - (inet:ip_address(), inet:port_number(), module(), [gen_tcp:listen_option()], - module(), any(), mfargs(), mfargs(), integer(), string()) -> - rabbit_types:ok_pid_or_error(). - -start_link(IPAddress, Port, Transport, SocketOpts, ProtoSup, ProtoOpts, OnStartup, OnShutdown, - ConcurrentAcceptorCount, Label) -> - supervisor:start_link( - ?MODULE, {IPAddress, Port, Transport, SocketOpts, ProtoSup, ProtoOpts, OnStartup, OnShutdown, - ConcurrentAcceptorCount, Label}). - -init({IPAddress, Port, Transport, SocketOpts, ProtoSup, ProtoOpts, OnStartup, OnShutdown, - ConcurrentAcceptorCount, Label}) -> - {ok, AckTimeout} = application:get_env(rabbit, ssl_handshake_timeout), - MaxConnections = rabbit_misc:get_env(rabbit, connection_max, infinity), - RanchListenerOpts = #{ - num_acceptors => ConcurrentAcceptorCount, - max_connections => MaxConnections, - handshake_timeout => AckTimeout, - connection_type => supervisor, - socket_opts => [{ip, IPAddress}, - {port, Port} | - SocketOpts] - }, - Flags = {one_for_all, 10, 10}, - OurChildSpecStart = {tcp_listener, start_link, [IPAddress, Port, OnStartup, OnShutdown, Label]}, - OurChildSpec = {tcp_listener, OurChildSpecStart, transient, 16#ffffffff, worker, [tcp_listener]}, - RanchChildSpec = ranch:child_spec(rabbit_networking:ranch_ref(IPAddress, Port), - Transport, RanchListenerOpts, - ProtoSup, ProtoOpts), - {ok, {Flags, [RanchChildSpec, OurChildSpec]}}. diff --git a/src/term_to_binary_compat.erl b/src/term_to_binary_compat.erl deleted file mode 100644 index 327a846d1f..0000000000 --- a/src/term_to_binary_compat.erl +++ /dev/null @@ -1,15 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2017-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(term_to_binary_compat). - --include("rabbit.hrl"). - --export([term_to_binary_1/1]). - -term_to_binary_1(Term) -> - term_to_binary(Term, [{minor_version, 1}]). diff --git a/src/vhost.erl b/src/vhost.erl deleted file mode 100644 index ca704183a0..0000000000 --- a/src/vhost.erl +++ /dev/null @@ -1,172 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(vhost). - --include_lib("rabbit_common/include/rabbit.hrl"). --include("vhost.hrl"). - --export([ - new/2, - new/3, - fields/0, - fields/1, - info_keys/0, - record_version_to_use/0, - upgrade/1, - upgrade_to/2, - pattern_match_all/0, - get_name/1, - get_limits/1, - get_metadata/1, - get_description/1, - get_tags/1, - set_limits/2 -]). - --define(record_version, vhost_v2). - --type(name() :: binary()). - --type(metadata_key() :: atom()). - --type(metadata() :: #{description => binary(), - tags => [atom()], - metadata_key() => any()} | undefined). - --type vhost() :: vhost_v1:vhost_v1() | vhost_v2(). - --record(vhost, { - %% name as a binary - virtual_host :: name() | '_', - %% proplist of limits configured, if any - limits :: list() | '_', - metadata :: metadata() | '_' -}). - --type vhost_v2() :: #vhost{ - virtual_host :: name(), - limits :: list(), - metadata :: metadata() - }. - --type vhost_pattern() :: vhost_v1:vhost_v1_pattern() | - vhost_v2_pattern(). --type vhost_v2_pattern() :: #vhost{ - virtual_host :: name() | '_', - limits :: '_', - metadata :: '_' - }. - --export_type([name/0, - metadata_key/0, - metadata/0, - vhost/0, - vhost_v2/0, - vhost_pattern/0, - vhost_v2_pattern/0]). - --spec new(name(), list()) -> vhost(). -new(Name, Limits) -> - case record_version_to_use() of - ?record_version -> - #vhost{virtual_host = Name, limits = Limits}; - _ -> - vhost_v1:new(Name, Limits) - end. - --spec new(name(), list(), map()) -> vhost(). -new(Name, Limits, Metadata) -> - case record_version_to_use() of - ?record_version -> - #vhost{virtual_host = Name, limits = Limits, metadata = Metadata}; - _ -> - vhost_v1:new(Name, Limits) - end. - --spec record_version_to_use() -> vhost_v1 | vhost_v2. - -record_version_to_use() -> - case rabbit_feature_flags:is_enabled(virtual_host_metadata) of - true -> ?record_version; - false -> vhost_v1:record_version_to_use() - end. - --spec upgrade(vhost()) -> vhost(). - -upgrade(#vhost{} = VHost) -> VHost; -upgrade(OldVHost) -> upgrade_to(record_version_to_use(), OldVHost). - --spec upgrade_to -(vhost_v2, vhost()) -> vhost_v2(); -(vhost_v1, vhost_v1:vhost_v1()) -> vhost_v1:vhost_v1(). - -upgrade_to(?record_version, #vhost{} = VHost) -> - VHost; -upgrade_to(?record_version, OldVHost) -> - Fields = erlang:tuple_to_list(OldVHost) ++ [#{description => <<"">>, tags => []}], - #vhost{} = erlang:list_to_tuple(Fields); -upgrade_to(Version, OldVHost) -> - vhost_v1:upgrade_to(Version, OldVHost). - - -fields() -> - case record_version_to_use() of - ?record_version -> fields(?record_version); - _ -> vhost_v1:fields() - end. - -fields(?record_version) -> record_info(fields, vhost); -fields(Version) -> vhost_v1:fields(Version). - -info_keys() -> - case record_version_to_use() of - %% note: this reports description and tags separately even though - %% they are stored in the metadata map. MK. - ?record_version -> [name, description, tags, metadata, tracing, cluster_state]; - _ -> vhost_v1:info_keys() - end. - --spec pattern_match_all() -> vhost_pattern(). - -pattern_match_all() -> - case record_version_to_use() of - ?record_version -> #vhost{_ = '_'}; - _ -> vhost_v1:pattern_match_all() - end. - --spec get_name(vhost()) -> name(). -get_name(#vhost{virtual_host = Value}) -> Value; -get_name(VHost) -> vhost_v1:get_name(VHost). - --spec get_limits(vhost()) -> list(). -get_limits(#vhost{limits = Value}) -> Value; -get_limits(VHost) -> vhost_v1:get_limits(VHost). - --spec get_metadata(vhost()) -> metadata(). -get_metadata(#vhost{metadata = Value}) -> Value; -get_metadata(VHost) -> vhost_v1:get_metadata(VHost). - --spec get_description(vhost()) -> binary(). -get_description(#vhost{} = VHost) -> - maps:get(description, get_metadata(VHost), undefined); -get_description(VHost) -> - vhost_v1:get_description(VHost). - --spec get_tags(vhost()) -> [atom()]. -get_tags(#vhost{} = VHost) -> - maps:get(tags, get_metadata(VHost), undefined); -get_tags(VHost) -> - vhost_v1:get_tags(VHost). - -set_limits(VHost, Value) -> - case record_version_to_use() of - ?record_version -> - VHost#vhost{limits = Value}; - _ -> - vhost_v1:set_limits(VHost, Value) - end. diff --git a/src/vhost_v1.erl b/src/vhost_v1.erl deleted file mode 100644 index 5b53eb148a..0000000000 --- a/src/vhost_v1.erl +++ /dev/null @@ -1,106 +0,0 @@ -%% This Source Code Form is subject to the terms of the Mozilla Public -%% License, v. 2.0. If a copy of the MPL was not distributed with this -%% file, You can obtain one at https://mozilla.org/MPL/2.0/. -%% -%% Copyright (c) 2018-2020 VMware, Inc. or its affiliates. All rights reserved. -%% - --module(vhost_v1). - --include("vhost.hrl"). - --export([new/2, - new/3, - upgrade/1, - upgrade_to/2, - fields/0, - fields/1, - info_keys/0, - field_name/0, - record_version_to_use/0, - pattern_match_all/0, - get_name/1, - get_limits/1, - get_metadata/1, - get_description/1, - get_tags/1, - set_limits/2 -]). - --define(record_version, ?MODULE). - -%% Represents a vhost. -%% -%% Historically this record had 2 arguments although the 2nd -%% was never used (`dummy`, always undefined). This is because -%% single field records were/are illegal in OTP. -%% -%% As of 3.6.x, the second argument is vhost limits, -%% which is actually used and has the same default. -%% Nonetheless, this required a migration, see rabbit_upgrade_functions. - --record(vhost, { - %% name as a binary - virtual_host :: vhost:name() | '_', - %% proplist of limits configured, if any - limits :: list() | '_'}). - --type vhost() :: vhost_v1(). --type vhost_v1() :: #vhost{ - virtual_host :: vhost:name(), - limits :: list() - }. - --export_type([vhost/0, - vhost_v1/0, - vhost_pattern/0, - vhost_v1_pattern/0]). - - --spec new(vhost:name(), list()) -> vhost(). -new(Name, Limits) -> - #vhost{virtual_host = Name, limits = Limits}. - --spec new(vhost:name(), list(), map()) -> vhost(). -new(Name, Limits, _Metadata) -> - #vhost{virtual_host = Name, limits = Limits}. - - --spec record_version_to_use() -> vhost_v1. -record_version_to_use() -> - ?record_version. - --spec upgrade(vhost()) -> vhost(). -upgrade(#vhost{} = VHost) -> VHost. - --spec upgrade_to(vhost_v1, vhost()) -> vhost(). -upgrade_to(?record_version, #vhost{} = VHost) -> - VHost. - -fields() -> fields(?record_version). - -fields(?record_version) -> record_info(fields, vhost). - -field_name() -> #vhost.virtual_host. - -info_keys() -> [name, tracing, cluster_state]. - --type vhost_pattern() :: vhost_v1_pattern(). --type vhost_v1_pattern() :: #vhost{ - virtual_host :: vhost:name() | '_', - limits :: '_' - }. - --spec pattern_match_all() -> vhost_pattern(). - -pattern_match_all() -> #vhost{_ = '_'}. - -get_name(#vhost{virtual_host = Value}) -> Value. -get_limits(#vhost{limits = Value}) -> Value. - -get_metadata(_VHost) -> undefined. -get_description(_VHost) -> undefined. -get_tags(_VHost) -> undefined. - -set_limits(VHost, Value) -> - VHost#vhost{limits = Value}. |