Merge branch 'master' into rabbitmq-server-567

16 files changed, 294 insertions, 89 deletions

diff --git a/LICENSE b/LICENSE
index 86b189fe04..8964048501 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,8 +1,5 @@
 This package, the RabbitMQ server is licensed under the MPL. For the
 MPL, please see LICENSE-MPL-RabbitMQ.
 
-The files `mochinum.erl' and `mochiweb_util.erl` are (c) 2007 Mochi Media, Inc and
-licensed under a MIT license, see LICENSE-MIT-Mochi.
-
 If you have any questions regarding licensing, please contact us at
 info@rabbitmq.com.
diff --git a/LICENSE-MIT-Mochi b/LICENSE-MIT-Mochi
deleted file mode 100644
index c85b65a4d3..0000000000
--- a/LICENSE-MIT-Mochi
+++ /dev/null
@@ -1,9 +0,0 @@
-This is the MIT license.
-
-Copyright (c) 2007 Mochi Media, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/Makefile b/Makefile
index f7a0c361bc..8c2ad5518a 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@ PROJECT = rabbit
 VERSION ?= $(call get_app_version,src/$(PROJECT).app.src)
 
 DEPS = ranch lager rabbit_common rabbitmq_cli
-TEST_DEPS = rabbitmq_ct_helpers amqp_client meck proper
+TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client meck proper
 
 dep_rabbitmq_cli = git_rmq rabbitmq-cli $(current_rmq_ref) $(base_rmq_ref) rabbitmq-cli-integration
 
diff --git a/docs/rabbitmq.conf.example b/docs/rabbitmq.conf.example
index 33685191a3..96d0967db7 100644
--- a/docs/rabbitmq.conf.example
+++ b/docs/rabbitmq.conf.example
@@ -380,6 +380,21 @@
 ##
 # queue_index_embed_msgs_below = 4kb
 
+## Whether or not to enable background periodic GC of all
+## Erlang processes in "waiting" state.
+##
+## Disabling background GC may reduce latency for client operations,
+## keeping it enabled may reduce median RAM usage.
+##
+# background_gc_enabled = true
+
+## Target (desired) interval (in milliseconds) at which we run background GC.
+## The actual interval will vary depending on how long it takes to execute
+## the operation (can be higher than this interval). Values less than
+## 30000 milliseconds are not recommended.
+##
+# background_gc_target_interval = 60000
+
 ## ----------------------------------------------------------------------------
 ## Advanced Erlang Networking/Clustering Options.
 ##
@@ -615,7 +630,7 @@
 
 ## File rotation config. No rotation by defualt.
 ## DO NOT SET rotation date to ''. Leave unset if require "" value
-# log.file.rotation.date = $D0 
+# log.file.rotation.date = $D0
 # log.file.rotation.size = 0
 
 
diff --git a/docs/rabbitmq.config.example b/docs/rabbitmq.config.example
index 3e1137aa8b..aaffcab2d8 100644
--- a/docs/rabbitmq.config.example
+++ b/docs/rabbitmq.config.example
@@ -331,7 +331,22 @@
    %% Size in bytes below which to embed messages in the queue index. See
    %% http://www.rabbitmq.com/persistence-conf.html
    %%
-   %% {queue_index_embed_msgs_below, 4096}
+   %% {queue_index_embed_msgs_below, 4096},
+
+   %% Whether or not to enable background periodic GC of all
+   %% Erlang processes in "waiting" state.
+   %%
+   %% Disabling background GC may reduce latency for client operations,
+   %% keeping it enabled may reduce median RAM usage.
+   %%
+   %% {background_gc_enabled, true},
+   %%
+   %% Target (desired) interval (in milliseconds) at which we run background GC.
+   %% The actual interval will vary depending on how long it takes to execute
+   %% the operation (can be higher than this interval). Values less than
+   %% 30000 milliseconds are not recommended.
+   %%
+   %% {background_gc_target_interval, 60000}
 
   ]},
 
diff --git a/priv/schema/rabbitmq.schema b/priv/schema/rabbitmq.schema
index f31ec5416c..187e77017c 100644
--- a/priv/schema/rabbitmq.schema
+++ b/priv/schema/rabbitmq.schema
@@ -910,6 +910,20 @@ end}.
 {mapping, "queue_index_embed_msgs_below", "rabbit.queue_index_embed_msgs_below",
     [{datatype, bytesize}]}.
 
+%% Whether or not to enable background GC.
+%%
+%% {background_gc_enabled, true}
+
+{mapping, "background_gc_enabled", "rabbit.background_gc_enabled",
+    [{datatype, {enum, [true, false]}}]}.
+
+%% Interval (in milliseconds) at which we run background GC.
+%%
+%% {background_gc_target_interval, 60000}
+
+{mapping, "background_gc_target_interval", "rabbit.background_gc_target_interval",
+    [{datatype, integer}]}.
+
 % ==========================
 % Lager section
 % ==========================
diff --git a/rabbitmq-components.mk b/rabbitmq-components.mk
index 4444a2b676..071385e8e7 100644
--- a/rabbitmq-components.mk
+++ b/rabbitmq-components.mk
@@ -29,6 +29,7 @@ dep_rabbitmq_clusterer                = git_rmq rabbitmq-clusterer $(current_rmq
 dep_rabbitmq_cli                      = git_rmq rabbitmq-cli $(current_rmq_ref) $(base_rmq_ref) master
 dep_rabbitmq_codegen                  = git_rmq rabbitmq-codegen $(current_rmq_ref) $(base_rmq_ref) master
 dep_rabbitmq_consistent_hash_exchange = git_rmq rabbitmq-consistent-hash-exchange $(current_rmq_ref) $(base_rmq_ref) master
+dep_rabbitmq_ct_client_helpers        = git_rmq rabbitmq-ct-client-helpers $(current_rmq_ref) $(base_rmq_ref) master
 dep_rabbitmq_ct_helpers               = git_rmq rabbitmq-ct-helpers $(current_rmq_ref) $(base_rmq_ref) master
 dep_rabbitmq_delayed_message_exchange = git_rmq rabbitmq-delayed-message-exchange $(current_rmq_ref) $(base_rmq_ref) master
 dep_rabbitmq_dotnet_client            = git_rmq rabbitmq-dotnet-client $(current_rmq_ref) $(base_rmq_ref) master
@@ -72,12 +73,16 @@ dep_toke                              = git_rmq toke $(current_rmq_ref) $(base_r
 
 dep_rabbitmq_public_umbrella          = git_rmq rabbitmq-public-umbrella $(current_rmq_ref) $(base_rmq_ref) master
 
-# FIXME: As of 2015-11-20, we depend on Ranch 1.2.1, but erlang.mk
-# defaults to Ranch 1.1.0. All projects depending indirectly on Ranch
-# needs to add "ranch" as a BUILD_DEPS. The list of projects needing
-# this workaround are:
-#     o  rabbitmq-web-stomp
-dep_ranch = git https://github.com/ninenines/ranch 1.2.1
+# Third-party dependencies version pinning.
+#
+# We do that in this file, which is copied in all projects, to ensure
+# all projects use the same versions. It avoids conflicts and makes it
+# possible to work with rabbitmq-public-umbrella.
+
+dep_cowboy_commit = 1.0.3
+dep_mochiweb = git git://github.com/basho/mochiweb.git v2.9.0p2
+dep_ranch_commit = 1.2.1
+dep_webmachine_commit = 1.10.8p2
 
 RABBITMQ_COMPONENTS = amqp_client \
 		      rabbit \
@@ -92,6 +97,7 @@ RABBITMQ_COMPONENTS = amqp_client \
 		      rabbitmq_cli \
 		      rabbitmq_codegen \
 		      rabbitmq_consistent_hash_exchange \
+		      rabbitmq_ct_client_helpers \
 		      rabbitmq_ct_helpers \
 		      rabbitmq_delayed_message_exchange \
 		      rabbitmq_dotnet_client \
diff --git a/scripts/rabbitmq-defaults b/scripts/rabbitmq-defaults
index 5342978694..fdcf624d1b 100755
--- a/scripts/rabbitmq-defaults
+++ b/scripts/rabbitmq-defaults
@@ -44,4 +44,15 @@ SCHEMA_DIR=${SYS_PREFIX}/var/lib/rabbitmq/schema
 
 PLUGINS_DIR="${RABBITMQ_HOME}/plugins"
 
+# RABBIT_HOME can contain a version number, so default plugins
+# directory can be hard to find if we want to package some plugin
+# separately. When RABBITMQ_HOME points to a standard location where
+# it's usally being installed by package managers, we add
+# "/usr/lib/rabbitmq/plugins" to plugin search path.
+case "$RABBITMQ_HOME" in
+    /usr/lib/rabbitmq/*)
+        PLUGINS_DIR="/usr/lib/rabbitmq/plugins:$PLUGINS_DIR"
+        ;;
+esac
+
 CONF_ENV_FILE=${SYS_PREFIX}/etc/rabbitmq/rabbitmq-env.conf
diff --git a/scripts/rabbitmq-env b/scripts/rabbitmq-env
index f1962e2b7b..d975f274b2 100755
--- a/scripts/rabbitmq-env
+++ b/scripts/rabbitmq-env
@@ -57,6 +57,20 @@ rmq_realpath() {
     fi
 }
 
+path_contains_existing_directory() {
+    local path="${1:?}"
+    local dir
+    local rc
+    local IFS="
+    "
+    for dir in $(echo "$path" | tr ':' '\n'); do
+        if [ -d "$dir" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
 RABBITMQ_HOME="$(rmq_realpath "${RABBITMQ_SCRIPTS_DIR}/..")"
 ESCRIPT_DIR="${RABBITMQ_HOME}/escript"
 
@@ -108,8 +122,9 @@ fi
 rmq_normalize_path() {
     local path=$1
 
-    # Remove redundant slashes and strip a trailing slash
-    echo "$path" | sed -e 's#/\{2,\}#/#g' -e 's#/$##'
+    # Remove redundant slashes and strip a trailing slash for a
+    # PATH-like vars - ':' is the delimiter
+    echo "$path" | sed -e 's#/\{2,\}#/#g' -e 's#/$##' -e 's#/:#:#g'
 }
 
 rmq_normalize_path_var() {
@@ -265,9 +280,8 @@ if [ "${RABBITMQ_DEV_ENV}" ]; then
             RABBITMQ_ENABLED_PLUGINS_FILE="${enabled_plugins_file}"
         fi
     fi
-    
-    
-    if [ -d "${RABBITMQ_PLUGINS_DIR}" ]; then
+
+    if path_contains_existing_directory "${RABBITMQ_PLUGINS_DIR}" ; then
         # RabbitMQ was started with "make run-broker" from its own
         # source tree. Take rabbit_common from the plugins directory.
         ERL_LIBS="${RABBITMQ_PLUGINS_DIR}:${ERL_LIBS}"
@@ -291,7 +305,7 @@ if [ "${RABBITMQ_DEV_ENV}" ]; then
         ERL_LIBS="${DEPS_DIR_norm}:${ERL_LIBS}"
     fi
 else
-    if [ -d "${RABBITMQ_PLUGINS_DIR}" ]; then
+    if path_contains_existing_directory "${RABBITMQ_PLUGINS_DIR}" ; then
         # RabbitMQ was started from its install directory. Take
         # rabbit_common from the plugins directory.
         ERL_LIBS="${RABBITMQ_PLUGINS_DIR}:${ERL_LIBS}"
diff --git a/src/background_gc.erl b/src/background_gc.erl
index 2986f356f5..835eaef4df 100644
--- a/src/background_gc.erl
+++ b/src/background_gc.erl
@@ -25,7 +25,6 @@
          terminate/2, code_change/3]).
 
 -define(MAX_RATIO, 0.01).
--define(IDEAL_INTERVAL, 60000).
 -define(MAX_INTERVAL, 240000).
 
 -record(state, {last_interval}).
@@ -45,7 +44,9 @@ run() -> gen_server2:cast(?MODULE, run).
 
 %%----------------------------------------------------------------------------
 
-init([]) -> {ok, interval_gc(#state{last_interval = ?IDEAL_INTERVAL})}.
+init([]) ->
+    {ok, IdealInterval} = application:get_env(rabbit, background_gc_target_interval),
+    {ok, interval_gc(#state{last_interval = IdealInterval})}.
 
 handle_call(Msg, _From, State) ->
     {stop, {unexpected_call, Msg}, {unexpected_call, Msg}, State}.
@@ -65,14 +66,22 @@ terminate(_Reason, State) -> State.
 %%----------------------------------------------------------------------------
 
 interval_gc(State = #state{last_interval = LastInterval}) ->
+    {ok, IdealInterval} = application:get_env(rabbit, background_gc_target_interval),
     {ok, Interval} = rabbit_misc:interval_operation(
                        {?MODULE, gc, []},
-                       ?MAX_RATIO, ?MAX_INTERVAL, ?IDEAL_INTERVAL, LastInterval),
+                       ?MAX_RATIO, ?MAX_INTERVAL, IdealInterval, LastInterval),
     erlang:send_after(Interval, self(), run),
     State#state{last_interval = Interval}.
 
 gc() ->
-    [garbage_collect(P) || P <- processes(),
-                           {status, waiting} == process_info(P, status)],
-    garbage_collect(), %% since we will never be waiting...
+    Enabled = rabbit_misc:get_env(rabbit, background_gc_enabled, true),
+    case Enabled of
+        true ->
+            [garbage_collect(P) || P <- processes(),
+                                   {status, waiting} == process_info(P, status)],
+            %% since we will never be waiting...
+            garbage_collect();
+         false ->
+            ok
+    end,
     ok.
diff --git a/src/rabbit.app.src b/src/rabbit.app.src
index 250b14bc32..5f3120b117 100644
--- a/src/rabbit.app.src
+++ b/src/rabbit.app.src
@@ -117,5 +117,7 @@
          ]},
 
          %% rabbitmq-server-973
-         {lazy_queue_explicit_gc_run_operation_threshold, 250}
+         {lazy_queue_explicit_gc_run_operation_threshold, 250},
+         {background_gc_enabled, true},
+         {background_gc_target_interval, 60000}
         ]}]}.
diff --git a/src/rabbit.erl b/src/rabbit.erl
index 6dde2aca88..ebc92150eb 100644
--- a/src/rabbit.erl
+++ b/src/rabbit.erl
@@ -22,7 +22,7 @@
          stop_and_halt/0, await_startup/0, status/0, is_running/0, alarms/0,
          is_running/1, environment/0, rotate_logs/0, force_event_refresh/1,
          start_fhc/0]).
--export([start/2, stop/1, prep_stop/1]).
+-export([start/2, stop/1]).
 -export([start_apps/1, stop_apps/1]).
 -export([log_locations/0, config_files/0, decrypt_config/2]). %% for testing and mgmt-agent
 
@@ -727,7 +727,7 @@ start(normal, []) ->
             Error
     end.
 
-prep_stop(_State) ->
+stop(_State) ->
     ok = rabbit_alarm:stop(),
     ok = case rabbit_mnesia:is_clustered() of
              true  -> ok;
@@ -735,9 +735,7 @@ prep_stop(_State) ->
          end,
     ok.
 
-stop(_) -> ok.
-
--spec boot_error(atom(), term()) -> no_return().
+-spec boot_error(term(), not_available | [tuple()]) -> no_return().
 
 boot_error(_, {could_not_start, rabbit, {{timeout_waiting_for_tables, _}, _}}) ->
     AllNodes = rabbit_mnesia:cluster_nodes(all),
diff --git a/src/rabbit_plugins.erl b/src/rabbit_plugins.erl
index 723f725303..3f65452bdf 100644
--- a/src/rabbit_plugins.erl
+++ b/src/rabbit_plugins.erl
@@ -134,46 +134,16 @@ active() ->
             lists:member(App, InstalledPlugins)].
 
 %% @doc Get the list of plugins which are ready to be enabled.
-list(PluginsDir) ->
-    list(PluginsDir, false).
+list(PluginsPath) ->
+    list(PluginsPath, false).
 
-list(PluginsDir, IncludeRequiredDeps) ->
-    EZs = [{ez, EZ} || EZ <- filelib:wildcard("*.ez", PluginsDir)],
-    FreeApps = [{app, App} ||
-                   App <- filelib:wildcard("*/ebin/*.app", PluginsDir)],
-    %% We load the "rabbit" application to be sure we can get the
-    %% "applications" key. This is required for rabbitmq-plugins for
-    %% instance.
-    application:load(rabbit),
-    {ok, RabbitDeps} = application:get_key(rabbit, applications),
-    AllPlugins = [plugin_info(PluginsDir, Plug) || Plug <- EZs ++ FreeApps],
-    {AvailablePlugins, Problems} =
-        lists:foldl(
-            fun ({error, EZ, Reason}, {Plugins1, Problems1}) ->
-                    {Plugins1, [{EZ, Reason} | Problems1]};
-                (Plugin = #plugin{name = Name},
-                 {Plugins1, Problems1}) ->
-                    %% Applications RabbitMQ depends on (eg.
-                    %% "rabbit_common") can't be considered
-                    %% plugins, otherwise rabbitmq-plugins would
-                    %% list them and the user may believe he can
-                    %% disable them.
-                    case IncludeRequiredDeps orelse
-                      not lists:member(Name, RabbitDeps) of
-                        true  -> {[Plugin|Plugins1], Problems1};
-                        false -> {Plugins1, Problems1}
-                    end
-            end, {[], []},
-            AllPlugins),
-    case Problems of
-        [] -> ok;
-        _  -> rabbit_log:warning(
-                "Problem reading some plugins: ~p~n", [Problems])
-    end,
-
-    Plugins = lists:filter(fun(P) -> not plugin_provided_by_otp(P) end,
-                           AvailablePlugins),
-    ensure_dependencies(Plugins).
+list(PluginsPath, IncludeRequiredDeps) ->
+    {AllPlugins, LoadingProblems} = discover_plugins(split_path(PluginsPath)),
+    {UniquePlugins, DuplicateProblems} = remove_duplicate_plugins(AllPlugins),
+    Plugins1 = maybe_keep_required_deps(IncludeRequiredDeps, UniquePlugins),
+    Plugins2 = remove_otp_overrideable_plugins(Plugins1),
+    maybe_report_plugin_loading_problems(LoadingProblems ++ DuplicateProblems),
+    ensure_dependencies(Plugins2).
 
 %% @doc Read the list of enabled plugins from the supplied term file.
 read_enabled(PluginsFile) ->
@@ -425,14 +395,12 @@ prepare_plugin(#plugin{type = dir, name = Name, location = Location},
                ExpandDir) ->
     rabbit_file:recursive_copy(Location, filename:join([ExpandDir, Name])).
 
-plugin_info(Base, {ez, EZ0}) ->
-    EZ = filename:join([Base, EZ0]),
+plugin_info({ez, EZ}) ->
     case read_app_file(EZ) of
         {application, Name, Props} -> mkplugin(Name, Props, ez, EZ);
         {error, Reason}            -> {error, EZ, Reason}
     end;
-plugin_info(Base, {app, App0}) ->
-    App = filename:join([Base, App0]),
+plugin_info({app, App}) ->
     case rabbit_file:read_term_file(App) of
         {ok, [{application, Name, Props}]} ->
             mkplugin(Name, Props, dir,
@@ -486,9 +454,99 @@ plugin_names(Plugins) ->
     [Name || #plugin{name = Name} <- Plugins].
 
 lookup_plugins(Names, AllPlugins) ->
-    % Preserve order of Names
+    %% Preserve order of Names
     lists:map(
         fun(Name) ->
             lists:keyfind(Name, #plugin.name, AllPlugins)
         end,
         Names).
+
+%% Split PATH-like value into its components.
+split_path(PathString) ->
+    Delimiters = case os:type() of
+                     {unix, _} -> ":";
+                     {win32, _} -> ";"
+                 end,
+    string:tokens(PathString, Delimiters).
+
+%% Search for files using glob in a given dir. Returns full filenames of those files.
+full_path_wildcard(Glob, Dir) ->
+    [filename:join([Dir, File]) || File <- filelib:wildcard(Glob, Dir)].
+
+%% Returns list off all .ez files in a given set of directories
+list_ezs([]) ->
+    [];
+list_ezs([Dir|Rest]) ->
+    [{ez, EZ} || EZ <- full_path_wildcard("*.ez", Dir)] ++ list_ezs(Rest).
+
+%% Returns list of all files that look like OTP applications in a
+%% given set of directories.
+list_free_apps([]) ->
+    [];
+list_free_apps([Dir|Rest]) ->
+    [{app, App} || App <- full_path_wildcard("*/ebin/*.app", Dir)]
+        ++ list_free_apps(Rest).
+
+compare_by_name_and_version(#plugin{name = Name, version = VersionA},
+                            #plugin{name = Name, version = VersionB}) ->
+    ec_semver:lte(VersionA, VersionB);
+compare_by_name_and_version(#plugin{name = NameA},
+                            #plugin{name = NameB}) ->
+    NameA =< NameB.
+
+-spec discover_plugins([Directory]) -> {[#plugin{}], [Problem]} when
+      Directory :: file:name(),
+      Problem :: {file:name(), term()}.
+discover_plugins(PluginsDirs) ->
+    EZs = list_ezs(PluginsDirs),
+    FreeApps = list_free_apps(PluginsDirs),
+    read_plugins_info(EZs ++ FreeApps, {[], []}).
+
+read_plugins_info([], Acc) ->
+    Acc;
+read_plugins_info([Path|Paths], {Plugins, Problems}) ->
+    case plugin_info(Path) of
+        #plugin{} = Plugin ->
+            read_plugins_info(Paths, {[Plugin|Plugins], Problems});
+        {error, Location, Reason} ->
+            read_plugins_info(Paths, {Plugins, [{Location, Reason}|Problems]})
+    end.
+
+remove_duplicate_plugins(Plugins) ->
+    %% Reverse order ensures that if there are several versions of the
+    %% same plugin, the most recent one comes first.
+    Sorted = lists:reverse(
+               lists:sort(fun compare_by_name_and_version/2, Plugins)),
+    remove_duplicate_plugins(Sorted, {[], []}).
+
+remove_duplicate_plugins([], Acc) ->
+    Acc;
+remove_duplicate_plugins([Best = #plugin{name = Name}, Offender = #plugin{name = Name} | Rest],
+                  {Plugins0, Problems0}) ->
+    Problems1 = [{Offender#plugin.location, duplicate_plugin}|Problems0],
+    remove_duplicate_plugins([Best|Rest], {Plugins0, Problems1});
+remove_duplicate_plugins([Plugin|Rest], {Plugins0, Problems0}) ->
+    Plugins1 = [Plugin|Plugins0],
+    remove_duplicate_plugins(Rest, {Plugins1, Problems0}).
+
+maybe_keep_required_deps(true, Plugins) ->
+    Plugins;
+maybe_keep_required_deps(false, Plugins) ->
+    %% We load the "rabbit" application to be sure we can get the
+    %% "applications" key. This is required for rabbitmq-plugins for
+    %% instance.
+    application:load(rabbit),
+    {ok, RabbitDeps} = application:get_key(rabbit, applications),
+    lists:filter(fun(#plugin{name = Name}) ->
+                         not lists:member(Name, RabbitDeps)
+                 end,
+                 Plugins).
+
+remove_otp_overrideable_plugins(Plugins) ->
+    lists:filter(fun(P) -> not plugin_provided_by_otp(P) end,
+                 Plugins).
+
+maybe_report_plugin_loading_problems([]) ->
+    ok;
+maybe_report_plugin_loading_problems(Problems) ->
+    rabbit_log:warning("Problem reading some plugins: ~p~n", [Problems]).
diff --git a/src/rabbit_plugins_main.erl b/src/rabbit_plugins_main.erl
index 4618249ee8..dccdd92969 100644
--- a/src/rabbit_plugins_main.erl
+++ b/src/rabbit_plugins_main.erl
@@ -191,9 +191,9 @@ format_plugins(Node, Pattern, Opts, #cli{all      = All,
 
     EnabledImplicitly = Implicit -- Enabled,
     {StatusMsg, Running} =
-        case rabbit_misc:rpc_call(Node, rabbit_plugins, active, []) of
-            {badrpc, _} -> {"[failed to contact ~s - status not shown]", []};
-            Active      -> {"* = running on ~s", Active}
+        case remote_running_plugins(Node) of
+            {ok, Active} -> {"* = running on ~s", Active};
+            error        -> {"[failed to contact ~s - status not shown]", []}
         end,
     {ok, RE} = re:compile(Pattern),
     Plugins = [ Plugin ||
@@ -218,7 +218,7 @@ format_plugins(Node, Pattern, Opts, #cli{all      = All,
                    Format, MaxWidth) || P <- Plugins1],
     ok.
 
-format_plugin(#plugin{name = Name, version = Version,
+format_plugin(#plugin{name = Name, version = OnDiskVersion,
                       description = Description, dependencies = Deps},
               Enabled, EnabledImplicitly, Running, Format,
               MaxWidth) ->
@@ -228,7 +228,7 @@ format_plugin(#plugin{name = Name, version = Version,
                        {false, true} -> "e";
                        _             -> " "
                    end,
-    RunningGlyph = case lists:member(Name, Running) of
+    RunningGlyph = case lists:keymember(Name, 1, Running) of
                        true  -> "*";
                        false -> " "
                    end,
@@ -236,6 +236,7 @@ format_plugin(#plugin{name = Name, version = Version,
     Opt = fun (_F, A, A) -> ok;
               ( F, A, _) -> io:format(F, [A])
           end,
+    Version = format_running_plugin_version(Name, OnDiskVersion, Running),
     case Format of
         minimal -> io:format("~s~n", [Name]);
         normal  -> io:format("~s ~-" ++ integer_to_list(MaxWidth) ++ "w ",
@@ -327,3 +328,31 @@ rpc_call(Node, Online, Mod, Fun, Args) ->
 
 plur([_]) -> "";
 plur(_)   -> "s".
+
+-spec remote_running_plugins(node()) -> [{atom(), Vsn :: string()}].
+remote_running_plugins(Node) ->
+    case rabbit_misc:rpc_call(Node, rabbit_plugins, active, []) of
+        {badrpc, _} -> error;
+        Active      -> maybe_augment_with_versions(Node, Active)
+    end.
+
+-spec maybe_augment_with_versions(node(), [atom()]) -> [{atom(), Vsn :: string()}].
+maybe_augment_with_versions(Node, Plugins) ->
+    case rabbit_misc:rpc_call(Node, rabbit_misc, which_applications, []) of
+        {badrpc, _} ->
+            error;
+        All ->
+            {ok, [{App, Vsn} || {App, _, Vsn} <- All,
+                                lists:member(App, Plugins)]}
+    end.
+
+-spec format_running_plugin_version(atom(), string(), [{atom(), Vsn :: string()}]) -> string().
+format_running_plugin_version(Name, OnDiskVersion, RunningPlugins) ->
+    case lists:keyfind(Name, 1, RunningPlugins) of
+        false ->
+            OnDiskVersion;
+        {_, OnDiskVersion} ->
+            OnDiskVersion;
+        {_, RunningVersion} ->
+            io_lib:format("~s (pending upgrade to ~s)", [RunningVersion, OnDiskVersion])
+    end.
diff --git a/test/config_schema_SUITE_data/snippets.config b/test/config_schema_SUITE_data/snippets.config
index a64b6e06c9..f5832981bc 100644
--- a/test/config_schema_SUITE_data/snippets.config
+++ b/test/config_schema_SUITE_data/snippets.config
@@ -772,5 +772,19 @@ autocluster.node_type = ram",
 [{rabbit, [
     {autocluster, [{peer_discovery_backend, rabbit_peer_discovery_classic_config},
                    {node_type, ram}]}
+]}],[]},
+{77,
+"background_gc_enabled = true
+background_gc_target_interval = 30000",
+[{rabbit, [
+    {background_gc_enabled, true},
+    {background_gc_target_interval, 30000}
+]}],[]},
+{77.1,
+"background_gc_enabled = false
+background_gc_target_interval = 30000",
+[{rabbit, [
+    {background_gc_enabled, false},
+    {background_gc_target_interval, 30000}
 ]}],[]}
 ].
diff --git a/test/unit_SUITE.erl b/test/unit_SUITE.erl
index 8ac3d41afe..8499fd2abc 100644
--- a/test/unit_SUITE.erl
+++ b/test/unit_SUITE.erl
@@ -32,7 +32,6 @@ groups() ->
     [
       {parallel_tests, [parallel], [
           arguments_parser,
-          mutually_exclusive_flags_parsing,
           {basic_header_handling, [parallel], [
               write_table_with_invalid_existing_type,
               invalid_existing_headers,
@@ -43,6 +42,8 @@ groups() ->
           content_framing,
           content_transcoding,
           decrypt_config,
+          listing_plugins_from_multiple_directories,
+          mutually_exclusive_flags_parsing,
           rabbitmqctl_encode,
           pg_local,
           pmerge,
@@ -968,3 +969,34 @@ unfold(_Config) ->
                                        (N) -> {true, N*2, N-1}
                                    end, 10),
     passed.
+
+listing_plugins_from_multiple_directories(Config) ->
+    %% Generate some fake plugins in .ez files
+    FirstDir = filename:join([?config(priv_dir, Config), "listing_plugins_from_multiple_directories-1"]),
+    SecondDir = filename:join([?config(priv_dir, Config), "listing_plugins_from_multiple_directories-2"]),
+    ok = file:make_dir(FirstDir),
+    ok = file:make_dir(SecondDir),
+    lists:foreach(fun({Dir, AppName, Vsn}) ->
+                          EzName = filename:join([Dir, io_lib:format("~s-~s.ez", [AppName, Vsn])]),
+                          AppFileName = lists:flatten(io_lib:format("~s-~s/ebin/~s.app", [AppName, Vsn, AppName])),
+                          AppFileContents = list_to_binary(io_lib:format("~p.", [{application, AppName, [{vsn, Vsn}]}])),
+                          {ok, {_, EzData}} = zip:zip(EzName, [{AppFileName, AppFileContents}], [memory]),
+                          ok = file:write_file(EzName, EzData)
+                  end,
+                  [{FirstDir, plugin_first_dir, "3"},
+                   {SecondDir, plugin_second_dir, "4"},
+                   {FirstDir, plugin_both, "1"},
+                   {SecondDir, plugin_both, "2"}]),
+
+    %% Everything was collected from both directories, plugin with higher version should take precedence
+    Path = FirstDir ++ ":" ++ SecondDir,
+    Got = lists:sort([{Name, Vsn} || #plugin{name = Name, version = Vsn} <- rabbit_plugins:list(Path)]),
+    Expected = [{plugin_both, "2"}, {plugin_first_dir, "3"}, {plugin_second_dir, "4"}],
+    case Got of
+        Expected ->
+            ok;
+        _ ->
+            ct:pal("Got ~p~nExpected: ~p", [Got, Expected]),
+            exit({wrong_plugins_list, Got})
+    end,
+    ok.