Refactor: Simplify boot/cleanup step handling and unify modules

Isolate boot steps to a set of applications and avoid using ets to
track which have run. This also simplifies cleanup step execution.

The force_reload code isn't needed, since the real issue preventing
updated versions of modules from being found lay in the proper
expansion of (the correct set of) .ez archives, which was fixed in
revision 1918e77.

Unify all the boot handling code under rabbit.erl once again and now
completely remove rabbit_boot, since it's no longer needed. Remove
unused exports and tidy.

3 files changed, 91 insertions, 175 deletions

diff --git a/src/rabbit.erl b/src/rabbit.erl
index 46838364d2..c703fedb35 100644
--- a/src/rabbit.erl
+++ b/src/rabbit.erl
@@ -24,7 +24,7 @@
          start_fhc/0]).
 -export([run_boot_steps/0, load_steps/1, run_step/3]).
 -export([start/2, stop/1]).
--export([handle_app_error/1, start_apps/1]).
+-export([start_apps/1, stop_apps/1]).
 -export([log_location/1]). %% for testing
 
 %%---------------------------------------------------------------------------
@@ -242,7 +242,6 @@
 -spec(maybe_insert_default_data/0 :: () -> 'ok').
 -spec(boot_delegate/0 :: () -> 'ok').
 -spec(recover/0 :: () -> 'ok').
--spec(handle_app_error/1 :: (term()) -> fun((atom(), term()) -> no_return())).
 
 -endif.
 
@@ -343,10 +342,10 @@ handle_app_error(Term) ->
     end.
 
 start_apps(Apps) ->
-    rabbit_boot:force_reload(Apps),
+    app_utils:load_applications(Apps),
     StartupApps = app_utils:app_dependency_order(Apps, false),
     case whereis(rabbit_boot) of
-        undefined -> rabbit:run_boot_steps();
+        undefined -> run_boot_steps(Apps);
         _         -> ok
     end,
     ok = app_utils:start_applications(StartupApps,
@@ -393,6 +392,29 @@ stop_and_halt() ->
     end,
     ok.
 
+stop_apps(Apps) ->
+    try
+        ok = app_utils:stop_applications(
+               Apps, handle_app_error(error_during_shutdown))
+    after
+        run_cleanup_steps(Apps),
+        [begin
+             {ok, Mods} = application:get_key(App, modules),
+             [begin
+                  code:soft_purge(Mod),
+                  code:delete(Mod),
+                  false = code:is_loaded(Mod)
+              end || Mod <- Mods],
+             application:unload(App)
+         end || App <- Apps]
+    end.
+
+run_cleanup_steps(Apps) ->
+    [run_step(Name, Attributes, cleanup) ||
+        {App, Name, Attributes} <- load_steps(Apps),
+        lists:member(App, Apps)],
+    ok.
+
 await_startup() ->
     app_utils:wait_for_applications(app_startup_order()).
 
@@ -463,7 +485,6 @@ start(normal, []) ->
             true = register(rabbit, self()),
             print_banner(),
             log_banner(),
-            rabbit_boot:prepare_boot_table(),
             run_boot_steps(),
             {ok, SupPid};
         Error ->
@@ -493,16 +514,12 @@ app_shutdown_order() ->
 %% boot step logic
 
 run_boot_steps() ->
-    Steps = load_steps(boot),
-    [ok = run_boot_step(Step) || Step <- Steps],
-    ok.
+    run_boot_steps([App || {App, _, _} <- application:loaded_applications()]).
 
-run_boot_step({_, StepName, Attributes}) ->
-    case catch rabbit_boot:already_run(StepName) of
-        false -> ok = run_step(StepName, Attributes, mfa),
-                 rabbit_boot:mark_complete(StepName);
-        _     -> ok
-    end,
+run_boot_steps(Apps) ->
+    Steps = load_steps(Apps),
+    [ok = run_step(StepName, Attributes, mfa) ||
+        {_, StepName, Attributes} <- Steps],
     ok.
 
 run_step(StepName, Attributes, AttributeName) ->
@@ -524,45 +541,71 @@ run_step(StepName, Attributes, AttributeName) ->
             ok
     end.
 
-load_steps(Type) ->
+load_steps(BaseApps) ->
+    Apps = BaseApps -- app_utils:which_applications(),  %% exclude running apps
     StepAttrs = rabbit_misc:all_module_attributes_with_app(rabbit_boot_step),
-    sort_boot_steps(
-      Type,
-      lists:usort(
-        [{Mod, {AppName, Steps}} || {AppName, Mod, Steps} <- StepAttrs])).
+    {AllSteps, StepsDict} =
+        lists:foldl(
+          fun({AppName, Mod, Steps}, {AccSteps, AccDict}) ->
+                  {[{Mod, {AppName, Steps}}|AccSteps],
+                   lists:foldl(
+                     fun({StepName, _}, Acc) ->
+                             dict:store(StepName, AppName, Acc)
+                     end, AccDict, Steps)}
+          end, {[], dict:new()}, StepAttrs),
+    Steps = lists:foldl(filter_steps(Apps, StepsDict), [], AllSteps),
+    sort_boot_steps(lists:usort(Steps)).
+
+filter_steps(Apps, Dict) ->
+    fun({Mod, {AppName, Steps}}, Acc) ->
+            Steps2 = [begin
+                          Filtered = lists:foldl(filter_attrs(Apps, Dict),
+                                                 [], Attrs),
+                          {Step, Filtered}
+                      end || {Step, Attrs} <- Steps,
+                             filter_app(Apps, Dict, Step)],
+            [{Mod, {AppName, Steps2}}|Acc]
+    end.
+
+filter_app(Apps, Dict, Step) ->
+    case dict:find(Step, Dict) of
+        {ok, App} -> lists:member(App, Apps);
+        error     -> false
+    end.
+
+filter_attrs(Apps, Dict) ->
+    fun(Attr={Type, Other}, AccAttrs) when Type =:= requires orelse
+                                           Type =:= enables ->
+            %% If we don't know about a dependency, we allow it through,
+            %% since we don't *know* that it should be ignored. If, on
+            %% the other hand, we recognise a dependency then we _only_
+            %% include it (i.e., the requires/enables attribute itself)
+            %% if the referenced step comes from one of the Apps we're
+            %% actively working with at this point.
+            case dict:find(Other, Dict) of
+                error     -> [Attr | AccAttrs];
+                {ok, App} -> case lists:member(App, Apps) of
+                                 true  -> [Attr | AccAttrs];
+                                 false -> AccAttrs
+                             end
+            end;
+       (Attr, AccAttrs) ->
+            [Attr | AccAttrs]
+    end.
 
 vertices(_Module, {AppName, Steps}) ->
     [{StepName, {AppName, StepName, Atts}} || {StepName, Atts} <- Steps].
 
-edges(Type) ->
-    %% When running "boot" steps, both hard _and_ soft dependencies are
-    %% considered equally. When running "cleanup" steps however, we only
-    %% consider /hard/ dependencies (i.e., of the form
-    %% {DependencyType, {hard, StepName}}) as dependencies.
-    fun (_Module, {_AppName, Steps}) ->
-            [case Key of
-                 requires -> {StepName, strip_type(OtherStep)};
-                 enables  -> {strip_type(OtherStep), StepName}
-             end || {StepName, Atts} <- Steps,
-                    {Key, OtherStep} <- Atts,
-                    filter_dependent_steps(Key, OtherStep, Type)]
-    end.
+edges(_Module, {_AppName, Steps}) ->
+    [case Key of
+         requires -> {StepName, OtherStep};
+         enables  -> {OtherStep, StepName}
+     end || {StepName, Atts} <- Steps,
+            {Key, OtherStep} <- Atts,
+            Key =:= requires orelse Key =:= enables].
 
-filter_dependent_steps(Key, Dependency, Type)
-  when Key =:= requires orelse Key =:= enables ->
-    case {Dependency, Type} of
-        {{hard, _}, cleanup} -> true;
-        {_SoftReqs, cleanup} -> false;
-        {_,         boot}    -> true
-    end;
-filter_dependent_steps(_, _, _) ->
-    false.
-
-strip_type({hard, Step}) -> Step;
-strip_type(Step)         -> Step.
-
-sort_boot_steps(Type, UnsortedSteps) ->
-    case rabbit_misc:build_acyclic_graph(fun vertices/2, edges(Type),
+sort_boot_steps(UnsortedSteps) ->
+    case rabbit_misc:build_acyclic_graph(fun vertices/2, fun edges/2,
                                          UnsortedSteps) of
         {ok, G} ->
             %% Use topological sort to find a consistent ordering (if
diff --git a/src/rabbit_boot.erl b/src/rabbit_boot.erl
deleted file mode 100644
index e9bf5457de..0000000000
--- a/src/rabbit_boot.erl
+++ /dev/null
@@ -1,127 +0,0 @@
-%% The contents of this file are subject to the Mozilla Public License
-%% Version 1.1 (the "License"); you may not use this file except in
-%% compliance with the License. You may obtain a copy of the License
-%% at http://www.mozilla.org/MPL/
-%%
-%% Software distributed under the License is distributed on an "AS IS"
-%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-%% the License for the specific language governing rights and
-%% limitations under the License.
-%%
-%% The Original Code is RabbitMQ.
-%%
-%% The Initial Developer of the Original Code is GoPivotal, Inc.
-%% Copyright (c) 2007-2013 GoPivotal, Inc.  All rights reserved.
-%%
-
--module(rabbit_boot).
-
--export([prepare_boot_table/0]).
--export([stop/1]).
--export([force_reload/1]).
--export([already_run/1, mark_complete/1]).
-
--ifdef(use_specs).
-
--spec(prepare_boot_table/0 :: () -> 'ok').
--spec(already_run/1        :: (atom()) -> boolean()).
--spec(mark_complete/1      :: (atom()) -> 'ok').
--spec(stop/1               :: ([atom()]) -> 'ok').
--spec(force_reload/1       :: ([atom()]) -> 'ok').
-
--endif.
-
-%% When the broker is starting, we must run all the boot steps within the
-%% rabbit:start/2 application callback, after rabbit_sup has started and
-%% before any plugin applications begin to start. To achieve this, we process
-%% the boot steps from all loaded applications.
-%%
-%% If the broker is already running however, we must run all boot steps for
-%% each application/plugin we're starting, plus any other (dependent) steps.
-%% To achieve this, we process boot steps as usual, but skip those that have
-%% already run (i.e., whilst, or even since the broker started).
-%%
-%% Tracking which boot steps have run is done via a shared ets table, owned
-%% by the "rabbit" process.
-
-%%---------------------------------------------------------------------------
-%% Public API
-
-prepare_boot_table() ->
-    ets:new(?MODULE, [named_table, public, ordered_set]).
-
-stop(Apps) ->
-    try
-        ok = app_utils:stop_applications(
-               Apps, rabbit:handle_app_error(error_during_shutdown))
-    after
-        BootSteps = rabbit:load_steps(boot),
-        ToDelete = [Step || {App, _, _}=Step <- BootSteps,
-                            lists:member(App, Apps)],
-        [ets:delete(?MODULE, Step) || {_, Step, _} <- ToDelete],
-        run_cleanup_steps(Apps),
-        [begin
-             {ok, Mods} = application:get_key(App, modules),
-             [begin
-                  code:soft_purge(Mod),
-                  code:delete(Mod),
-                  false = code:is_loaded(Mod)
-              end || Mod <- Mods],
-             application:unload(App)
-         end || App <- Apps]
-    end.
-
-run_cleanup_steps(Apps) ->
-    Completed = sets:new(),
-    lists:foldl(
-      fun({_, Name, _}=Step, Acc) ->
-              case sets:is_element(Name, Completed) of
-                  true  -> Acc;
-                  false -> run_cleanup_step(Step),
-                           sets:add_element(Name, Acc)
-              end
-      end,
-      Completed,
-      [Step || {App, _, _}=Step <- rabbit:load_steps(cleanup),
-               lists:member(App, Apps)]),
-    ok.
-
-%%---------------------------------------------------------------------------
-%% Private API
-
-force_reload(Apps) ->
-    ok = app_utils:load_applications(Apps),
-    ok = do_reload(Apps).
-
-do_reload([App|Apps]) ->
-    case application:get_key(App, modules) of
-        {ok, Modules} -> reload_all(Modules);
-        _             -> ok
-    end,
-    force_reload(Apps);
-do_reload([]) ->
-    ok.
-
-reload_all(Modules) ->
-    [begin
-         case code:soft_purge(Mod) of
-             true  -> load_mod(Mod);
-             false -> ok
-         end
-     end || Mod <- Modules].
-
-load_mod(Mod) ->
-    case code:is_loaded(Mod) of
-        {file, Path} when Path /= 'preloaded' -> code:load_abs(Path);
-        _                                     -> code:load_file(Mod)
-    end.
-
-run_cleanup_step({_, StepName, Attributes}) ->
-    rabbit:run_step(StepName, Attributes, cleanup).
-
-already_run(StepName) ->
-    ets:member(?MODULE, StepName).
-
-mark_complete(StepName) ->
-    ets:insert(?MODULE, {StepName}).
-
diff --git a/src/rabbit_plugins.erl b/src/rabbit_plugins.erl
index 6572a625bd..1f36ce4ce0 100644
--- a/src/rabbit_plugins.erl
+++ b/src/rabbit_plugins.erl
@@ -48,7 +48,7 @@ enable(Plugins) ->
 
 disable(Plugins) ->
     app_utils:update_running_apps(
-      fun() -> rabbit_boot:stop(Plugins) end,
+      fun() -> rabbit:stop_apps(Plugins) end,
       fun(Diff) ->
               ok = rabbit_event:notify(plugins_changed, [{disabled, Diff}])
       end).