Restructure our boot procedure for more flexible boot-step handling

6 files changed, 245 insertions, 147 deletions

diff --git a/src/app_utils.erl b/src/app_utils.erl
index 5ae2d2954e..24c6165efd 100644
--- a/src/app_utils.erl
+++ b/src/app_utils.erl
@@ -17,7 +17,7 @@
 
 -export([load_applications/1, start_applications/1, start_applications/2,
          stop_applications/1, stop_applications/2, app_dependency_order/2,
-         wait_for_applications/1]).
+         wait_for_applications/1, app_dependencies/1]).
 
 -ifdef(use_specs).
 
@@ -30,6 +30,7 @@
 -spec stop_applications([atom()], error_handler())  -> 'ok'.
 -spec wait_for_applications([atom()])               -> 'ok'.
 -spec app_dependency_order([atom()], boolean())     -> [digraph:vertex()].
+-spec app_dependencies(atom())                      -> [atom()].
 
 -endif.
 
@@ -54,7 +55,7 @@ stop_applications(Apps) ->
 
 start_applications(Apps, ErrorHandler) ->
     manage_applications(fun lists:foldl/3,
-                        fun application:start/1,
+                        fun start/1,
                         fun application:stop/1,
                         already_started,
                         ErrorHandler,
@@ -68,7 +69,6 @@ stop_applications(Apps, ErrorHandler) ->
                         ErrorHandler,
                         Apps).
 
-
 wait_for_applications(Apps) ->
     [wait_for_application(App) || App <- Apps], ok.
 
@@ -80,8 +80,9 @@ app_dependency_order(RootApps, StripUnreachable) ->
                     {App, _Desc, _Vsn} <- application:loaded_applications()]),
     try
         case StripUnreachable of
-            true -> digraph:del_vertices(G, digraph:vertices(G) --
-                     digraph_utils:reachable(RootApps, G));
+            true  -> digraph:del_vertices(
+                       G, digraph:vertices(G) --
+                           digraph_utils:reachable(RootApps, G));
             false -> ok
         end,
         digraph_utils:topsort(G)
@@ -92,6 +93,15 @@ app_dependency_order(RootApps, StripUnreachable) ->
 %%---------------------------------------------------------------------------
 %% Private API
 
+start(rabbit) ->
+    case application:start(rabbit) of
+        ok  -> rabbit_boot:run_boot_steps(rabbit), ok;
+        Err -> Err
+    end;
+start(App) ->
+    rabbit_boot:run_boot_steps(App),
+    application:start(App).
+
 wait_for_application(Application) ->
     case lists:keymember(Application, 1, rabbit_misc:which_applications()) of
          true  -> ok;
diff --git a/src/rabbit.erl b/src/rabbit.erl
index 1b7fe6dada..562497b326 100644
--- a/src/rabbit.erl
+++ b/src/rabbit.erl
@@ -312,8 +312,7 @@ start() ->
                      ok = ensure_working_log_handlers(),
                      rabbit_node_monitor:prepare_cluster_status_files(),
                      rabbit_mnesia:check_cluster_consistency(),
-                     ok = app_utils:start_applications(
-                            app_startup_order(), fun handle_app_error/2),
+                     ok = rabbit_boot:start(app_startup_order()),
                      ok = log_broker_started(rabbit_plugins:active())
              end).
 
@@ -331,20 +330,10 @@ boot() ->
                      rabbit_mnesia:check_cluster_consistency(),
                      Plugins = rabbit_plugins:setup(),
                      ToBeLoaded = Plugins ++ ?APPS,
-                     ok = app_utils:load_applications(ToBeLoaded),
-                     StartupApps = app_utils:app_dependency_order(ToBeLoaded,
-                                                                  false),
-                     ok = app_utils:start_applications(
-                            StartupApps, fun handle_app_error/2),
+                     ok = rabbit_boot:start(ToBeLoaded),
                      ok = log_broker_started(Plugins)
              end).
 
-handle_app_error(App, {bad_return, {_MFA, {'EXIT', {Reason, _}}}}) ->
-    throw({could_not_start, App, Reason});
-
-handle_app_error(App, Reason) ->
-    throw({could_not_start, App, Reason}).
-
 start_it(StartFun) ->
     Marker = spawn_link(fun() -> receive stop -> ok end end),
     register(rabbit_boot, Marker),
@@ -352,9 +341,9 @@ start_it(StartFun) ->
         StartFun()
     catch
         throw:{could_not_start, _App, _Reason}=Err ->
-            boot_error(Err, not_available);
+            rabbit_boot:boot_error(Err, not_available);
          _:Reason ->
-            boot_error(Reason, erlang:get_stacktrace())
+            rabbit_boot:boot_error(Reason, erlang:get_stacktrace())
     after
         unlink(Marker),
         Marker ! stop,
@@ -367,7 +356,7 @@ stop() ->
         undefined -> ok;
         _         -> await_startup()
     end,
-    rabbit_log:info("Stopping RabbitMQ~n"),
+    rabbit_log:info("Stopping RabbitMQ~n"), %% TODO: move this to boot:stop/1
     ok = app_utils:stop_applications(app_shutdown_order()).
 
 stop_and_halt() ->
@@ -441,7 +430,6 @@ start(normal, []) ->
             true = register(rabbit, self()),
             print_banner(),
             log_banner(),
-            [ok = run_boot_step(Step) || Step <- boot_steps()],
             {ok, SupPid};
         Error ->
             Error
@@ -467,120 +455,6 @@ app_shutdown_order() ->
     app_utils:app_dependency_order(Apps, true).
 
 %%---------------------------------------------------------------------------
-%% boot step logic
-
-run_boot_step({_StepName, Attributes}) ->
-    case [MFA || {mfa, MFA} <- Attributes] of
-        [] ->
-            ok;
-        MFAs ->
-            [try
-                 apply(M,F,A)
-             of
-                 ok ->              ok;
-                 {error, Reason} -> boot_error(Reason, not_available)
-             catch
-                 _:Reason -> boot_error(Reason, erlang:get_stacktrace())
-             end || {M,F,A} <- MFAs],
-            ok
-    end.
-
-boot_steps() ->
-    sort_boot_steps(rabbit_misc:all_module_attributes(rabbit_boot_step)).
-
-vertices(_Module, Steps) ->
-    [{StepName, {StepName, Atts}} || {StepName, Atts} <- Steps].
-
-edges(_Module, Steps) ->
-    [case Key of
-         requires -> {StepName, OtherStep};
-         enables  -> {OtherStep, StepName}
-     end || {StepName, Atts} <- Steps,
-            {Key, OtherStep} <- Atts,
-            Key =:= requires orelse Key =:= enables].
-
-sort_boot_steps(UnsortedSteps) ->
-    case rabbit_misc:build_acyclic_graph(fun vertices/2, fun edges/2,
-                                         UnsortedSteps) of
-        {ok, G} ->
-            %% Use topological sort to find a consistent ordering (if
-            %% there is one, otherwise fail).
-            SortedSteps = lists:reverse(
-                            [begin
-                                 {StepName, Step} = digraph:vertex(G,
-                                                                   StepName),
-                                 Step
-                             end || StepName <- digraph_utils:topsort(G)]),
-            digraph:delete(G),
-            %% Check that all mentioned {M,F,A} triples are exported.
-            case [{StepName, {M,F,A}} ||
-                     {StepName, Attributes} <- SortedSteps,
-                     {mfa, {M,F,A}}         <- Attributes,
-                     not erlang:function_exported(M, F, length(A))] of
-                []               -> SortedSteps;
-                MissingFunctions -> basic_boot_error(
-                                      {missing_functions, MissingFunctions},
-                                      "Boot step functions not exported: ~p~n",
-                                      [MissingFunctions])
-            end;
-        {error, {vertex, duplicate, StepName}} ->
-            basic_boot_error({duplicate_boot_step, StepName},
-                             "Duplicate boot step name: ~w~n", [StepName]);
-        {error, {edge, Reason, From, To}} ->
-            basic_boot_error(
-              {invalid_boot_step_dependency, From, To},
-              "Could not add boot step dependency of ~w on ~w:~n~s",
-              [To, From,
-               case Reason of
-                   {bad_vertex, V} ->
-                       io_lib:format("Boot step not registered: ~w~n", [V]);
-                   {bad_edge, [First | Rest]} ->
-                       [io_lib:format("Cyclic dependency: ~w", [First]),
-                        [io_lib:format(" depends on ~w", [Next]) ||
-                            Next <- Rest],
-                        io_lib:format(" depends on ~w~n", [First])]
-               end])
-    end.
-
--ifdef(use_specs).
--spec(boot_error/2 :: (term(), not_available | [tuple()]) -> no_return()).
--endif.
-boot_error(Term={error, {timeout_waiting_for_tables, _}}, _Stacktrace) ->
-    AllNodes = rabbit_mnesia:cluster_nodes(all),
-    {Err, Nodes} =
-        case AllNodes -- [node()] of
-            [] -> {"Timeout contacting cluster nodes. Since RabbitMQ was"
-                   " shut down forcefully~nit cannot determine which nodes"
-                   " are timing out.~n", []};
-            Ns -> {rabbit_misc:format(
-                     "Timeout contacting cluster nodes: ~p.~n", [Ns]),
-                   Ns}
-        end,
-    basic_boot_error(Term,
-                     Err ++ rabbit_nodes:diagnostics(Nodes) ++ "~n~n", []);
-boot_error(Reason, Stacktrace) ->
-    Fmt = "Error description:~n   ~p~n~n" ++
-        "Log files (may contain more information):~n   ~s~n   ~s~n~n",
-    Args = [Reason, log_location(kernel), log_location(sasl)],
-    boot_error(Reason, Fmt, Args, Stacktrace).
-
--ifdef(use_specs).
--spec(boot_error/4 :: (term(), string(), [any()], not_available | [tuple()])
-                      -> no_return()).
--endif.
-boot_error(Reason, Fmt, Args, not_available) ->
-    basic_boot_error(Reason, Fmt, Args);
-boot_error(Reason, Fmt, Args, Stacktrace) ->
-    basic_boot_error(Reason, Fmt ++ "Stack trace:~n   ~p~n~n",
-                     Args ++ [Stacktrace]).
-
-basic_boot_error(Reason, Format, Args) ->
-    io:format("~n~nBOOT FAILED~n===========~n~n" ++ Format, Args),
-    rabbit_misc:local_info_msg(Format, Args),
-    timer:sleep(1000),
-    exit({?MODULE, failure_during_boot, Reason}).
-
-%%---------------------------------------------------------------------------
 %% boot step functions
 
 boot_delegate() ->
diff --git a/src/rabbit_alarm.erl b/src/rabbit_alarm.erl
index cd1d125bd5..ac2fb42f11 100644
--- a/src/rabbit_alarm.erl
+++ b/src/rabbit_alarm.erl
@@ -53,7 +53,8 @@ start_link() ->
 start() ->
     ok = rabbit_sup:start_restartable_child(?MODULE),
     ok = gen_event:add_handler(?SERVER, ?MODULE, []),
-    {ok, MemoryWatermark} = application:get_env(vm_memory_high_watermark),
+    {ok, MemoryWatermark} = application:get_env(rabbit,
+                                                vm_memory_high_watermark),
     rabbit_sup:start_restartable_child(
       vm_memory_monitor, [MemoryWatermark,
                           fun (Alarm) ->
@@ -61,7 +62,8 @@ start() ->
                                   set_alarm(Alarm)
                           end,
                           fun clear_alarm/1]),
-    {ok, DiskLimit} = application:get_env(disk_free_limit),
+    {ok, DiskLimit} = application:get_env(rabbit,
+                                          disk_free_limit),
     rabbit_sup:start_restartable_child(rabbit_disk_monitor, [DiskLimit]),
     ok.
 
diff --git a/src/rabbit_boot.erl b/src/rabbit_boot.erl
new file mode 100644
index 0000000000..5426301408
--- /dev/null
+++ b/src/rabbit_boot.erl
@@ -0,0 +1,210 @@
+%% The contents of this file are subject to the Mozilla Public License
+%% Version 1.1 (the "License"); you may not use this file except in
+%% compliance with the License. You may obtain a copy of the License
+%% at http://www.mozilla.org/MPL/
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and
+%% limitations under the License.
+%%
+%% The Original Code is RabbitMQ.
+%%
+%% The Initial Developer of the Original Code is GoPivotal, Inc.
+%% Copyright (c) 2007-2013 GoPivotal, Inc.  All rights reserved.
+%%
+
+-module(rabbit_boot).
+
+-export([start/1, stop/1]).
+-export([run_boot_steps/1]).
+-export([boot_error/2, boot_error/4]).
+
+-ifdef(use_specs).
+
+-spec(start/1          :: ([atom()]) -> 'ok').
+-spec(stop/1           :: ([atom()]) -> 'ok').
+-spec(run_boot_steps/1 :: (atom())   -> 'ok').
+-spec(boot_error/2     :: (term(), not_available | [tuple()]) -> no_return()).
+-spec(boot_error/4     :: (term(), string(), [any()], not_available | [tuple()])
+                            -> no_return()).
+
+-endif.
+
+%%---------------------------------------------------------------------------
+%% Public API
+
+start(Apps) ->
+    try
+        ets:new(boot_steps, [named_table, public, ordered_set]),
+        ok = app_utils:load_applications(Apps),
+        StartupApps = app_utils:app_dependency_order(Apps, false),
+        ok = app_utils:start_applications(StartupApps,
+                                          handle_app_error(could_not_start))
+    after
+        ets:delete(boot_steps)
+    end.
+
+stop(Apps) ->
+    ShutdownApps = app_utils:app_dependency_order(Apps, true),
+    try
+        ok = app_utils:stop_applications(
+               ShutdownApps, handle_app_error(error_during_shutdown))
+    after
+        [begin
+             Steps =
+                 sort_boot_steps(rabbit_misc:all_module_attributes(
+                                   App, rabbit_cleanup_step)),
+             [run_boot_step(Step) || Step <- Steps]
+         end || App <- ShutdownApps]
+    end.
+
+run_boot_steps(App) ->
+    RootApps = app_utils:app_dependencies(App),
+    Steps =
+        sort_boot_steps(
+          lists:usort(
+            lists:append(
+              [rabbit_misc:all_module_attributes(A, rabbit_boot_step) ||
+                  A <- [App|RootApps]]))),
+    [ok = run_boot_step(Step) || Step <- Steps],
+    ok.
+
+boot_error(Term={error, {timeout_waiting_for_tables, _}}, _Stacktrace) ->
+    AllNodes = rabbit_mnesia:cluster_nodes(all),
+    {Err, Nodes} =
+        case AllNodes -- [node()] of
+            [] -> {"Timeout contacting cluster nodes. Since RabbitMQ was"
+                   " shut down forcefully~nit cannot determine which nodes"
+                   " are timing out.~n", []};
+            Ns -> {rabbit_misc:format(
+                     "Timeout contacting cluster nodes: ~p.~n", [Ns]),
+                   Ns}
+        end,
+    basic_boot_error(Term,
+                     Err ++ rabbit_nodes:diagnostics(Nodes) ++ "~n~n", []);
+boot_error(Reason, Stacktrace) ->
+    Fmt = "Error description:~n   ~p~n~n" ++
+        "Log files (may contain more information):~n   ~s~n   ~s~n~n",
+    Args = [Reason, log_location(kernel), log_location(sasl)],
+    boot_error(Reason, Fmt, Args, Stacktrace).
+
+boot_error(Reason, Fmt, Args, not_available) ->
+    basic_boot_error(Reason, Fmt, Args);
+boot_error(Reason, Fmt, Args, Stacktrace) ->
+    basic_boot_error(Reason, Fmt ++ "Stack trace:~n   ~p~n~n",
+                     Args ++ [Stacktrace]).
+
+%%---------------------------------------------------------------------------
+%% Private API
+
+handle_app_error(Term) ->
+    fun(App, {bad_return, {_MFA, {'EXIT', {ExitReason, _}}}}) ->
+            throw({Term, App, ExitReason});
+       (App, Reason) ->
+            throw({Term, App, Reason})
+    end.
+
+run_boot_step({StepName, Attributes}) ->
+    case already_run(StepName) of
+        false -> run_it(StepName, Attributes);
+        true  -> ok
+    end.
+
+run_it(StepName, Attributes) ->
+    case [MFA || {mfa, MFA} <- Attributes] of
+        [] ->
+            ok;
+        MFAs ->
+            [try
+                 apply(M,F,A)
+             of
+                 ok              -> mark_complete(StepName);
+                 {error, Reason} -> boot_error(Reason, not_available)
+             catch
+                 _:Reason -> boot_error(Reason, erlang:get_stacktrace())
+             end || {M,F,A} <- MFAs],
+            ok
+    end.
+
+already_run(StepName) ->
+    ets:member(boot_steps, StepName).
+
+mark_complete(StepName) ->
+    ets:insert(boot_steps, {StepName}).
+
+basic_boot_error(Reason, Format, Args) ->
+    io:format("~n~nBOOT FAILED~n===========~n~n" ++ Format, Args),
+    rabbit_misc:local_info_msg(Format, Args),
+    timer:sleep(1000),
+    exit({?MODULE, failure_during_boot, Reason}).
+
+%% TODO: move me to rabbit_misc
+log_location(Type) ->
+    case application:get_env(rabbit, case Type of
+                                         kernel -> error_logger;
+                                         sasl   -> sasl_error_logger
+                                     end) of
+        {ok, {file, File}} -> File;
+        {ok, false}        -> undefined;
+        {ok, tty}          -> tty;
+        {ok, silent}       -> undefined;
+        {ok, Bad}          -> throw({error, {cannot_log_to_file, Bad}});
+        _                  -> undefined
+    end.
+
+vertices(_Module, Steps) ->
+    [{StepName, {StepName, Atts}} || {StepName, Atts} <- Steps].
+
+edges(_Module, Steps) ->
+    [case Key of
+         requires -> {StepName, OtherStep};
+         enables  -> {OtherStep, StepName}
+     end || {StepName, Atts} <- Steps,
+            {Key, OtherStep} <- Atts,
+            Key =:= requires orelse Key =:= enables].
+
+sort_boot_steps(UnsortedSteps) ->
+    case rabbit_misc:build_acyclic_graph(fun vertices/2, fun edges/2,
+                                         UnsortedSteps) of
+        {ok, G} ->
+            %% Use topological sort to find a consistent ordering (if
+            %% there is one, otherwise fail).
+            SortedSteps = lists:reverse(
+                            [begin
+                                 {StepName, Step} = digraph:vertex(G,
+                                                                   StepName),
+                                 Step
+                             end || StepName <- digraph_utils:topsort(G)]),
+            digraph:delete(G),
+            %% Check that all mentioned {M,F,A} triples are exported.
+            case [{StepName, {M,F,A}} ||
+                     {StepName, Attributes} <- SortedSteps,
+                     {mfa, {M,F,A}}         <- Attributes,
+                     not erlang:function_exported(M, F, length(A))] of
+                []               -> SortedSteps;
+                MissingFunctions -> basic_boot_error(
+                                      {missing_functions, MissingFunctions},
+                                      "Boot step functions not exported: ~p~n",
+                                      [MissingFunctions])
+            end;
+        {error, {vertex, duplicate, StepName}} ->
+            basic_boot_error({duplicate_boot_step, StepName},
+                             "Duplicate boot step name: ~w~n", [StepName]);
+        {error, {edge, Reason, From, To}} ->
+            basic_boot_error(
+              {invalid_boot_step_dependency, From, To},
+              "Could not add boot step dependency of ~w on ~w:~n~s",
+              [To, From,
+               case Reason of
+                   {bad_vertex, V} ->
+                       io_lib:format("Boot step not registered: ~w~n", [V]);
+                   {bad_edge, [First | Rest]} ->
+                       [io_lib:format("Cyclic dependency: ~w", [First]),
+                        [io_lib:format(" depends on ~w", [Next]) ||
+                            Next <- Rest],
+                        io_lib:format(" depends on ~w~n", [First])]
+               end])
+    end.
+
+
diff --git a/src/rabbit_misc.erl b/src/rabbit_misc.erl
index 00c4eaf3df..0fa3190a69 100644
--- a/src/rabbit_misc.erl
+++ b/src/rabbit_misc.erl
@@ -51,7 +51,8 @@
 -export([dict_cons/3, orddict_cons/3, gb_trees_cons/3]).
 -export([gb_trees_fold/3, gb_trees_foreach/2]).
 -export([parse_arguments/3]).
--export([all_module_attributes/1, build_acyclic_graph/3]).
+-export([all_module_attributes/1, all_module_attributes/2]).
+-export([build_acyclic_graph/3]).
 -export([now_ms/0]).
 -export([const_ok/0, const/1]).
 -export([ntoa/1, ntoab/1]).
@@ -209,6 +210,7 @@
         -> {'ok', {atom(), [{string(), string()}], [string()]}} |
            'no_command').
 -spec(all_module_attributes/1 :: (atom()) -> [{atom(), [term()]}]).
+-spec(all_module_attributes/2 :: (atom(), atom()) -> [{atom(), [term()]}]).
 -spec(build_acyclic_graph/3 ::
         (graph_vertex_fun(), graph_edge_fun(), [{atom(), [term()]}])
         -> rabbit_types:ok_or_error2(digraph(),
@@ -850,12 +852,8 @@ module_attributes(Module) ->
             V
     end.
 
-all_module_attributes(Name) ->
-    Modules =
-        lists:usort(
-          lists:append(
-            [Modules || {App, _, _}   <- application:loaded_applications(),
-                        {ok, Modules} <- [application:get_key(App, modules)]])),
+all_module_attributes(App, Name) ->
+    {ok, Modules} = application:get_key(App, modules),
     lists:foldl(
       fun (Module, Acc) ->
               case lists:append([Atts || {N, Atts} <- module_attributes(Module),
@@ -865,6 +863,10 @@ all_module_attributes(Name) ->
               end
       end, [], Modules).
 
+all_module_attributes(Name) ->
+    lists:usort(lists:append(
+                  [all_module_attributes(App, Name) ||
+                      {App, _, _}   <- application:loaded_applications()])).
 
 build_acyclic_graph(VertexFun, EdgeFun, Graph) ->
     G = digraph:new([acyclic]),
diff --git a/src/rabbit_networking.erl b/src/rabbit_networking.erl
index 91be4dcb07..5960c00075 100644
--- a/src/rabbit_networking.erl
+++ b/src/rabbit_networking.erl
@@ -125,12 +125,12 @@ boot() ->
     ok = boot_ssl().
 
 boot_tcp() ->
-    {ok, TcpListeners} = application:get_env(tcp_listeners),
+    {ok, TcpListeners} = application:get_env(rabbit, tcp_listeners),
     [ok = start_tcp_listener(Listener) || Listener <- TcpListeners],
     ok.
 
 boot_ssl() ->
-    case application:get_env(ssl_listeners) of
+    case application:get_env(rabbit, ssl_listeners) of
         {ok, []} ->
             ok;
         {ok, SslListeners} ->