merge default into bug20399

7 files changed, 205 insertions, 101 deletions

diff --git a/src/rabbit.erl b/src/rabbit.erl
index ef1e004996..b098fd5a27 100644
--- a/src/rabbit.erl
+++ b/src/rabbit.erl
@@ -215,6 +215,16 @@ log_location(Type) ->
         _                  -> undefined
     end.
 
+app_location() ->
+    {ok, Application} = application:get_application(),
+    filename:absname(code:where_is_file(atom_to_list(Application) ++ ".app")).
+
+home_dir() ->
+    case init:get_argument(home) of
+        {ok, [[Home]]} -> Home;
+        Other          -> Other
+    end.
+
 %---------------------------------------------------------------------------
 
 print_banner() ->
@@ -237,10 +247,13 @@ print_banner() ->
               [Product, string:right([$v|Version], ProductLen),
                ?PROTOCOL_VERSION_MAJOR, ?PROTOCOL_VERSION_MINOR,
                ?COPYRIGHT_MESSAGE, ?INFORMATION_MESSAGE]),
-    Settings = [{"node",         node()},
-                {"log",          log_location(kernel)},
-                {"sasl log",     log_location(sasl)},
-                {"database dir", rabbit_mnesia:dir()}],
+    Settings = [{"node",           node()},
+                {"app descriptor", app_location()},
+                {"home dir",       home_dir()},
+                {"cookie hash",    rabbit_misc:cookie_hash()},
+                {"log",            log_location(kernel)},
+                {"sasl log",       log_location(sasl)},
+                {"database dir",   rabbit_mnesia:dir()}],
     DescrLen = lists:max([length(K) || {K, _V} <- Settings]),
     Format = "~-" ++ integer_to_list(DescrLen) ++ "s: ~s~n",
     lists:foreach(fun ({K, V}) -> io:format(Format, [K, V]) end, Settings),
diff --git a/src/rabbit_control.erl b/src/rabbit_control.erl
index 69e91803d6..a15c7fd84e 100644
--- a/src/rabbit_control.erl
+++ b/src/rabbit_control.erl
@@ -80,13 +80,38 @@ start() ->
         {error, Reason} ->
             error("~p", [Reason]),
             halt(2);
+        {badrpc, Reason} ->
+            error("unable to connect to node ~w: ~w", [Node, Reason]),
+            print_badrpc_diagnostics(Node),
+            halt(2);
         Other ->
             error("~p", [Other]),
             halt(2)
     end.
 
-error(Format, Args) ->
-    rabbit_misc:format_stderr("Error: " ++ Format ++ "~n", Args).
+fmt_stderr(Format, Args) -> rabbit_misc:format_stderr(Format ++ "~n", Args).
+
+error(Format, Args) -> fmt_stderr("Error: " ++ Format, Args).
+
+print_badrpc_diagnostics(Node) ->
+    fmt_stderr("diagnostics:", []),
+    NodeHost = rabbit_misc:nodehost(Node),
+    case net_adm:names(NodeHost) of
+        {error, EpmdReason} ->
+            fmt_stderr("- unable to connect to epmd on ~s: ~w",
+                       [NodeHost, EpmdReason]);
+                {ok, NamePorts} ->
+            fmt_stderr("- nodes and their ports on ~s: ~p",
+                       [NodeHost, [{list_to_atom(Name), Port} ||
+                                      {Name, Port} <- NamePorts]])
+            end,
+    fmt_stderr("- current node: ~w", [node()]),
+    case init:get_argument(home) of
+        {ok, [[Home]]} -> fmt_stderr("- current node home dir: ~s", [Home]);
+        Other          -> fmt_stderr("- no current node home dir: ~p", [Other])
+    end,
+    fmt_stderr("- current node cookie hash: ~s", [rabbit_misc:cookie_hash()]),
+    ok.
 
 parse_args(["-n", NodeS | Args], Params) ->
     Node = case lists:member($@, NodeS) of
@@ -197,9 +222,11 @@ action(cluster, Node, ClusterNodeSs, Inform) ->
 
 action(status, Node, [], Inform) ->
     Inform("Status of node ~p", [Node]),
-    Res = call(Node, {rabbit, status, []}),
-    io:format("~p~n", [Res]),
-    ok;
+    case call(Node, {rabbit, status, []}) of
+        {badrpc, _} = Res -> Res;
+        Res               -> io:format("~p~n", [Res]),
+                             ok
+    end;
 
 action(rotate_logs, Node, [], Inform) ->
     Inform("Reopening logs for node ~p", [Node]),
diff --git a/src/rabbit_dialyzer.erl b/src/rabbit_dialyzer.erl
new file mode 100644
index 0000000000..23e6fc4432
--- /dev/null
+++ b/src/rabbit_dialyzer.erl
@@ -0,0 +1,91 @@
+%%   The contents of this file are subject to the Mozilla Public License
+%%   Version 1.1 (the "License"); you may not use this file except in
+%%   compliance with the License. You may obtain a copy of the License at
+%%   http://www.mozilla.org/MPL/
+%%
+%%   Software distributed under the License is distributed on an "AS IS"
+%%   basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+%%   License for the specific language governing rights and limitations
+%%   under the License.
+%%
+%%   The Original Code is RabbitMQ.
+%%
+%%   The Initial Developers of the Original Code are LShift Ltd,
+%%   Cohesive Financial Technologies LLC, and Rabbit Technologies Ltd.
+%%
+%%   Portions created before 22-Nov-2008 00:00:00 GMT by LShift Ltd,
+%%   Cohesive Financial Technologies LLC, or Rabbit Technologies Ltd
+%%   are Copyright (C) 2007-2008 LShift Ltd, Cohesive Financial
+%%   Technologies LLC, and Rabbit Technologies Ltd.
+%%
+%%   Portions created by LShift Ltd are Copyright (C) 2007-2009 LShift
+%%   Ltd. Portions created by Cohesive Financial Technologies LLC are
+%%   Copyright (C) 2007-2009 Cohesive Financial Technologies
+%%   LLC. Portions created by Rabbit Technologies Ltd are Copyright
+%%   (C) 2007-2009 Rabbit Technologies Ltd.
+%%
+%%   All Rights Reserved.
+%%
+%%   Contributor(s): ______________________________________.
+%%
+
+-module(rabbit_dialyzer).
+-include("rabbit.hrl").
+
+-export([create_basic_plt/1, add_to_plt/2, dialyze_files/2, halt_with_code/1]).
+
+%%----------------------------------------------------------------------------
+
+-ifdef(use_specs).
+
+-spec(create_basic_plt/1 :: (string()) -> 'ok').
+-spec(add_to_plt/2 :: (string(), string()) -> 'ok').
+-spec(dialyze_files/2 :: (string(), string()) -> 'ok').
+-spec(halt_with_code/1 :: (atom()) -> no_return()).
+
+-endif.
+
+%%----------------------------------------------------------------------------
+
+create_basic_plt(BasicPltPath) ->
+    OptsRecord = dialyzer_options:build(
+                   [{analysis_type, plt_build},
+                    {output_plt, BasicPltPath},
+                    {files_rec, otp_apps_dependencies_paths()}]),
+    dialyzer_cl:start(OptsRecord),
+    ok.
+
+add_to_plt(PltPath, FilesString) ->
+    {ok, Files} = regexp:split(FilesString, " "),
+    DialyzerWarnings = dialyzer:run([{analysis_type, plt_add},
+                                     {init_plt, PltPath},
+                                     {output_plt, PltPath},
+                                     {files, Files}]),
+    print_warnings(DialyzerWarnings),
+    ok.
+
+dialyze_files(PltPath, ModifiedFiles) ->
+    {ok, Files} = regexp:split(ModifiedFiles, " "),
+    DialyzerWarnings = dialyzer:run([{init_plt, PltPath},
+                                     {files, Files}]),
+    case DialyzerWarnings of
+        [] -> io:format("~nOk~n"),
+              ok;
+        _  -> io:format("~nFAILED with the following warnings:~n"),
+              print_warnings(DialyzerWarnings),
+              fail
+    end.
+
+print_warnings(Warnings) ->
+    [io:format("~s", [dialyzer:format_warning(W)]) || W <- Warnings],
+    io:format("~n"),
+    ok.
+
+otp_apps_dependencies_paths() ->
+    [code:lib_dir(App, ebin) ||
+        App <- [kernel, stdlib, sasl, mnesia, os_mon, ssl, eunit, tools]].
+
+halt_with_code(ok) ->
+    halt();
+halt_with_code(fail) ->
+    halt(1).
diff --git a/src/rabbit_load.erl b/src/rabbit_load.erl
index 7bf85347fb..6ef638cb59 100644
--- a/src/rabbit_load.erl
+++ b/src/rabbit_load.erl
@@ -41,7 +41,7 @@
 -ifdef(use_specs).
 
 -type(erlang_node() :: atom()).
--type(load() :: {{non_neg_integer(), float()}, erlang_node()}).
+-type(load() :: {{non_neg_integer(), integer() | 'unknown'}, erlang_node()}).
 -spec(local_load/0 :: () -> load()).
 -spec(remote_loads/0 :: () -> [load()]).
 -spec(pick/0 :: () -> erlang_node()).
@@ -52,8 +52,11 @@
 
 local_load() ->
     LoadAvg = case whereis(cpu_sup) of
-                  undefined -> 0.0;
-                  _Other -> cpu_sup:avg1()
+                  undefined -> unknown;
+                  _         -> case cpu_sup:avg1() of
+                                   L when is_integer(L) -> L;
+                                   {error, timeout}     -> unknown
+                               end
               end,
     {{statistics(run_queue), LoadAvg}, node()}.
 
@@ -65,8 +68,12 @@ remote_loads() ->
 pick() ->
     RemoteLoads = remote_loads(),
     {{RunQ, LoadAvg}, Node} = local_load(),
-    %% add bias towards current node
-    AdjustedLoadAvg = LoadAvg * ?FUDGE_FACTOR,
+    %% add bias towards current node; we rely on Erlang's term order
+    %% of SomeFloat < local_unknown < unknown.
+    AdjustedLoadAvg = case LoadAvg of
+                          unknown -> local_unknown;
+                          _       -> LoadAvg * ?FUDGE_FACTOR
+                      end,
     Loads = [{{RunQ, AdjustedLoadAvg}, Node} | RemoteLoads],
     {_, SelectedNode} = lists:min(Loads),
     SelectedNode.
diff --git a/src/rabbit_misc.erl b/src/rabbit_misc.erl
index 5843143839..b20e9a86b6 100644
--- a/src/rabbit_misc.erl
+++ b/src/rabbit_misc.erl
@@ -47,7 +47,7 @@
 -export([with_user/2, with_vhost/2, with_user_and_vhost/3]).
 -export([execute_mnesia_transaction/1]).
 -export([ensure_ok/2]).
--export([localnode/1, tcp_name/3]).
+-export([localnode/1, nodehost/1, cookie_hash/0, tcp_name/3]).
 -export([intersperse/2, upmap/2, map_in_order/2]).
 -export([table_foreach/2]).
 -export([dirty_read_all/1, dirty_foreach_key/2, dirty_dump_log/1]).
@@ -106,6 +106,8 @@
 -spec(execute_mnesia_transaction/1 :: (thunk(A)) -> A).
 -spec(ensure_ok/2 :: (ok_or_error(), atom()) -> 'ok').
 -spec(localnode/1 :: (atom()) -> erlang_node()).
+-spec(nodehost/1 :: (erlang_node()) -> string()).
+-spec(cookie_hash/0 :: () -> string()).
 -spec(tcp_name/3 :: (atom(), ip_address(), ip_port()) -> atom()).
 -spec(intersperse/2 :: (A, [A]) -> [A]).
 -spec(upmap/2 :: (fun ((A) -> B), [A]) -> [B]).
@@ -307,11 +309,15 @@ ensure_ok(ok, _) -> ok;
 ensure_ok({error, Reason}, ErrorTag) -> throw({error, {ErrorTag, Reason}}).
 
 localnode(Name) ->
+    list_to_atom(lists:append([atom_to_list(Name), "@", nodehost(node())])).
+
+nodehost(Node) ->
     %% This is horrible, but there doesn't seem to be a way to split a
     %% nodename into its constituent parts.
-    list_to_atom(lists:append(atom_to_list(Name),
-                              lists:dropwhile(fun (E) -> E =/= $@ end,
-                                              atom_to_list(node())))).
+    tl(lists:dropwhile(fun (E) -> E =/= $@ end, atom_to_list(Node))).
+
+cookie_hash() ->
+    ssl_base64:encode(erlang:md5(atom_to_list(erlang:get_cookie()))).
 
 tcp_name(Prefix, IPAddress, Port)
   when is_atom(Prefix) andalso is_number(Port) ->
diff --git a/src/rabbit_networking.erl b/src/rabbit_networking.erl
index eed21a01cd..4bbdb65eb5 100644
--- a/src/rabbit_networking.erl
+++ b/src/rabbit_networking.erl
@@ -39,8 +39,8 @@
 %%used by TCP-based transports, e.g. STOMP adapter
 -export([check_tcp_listener_address/3]).
 
--export([tcp_listener_started/2, ssl_connection_upgrade/2, 
-        tcp_listener_stopped/2, start_client/1]).
+-export([tcp_listener_started/2, tcp_listener_stopped/2,
+         start_client/1, start_ssl_client/2]).
 
 -include("rabbit.hrl").
 -include_lib("kernel/include/inet.hrl").
@@ -101,7 +101,7 @@ check_tcp_listener_address(NamePrefix, Host, Port) ->
     if is_integer(Port) andalso (Port >= 0) andalso (Port =< 65535) -> ok;
        true -> error_logger:error_msg("invalid port ~p - not 0..65535~n",
                                       [Port]),
-               throw({error, invalid_port, Port})
+               throw({error, {invalid_port, Port}})
     end,
     Name = rabbit_misc:tcp_name(NamePrefix, IPAddress, Port),
     {IPAddress, Name}.
@@ -112,7 +112,7 @@ start_tcp_listener(Host, Port) ->
 
 start_ssl_listener(Host, Port, SslOpts) ->
     start_listener(Host, Port, "SSL Listener",
-                   {?MODULE, ssl_connection_upgrade, [SslOpts]}).
+                   {?MODULE, start_ssl_client, [SslOpts]}).
 
 start_listener(Host, Port, Label, OnConnect) ->
     {IPAddress, Name} =
@@ -166,7 +166,7 @@ start_client(Sock) ->
     Child ! {go, Sock},
     Child.
 
-ssl_connection_upgrade(SslOpts, Sock) ->
+start_ssl_client(SslOpts, Sock) ->
     {ok, {PeerAddress, PeerPort}} = rabbit_net:peername(Sock),
     PeerIp = inet_parse:ntoa(PeerAddress),
 
diff --git a/src/rabbit_reader.erl b/src/rabbit_reader.erl
index 677144a5cf..5cc989929f 100644
--- a/src/rabbit_reader.erl
+++ b/src/rabbit_reader.erl
@@ -49,7 +49,6 @@
 -define(HANDSHAKE_TIMEOUT, 10).
 -define(NORMAL_TIMEOUT, 3).
 -define(CLOSING_TIMEOUT, 1).
--define(CHANNEL_CLOSING_TIMEOUT, 1).
 -define(CHANNEL_TERMINATION_TIMEOUT, 3).
 
 %---------------------------------------------------------------------------
@@ -94,23 +93,19 @@
 %%   -> log error, wait for channels to terminate forcefully, start
 %%      terminate_connection timer, send close, *closed*
 %%   channel exit with soft error
-%%   -> log error, start terminate_channel timer, mark channel as
-%%      closing, *running*
-%%   terminate_channel timeout -> remove 'closing' mark, *running*
+%%   -> log error, mark channel as closing, *running*
 %%   handshake_timeout -> ignore, *running*
 %%   heartbeat timeout -> *throw*
 %% closing:
 %%   socket close -> *terminate*
 %%   receive frame -> ignore, *closing*
-%%   terminate_channel timeout -> remove 'closing' mark, *closing*
 %%   handshake_timeout -> ignore, *closing*
 %%   heartbeat timeout -> *throw*
 %%   channel exit with hard error
 %%   -> log error, wait for channels to terminate forcefully, start
 %%      terminate_connection timer, send close, *closed*
 %%   channel exit with soft error
-%%   -> log error, start terminate_channel timer, mark channel as
-%%      closing
+%%   -> log error, mark channel as closing
 %%      if last channel to exit then send connection.close_ok,
 %%         start terminate_connection timer, *closed*
 %%      else *closing*
@@ -123,7 +118,6 @@
 %%     *closed*
 %%   receive frame -> ignore, *closed*
 %%   terminate_connection timeout -> *terminate*
-%%   terminate_channel timeout -> remove 'closing' mark, *closed*
 %%   handshake_timeout -> ignore, *closed*
 %%   heartbeat timeout -> *throw*
 %%   channel exit -> log error, *closed*
@@ -290,8 +284,6 @@ mainloop(Parent, Deb, State = #v1{sock= Sock, recv_ref = Ref}) ->
             mainloop(Parent, Deb, handle_channel_exit(Channel, Reason, State));
         {'EXIT', Pid, Reason} ->
             mainloop(Parent, Deb, handle_dependent_exit(Pid, Reason, State));
-        {terminate_channel, Channel, Ref1} ->
-            mainloop(Parent, Deb, terminate_channel(Channel, Ref1, State));
         terminate_connection ->
             State;
         handshake_timeout ->
@@ -339,32 +331,14 @@ close_connection(State = #v1{connection = #connection{
     State#v1{connection_state = closed}.
 
 close_channel(Channel, State) ->
-    Ref = make_ref(),
-    TRef = erlang:send_after(1000 * ?CHANNEL_CLOSING_TIMEOUT,
-                             self(),
-                             {terminate_channel, Channel, Ref}),
-    put({closing_channel, Channel}, {Ref, TRef}),
-    State.
-
-terminate_channel(Channel, Ref, State) ->
-    case get({closing_channel, Channel}) of
-        undefined -> ok; %% got close_ok in the meantime
-        {Ref, _}  -> erase({closing_channel, Channel}),
-                     ok;
-        {_Ref, _} -> ok %% got close_ok, and have new closing channel
-    end,
+    put({channel, Channel}, closing),
     State.
 
 handle_channel_exit(Channel, Reason, State) ->
-    %% We remove the channel from the inbound map only. That allows
-    %% the channel to be re-opened, but also means the remaining
-    %% cleanup, including possibly closing the connection, is deferred
-    %% until we get the (normal) exit signal.
-    erase({channel, Channel}),
     handle_exception(State, Channel, Reason).
 
 handle_dependent_exit(Pid, normal, State) ->
-    channel_cleanup(Pid),
+    erase({chpid, Pid}),
     maybe_close(State);
 handle_dependent_exit(Pid, Reason, State) ->
     case channel_cleanup(Pid) of
@@ -374,17 +348,10 @@ handle_dependent_exit(Pid, Reason, State) ->
 
 channel_cleanup(Pid) ->
     case get({chpid, Pid}) of
-        undefined ->
-            case get({closing_chpid, Pid}) of
-                undefined -> undefined;
-                {channel, Channel} ->
-                    erase({closing_chpid, Pid}),
-                    Channel
-            end;
-        {channel, Channel} ->
-            erase({channel, Channel}),
-            erase({chpid, Pid}),
-            Channel
+        undefined          -> undefined;
+        {channel, Channel} -> erase({channel, Channel}),
+                              erase({chpid, Pid}),
+                              Channel
     end.
 
 all_channels() -> [Pid || {{chpid, Pid},_} <- get()].
@@ -449,7 +416,7 @@ handle_frame(_Type, _Channel, _Payload, State = #v1{connection_state = CS})
     State;
 handle_frame(Type, 0, Payload, State) ->
     case analyze_frame(Type, Payload) of
-        error     -> throw({unknown_frame, Type, Payload});
+        error     -> throw({unknown_frame, 0, Type, Payload});
         heartbeat -> State;
         trace     -> State;
         {method, MethodName, FieldsBin} ->
@@ -458,20 +425,34 @@ handle_frame(Type, 0, Payload, State) ->
     end;
 handle_frame(Type, Channel, Payload, State) ->
     case analyze_frame(Type, Payload) of
-        error         -> throw({unknown_frame, Type, Payload});
+        error         -> throw({unknown_frame, Channel, Type, Payload});
         heartbeat     -> throw({unexpected_heartbeat_frame, Channel});
         trace         -> throw({unexpected_trace_frame, Channel});
         AnalyzedFrame ->
             %%?LOGDEBUG("Ch ~p Frame ~p~n", [Channel, AnalyzedFrame]),
             case get({channel, Channel}) of
                 {chpid, ChPid} ->
-                    ok = check_for_close(Channel, ChPid, AnalyzedFrame),
+                    case AnalyzedFrame of
+                        {method, 'channel.close', _} ->
+                            erase({channel, Channel});
+                        _ -> ok
+                    end,
                     ok = rabbit_framing_channel:process(ChPid, AnalyzedFrame),
                     State;
+                closing ->
+                    %% According to the spec, after sending a
+                    %% channel.close we must ignore all frames except
+                    %% channel.close_ok.
+                    case AnalyzedFrame of
+                        {method, 'channel.close_ok', _} ->
+                            erase({channel, Channel});
+                        _ -> ok
+                    end,
+                    State;
                 undefined ->
                     case State#v1.connection_state of
-                        running -> send_to_new_channel(
-                                     Channel, AnalyzedFrame, State),
+                        running -> ok = send_to_new_channel(
+                                          Channel, AnalyzedFrame, State),
                                    State;
                         Other   -> throw({channel_frame_while_starting,
                                           Channel, Other, AnalyzedFrame})
@@ -714,38 +695,17 @@ i(Item, #v1{}) ->
 %%--------------------------------------------------------------------------
 
 send_to_new_channel(Channel, AnalyzedFrame, State) ->
-    case get({closing_channel, Channel}) of
-        undefined ->
-            #v1{sock = Sock,
-                connection = #connection{
-                  frame_max = FrameMax,
-                  user = #user{username = Username},
-                  vhost = VHost}} = State,
-            WriterPid = rabbit_writer:start(Sock, Channel, FrameMax),
-            ChPid = rabbit_framing_channel:start_link(
-                      fun rabbit_channel:start_link/5,
-                      [Channel, self(), WriterPid, Username, VHost]),
-            put({channel, Channel}, {chpid, ChPid}),
-            put({chpid, ChPid}, {channel, Channel}),
-            ok = rabbit_framing_channel:process(ChPid, AnalyzedFrame);
-        {_, TRef} ->
-            %% According to the spec, after sending a channel.close we
-            %% must ignore all frames except channel.close_ok.
-            case AnalyzedFrame of
-                {method, 'channel.close_ok', _} ->
-                    erlang:cancel_timer(TRef),
-                    erase({closing_channel, Channel}),
-                    ok;
-                _Other -> ok
-            end
-    end.
-
-check_for_close(Channel, ChPid, {method, 'channel.close', _}) ->
-    channel_cleanup(ChPid),
-    put({closing_chpid, ChPid}, {channel, Channel}),
-    ok;
-check_for_close(_Channel, _ChPid, _Frame) ->
-    ok.
+    #v1{sock = Sock, connection = #connection{
+                       frame_max = FrameMax,
+                       user = #user{username = Username},
+                       vhost = VHost}} = State,
+    WriterPid = rabbit_writer:start(Sock, Channel, FrameMax),
+    ChPid = rabbit_framing_channel:start_link(
+              fun rabbit_channel:start_link/5,
+              [Channel, self(), WriterPid, Username, VHost]),
+    put({channel, Channel}, {chpid, ChPid}),
+    put({chpid, ChPid}, {channel, Channel}),
+    ok = rabbit_framing_channel:process(ChPid, AnalyzedFrame).
 
 log_channel_error(ConnectionState, Channel, Reason) ->
     rabbit_log:error("connection ~p (~p), channel ~p - error:~n~p~n",