Fix a race condition that can occur on queue deletion. Also change boot order to make sure the msg_stores are started before us (and thus stopped after us)

4 files changed, 34 insertions, 25 deletions

diff --git a/src/rabbit_mirror_queue_coordinator.erl b/src/rabbit_mirror_queue_coordinator.erl
index fb65014468..6303952d1f 100644
--- a/src/rabbit_mirror_queue_coordinator.erl
+++ b/src/rabbit_mirror_queue_coordinator.erl
@@ -79,9 +79,12 @@ handle_cast({gm_deaths, Deaths},
             State = #state { q  = #amqqueue { name = QueueName } }) ->
     rabbit_log:info("Master ~p saw deaths ~p for queue ~p~n",
                     [self(), Deaths, QueueName]),
-    Node = node(),
-    Node = node(rabbit_mirror_queue_misc:remove_from_queue(QueueName, Deaths)),
-    noreply(State).
+    case rabbit_mirror_queue_misc:remove_from_queue(QueueName, Deaths) of
+        {ok, Pid} when node(Pid) =:= node() ->
+            noreply(State);
+        {error, not_found} ->
+            {stop, normal, State}
+    end.
 
 handle_info(Msg, State) ->
     {stop, {unexpected_info, Msg}, State}.
diff --git a/src/rabbit_mirror_queue_misc.erl b/src/rabbit_mirror_queue_misc.erl
index 237bf080a9..0560207662 100644
--- a/src/rabbit_mirror_queue_misc.erl
+++ b/src/rabbit_mirror_queue_misc.erl
@@ -24,19 +24,23 @@ remove_from_queue(QueueName, DeadPids) ->
     DeadNodes = [node(DeadPid) || DeadPid <- DeadPids],
     rabbit_misc:execute_mnesia_transaction(
       fun () ->
-              [Q = #amqqueue { pid         = QPid,
-                               mirror_pids = MPids }] =
-                  mnesia:read({rabbit_queue, QueueName}),
-              [QPid1 | MPids1] =
-                  [Pid || Pid <- [QPid | MPids],
-                          not lists:member(node(Pid), DeadNodes)],
-              case {{QPid, MPids}, {QPid1, MPids1}} of
-                  {Same, Same} ->
-                      QPid;
-                  _ ->
-                      Q1 = Q #amqqueue { pid         = QPid1,
-                                         mirror_pids = MPids1 },
-                      mnesia:write(rabbit_queue, Q1, write),
-                      QPid1
+              %% Someone else could have deleted the queue before we
+              %% get here.
+              case mnesia:read({rabbit_queue, QueueName}) of
+                  [] -> {error, not_found};
+                  [Q = #amqqueue { pid         = QPid,
+                                   mirror_pids = MPids }] ->
+                      [QPid1 | MPids1] =
+                          [Pid || Pid <- [QPid | MPids],
+                                  not lists:member(node(Pid), DeadNodes)],
+                      case {{QPid, MPids}, {QPid1, MPids1}} of
+                          {Same, Same} ->
+                              {ok, QPid};
+                          _ ->
+                              Q1 = Q #amqqueue { pid         = QPid1,
+                                                 mirror_pids = MPids1 },
+                              mnesia:write(rabbit_queue, Q1, write),
+                              {ok, QPid1}
+                      end
               end
       end).
diff --git a/src/rabbit_mirror_queue_slave.erl b/src/rabbit_mirror_queue_slave.erl
index 166f473a71..f124bc9eb0 100644
--- a/src/rabbit_mirror_queue_slave.erl
+++ b/src/rabbit_mirror_queue_slave.erl
@@ -166,16 +166,18 @@ handle_call({gm_deaths, Deaths}, From,
                              master_node = MNode }) ->
     rabbit_log:info("Slave ~p saw deaths ~p for queue ~p~n",
                     [self(), Deaths, QueueName]),
-    case {node(), node(rabbit_mirror_queue_misc:remove_from_queue(
-                         QueueName, Deaths))} of
-        {_Node, MNode} ->
+    case rabbit_mirror_queue_misc:remove_from_queue(QueueName, Deaths) of
+        {ok, Pid} when node(Pid) =:= MNode ->
             reply(ok, State);
-        {Node, Node} ->
+        {ok, Pid} when node(Pid) =:= node() ->
             promote_me(From, State);
-        {_Node, MNode1} ->
+        {ok, Pid} ->
             gen_server2:reply(From, ok),
             ok = gm:broadcast(GM, heartbeat),
-            noreply(State #state { master_node = MNode1 })
+            noreply(State #state { master_node = node(Pid) });
+        {error, not_found} ->
+            gen_server2:reply(From, ok),
+            {stop, normal, State}
     end;
 
 handle_call({maybe_run_queue_via_backing_queue, Fun}, _From, State) ->
diff --git a/src/rabbit_mirror_queue_slave_sup.erl b/src/rabbit_mirror_queue_slave_sup.erl
index 6658e6c30f..80c0520c08 100644
--- a/src/rabbit_mirror_queue_slave_sup.erl
+++ b/src/rabbit_mirror_queue_slave_sup.erl
@@ -19,8 +19,8 @@
 -rabbit_boot_step({mirror_queue_slave_sup,
                    [{description, "mirror queue slave sup"},
                     {mfa,         {rabbit_mirror_queue_slave_sup, start, []}},
-                    {requires,    core_initialized},
-                    {enables,     queue_sup_queue_recovery}]}).
+                    {requires,    queue_sup_queue_recovery},
+                    {enables,     routing_ready}]}).
 
 -behaviour(supervisor2).