Merge branch 'master' into rabbitmq-server-1310

Conflicts:
	test/vhost_SUITE.erl

2 files changed, 46 insertions, 11 deletions

diff --git a/test/rabbitmqctl_integration_SUITE.erl b/test/rabbitmqctl_integration_SUITE.erl
index 535725d585..71b74ea104 100644
--- a/test/rabbitmqctl_integration_SUITE.erl
+++ b/test/rabbitmqctl_integration_SUITE.erl
@@ -31,6 +31,7 @@
 -export([list_queues_local/1
         ,list_queues_offline/1
         ,list_queues_online/1
+        ,list_queues_stopped/1
         ]).
 
 all() ->
@@ -44,6 +45,7 @@ groups() ->
             [list_queues_local
             ,list_queues_online
             ,list_queues_offline
+            ,list_queues_stopped
             ]}
     ].
 
@@ -96,13 +98,19 @@ end_per_group(list_queues, Config0) ->
     rabbit_ct_helpers:run_steps(Config1,
                                 rabbit_ct_client_helpers:teardown_steps() ++
                                     rabbit_ct_broker_helpers:teardown_steps());
-end_per_group(global_parameters, Config) ->
-    rabbit_ct_helpers:run_teardown_steps(Config,
-        rabbit_ct_client_helpers:teardown_steps() ++
-        rabbit_ct_broker_helpers:teardown_steps());
 end_per_group(_, Config) ->
     Config.
 
+init_per_testcase(list_queues_stopped, Config0) ->
+    %% Start node 3 to crash it's queues
+    rabbit_ct_broker_helpers:start_node(Config0, 2),
+    %% Make vhost "down" on nodes 2 and 3
+    rabbit_ct_broker_helpers:force_vhost_failure(Config0, 1, <<"/">>),
+    rabbit_ct_broker_helpers:force_vhost_failure(Config0, 2, <<"/">>),
+
+    rabbit_ct_broker_helpers:stop_node(Config0, 2),
+    rabbit_ct_helpers:testcase_started(Config0, list_queues_stopped);
+
 init_per_testcase(Testcase, Config0) ->
     rabbit_ct_helpers:testcase_started(Config0, Testcase).
 
@@ -134,6 +142,23 @@ list_queues_offline(Config) ->
     assert_ctl_queues(Config, 1, ["--offline"], OfflineQueues),
     ok.
 
+list_queues_stopped(Config) ->
+    Node1Queues = lists:sort(lists:nth(1, ?config(per_node_queues, Config))),
+    Node2Queues = lists:sort(lists:nth(2, ?config(per_node_queues, Config))),
+    Node3Queues = lists:sort(lists:nth(3, ?config(per_node_queues, Config))),
+
+    %% All queues are listed
+    ListedQueues =
+        [ {Name, State}
+          || [Name, State] <- rabbit_ct_broker_helpers:rabbitmqctl_list(
+                                Config, 0, ["list_queues", "name", "state"]) ],
+
+    [ <<"running">> = proplists:get_value(Q, ListedQueues) || Q <- Node1Queues ],
+    %% Node is running. Vhost is down
+    [ <<"stopped">> = proplists:get_value(Q, ListedQueues) || Q <- Node2Queues ],
+    %% Node is not running. Vhost is down
+    [ <<"down">> = proplists:get_value(Q, ListedQueues) || Q <- Node3Queues ].
+
 %%----------------------------------------------------------------------------
 %% Helpers
 %%----------------------------------------------------------------------------
diff --git a/test/vhost_SUITE.erl b/test/vhost_SUITE.erl
index 1e1c03bd72..0418312afb 100644
--- a/test/vhost_SUITE.erl
+++ b/test/vhost_SUITE.erl
@@ -87,7 +87,7 @@ init_per_group(cluster_size_2_direct, Config) ->
     Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]),
     init_per_multinode_group(cluster_size_2_direct, Config1, 2).
 
-init_per_multinode_group(Group, Config, NodeCount) ->
+init_per_multinode_group(_Group, Config, NodeCount) ->
     Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"),
     Config1 = rabbit_ct_helpers:set_config(Config, [
                                                     {rmq_nodes_count, NodeCount},
@@ -115,12 +115,18 @@ end_per_testcase(Testcase, Config) ->
         cluster_vhost_deletion_forces_connection_closure -> ok;
         single_node_vhost_deletion_forces_connection_closure -> ok;
         _ ->
-            ok = rabbit_ct_broker_helpers:delete_vhost(Config, VHost2)
+            delete_vhost(Config, VHost2)
     end,
-    ok = rabbit_ct_broker_helpers:delete_vhost(Config, VHost1),
+    delete_vhost(Config, VHost1),
     clear_all_connection_tracking_tables(Config),
     rabbit_ct_helpers:testcase_finished(Config, Testcase).
 
+delete_vhost(Config, VHost) ->
+    case rabbit_ct_broker_helpers:delete_vhost(Config, VHost) of
+        ok                          -> ok;
+        {error, {no_such_vhost, _}} -> ok
+    end.
+
 clear_all_connection_tracking_tables(Config) ->
     [rabbit_ct_broker_helpers:rpc(Config,
         N,
@@ -131,6 +137,7 @@ clear_all_connection_tracking_tables(Config) ->
 %% -------------------------------------------------------------------
 %% Test cases.
 %% -------------------------------------------------------------------
+
 single_node_vhost_deletion_forces_connection_closure(Config) ->
     VHost1 = <<"vhost1">>,
     VHost2 = <<"vhost2">>,
@@ -252,7 +259,10 @@ dead_vhost_connection_refused_on_failure_node(Config) ->
 
     %% Can open connections to vhost2 on node 1
     [_Conn21] = open_connections(Config, [{1, VHost2}]),
-    ?assertEqual(1, count_connections_in(Config, VHost2)).
+    ?assertEqual(1, count_connections_in(Config, VHost2)),
+
+    rabbit_ct_broker_helpers:delete_vhost(Config, VHost2),
+    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
 
 cluster_vhost_deletion_forces_connection_closure(Config) ->
     VHost1 = <<"vhost1">>,
@@ -380,7 +390,7 @@ open_connections(Config, NodesAndVHosts) ->
       (Node) ->
           rabbit_ct_client_helpers:OpenConnectionFun(Config, Node)
       end, NodesAndVHosts),
-    timer:sleep(500),
+    timer:sleep(700),
     Conns.
 
 close_connections(Conns) ->
@@ -388,12 +398,12 @@ close_connections(Conns) ->
       (Conn) ->
           rabbit_ct_client_helpers:close_connection(Conn)
       end, Conns),
-    timer:sleep(500).
+    timer:sleep(700).
 
 count_connections_in(Config, VHost) ->
     count_connections_in(Config, VHost, 0).
 count_connections_in(Config, VHost, NodeIndex) ->
-    timer:sleep(200),
+    timer:sleep(300),
     rabbit_ct_broker_helpers:rpc(Config, NodeIndex,
                                  rabbit_connection_tracking,
                                  count_connections_in, [VHost]).