Test that a node can start with a dead vhost and potential slaves on it

1 files changed, 107 insertions, 57 deletions

diff --git a/test/vhost_SUITE.erl b/test/vhost_SUITE.erl
index c27f79d757..1e1c03bd72 100644
--- a/test/vhost_SUITE.erl
+++ b/test/vhost_SUITE.erl
@@ -41,7 +41,9 @@ groups() ->
         vhost_failure_forces_connection_closure,
         dead_vhost_connection_refused,
         vhost_failure_forces_connection_closure_on_failure_node,
-        dead_vhost_connection_refused_on_failure_node
+        dead_vhost_connection_refused_on_failure_node,
+        node_starts_with_dead_vhosts,
+        node_starts_with_dead_vhosts_and_ignore_slaves
     ],
     [
       {cluster_size_1_network, [], ClusterSize1Tests},
@@ -107,6 +109,15 @@ init_per_testcase(Testcase, Config) ->
     Config.
 
 end_per_testcase(Testcase, Config) ->
+    VHost1 = <<"vhost1">>,
+    VHost2 = <<"vhost2">>,
+    case Testcase of
+        cluster_vhost_deletion_forces_connection_closure -> ok;
+        single_node_vhost_deletion_forces_connection_closure -> ok;
+        _ ->
+            ok = rabbit_ct_broker_helpers:delete_vhost(Config, VHost2)
+    end,
+    ok = rabbit_ct_broker_helpers:delete_vhost(Config, VHost1),
     clear_all_connection_tracking_tables(Config),
     rabbit_ct_helpers:testcase_finished(Config, Testcase).
 
@@ -141,9 +152,7 @@ single_node_vhost_deletion_forces_connection_closure(Config) ->
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
     close_connections([Conn1]),
-    ?assertEqual(0, count_connections_in(Config, VHost1)),
-
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
+    ?assertEqual(0, count_connections_in(Config, VHost1)).
 
 vhost_failure_forces_connection_closure(Config) ->
     VHost1 = <<"vhost1">>,
@@ -161,15 +170,12 @@ vhost_failure_forces_connection_closure(Config) ->
     [_Conn2] = open_connections(Config, [{0, VHost2}]),
     ?assertEqual(1, count_connections_in(Config, VHost2)),
 
-    force_vhost_failure(Config, VHost2),
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, VHost2),
     timer:sleep(200),
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
     close_connections([Conn1]),
-    ?assertEqual(0, count_connections_in(Config, VHost1)),
-
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost2),
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
+    ?assertEqual(0, count_connections_in(Config, VHost1)).
 
 dead_vhost_connection_refused(Config) ->
     VHost1 = <<"vhost1">>,
@@ -181,7 +187,7 @@ dead_vhost_connection_refused(Config) ->
     ?assertEqual(0, count_connections_in(Config, VHost1)),
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
-    force_vhost_failure(Config, VHost2),
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, VHost2),
     timer:sleep(200),
 
     [_Conn1] = open_connections(Config, [{0, VHost1}]),
@@ -190,10 +196,7 @@ dead_vhost_connection_refused(Config) ->
     [_Conn2] = open_connections(Config, [{0, VHost2}]),
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
-    expect_that_client_connection_is_rejected(Config, 0, VHost2),
-
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost2),
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
+    expect_that_client_connection_is_rejected(Config, 0, VHost2).
 
 
 vhost_failure_forces_connection_closure_on_failure_node(Config) ->
@@ -213,7 +216,7 @@ vhost_failure_forces_connection_closure_on_failure_node(Config) ->
     [_Conn21] = open_connections(Config, [{1, VHost2}]),
     ?assertEqual(2, count_connections_in(Config, VHost2)),
 
-    force_vhost_failure(Config, 0, VHost2),
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, 0, VHost2),
     timer:sleep(200),
     %% Vhost2 connection on node 1 is still alive
     ?assertEqual(1, count_connections_in(Config, VHost2)),
@@ -221,10 +224,7 @@ vhost_failure_forces_connection_closure_on_failure_node(Config) ->
     ?assertEqual(1, count_connections_in(Config, VHost1)),
 
     close_connections([Conn1]),
-    ?assertEqual(0, count_connections_in(Config, VHost1)),
-
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost2),
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
+    ?assertEqual(0, count_connections_in(Config, VHost1)).
 
 dead_vhost_connection_refused_on_failure_node(Config) ->
     VHost1 = <<"vhost1">>,
@@ -236,7 +236,7 @@ dead_vhost_connection_refused_on_failure_node(Config) ->
     ?assertEqual(0, count_connections_in(Config, VHost1)),
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
-    force_vhost_failure(Config, 0, VHost2),
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, 0, VHost2),
     timer:sleep(200),
     %% Can open connections to vhost1 on node 0 and 1
     [_Conn10] = open_connections(Config, [{0, VHost1}]),
@@ -252,41 +252,7 @@ dead_vhost_connection_refused_on_failure_node(Config) ->
 
     %% Can open connections to vhost2 on node 1
     [_Conn21] = open_connections(Config, [{1, VHost2}]),
-    ?assertEqual(1, count_connections_in(Config, VHost2)),
-
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost2),
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
-
-force_vhost_failure(Config, VHost) -> force_vhost_failure(Config, 0, VHost).
-
-force_vhost_failure(Config, Node, VHost) ->
-    force_vhost_failure(Config, Node, VHost, 10).
-
-force_vhost_failure(_Config, _Node, VHost, 0) ->
-    error({failed_to_force_vhost_failure, no_more_attempts_left, VHost});
-force_vhost_failure(Config, Node, VHost, Attempts) ->
-    MessageStorePid = get_message_store_pid(Config, VHost),
-    rabbit_ct_broker_helpers:rpc(Config, Node,
-                                 erlang, exit,
-                                 [MessageStorePid, force_vhost_failure]),
-    %% Give it a time to fail
-    timer:sleep(200),
-    case rabbit_ct_broker_helpers:rpc(Config, 0,
-                                      rabbit_vhost_sup_sup, is_vhost_alive,
-                                      [VHost]) of
-        true  -> force_vhost_failure(Config, Node, VHost, Attempts - 1);
-        false -> ok
-    end.
-
-get_message_store_pid(Config, VHost) ->
-    {ok, VHostSup} = rabbit_ct_broker_helpers:rpc(Config, 0,
-        rabbit_vhost_sup_sup, get_vhost_sup, [VHost]),
-    Children = rabbit_ct_broker_helpers:rpc(Config, 0,
-                                            supervisor, which_children,
-                                            [VHostSup]),
-    [MsgStorePid] = [Pid || {Name, Pid, _, _} <- Children,
-                            Name == msg_store_persistent],
-    MsgStorePid.
+    ?assertEqual(1, count_connections_in(Config, VHost2)).
 
 cluster_vhost_deletion_forces_connection_closure(Config) ->
     VHost1 = <<"vhost1">>,
@@ -309,9 +275,93 @@ cluster_vhost_deletion_forces_connection_closure(Config) ->
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
     close_connections([Conn1]),
-    ?assertEqual(0, count_connections_in(Config, VHost1)),
+    ?assertEqual(0, count_connections_in(Config, VHost1)).
+
+node_starts_with_dead_vhosts(Config) ->
+    VHost1 = <<"vhost1">>,
+    VHost2 = <<"vhost2">>,
+
+    set_up_vhost(Config, VHost1),
+    set_up_vhost(Config, VHost2),
+
+    Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config, 1, VHost1),
+    {ok, Chan} = amqp_connection:open_channel(Conn),
+
+    QName = <<"node_starts_with_dead_vhosts-q-1">>,
+    amqp_channel:call(Chan, #'queue.declare'{queue = QName, durable = true}),
+    rabbit_ct_client_helpers:publish(Chan, QName, 10),
+
+    DataStore1 = rabbit_ct_broker_helpers:rpc(
+        Config, 1, rabbit_vhost, msg_store_dir_path, [VHost1]),
+
+    rabbit_ct_broker_helpers:stop_node(Config, 1),
+
+    file:write_file(filename:join(DataStore1, "recovery.dets"), <<"garbage">>),
+
+    %% The node should start without a vhost
+    ok = rabbit_ct_broker_helpers:start_node(Config, 1),
+
+    timer:sleep(500),
+
+    false = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost1]),
+    true = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost2]).
+
+node_starts_with_dead_vhosts_and_ignore_slaves(Config) ->
+    VHost1 = <<"vhost1">>,
+    VHost2 = <<"vhost2">>,
+
+    set_up_vhost(Config, VHost1),
+    set_up_vhost(Config, VHost2),
+
+    true = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost1]),
+    true = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost2]),
+
+    Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config, 0, VHost1),
+    {ok, Chan} = amqp_connection:open_channel(Conn),
+
+    QName = <<"node_starts_with_dead_vhosts_and_ignore_slaves-q-0">>,
+    amqp_channel:call(Chan, #'queue.declare'{queue = QName, durable = true}),
+    ok = rabbit_ct_broker_helpers:rpc(Config, 0,
+             rabbit_policy, set,
+             [VHost1, <<"mirror">>, <<".*">>, [{<<"ha-mode">>, <<"all">>}],
+              0, <<"queues">>, <<"acting-user">>]),
+
+    %% Wait for the queue to create a slave
+    timer:sleep(300),
+
+    rabbit_ct_client_helpers:publish(Chan, QName, 10),
+
+    {ok, Q} = rabbit_ct_broker_helpers:rpc(
+                Config, 0,
+                rabbit_amqqueue, lookup,
+                [rabbit_misc:r(VHost1, queue, QName)], infinity),
+
+    Node1 = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename),
+
+    #amqqueue{sync_slave_pids = [Pid]} = Q,
+
+    Node1 = node(Pid),
+
+    DataStore1 = rabbit_ct_broker_helpers:rpc(
+        Config, 1, rabbit_vhost, msg_store_dir_path, [VHost1]),
+
+    rabbit_ct_broker_helpers:stop_node(Config, 1),
+
+    file:write_file(filename:join(DataStore1, "recovery.dets"), <<"garbage">>),
+
+    %% The node should start without a vhost
+    ok = rabbit_ct_broker_helpers:start_node(Config, 1),
+
+    timer:sleep(500),
 
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
+    false = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost1]),
+    true = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost2]).
 
 %% -------------------------------------------------------------------
 %% Helpers