Merge branch 'master' into rabbitmq-cli-207

3 files changed, 242 insertions, 65 deletions

diff --git a/test/dynamic_ha_SUITE.erl b/test/dynamic_ha_SUITE.erl
index b2f212fe75..c70f23c066 100644
--- a/test/dynamic_ha_SUITE.erl
+++ b/test/dynamic_ha_SUITE.erl
@@ -57,13 +57,19 @@ groups() ->
       {clustered, [], [
           {cluster_size_2, [], [
               vhost_deletion,
-              promote_on_shutdown
+              promote_on_shutdown,
+              slave_recovers_after_vhost_failure,
+              slave_recovers_after_vhost_down_an_up,
+              master_migrates_on_vhost_down,
+              slave_recovers_after_vhost_down_and_master_migrated,
+              queue_survive_adding_dead_vhost_mirror
             ]},
           {cluster_size_3, [], [
               change_policy,
               rapid_change,
               nodes_policy_should_pick_master_from_its_params,
-              promote_slave_after_standalone_restart
+              promote_slave_after_standalone_restart,
+              queue_survive_adding_dead_vhost_mirror
               % FIXME: Re-enable those tests when the know issues are
               % fixed.
               % failing_random_policies,
@@ -218,6 +224,19 @@ rapid_loop(Config, Node, MRef) ->
             rapid_loop(Config, Node, MRef)
     end.
 
+queue_survive_adding_dead_vhost_mirror(Config) ->
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, 1, <<"/">>),
+    NodeA = rabbit_ct_broker_helpers:get_node_config(Config, 0, nodename),
+    ChA = rabbit_ct_client_helpers:open_channel(Config, NodeA),
+    QName = <<"queue_survive_adding_dead_vhost_mirror-q-1">>,
+    amqp_channel:call(ChA, #'queue.declare'{queue = QName}),
+    Q = find_queue(QName, NodeA),
+    Pid = proplists:get_value(pid, Q),
+    rabbit_ct_broker_helpers:set_ha_policy_all(Config),
+    %% Queue should not fail
+    Q1 = find_queue(QName, NodeA),
+    Pid = proplists:get_value(pid, Q1).
+
 %% Vhost deletion needs to successfully tear down policies and queues
 %% with policies. At least smoke-test that it doesn't blow up.
 vhost_deletion(Config) ->
@@ -303,6 +322,72 @@ nodes_policy_should_pick_master_from_its_params(Config) ->
     amqp_channel:call(Ch, #'queue.delete'{queue = ?QNAME}),
     _ = rabbit_ct_broker_helpers:clear_policy(Config, A, ?POLICY).
 
+slave_recovers_after_vhost_failure(Config) ->
+    [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
+    rabbit_ct_broker_helpers:set_ha_policy_all(Config),
+    ACh = rabbit_ct_client_helpers:open_channel(Config, A),
+    QName = <<"slave_recovers_after_vhost_failure-q">>,
+    amqp_channel:call(ACh, #'queue.declare'{queue = QName}),
+    timer:sleep(300),
+    assert_slaves(A, QName, {A, [B]}, [{A, []}]),
+
+    %% Crash vhost on a node hosting a mirror
+    {ok, Sup} = rabbit_ct_broker_helpers:rpc(Config, B, rabbit_vhost_sup_sup, get_vhost_sup, [<<"/">>]),
+    exit(Sup, foo),
+
+    assert_slaves(A, QName, {A, [B]}, [{A, []}]).
+
+slave_recovers_after_vhost_down_an_up(Config) ->
+    [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
+    rabbit_ct_broker_helpers:set_ha_policy_all(Config),
+    ACh = rabbit_ct_client_helpers:open_channel(Config, A),
+    QName = <<"slave_recovers_after_vhost_down_an_up-q">>,
+    amqp_channel:call(ACh, #'queue.declare'{queue = QName}),
+    timer:sleep(100),
+    assert_slaves(A, QName, {A, [B]}, [{A, []}]),
+
+    %% Crash vhost on a node hosting a mirror
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, B, <<"/">>),
+    %% Vhost is down now
+    false = rabbit_ct_broker_helpers:rpc(Config, B, rabbit_vhost_sup_sup, is_vhost_alive, [<<"/">>]),
+    timer:sleep(300),
+    %% Vhost is back up
+    {ok, _Sup} = rabbit_ct_broker_helpers:rpc(Config, B, rabbit_vhost_sup_sup, start_vhost, [<<"/">>]),
+
+    assert_slaves(A, QName, {A, [B]}, [{A, []}]).
+
+master_migrates_on_vhost_down(Config) ->
+    [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
+    rabbit_ct_broker_helpers:set_ha_policy_all(Config),
+    ACh = rabbit_ct_client_helpers:open_channel(Config, A),
+    QName = <<"master_migrates_on_vhost_down-q">>,
+    amqp_channel:call(ACh, #'queue.declare'{queue = QName}),
+    timer:sleep(100),
+    assert_slaves(A, QName, {A, [B]}, [{A, []}]),
+
+    %% Crash vhost on the node hosting queue master
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, A, <<"/">>),
+    timer:sleep(300),
+    assert_slaves(A, QName, {B, []}).
+
+slave_recovers_after_vhost_down_and_master_migrated(Config) ->
+    [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
+    rabbit_ct_broker_helpers:set_ha_policy_all(Config),
+    ACh = rabbit_ct_client_helpers:open_channel(Config, A),
+    QName = <<"slave_recovers_after_vhost_down_and_master_migrated-q">>,
+    amqp_channel:call(ACh, #'queue.declare'{queue = QName}),
+    timer:sleep(100),
+    assert_slaves(A, QName, {A, [B]}, [{A, []}]),
+    %% Crash vhost on the node hosting queue master
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, A, <<"/">>),
+    timer:sleep(300),
+    assert_slaves(B, QName, {B, []}),
+
+    %% Restart the vhost on the node (previously) hosting queue master
+    {ok, _Sup} = rabbit_ct_broker_helpers:rpc(Config, A, rabbit_vhost_sup_sup, start_vhost, [<<"/">>]),
+    timer:sleep(300),
+    assert_slaves(B, QName, {B, [A]}, [{B, []}]).
+
 random_policy(Config) ->
     run_proper(fun prop_random_policy/1, [Config]).
 
@@ -341,7 +426,7 @@ promote_slave_after_standalone_restart(Config) ->
     rabbit_ct_broker_helpers:stop_node(Config, B),
     rabbit_ct_broker_helpers:stop_node(Config, A),
 
-    %% Restart one slave
+    %% Restart one mirror
     forget_cluster_node(Config, B, C),
     forget_cluster_node(Config, B, A),
 
@@ -367,9 +452,11 @@ assert_slaves(RPCNode, QName, Exp) ->
 assert_slaves(RPCNode, QName, Exp, PermittedIntermediate) ->
     assert_slaves0(RPCNode, QName, Exp,
                   [{get(previous_exp_m_node), get(previous_exp_s_nodes)} |
-                   PermittedIntermediate]).
+                   PermittedIntermediate], 1000).
 
-assert_slaves0(RPCNode, QName, {ExpMNode, ExpSNodes}, PermittedIntermediate) ->
+assert_slaves0(_, _, _, _, 0) ->
+    error(give_up_waiting_for_slaves);
+assert_slaves0(RPCNode, QName, {ExpMNode, ExpSNodes}, PermittedIntermediate, Attempts) ->
     Q = find_queue(QName, RPCNode),
     Pid = proplists:get_value(pid, Q),
     SPids = proplists:get_value(slave_pids, Q),
@@ -395,7 +482,8 @@ assert_slaves0(RPCNode, QName, {ExpMNode, ExpSNodes}, PermittedIntermediate) ->
                            [State, {ExpMNode, ExpSNodes}]),
                     timer:sleep(100),
                     assert_slaves0(RPCNode, QName, {ExpMNode, ExpSNodes},
-                                   PermittedIntermediate)
+                                   PermittedIntermediate,
+                                   Attempts - 1)
             end;
         true ->
             put(previous_exp_m_node, ExpMNode),
@@ -415,10 +503,14 @@ equal_list([H|T], Act)  -> case lists:member(H, Act) of
                            end.
 
 find_queue(QName, RPCNode) ->
+    find_queue(QName, RPCNode, 1000).
+
+find_queue(QName, RPCNode, 0) -> error({did_not_find_queue, QName, RPCNode});
+find_queue(QName, RPCNode, Attempts) ->
     Qs = rpc:call(RPCNode, rabbit_amqqueue, info_all, [?VHOST], infinity),
     case find_queue0(QName, Qs) of
         did_not_find_queue -> timer:sleep(100),
-                              find_queue(QName, RPCNode);
+                              find_queue(QName, RPCNode, Attempts - 1);
         Q -> Q
     end.
 
diff --git a/test/rabbitmqctl_integration_SUITE.erl b/test/rabbitmqctl_integration_SUITE.erl
index 535725d585..71b74ea104 100644
--- a/test/rabbitmqctl_integration_SUITE.erl
+++ b/test/rabbitmqctl_integration_SUITE.erl
@@ -31,6 +31,7 @@
 -export([list_queues_local/1
         ,list_queues_offline/1
         ,list_queues_online/1
+        ,list_queues_stopped/1
         ]).
 
 all() ->
@@ -44,6 +45,7 @@ groups() ->
             [list_queues_local
             ,list_queues_online
             ,list_queues_offline
+            ,list_queues_stopped
             ]}
     ].
 
@@ -96,13 +98,19 @@ end_per_group(list_queues, Config0) ->
     rabbit_ct_helpers:run_steps(Config1,
                                 rabbit_ct_client_helpers:teardown_steps() ++
                                     rabbit_ct_broker_helpers:teardown_steps());
-end_per_group(global_parameters, Config) ->
-    rabbit_ct_helpers:run_teardown_steps(Config,
-        rabbit_ct_client_helpers:teardown_steps() ++
-        rabbit_ct_broker_helpers:teardown_steps());
 end_per_group(_, Config) ->
     Config.
 
+init_per_testcase(list_queues_stopped, Config0) ->
+    %% Start node 3 to crash it's queues
+    rabbit_ct_broker_helpers:start_node(Config0, 2),
+    %% Make vhost "down" on nodes 2 and 3
+    rabbit_ct_broker_helpers:force_vhost_failure(Config0, 1, <<"/">>),
+    rabbit_ct_broker_helpers:force_vhost_failure(Config0, 2, <<"/">>),
+
+    rabbit_ct_broker_helpers:stop_node(Config0, 2),
+    rabbit_ct_helpers:testcase_started(Config0, list_queues_stopped);
+
 init_per_testcase(Testcase, Config0) ->
     rabbit_ct_helpers:testcase_started(Config0, Testcase).
 
@@ -134,6 +142,23 @@ list_queues_offline(Config) ->
     assert_ctl_queues(Config, 1, ["--offline"], OfflineQueues),
     ok.
 
+list_queues_stopped(Config) ->
+    Node1Queues = lists:sort(lists:nth(1, ?config(per_node_queues, Config))),
+    Node2Queues = lists:sort(lists:nth(2, ?config(per_node_queues, Config))),
+    Node3Queues = lists:sort(lists:nth(3, ?config(per_node_queues, Config))),
+
+    %% All queues are listed
+    ListedQueues =
+        [ {Name, State}
+          || [Name, State] <- rabbit_ct_broker_helpers:rabbitmqctl_list(
+                                Config, 0, ["list_queues", "name", "state"]) ],
+
+    [ <<"running">> = proplists:get_value(Q, ListedQueues) || Q <- Node1Queues ],
+    %% Node is running. Vhost is down
+    [ <<"stopped">> = proplists:get_value(Q, ListedQueues) || Q <- Node2Queues ],
+    %% Node is not running. Vhost is down
+    [ <<"down">> = proplists:get_value(Q, ListedQueues) || Q <- Node3Queues ].
+
 %%----------------------------------------------------------------------------
 %% Helpers
 %%----------------------------------------------------------------------------
diff --git a/test/vhost_SUITE.erl b/test/vhost_SUITE.erl
index a519d01af5..6ed84dcfe3 100644
--- a/test/vhost_SUITE.erl
+++ b/test/vhost_SUITE.erl
@@ -41,7 +41,9 @@ groups() ->
         vhost_failure_forces_connection_closure,
         dead_vhost_connection_refused,
         vhost_failure_forces_connection_closure_on_failure_node,
-        dead_vhost_connection_refused_on_failure_node
+        dead_vhost_connection_refused_on_failure_node,
+        node_starts_with_dead_vhosts,
+        node_starts_with_dead_vhosts_and_ignore_slaves
     ],
     [
       {cluster_size_1_network, [], ClusterSize1Tests},
@@ -85,7 +87,7 @@ init_per_group(cluster_size_2_direct, Config) ->
     Config1 = rabbit_ct_helpers:set_config(Config, [{connection_type, direct}]),
     init_per_multinode_group(cluster_size_2_direct, Config1, 2).
 
-init_per_multinode_group(Group, Config, NodeCount) ->
+init_per_multinode_group(_Group, Config, NodeCount) ->
     Suffix = rabbit_ct_helpers:testcase_absname(Config, "", "-"),
     Config1 = rabbit_ct_helpers:set_config(Config, [
                                                     {rmq_nodes_count, NodeCount},
@@ -107,9 +109,24 @@ init_per_testcase(Testcase, Config) ->
     Config.
 
 end_per_testcase(Testcase, Config) ->
+    VHost1 = <<"vhost1">>,
+    VHost2 = <<"vhost2">>,
+    case Testcase of
+        cluster_vhost_deletion_forces_connection_closure -> ok;
+        single_node_vhost_deletion_forces_connection_closure -> ok;
+        _ ->
+            delete_vhost(Config, VHost2)
+    end,
+    delete_vhost(Config, VHost1),
     clear_all_connection_tracking_tables(Config),
     rabbit_ct_helpers:testcase_finished(Config, Testcase).
 
+delete_vhost(Config, VHost) ->
+    case rabbit_ct_broker_helpers:delete_vhost(Config, VHost) of
+        ok                          -> ok;
+        {error, {no_such_vhost, _}} -> ok
+    end.
+
 clear_all_connection_tracking_tables(Config) ->
     [rabbit_ct_broker_helpers:rpc(Config,
         N,
@@ -120,6 +137,7 @@ clear_all_connection_tracking_tables(Config) ->
 %% -------------------------------------------------------------------
 %% Test cases.
 %% -------------------------------------------------------------------
+
 single_node_vhost_deletion_forces_connection_closure(Config) ->
     VHost1 = <<"vhost1">>,
     VHost2 = <<"vhost2">>,
@@ -141,9 +159,7 @@ single_node_vhost_deletion_forces_connection_closure(Config) ->
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
     close_connections([Conn1]),
-    ?assertEqual(0, count_connections_in(Config, VHost1)),
-
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
+    ?assertEqual(0, count_connections_in(Config, VHost1)).
 
 vhost_failure_forces_connection_closure(Config) ->
     VHost1 = <<"vhost1">>,
@@ -161,15 +177,12 @@ vhost_failure_forces_connection_closure(Config) ->
     [_Conn2] = open_connections(Config, [{0, VHost2}]),
     ?assertEqual(1, count_connections_in(Config, VHost2)),
 
-    force_vhost_failure(Config, VHost2),
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, VHost2),
     timer:sleep(200),
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
     close_connections([Conn1]),
-    ?assertEqual(0, count_connections_in(Config, VHost1)),
-
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost2),
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
+    ?assertEqual(0, count_connections_in(Config, VHost1)).
 
 dead_vhost_connection_refused(Config) ->
     VHost1 = <<"vhost1">>,
@@ -181,7 +194,7 @@ dead_vhost_connection_refused(Config) ->
     ?assertEqual(0, count_connections_in(Config, VHost1)),
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
-    force_vhost_failure(Config, VHost2),
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, VHost2),
     timer:sleep(200),
 
     [_Conn1] = open_connections(Config, [{0, VHost1}]),
@@ -190,10 +203,7 @@ dead_vhost_connection_refused(Config) ->
     [_Conn2] = open_connections(Config, [{0, VHost2}]),
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
-    expect_that_client_connection_is_rejected(Config, 0, VHost2),
-
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost2),
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
+    expect_that_client_connection_is_rejected(Config, 0, VHost2).
 
 
 vhost_failure_forces_connection_closure_on_failure_node(Config) ->
@@ -213,7 +223,7 @@ vhost_failure_forces_connection_closure_on_failure_node(Config) ->
     [_Conn21] = open_connections(Config, [{1, VHost2}]),
     ?assertEqual(2, count_connections_in(Config, VHost2)),
 
-    force_vhost_failure(Config, 0, VHost2),
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, 0, VHost2),
     timer:sleep(200),
     %% Vhost2 connection on node 1 is still alive
     ?assertEqual(1, count_connections_in(Config, VHost2)),
@@ -221,10 +231,7 @@ vhost_failure_forces_connection_closure_on_failure_node(Config) ->
     ?assertEqual(1, count_connections_in(Config, VHost1)),
 
     close_connections([Conn1]),
-    ?assertEqual(0, count_connections_in(Config, VHost1)),
-
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost2),
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
+    ?assertEqual(0, count_connections_in(Config, VHost1)).
 
 dead_vhost_connection_refused_on_failure_node(Config) ->
     VHost1 = <<"vhost1">>,
@@ -236,7 +243,7 @@ dead_vhost_connection_refused_on_failure_node(Config) ->
     ?assertEqual(0, count_connections_in(Config, VHost1)),
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
-    force_vhost_failure(Config, 0, VHost2),
+    rabbit_ct_broker_helpers:force_vhost_failure(Config, 0, VHost2),
     timer:sleep(200),
     %% Can open connections to vhost1 on node 0 and 1
     [_Conn10] = open_connections(Config, [{0, VHost1}]),
@@ -257,37 +264,6 @@ dead_vhost_connection_refused_on_failure_node(Config) ->
     rabbit_ct_broker_helpers:delete_vhost(Config, VHost2),
     rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
 
-force_vhost_failure(Config, VHost) -> force_vhost_failure(Config, 0, VHost).
-
-force_vhost_failure(Config, Node, VHost) ->
-    force_vhost_failure(Config, Node, VHost, 10).
-
-force_vhost_failure(_Config, _Node, VHost, 0) ->
-    error({failed_to_force_vhost_failure, no_more_attempts_left, VHost});
-force_vhost_failure(Config, Node, VHost, Attempts) ->
-    MessageStorePid = get_message_store_pid(Config, VHost),
-    rabbit_ct_broker_helpers:rpc(Config, Node,
-                                 erlang, exit,
-                                 [MessageStorePid, force_vhost_failure]),
-    %% Give it a time to fail
-    timer:sleep(200),
-    case rabbit_ct_broker_helpers:rpc(Config, 0,
-                                      rabbit_vhost_sup_sup, is_vhost_alive,
-                                      [VHost]) of
-        true  -> force_vhost_failure(Config, Node, VHost, Attempts - 1);
-        false -> ok
-    end.
-
-get_message_store_pid(Config, VHost) ->
-    {ok, VHostSup} = rabbit_ct_broker_helpers:rpc(Config, 0,
-        rabbit_vhost_sup_sup, vhost_sup, [VHost]),
-    Children = rabbit_ct_broker_helpers:rpc(Config, 0,
-                                            supervisor, which_children,
-                                            [VHostSup]),
-    [MsgStorePid] = [Pid || {Name, Pid, _, _} <- Children,
-                            Name == msg_store_persistent],
-    MsgStorePid.
-
 cluster_vhost_deletion_forces_connection_closure(Config) ->
     VHost1 = <<"vhost1">>,
     VHost2 = <<"vhost2">>,
@@ -309,9 +285,93 @@ cluster_vhost_deletion_forces_connection_closure(Config) ->
     ?assertEqual(0, count_connections_in(Config, VHost2)),
 
     close_connections([Conn1]),
-    ?assertEqual(0, count_connections_in(Config, VHost1)),
+    ?assertEqual(0, count_connections_in(Config, VHost1)).
 
-    rabbit_ct_broker_helpers:delete_vhost(Config, VHost1).
+node_starts_with_dead_vhosts(Config) ->
+    VHost1 = <<"vhost1">>,
+    VHost2 = <<"vhost2">>,
+
+    set_up_vhost(Config, VHost1),
+    set_up_vhost(Config, VHost2),
+
+    Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config, 1, VHost1),
+    {ok, Chan} = amqp_connection:open_channel(Conn),
+
+    QName = <<"node_starts_with_dead_vhosts-q-1">>,
+    amqp_channel:call(Chan, #'queue.declare'{queue = QName, durable = true}),
+    rabbit_ct_client_helpers:publish(Chan, QName, 10),
+
+    DataStore1 = rabbit_ct_broker_helpers:rpc(
+        Config, 1, rabbit_vhost, msg_store_dir_path, [VHost1]),
+
+    rabbit_ct_broker_helpers:stop_node(Config, 1),
+
+    file:write_file(filename:join(DataStore1, "recovery.dets"), <<"garbage">>),
+
+    %% The node should start without a vhost
+    ok = rabbit_ct_broker_helpers:start_node(Config, 1),
+
+    timer:sleep(500),
+
+    false = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost1]),
+    true = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost2]).
+
+node_starts_with_dead_vhosts_and_ignore_slaves(Config) ->
+    VHost1 = <<"vhost1">>,
+    VHost2 = <<"vhost2">>,
+
+    set_up_vhost(Config, VHost1),
+    set_up_vhost(Config, VHost2),
+
+    true = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost1]),
+    true = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost2]),
+
+    Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config, 0, VHost1),
+    {ok, Chan} = amqp_connection:open_channel(Conn),
+
+    QName = <<"node_starts_with_dead_vhosts_and_ignore_slaves-q-0">>,
+    amqp_channel:call(Chan, #'queue.declare'{queue = QName, durable = true}),
+    ok = rabbit_ct_broker_helpers:rpc(Config, 0,
+             rabbit_policy, set,
+             [VHost1, <<"mirror">>, <<".*">>, [{<<"ha-mode">>, <<"all">>}],
+              0, <<"queues">>, <<"acting-user">>]),
+
+    %% Wait for the queue to create a slave
+    timer:sleep(300),
+
+    rabbit_ct_client_helpers:publish(Chan, QName, 10),
+
+    {ok, Q} = rabbit_ct_broker_helpers:rpc(
+                Config, 0,
+                rabbit_amqqueue, lookup,
+                [rabbit_misc:r(VHost1, queue, QName)], infinity),
+
+    Node1 = rabbit_ct_broker_helpers:get_node_config(Config, 1, nodename),
+
+    #amqqueue{sync_slave_pids = [Pid]} = Q,
+
+    Node1 = node(Pid),
+
+    DataStore1 = rabbit_ct_broker_helpers:rpc(
+        Config, 1, rabbit_vhost, msg_store_dir_path, [VHost1]),
+
+    rabbit_ct_broker_helpers:stop_node(Config, 1),
+
+    file:write_file(filename:join(DataStore1, "recovery.dets"), <<"garbage">>),
+
+    %% The node should start without a vhost
+    ok = rabbit_ct_broker_helpers:start_node(Config, 1),
+
+    timer:sleep(500),
+
+    false = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost1]),
+    true = rabbit_ct_broker_helpers:rpc(Config, 1,
+                rabbit_vhost_sup_sup, is_vhost_alive, [VHost2]).
 
 %% -------------------------------------------------------------------
 %% Helpers