peer_discovery_classic_config_SUITE: Handle dead-locks

... when nodes are waiting for each other to finish Mnesia
initialization.

So if the success condition is not met, we reset and restart all nodes
except the first one to trigger peer discovery again. We check the
success condition after that.

1 files changed, 38 insertions, 6 deletions

diff --git a/test/peer_discovery_classic_config_SUITE.erl b/test/peer_discovery_classic_config_SUITE.erl
index d1c79ec923..997a3af134 100644
--- a/test/peer_discovery_classic_config_SUITE.erl
+++ b/test/peer_discovery_classic_config_SUITE.erl
@@ -21,8 +21,7 @@
 -include_lib("eunit/include/eunit.hrl").
 
 -import(rabbit_ct_broker_helpers, [
-    stop_node/2, reset_node/1, start_node/2,
-    rewrite_node_config_file/2, cluster_members_online/2
+    cluster_members_online/2
 ]).
 
 -compile(export_all).
@@ -160,16 +159,49 @@ successful_discovery(Config) ->
                     3 =:= length(cluster_members_online(Config, 0)) andalso
                     3 =:= length(cluster_members_online(Config, 1))
                 end,
-    rabbit_ct_helpers:await_condition(Condition, 90000).
+    try
+        rabbit_ct_helpers:await_condition(Condition, 30000)
+    catch
+        exit:{test_case_failed, _} ->
+            ct:pal(
+              ?LOW_IMPORTANCE,
+              "Possible dead-lock; resetting/restarting nodes 2 and 3"),
+            reset_and_restart_node(Config, 1),
+            reset_and_restart_node(Config, 2),
+            rabbit_ct_helpers:await_condition(Condition, 30000)
+    end.
 
 successful_discovery_with_a_subset_of_nodes_coming_online(Config) ->
     Condition = fun() ->
                     2 =:= length(cluster_members_online(Config, 0)) andalso
                     2 =:= length(cluster_members_online(Config, 1))
                 end,
-    rabbit_ct_helpers:await_condition(Condition, 90000).
+    try
+        rabbit_ct_helpers:await_condition(Condition, 30000)
+    catch
+        exit:{test_case_failed, _} ->
+            ct:pal(
+              ?LOW_IMPORTANCE,
+              "Possible dead-lock; resetting/restarting node 2"),
+            reset_and_restart_node(Config, 1),
+            rabbit_ct_helpers:await_condition(Condition, 30000)
+    end.
 
 no_nodes_configured(Config) ->
     Condition = fun() -> length(cluster_members_online(Config, 0)) < 2 end,
-    rabbit_ct_helpers:await_condition(Condition, 10000).
-
+    try
+        rabbit_ct_helpers:await_condition(Condition, 10000)
+    catch
+        exit:{test_case_failed, _} ->
+            ct:pal(
+              ?LOW_IMPORTANCE,
+              "Possible dead-lock; resetting/restarting node 2"),
+            reset_and_restart_node(Config, 1),
+            rabbit_ct_helpers:await_condition(Condition, 10000)
+    end.
+
+reset_and_restart_node(Config, I) when is_integer(I) andalso I >= 0 ->
+    Name = rabbit_ct_broker_helpers:get_node_config(Config, I, nodename),
+    rabbit_control_helper:command(stop_app, Name),
+    rabbit_ct_broker_helpers:reset_node(Config, Name),
+    rabbit_control_helper:command(start_app, Name).