Workaround "global" hang

In rabbit_node_monitor:disconnect/1, we change the "dist_auto_connect"
kernel parameter to force a disconnection and give some time to all
components to handle the subsequent "nodedown" event.

"global" doesn't handle this situation very well. With an unfortunate
sequence of messages and bad timings, this can trigger an inconsistency
in its internal state. When this happens, global:sync() never returns.
See bug 26556, comment #5 for a detailed description.

The workaround consists of a process who parses the "global" internal
state if global:sync/0 didn't return in 15 seconds. If the state
contains in-progress synchronisation older than 10 seconds, the
spawned process sends fake nodedown/nodeup events to "global" on both
inconsistent nodes so they restart their synchronisation.

This workaround will be removed once the real bugs are fixed and
"dist_auto_connect" is left untouched.

2 files changed, 67 insertions, 1 deletions

diff --git a/src/rabbit_mnesia.erl b/src/rabbit_mnesia.erl
index f9110e5868..bde4221f0c 100644
--- a/src/rabbit_mnesia.erl
+++ b/src/rabbit_mnesia.erl
@@ -109,7 +109,7 @@ init() ->
     %% We intuitively expect the global name server to be synced when
     %% Mnesia is up. In fact that's not guaranteed to be the case -
     %% let's make it so.
-    ok = global:sync(),
+    ok = rabbit_node_monitor:global_sync(),
     ok.
 
 init_from_config() ->
diff --git a/src/rabbit_node_monitor.erl b/src/rabbit_node_monitor.erl
index 366aac84d3..0eb0882316 100644
--- a/src/rabbit_node_monitor.erl
+++ b/src/rabbit_node_monitor.erl
@@ -26,6 +26,7 @@
 -export([notify_node_up/0, notify_joined_cluster/0, notify_left_cluster/1]).
 -export([partitions/0, partitions/1, status/1, subscribe/1]).
 -export([pause_partition_guard/0]).
+-export([global_sync/0]).
 
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
@@ -259,6 +260,71 @@ pause_if_all_down_guard(PreferredNodes, LastNodes, LastState) ->
     end.
 
 %%----------------------------------------------------------------------------
+%% "global" hang workaround.
+%%----------------------------------------------------------------------------
+
+%% This code works around a possible inconsistency in the "global"
+%% state, causing global:sync/0 to never return.
+%%
+%%     1. A process is spawned.
+%%     2. If after 15", global:sync() didn't return, the "global"
+%%        state is parsed.
+%%     3. If it detects that a sync is blocked for more than 10",
+%%        the process sends fake nodedown/nodeup events to the two
+%%        nodes involved (one local, one remote).
+%%     4. Both "global" instances restart their synchronisation.
+%%     5. globao:sync() finally returns.
+%%
+%% FIXME: Remove this workaround, once we got rid of the change to
+%% "dist_auto_connect" and fixed the bugs uncovered.
+
+global_sync() ->
+    Pid = spawn(fun workaround_global_hang/0),
+    ok = global:sync(),
+    Pid ! global_sync_done,
+    ok.
+
+workaround_global_hang() ->
+    receive
+        global_sync_done ->
+            ok
+    after 15000 ->
+            find_blocked_global_peers()
+    end.
+
+find_blocked_global_peers() ->
+    {status, _, _, [Dict | _]} = sys:get_status(global_name_server),
+    find_blocked_global_peers1(Dict).
+
+find_blocked_global_peers1([{{sync_tag_his, Peer}, Timestamp} | Rest]) ->
+    Diff = timer:now_diff(erlang:now(), Timestamp),
+    if
+        Diff >= 10000 -> unblock_global_peer(Peer);
+        true          -> ok
+    end,
+    find_blocked_global_peers1(Rest);
+find_blocked_global_peers1([_ | Rest]) ->
+    find_blocked_global_peers1(Rest);
+find_blocked_global_peers1([]) ->
+    ok.
+
+unblock_global_peer(PeerNode) ->
+    ThisNode = node(),
+    PeerState = rpc:call(PeerNode, sys, get_status, [global_name_server]),
+    error_logger:info_msg(
+      "Global hang workaround: global state on ~s seems broken~n"
+      " * Peer global state:  ~p~n"
+      " * Local global state: ~p~n"
+      "Faking nodedown/nodeup between ~s and ~s~n",
+      [PeerNode, PeerState, sys:get_status(global_name_server),
+       PeerNode, ThisNode]),
+    {global_name_server, ThisNode} ! {nodedown, PeerNode},
+    {global_name_server, PeerNode} ! {nodedown, ThisNode},
+    {global_name_server, ThisNode} ! {nodeup, PeerNode},
+    {global_name_server, PeerNode} ! {nodeup, ThisNode},
+    ok.
+
+%%----------------------------------------------------------------------------
 %% gen_server callbacks
 %%----------------------------------------------------------------------------