Resolve direct_exchange_routing_v2 deadlock

Before this commit, when importing definitions with many bindings
and then enabling the feature flag direct_exchange_routing_v2 in
parallel a dead lock occurred.

The process that did the migration was hanging in:
```
(rabbit@r1-server-2.r1-nodes.default)42> erlang:process_info(<0.1447.0>, [current_stacktrace]).
[{current_stacktrace,[{gen,do_call,4,
                           [{file,"gen.erl"},{line,214}]},
                      {gen_server2,call,3,[{file,"gen_server2.erl"},{line,342}]},
                      {rabbit_misc,execute_mnesia_transaction,1,
                                   [{file,"rabbit_misc.erl"},{line,561}]},
                      {rabbit_core_ff,direct_exchange_routing_v2_migration,3,
                                      [{file,"rabbit_core_ff.erl"},{line,213}]},
                      {rabbit_feature_flags,run_migration_fun,3,
                                            [{file,"rabbit_feature_flags.erl"},{line,1621}]},
                      {rabbit_feature_flags,do_enable,1,
                                            [{file,"rabbit_feature_flags.erl"},{line,1523}]},
                      {erpc,execute_call,4,[{file,"erpc.erl"},{line,392}]}]}]
```

while all 8 worker processes under process `definition_import_pool_sup`
were hanging in:
```
(rabbit@r1-server-2.r1-nodes.default)38> erlang:process_info(<0.592.0>, [current_stacktrace]).
[{current_stacktrace,[{global,random_sleep,1,
                              [{file,"global.erl"},{line,2505}]},
                      {global,set_lock,4,[{file,"global.erl"},{line,421}]},
                      {rabbit_feature_flags,is_enabled,2,
                                            [{file,"rabbit_feature_flags.erl"},{line,581}]},
                      {rabbit_binding,sync_index_route,3,
                                      [{file,"rabbit_binding.erl"},{line,469}]},
                      {rabbit_binding,add,4,
                                      [{file,"rabbit_binding.erl"},{line,182}]},
                      {mnesia_tm,apply_fun,3,[{file,"mnesia_tm.erl"},{line,842}]},
                      {mnesia_tm,execute_transaction,5,
                                 [{file,"mnesia_tm.erl"},{line,818}]},
                      {rabbit_misc,'-execute_mnesia_transaction/1-fun-0-',1,
                                   [{file,"rabbit_misc.erl"},{line,565}]}]}]
```

So, all worker processes which imported the bindings were blocked
because rabbit_feature_flags:is_enabled/1 blocks because the feature
flag is in state_changing.

At the same time, the process which enables the feature flag could not
proceed because it was waiting for a lock held by the worker processes.

In this commit, we resolve the dead lock by not blocking in
rabbit_feature_flags:is_enabled/2.

This should be safe because we still rely on Mnesia locking to get a
consistent migration:
1. The migration process sets both a read lock on the rabbit_route table and a
write lock on the rabbit_index_route table. Therefore, once it acquired
both locks, it can populate the new rabbit_index_route table without any
import binding processes interfering.
2. Mnesia does a good job in avoiding dead locks by restarting
transactions (as long as we don't set out own locks outside of Mnesia,
as done prior to this commit).

1 files changed, 4 insertions, 1 deletions

diff --git a/deps/rabbit/src/rabbit_binding.erl b/deps/rabbit/src/rabbit_binding.erl
index 9f0cbd792c..8a15b302d3 100644
--- a/deps/rabbit/src/rabbit_binding.erl
+++ b/deps/rabbit/src/rabbit_binding.erl
@@ -466,7 +466,10 @@ sync_transient_route(Route, ShouldIndexTable, Fun) ->
     sync_index_route(Route, ShouldIndexTable, Fun).
 
 sync_index_route(Route, true, Fun) ->
-    case rabbit_feature_flags:is_enabled(direct_exchange_routing_v2) of
+    %% Do not block as blocking will cause a dead lock when
+    %% function rabbit_binding:populate_index_route_table/0
+    %% (i.e. feature flag migration) runs in parallel.
+    case rabbit_feature_flags:is_enabled(direct_exchange_routing_v2, non_blocking) of
         true ->
             ok = Fun(rabbit_index_route, index_route(Route), write);
         _ ->