Bump gen_server:call timeuts for rabbitmq_auth_backend_cache operationsmk-bump-cache-gen-server-call-timeout

as we have seen with channel operation timeouts, 5s is too low
under load spikes.

5 files changed, 35 insertions, 10 deletions

diff --git a/deps/rabbitmq_auth_backend_cache/include/rabbit_auth_backend_cache.hrl b/deps/rabbitmq_auth_backend_cache/include/rabbit_auth_backend_cache.hrl
new file mode 100644
index 0000000000..e59e65c14c
--- /dev/null
+++ b/deps/rabbitmq_auth_backend_cache/include/rabbit_auth_backend_cache.hrl
@@ -0,0 +1,9 @@
+%% This Source Code Form is subject to the terms of the Mozilla Public
+%% License, v. 2.0. If a copy of the MPL was not distributed with this
+%% file, You can obtain one at https://mozilla.org/MPL/2.0/.
+%%
+%% Copyright (c) 2007-2021 VMware, Inc. or its affiliates.  All rights reserved.
+%%
+
+%% Same as default channel operation timeout.
+-define(CACHE_OPERATION_TIMEOUT, 15000).
diff --git a/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_dict.erl b/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_dict.erl
index 3a769be338..a3ff7feb37 100644
--- a/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_dict.erl
+++ b/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_dict.erl
@@ -12,6 +12,8 @@
 
 -behaviour(rabbit_auth_cache).
 
+-include("include/rabbit_auth_backend_cache.hrl").
+
 -export([start_link/0,
          get/1, put/3, delete/1]).
 
@@ -20,9 +22,11 @@
 
 start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
-get(Key) -> gen_server:call(?MODULE, {get, Key}).
+get(Key) -> gen_server:call(?MODULE, {get, Key}, ?CACHE_OPERATION_TIMEOUT).
+
 put(Key, Value, TTL) -> gen_server:cast(?MODULE, {put, Key, Value, TTL}).
-delete(Key) -> gen_server:call(?MODULE, {delete, Key}).
+
+delete(Key) -> gen_server:call(?MODULE, {delete, Key}, ?CACHE_OPERATION_TIMEOUT).
 
 init(_Args) -> {ok, nostate}.
 
diff --git a/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets.erl b/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets.erl
index dc5320f924..e988686e8e 100644
--- a/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets.erl
+++ b/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets.erl
@@ -10,6 +10,8 @@
 -compile({no_auto_import,[get/1]}).
 -compile({no_auto_import,[put/2]}).
 
+-include("include/rabbit_auth_backend_cache.hrl").
+
 -behaviour(rabbit_auth_cache).
 
 -export([start_link/0,
@@ -18,17 +20,23 @@
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
--record(state, {cache, timers, ttl}).
+-record(state, {
+    cache,
+    timers,
+    ttl
+}).
 
 start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
-get(Key) -> gen_server:call(?MODULE, {get, Key}).
+get(Key) -> gen_server:call(?MODULE, {get, Key}, ?CACHE_OPERATION_TIMEOUT).
+
 put(Key, Value, TTL) ->
     Expiration = rabbit_auth_cache:expiration(TTL),
     gen_server:cast(?MODULE, {put, Key, Value, TTL, Expiration}).
-delete(Key) -> gen_server:call(?MODULE, {delete, Key}).
 
-init(_Args) ->
+delete(Key) -> gen_server:call(?MODULE, {delete, Key}, ?CACHE_OPERATION_TIMEOUT).
+
+init([]) ->
     {ok, #state{cache = ets:new(?MODULE, [set, private]),
                 timers = ets:new(auth_cache_ets_timers, [set, private])}}.
 
diff --git a/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets_segmented.erl b/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets_segmented.erl
index 0db7fe6ded..230dd6c2ac 100644
--- a/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets_segmented.erl
+++ b/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets_segmented.erl
@@ -16,6 +16,8 @@
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
+-include("include/rabbit_auth_backend_cache.hrl").
+
 -record(state, {
     segments = [],
     gc_timer,
@@ -32,13 +34,13 @@ get(Key) ->
 
 put(Key, Value, TTL) ->
     Expiration = rabbit_auth_cache:expiration(TTL),
-    Segment = gen_server:call(?MODULE, {get_write_segment, Expiration}),
+    Segment = gen_server:call(?MODULE, {get_write_segment, Expiration}, ?CACHE_OPERATION_TIMEOUT),
     ets:insert(Segment, {Key, {Expiration, Value}}),
     ok.
 
 delete(Key) ->
     [ets:delete(Table, Key)
-     || Table <- gen_server:call(?MODULE, get_segment_tables)].
+     || Table <- gen_server:call(?MODULE, get_segment_tables, ?CACHE_OPERATION_TIMEOUT)].
 
 gc() ->
     case whereis(?MODULE) of
@@ -96,7 +98,7 @@ maybe_add_segment(Expiration, SegmentSize, OldSegments) ->
     end.
 
 get_from_segments(Key) ->
-    Tables = gen_server:call(?MODULE, get_segment_tables),
+    Tables = gen_server:call(?MODULE, get_segment_tables, ?CACHE_OPERATION_TIMEOUT),
     lists:flatmap(
         fun(undefined) -> [];
            (T) ->
diff --git a/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets_segmented_stateless.erl b/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets_segmented_stateless.erl
index afea5d6561..79f60f1057 100644
--- a/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets_segmented_stateless.erl
+++ b/deps/rabbitmq_auth_backend_cache/src/rabbit_auth_cache_ets_segmented_stateless.erl
@@ -9,6 +9,8 @@
 -behaviour(gen_server).
 -behaviour(rabbit_auth_cache).
 
+-include("include/rabbit_auth_backend_cache.hrl").
+
 -export([start_link/1,
          get/1, put/3, delete/1]).
 -export([gc/0]).
@@ -90,7 +92,7 @@ segment(Expiration, SegmentSize) ->
     End.
 
 add_segment(Segment) ->
-    gen_server:call(?MODULE, {add_segment, Segment}).
+    gen_server:call(?MODULE, {add_segment, Segment}, ?CACHE_OPERATION_TIMEOUT).
 
 do_add_segment(Segment) ->
     case ets:lookup(?SEGMENT_TABLE, Segment) of