merging in from 21087. In testing, observed oscillation - with lots of queues, give 2 queues the same length such that they can't both fit in memory, and slowly trickle in messages. As each one gets a message, it'll force the other one out to disk (the other one will either be in the hibernating or lowrate groups). This is bad. Therefore, adjusted the conditions under which we bring a queue back in from disk to exclude queues that are either hibernating or low rate (don't forget, even a list_queues will wake up a queue and cause it to report memory). If you have two fast queues then neither of them will be in the groups of low rate or hibernating queues, so neither will be candidates for eviction so the problem doesn't exist there, instead, if they need more memory and can't fit in ram then they'll evict themselves to disk rather than anyone else.

Also realised that a million queues isn't unreasonable, so minimum number of tokens in the system should be more like 1e7 if not higher.

2 files changed, 17 insertions, 16 deletions

diff --git a/src/gen_server2.erl b/src/gen_server2.erl
index ba42c18fab..e46f2645bd 100644
--- a/src/gen_server2.erl
+++ b/src/gen_server2.erl
@@ -335,12 +335,8 @@ enter_loop(Mod, Options, State, ServerName, Timeout) ->
     Parent = get_parent(),
     Debug = debug_options(Name, Options),
     Queue = priority_queue:new(),
-    TimeoutState = case Timeout of
-                       {binary, Min} ->
-                           {Min, Min, undefined};
-                       _ -> undefined
-                   end,
-    loop(Parent, Name, State, Mod, Timeout, TimeoutState, Queue, Debug).
+    {Timeout1, TimeoutState} = build_timeout_state(Timeout),
+    loop(Parent, Name, State, Mod, Timeout1, TimeoutState, Queue, Debug).
 
 %%%========================================================================
 %%% Gen-callback functions
@@ -365,12 +361,9 @@ init_it(Starter, Parent, Name0, Mod, Args, Options) ->
 	    loop(Parent, Name, State, Mod, infinity, undefined, Queue, Debug);
 	{ok, State, Timeout} ->
 	    proc_lib:init_ack(Starter, {ok, self()}),
-            TimeoutState = case Timeout of
-                               {binary, Min} ->
-                                   {Min, Min, undefined};
-                               _ -> undefined
-                           end,
-	    loop(Parent, Name, State, Mod, binary, TimeoutState, Queue, Debug);
+            {Timeout1, TimeoutState} = build_timeout_state(Timeout),
+	    loop(Parent, Name, State, Mod, Timeout1, TimeoutState, Queue,
+                 Debug);
 	{stop, Reason} ->
 	    %% For consistency, we must make sure that the
 	    %% registered name (if any) is unregistered before
@@ -408,6 +401,12 @@ unregister_name({global,Name}) ->
 unregister_name(Pid) when is_pid(Pid) ->
     Pid.
 
+build_timeout_state(Timeout) ->
+    case Timeout of
+        {binary, Min} -> {binary,  {Min, Min, undefined}};
+        _             -> {Timeout, undefined}
+    end.
+
 %%%========================================================================
 %%% Internal functions
 %%%========================================================================
diff --git a/src/rabbit_queue_mode_manager.erl b/src/rabbit_queue_mode_manager.erl
index f5cc32b445..a5e9610a6b 100644
--- a/src/rabbit_queue_mode_manager.erl
+++ b/src/rabbit_queue_mode_manager.erl
@@ -41,9 +41,9 @@
 -export([register/4, report_memory/3, report_memory/5, info/0,
          pin_to_disk/1, unpin_to_disk/1]).
 
--define(TOTAL_TOKENS, 1000).
+-define(TOTAL_TOKENS, 10000000).
 -define(ACTIVITY_THRESHOLD, 25).
--define(INITIAL_TOKEN_ALLOCATION, 10).
+-define(INITIAL_TOKEN_ALLOCATION, 100).
 
 -define(SERVER, ?MODULE).
 
@@ -286,8 +286,10 @@ handle_cast({report_memory, Pid, Memory, BytesGained, BytesLost, Hibernating},
                         State1 = #state { available_tokens = Avail1,
                                           mixed_queues = Mixed1 } =
                             free_upto(Pid, Req, State),
-                        case Req > Avail1 of
-                            true -> %% not enough space, stay as disk
+                        case Req > Avail1 orelse Hibernating orelse LowRate of
+                            true ->
+                                %% not enough space, or no compelling
+                                %% reason, so stay as disk
                                 {State1, disk};
                             false -> %% can go to mixed mode
                                 {Module, Function, Args} =