Found the bug. What was happening was:

1) γ contains a range, say a to b, where a is a segment boundary but b-a < segmentsize
2) γ -> β happens, so all of γ gets loaded into q3 as betas. γ is empty
3) then push β to γ happens. There was a logic failure in there, which meant that with γ empty, _all_ of q3 would be pushed to γ, but incorrectly accounted for, making the vq think there is less in γ than there really is
4) now γ -> β happens again, vq is amazed to find more in the segment in γ than it was expecting. cue explosion
The bug was simply not ensuring that if γ was empty, only push out enough to leave q3 with a maximum of one segment, ending on a segment boundary

1 files changed, 8 insertions, 5 deletions

diff --git a/src/rabbit_variable_queue.erl b/src/rabbit_variable_queue.erl
index 79fd24b8d3..0bce4c2b1a 100644
--- a/src/rabbit_variable_queue.erl
+++ b/src/rabbit_variable_queue.erl
@@ -227,7 +227,7 @@ set_queue_ram_duration_target(
             State1;
        TargetRamMsgCount1 == undefined orelse
        TargetRamMsgCount < TargetRamMsgCount1 ->
-            State1;
+            maybe_start_prefetcher(State1);
        true ->
             reduce_memory_use(State1)
     end.
@@ -964,12 +964,15 @@ push_betas_to_gammas(State = #vqstate { q2 = Q2, gamma = Gamma, q3 = Q3,
     case queue:out(Q3) of
         {empty, _Q3} -> State1;
         {{value, #beta { seq_id = SeqId }}, _Q3a} -> 
+            {{value, #beta { seq_id = SeqIdMax }}, _Q3b} = queue:out_r(Q3),
             Limit = rabbit_queue_index:next_segment_boundary(SeqId),
-            case Gamma1SeqId of
-                Limit -> %% already only holding the minimum, nothing to do
+            %% ASSERTION
+            true = Gamma1SeqId == undefined orelse Gamma1SeqId > SeqIdMax,
+            case (Gamma1SeqId == undefined andalso SeqIdMax < Limit) orelse
+                Gamma1SeqId == Limit of
+                true -> %% already only holding LTE one segment indices in q3
                     State1;
-                _ when Gamma1SeqId == undefined orelse
-                (is_integer(Gamma1SeqId) andalso Gamma1SeqId > Limit) ->
+                false ->
                     %% ASSERTION (sadly large!)
                     %% This says that if Gamma1SeqId /= undefined then
                     %% the gap from Limit to Gamma1SeqId is an integer