QQ: tweaks to checkpointing for use cases with fewer larger messages.

Lower the min_checkpoint_interval substantially to allow quorum queues
better control over when checkpoints are taken.

Implement some tweaks to the checkpoint decision logic to be more likey
to take a checkpoint for low throughput queues with large messages.

Author: kjnilsson

deps/rabbit/src/rabbit_fifo.erl

@@ -2934,21 +2934,39 @@ do_checkpoints(Ts,
                           Smallest
                   end,
     MsgsTot = messages_total(MacState),
+    #{segments_range := SegRange,
+      num_segments := NumSegments} = ra_aux:log_stats(RaAux),
+    SegmentsMightBeDeletable = case SegRange of
+                            undefined ->
+                                false;
+                            {S, _E} ->
+                                NewSmallest > S
+                        end,
+    CanReclaimSegments = NumSegments > 1 andalso
+                         SegmentsMightBeDeletable,
+
     {CheckMinInterval, CheckMinIndexes, CheckMaxIndexes} =
         persistent_term:get(quorum_queue_checkpoint_config,
                             {?CHECK_MIN_INTERVAL_MS, ?CHECK_MIN_INDEXES,
                              ?CHECK_MAX_INDEXES}),
     EnoughTimeHasPassed = TimeSince > CheckMinInterval,
 
-    %% enough time has passed and enough indexes have been committed
-    case (IndexesSince > MinIndexes andalso
-          EnoughTimeHasPassed) orelse
-         %% the queue is empty and some commands have been
-         %% applied since the last checkpoint
-         (MsgsTot == 0 andalso
-          IndexesSince > CheckMinIndexes andalso
-          EnoughTimeHasPassed) orelse
-         Force of
+    case EnoughTimeHasPassed andalso
+         (
+          %% condition 1: enough indexes have been comitted since the last
+          %% checkpoint
+          (IndexesSince > MinIndexes) orelse
+          %% condition 2 the queue is empty and _some_ commands (more than 64)
+          %% have been applied since the last checkpoint
+          (MsgsTot == 0 andalso
+           IndexesSince > 64) orelse
+          %% condition 3: there are segments and the number of entries in
+          %% segments is considerably larger than the number of messages
+          %% in the queue
+          CanReclaimSegments orelse
+          %% doing it anyway
+          Force
+         ) of
         true ->
             %% take fewer checkpoints the more messages there are on queue
             NextIndexes = min(max(MsgsTot, CheckMinIndexes), CheckMaxIndexes),

deps/rabbit/src/rabbit_fifo.hrl

@@ -100,7 +100,7 @@
 % represents a partially applied module call
 
 -define(CHECK_MIN_INTERVAL_MS, 1000).
--define(CHECK_MIN_INDEXES, 4096).
+-define(CHECK_MIN_INDEXES, 4096 * 2).
 -define(CHECK_MAX_INDEXES, 666_667).
 
 -define(USE_AVG_HALF_LIFE, 10000.0).

deps/rabbit/src/rabbit_quorum_queue.erl

@@ -145,8 +145,9 @@
 -define(DELETE_TIMEOUT, 5000).
 -define(MEMBER_CHANGE_TIMEOUT, 20_000).
 -define(SNAPSHOT_INTERVAL, 8192). %% the ra default is 4096
-% -define(UNLIMITED_PREFETCH_COUNT, 2000). %% something large for ra
--define(MIN_CHECKPOINT_INTERVAL, 8192). %% the ra default is 16384
+%% setting a low default here to allow quorum queues to better chose themselves
+%% when to take a checkpoint
+-define(MIN_CHECKPOINT_INTERVAL, 64).
 -define(LEADER_HEALTH_CHECK_TIMEOUT, 5_000).
 -define(GLOBAL_LEADER_HEALTH_CHECK_TIMEOUT, 60_000).