Add 5 min initial delay for undesired allocation warning (#112427)

When a cluster just starts up, all shards may reside on a single node.
When a new node joins, it is likely half of the shards needs to relocate
to the new node. This temporary undesired allocation is expected and
should quickly resolve itself. This PR adds a 5 min initial delay so
that the cluster does not log warning in such situation.

Resolves: ES-9174

Author: ywangd

server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java

@@ -89,7 +89,10 @@ public class DesiredBalanceReconciler {
     private final DoubleGauge undesiredAllocationsRatio;
 
     public DesiredBalanceReconciler(ClusterSettings clusterSettings, ThreadPool threadPool, MeterRegistry meterRegistry) {
-        this.undesiredAllocationLogInterval = new FrequencyCappedAction(threadPool);
+        this.undesiredAllocationLogInterval = new FrequencyCappedAction(
+            threadPool.relativeTimeInMillisSupplier(),
+            TimeValue.timeValueMinutes(5)
+        );
         clusterSettings.initializeAndWatch(UNDESIRED_ALLOCATIONS_LOG_INTERVAL_SETTING, this.undesiredAllocationLogInterval::setMinInterval);
         clusterSettings.initializeAndWatch(
             UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING,

server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/FrequencyCappedAction.java

@@ -9,7 +9,6 @@
 package org.elasticsearch.cluster.routing.allocation.allocator;
 
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.threadpool.ThreadPool;
 
 import java.util.function.LongSupplier;
 
@@ -21,15 +20,12 @@ public class FrequencyCappedAction {
     private final LongSupplier currentTimeMillisSupplier;
     private TimeValue minInterval;
 
-    private long next = -1;
+    private long next;
 
-    public FrequencyCappedAction(ThreadPool threadPool) {
-        this(threadPool.relativeTimeInMillisSupplier());
-    }
-
-    public FrequencyCappedAction(LongSupplier currentTimeMillisSupplier) {
+    public FrequencyCappedAction(LongSupplier currentTimeMillisSupplier, TimeValue initialDelay) {
         this.currentTimeMillisSupplier = currentTimeMillisSupplier;
         this.minInterval = TimeValue.MAX_VALUE;
+        this.next = currentTimeMillisSupplier.getAsLong() + initialDelay.getMillis();
     }
 
     public void setMinInterval(TimeValue minInterval) {

server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconcilerTests.java

@@ -55,6 +55,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.concurrent.DeterministicTaskQueue;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.gateway.GatewayAllocator;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.shard.ShardId;
@@ -1281,9 +1282,12 @@ public void testShouldLogOnTooManyUndesiredAllocations() {
             .build();
 
         var threadPool = mock(ThreadPool.class);
-        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(new AtomicLong()::incrementAndGet);
+        final var timeInMillisSupplier = new AtomicLong();
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(timeInMillisSupplier::incrementAndGet);
 
         var reconciler = new DesiredBalanceReconciler(createBuiltInClusterSettings(), threadPool, mock(MeterRegistry.class));
+        final long initialDelayInMillis = TimeValue.timeValueMinutes(5).getMillis();
+        timeInMillisSupplier.addAndGet(randomLongBetween(initialDelayInMillis, 2 * initialDelayInMillis));
 
         var expectedWarningMessage = "[100%] of assigned shards ("
             + shardCount
@@ -1323,7 +1327,9 @@ public void testShouldLogOnTooManyUndesiredAllocations() {
     }
 
     private static void reconcile(RoutingAllocation routingAllocation, DesiredBalance desiredBalance) {
-        new DesiredBalanceReconciler(createBuiltInClusterSettings(), mock(ThreadPool.class), mock(MeterRegistry.class)).reconcile(
+        final var threadPool = mock(ThreadPool.class);
+        when(threadPool.relativeTimeInMillisSupplier()).thenReturn(new AtomicLong()::incrementAndGet);
+        new DesiredBalanceReconciler(createBuiltInClusterSettings(), threadPool, mock(MeterRegistry.class)).reconcile(
             desiredBalance,
             routingAllocation
         );

server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/FrequencyCappedActionTests.java

@@ -8,6 +8,7 @@
 
 package org.elasticsearch.cluster.routing.allocation.allocator;
 
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.concurrent.atomic.AtomicLong;
@@ -21,22 +22,29 @@ public void testFrequencyCapExecution() {
 
         var executions = new AtomicLong(0);
         var currentTime = new AtomicLong();
-        var action = new FrequencyCappedAction(currentTime::get);
+        final TimeValue initialDelay = randomBoolean() ? TimeValue.ZERO : TimeValue.timeValueSeconds(between(1, 300));
+        var action = new FrequencyCappedAction(currentTime::get, initialDelay);
 
         var minInterval = timeValueMillis(randomNonNegativeInt());
         action.setMinInterval(minInterval);
 
-        // initial execution should happen
         action.maybeExecute(executions::incrementAndGet);
+        if (initialDelay != TimeValue.ZERO) {
+            // Not executing due to initial delay
+            assertThat(executions.get(), equalTo(0L));
+            currentTime.addAndGet(randomLongBetween(initialDelay.millis(), initialDelay.millis() * 2));
+            action.maybeExecute(executions::incrementAndGet);
+        }
+        // initial execution should happen
         assertThat(executions.get(), equalTo(1L));
 
         // should not execute again too soon
-        currentTime.set(randomLongBetween(0, minInterval.millis() - 1));
+        currentTime.addAndGet(randomLongBetween(0, minInterval.millis() - 1));
         action.maybeExecute(executions::incrementAndGet);
         assertThat(executions.get(), equalTo(1L));
 
         // should execute min interval elapsed
-        currentTime.set(randomLongBetween(minInterval.millis(), Long.MAX_VALUE));
+        currentTime.addAndGet(randomLongBetween(minInterval.millis(), Long.MAX_VALUE));
         action.maybeExecute(executions::incrementAndGet);
         assertThat(executions.get(), equalTo(2L));
     }