Rate limit write load decider logging (elastic#136580)

Author: nicktindall

server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java

@@ -109,6 +109,16 @@ public boolean disabled() {
         Setting.Property.NodeScope
     );
 
+    /**
+     * The minimum amount of time between logging messages about write load decider interventions
+     */
+    public static final Setting<TimeValue> WRITE_LOAD_DECIDER_MINIMUM_LOGGING_INTERVAL = Setting.timeSetting(
+        SETTING_PREFIX + "log_interval",
+        TimeValue.timeValueMinutes(1),
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
     private volatile WriteLoadDeciderStatus writeLoadDeciderStatus;
     private volatile TimeValue minimumRerouteInterval;
     private volatile double highUtilizationThreshold;

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/WriteLoadConstraintDecider.java

@@ -18,8 +18,10 @@
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
 import org.elasticsearch.cluster.routing.allocation.WriteLoadConstraintSettings;
+import org.elasticsearch.common.FrequencyCappedAction;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.threadpool.ThreadPool;
 
 /**
@@ -31,10 +33,16 @@ public class WriteLoadConstraintDecider extends AllocationDecider {
 
     public static final String NAME = "write_load";
 
+    private final FrequencyCappedAction logInterventionMessage;
     private final WriteLoadConstraintSettings writeLoadConstraintSettings;
 
     public WriteLoadConstraintDecider(ClusterSettings clusterSettings) {
         this.writeLoadConstraintSettings = new WriteLoadConstraintSettings(clusterSettings);
+        logInterventionMessage = new FrequencyCappedAction(System::currentTimeMillis, TimeValue.ZERO);
+        clusterSettings.initializeAndWatch(
+            WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_MINIMUM_LOGGING_INTERVAL,
+            logInterventionMessage::setMinInterval
+        );
     }
 
     @Override
@@ -77,7 +85,9 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                 nodeWriteThreadPoolLoadThreshold,
                 shardRouting.shardId()
             );
-            logger.debug(explain);
+            if (logger.isDebugEnabled()) {
+                logInterventionMessage.maybeExecute(() -> logger.debug(explain));
+            }
             return Decision.single(Decision.Type.NOT_PREFERRED, NAME, explain);
         }
 
@@ -97,7 +107,9 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                 shardWriteLoad,
                 nodeWriteThreadPoolStats.totalThreadPoolThreads()
             );
-            logger.debug(explain);
+            if (logger.isDebugEnabled()) {
+                logInterventionMessage.maybeExecute(() -> logger.debug(explain));
+            }
             return Decision.single(Decision.Type.NOT_PREFERRED, NAME, explain);
         }
 
@@ -108,7 +120,6 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             node.nodeId(),
             newWriteThreadPoolUtilization
         );
-        logger.trace(explanation);
         return allocation.decision(Decision.YES, NAME, explanation);
     }
 
@@ -146,7 +157,9 @@ public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting
                 nodeWriteThreadPoolQueueLatencyThreshold.toHumanReadableString(2),
                 nodeWriteThreadPoolStats.averageThreadPoolUtilization()
             );
-            logger.debug(explain);
+            if (logger.isDebugEnabled()) {
+                logInterventionMessage.maybeExecute(() -> logger.debug(explain));
+            }
             return Decision.single(Decision.Type.NOT_PREFERRED, NAME, explain);
         }
 
@@ -156,7 +169,6 @@ public Decision canRemain(IndexMetadata indexMetadata, ShardRouting shardRouting
             nodeWriteThreadPoolStats.maxThreadPoolQueueLatencyMillis(),
             nodeWriteThreadPoolQueueLatencyThreshold.toHumanReadableString(2)
         );
-        logger.trace(explanation);
         return allocation.decision(Decision.YES, NAME, explanation);
     }
 

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -652,6 +652,7 @@ public void apply(Settings value, Settings current, Settings previous) {
         WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_HIGH_UTILIZATION_DURATION_SETTING,
         WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_QUEUE_LATENCY_THRESHOLD_SETTING,
         WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_REROUTE_INTERVAL_SETTING,
+        WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_MINIMUM_LOGGING_INTERVAL,
         SamplingService.TTL_POLL_INTERVAL_SETTING
     );
 }