Add settings for health indicator shard_capacity thresholds

samxbr · samxbr · commit 8f929a844b20 · 2025-10-07T19:01:22.000-04:00
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/30_feature.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/30_feature.yml
@@ -23,3 +23,56 @@
   - match:   { indicators.master_is_stable.status: "green" }
   - match:   { indicators.master_is_stable.symptom: "The cluster has a stable master node" }
   - is_false: indicators.master_is_stable.details
+---
+"cluster health test for shard capacity settings":
+  - do:
+        health_report:
+            feature: shards_capacity
+  - is_true: cluster_name
+  - match:   { indicators.shards_capacity.status: "green" }
+
+  # set large threshold to force the indicator to go yellow
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            health.shard_capacity.unhealthy_threshold.yellow: 10000
+        flat_settings: true
+  - match:  { persistent: { health.shard_capacity.unhealthy_threshold.yellow: "10000" } }
+
+  - do:
+      health_report:
+        feature: shards_capacity
+  - is_true: cluster_name
+  - match:  { indicators.shards_capacity.status: "yellow" }
+
+  # set large threshold to force the indicator to go red
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            health.shard_capacity.unhealthy_threshold.red: 9000
+        flat_settings: true
+  - match:  { persistent: { health.shard_capacity.unhealthy_threshold.red: "9000" } }
+
+  - do:
+      health_report:
+        feature: shards_capacity
+  - is_true: cluster_name
+  - match:  { indicators.shards_capacity.status: "red" }
+
+  # set back to default
+  - do:
+      cluster.put_settings:
+        body:
+          persistent:
+            health.shard_capacity.unhealthy_threshold.yellow: 10
+            health.shard_capacity.unhealthy_threshold.red: 5
+        flat_settings: true
+  - match:  { acknowledged: true }
+
+  - do:
+      health_report:
+        feature: shards_capacity
+  - is_true: cluster_name
+  - match:   { indicators.shards_capacity.status: "green" }
diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -82,6 +82,7 @@
 import org.elasticsearch.gateway.PersistedClusterStateService;
 import org.elasticsearch.health.HealthPeriodicLogger;
 import org.elasticsearch.health.node.LocalHealthMonitor;
+import org.elasticsearch.health.node.ShardsCapacityHealthIndicatorService;
 import org.elasticsearch.health.node.action.TransportHealthNodeAction;
 import org.elasticsearch.health.node.selection.HealthNodeTaskExecutor;
 import org.elasticsearch.http.HttpTransportSettings;
@@ -650,6 +651,8 @@ public void apply(Settings value, Settings current, Settings previous) {
         WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_HIGH_UTILIZATION_THRESHOLD_SETTING,
         WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_HIGH_UTILIZATION_DURATION_SETTING,
         WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_QUEUE_LATENCY_THRESHOLD_SETTING,
-        WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_REROUTE_INTERVAL_SETTING
+        WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_REROUTE_INTERVAL_SETTING,
+        ShardsCapacityHealthIndicatorService.SHARD_CAPACITY_UNHEALTHY_THRESHOLD_YELLOW,
+        ShardsCapacityHealthIndicatorService.SHARD_CAPACITY_UNHEALTHY_THRESHOLD_RED
     );
 }
diff --git a/server/src/main/java/org/elasticsearch/health/node/ShardsCapacityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/health/node/ShardsCapacityHealthIndicatorService.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.ReferenceDocs;
 import org.elasticsearch.common.TriFunction;
 import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.health.Diagnosis;
 import org.elasticsearch.health.HealthIndicatorDetails;
 import org.elasticsearch.health.HealthIndicatorImpact;
@@ -25,18 +26,24 @@
 import org.elasticsearch.health.metadata.HealthMetadata;
 import org.elasticsearch.indices.ShardLimitValidator;
 
+import java.util.Iterator;
 import java.util.List;
+import java.util.Locale;
+import java.util.Map;
 import java.util.stream.Stream;
 
 /**
  *  This indicator reports health data about the shard capacity across the cluster.
  *
- * <p>
+
  * The indicator will report:
- * * RED when there's room for less than 5 shards (either data or frozen nodes)
- * * YELLOW when there's room for less than 10 shards (either data or frozen nodes)
- * * GREEN otherwise
- * </p>
+ * <ul>
+ * <li> {@code RED} when there's room for less than the configured {@code health.shard_capacity.unhealthy_threshold.red} (default 5) shards
+ * (either data or frozen nodes)
+ * <li> {@code YELLOW} when there's room for less than the configured {@code health.shard_capacity.unhealthy_threshold.yellow} (default 10)
+ * shards (either data or frozen nodes)
+ * <li> {@code GREEN} otherwise
+ * </ul>
  *
  *  Although the `max_shard_per_node(.frozen)?` information is scoped by Node, we use the information from master because there is where
  *  the available room for new shards is checked before creating new indices.
@@ -89,10 +96,90 @@ public class ShardsCapacityHealthIndicatorService implements HealthIndicatorServ
         "frozen"
     );
 
+    public static final Setting<Integer> SHARD_CAPACITY_UNHEALTHY_THRESHOLD_YELLOW = Setting.intSetting(
+        "health.shard_capacity.unhealthy_threshold.yellow",
+        10,
+        0,
+        new Setting.Validator<>() {
+            @Override
+            public void validate(Integer value) {}
+
+            @Override
+            public void validate(Integer value, Map<Setting<?>, Object> settings) {
+                Integer redThreshold = (Integer) settings.get(SHARD_CAPACITY_UNHEALTHY_THRESHOLD_RED);
+                if (value < redThreshold) {
+                    throw new IllegalArgumentException(
+                        String.format(
+                            Locale.ROOT,
+                            "Setting [%s] (%d) must be greater than or equal to [%s] (%d)",
+                            SHARD_CAPACITY_UNHEALTHY_THRESHOLD_YELLOW.getKey(),
+                            value,
+                            SHARD_CAPACITY_UNHEALTHY_THRESHOLD_RED.getKey(),
+                            redThreshold
+                        )
+                    );
+                }
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final List<Setting<?>> settings = List.of(SHARD_CAPACITY_UNHEALTHY_THRESHOLD_RED);
+                return settings.iterator();
+            }
+        },
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    public static final Setting<Integer> SHARD_CAPACITY_UNHEALTHY_THRESHOLD_RED = Setting.intSetting(
+        "health.shard_capacity.unhealthy_threshold.red",
+        5,
+        0,
+        new Setting.Validator<>() {
+            @Override
+            public void validate(Integer value) {}
+
+            @Override
+            public void validate(Integer value, Map<Setting<?>, Object> settings) {
+                Integer yellowThreshold = (Integer) settings.get(SHARD_CAPACITY_UNHEALTHY_THRESHOLD_YELLOW);
+                if (value > yellowThreshold) {
+                    throw new IllegalArgumentException(
+                        String.format(
+                            Locale.ROOT,
+                            "Setting [%s] (%d) must be less than or equal to [%s] (%d)",
+                            SHARD_CAPACITY_UNHEALTHY_THRESHOLD_RED.getKey(),
+                            value,
+                            SHARD_CAPACITY_UNHEALTHY_THRESHOLD_YELLOW.getKey(),
+                            yellowThreshold
+                        )
+                    );
+                }
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final List<Setting<?>> settings = List.of(SHARD_CAPACITY_UNHEALTHY_THRESHOLD_YELLOW);
+                return settings.iterator();
+            }
+        },
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
     private final ClusterService clusterService;
 
-    public ShardsCapacityHealthIndicatorService(ClusterService clusterService) {
+    private int unhealthyThresholdYellow;
+    private int unhealthyThresholdRed;
+
+    public ShardsCapacityHealthIndicatorService(ClusterService clusterService, Settings settings) {
         this.clusterService = clusterService;
+        this.unhealthyThresholdYellow = SHARD_CAPACITY_UNHEALTHY_THRESHOLD_YELLOW.get(settings);
+        this.unhealthyThresholdRed = SHARD_CAPACITY_UNHEALTHY_THRESHOLD_RED.get(settings);
+
+        clusterService.getClusterSettings()
+            .addSettingsUpdateConsumer(SHARD_CAPACITY_UNHEALTHY_THRESHOLD_YELLOW, this::setUnhealthyThresholdYellow);
+        clusterService.getClusterSettings()
+            .addSettingsUpdateConsumer(SHARD_CAPACITY_UNHEALTHY_THRESHOLD_RED, this::setUnhealthyThresholdRed);
     }
 
     @Override
@@ -115,13 +202,17 @@ public HealthIndicatorResult calculate(boolean verbose, int maxAffectedResources
                 shardLimitsMetadata.maxShardsPerNode(),
                 state.nodes(),
                 state.metadata(),
-                ShardLimitValidator::checkShardLimitForNormalNodes
+                ShardLimitValidator::checkShardLimitForNormalNodes,
+                unhealthyThresholdYellow,
+                unhealthyThresholdRed
             ),
             calculateFrom(
                 shardLimitsMetadata.maxShardsPerNodeFrozen(),
                 state.nodes(),
                 state.metadata(),
-                ShardLimitValidator::checkShardLimitForFrozenNodes
+                ShardLimitValidator::checkShardLimitForFrozenNodes,
+                unhealthyThresholdYellow,
+                unhealthyThresholdRed
             )
         );
     }
@@ -164,7 +255,9 @@ private HealthIndicatorResult mergeIndicators(boolean verbose, StatusResult data
         return createIndicator(
             finalStatus,
             symptomBuilder.toString(),
-            verbose ? buildDetails(dataNodes.result, frozenNodes.result) : HealthIndicatorDetails.EMPTY,
+            verbose
+                ? buildDetails(dataNodes.result, frozenNodes.result, unhealthyThresholdYellow, unhealthyThresholdRed)
+                : HealthIndicatorDetails.EMPTY,
             indicatorImpacts,
             verbose ? diagnoses : List.of()
         );
@@ -174,22 +267,29 @@ static StatusResult calculateFrom(
         int maxShardsPerNodeSetting,
         DiscoveryNodes discoveryNodes,
         Metadata metadata,
-        ShardsCapacityChecker checker
+        ShardsCapacityChecker checker,
+        int shardThresholdYellow,
+        int shardThresholdRed
     ) {
-        var result = checker.check(maxShardsPerNodeSetting, 5, 1, discoveryNodes, metadata);
+        var result = checker.check(maxShardsPerNodeSetting, shardThresholdRed, 1, discoveryNodes, metadata);
         if (result.canAddShards() == false) {
             return new StatusResult(HealthStatus.RED, result);
         }
 
-        result = checker.check(maxShardsPerNodeSetting, 10, 1, discoveryNodes, metadata);
+        result = checker.check(maxShardsPerNodeSetting, shardThresholdYellow, 1, discoveryNodes, metadata);
         if (result.canAddShards() == false) {
             return new StatusResult(HealthStatus.YELLOW, result);
         }
 
         return new StatusResult(HealthStatus.GREEN, result);
     }
 
-    static HealthIndicatorDetails buildDetails(ShardLimitValidator.Result dataNodes, ShardLimitValidator.Result frozenNodes) {
+    static HealthIndicatorDetails buildDetails(
+        ShardLimitValidator.Result dataNodes,
+        ShardLimitValidator.Result frozenNodes,
+        int unhealthyThresholdYellow,
+        int unhealthyThresholdRed
+    ) {
         return (builder, params) -> {
             builder.startObject();
             {
@@ -208,6 +308,12 @@ static HealthIndicatorDetails buildDetails(ShardLimitValidator.Result dataNodes,
                 }
                 builder.endObject();
             }
+            {
+                builder.startObject("settings");
+                builder.field(SHARD_CAPACITY_UNHEALTHY_THRESHOLD_YELLOW.getKey(), unhealthyThresholdYellow);
+                builder.field(SHARD_CAPACITY_UNHEALTHY_THRESHOLD_RED.getKey(), unhealthyThresholdRed);
+                builder.endObject();
+            }
             builder.endObject();
             return builder;
         };
@@ -223,6 +329,14 @@ private HealthIndicatorResult unknownIndicator() {
         );
     }
 
+    private void setUnhealthyThresholdYellow(int value) {
+        this.unhealthyThresholdYellow = value;
+    }
+
+    private void setUnhealthyThresholdRed(int value) {
+        this.unhealthyThresholdRed = value;
+    }
+
     record StatusResult(HealthStatus status, ShardLimitValidator.Result result) {}
 
     @FunctionalInterface
diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
@@ -1469,7 +1469,7 @@ private Module loadDiagnosticServices(
             new StableMasterHealthIndicatorService(coordinationDiagnosticsService, clusterService),
             new RepositoryIntegrityHealthIndicatorService(clusterService),
             new DiskHealthIndicatorService(clusterService),
-            new ShardsCapacityHealthIndicatorService(clusterService),
+            new ShardsCapacityHealthIndicatorService(clusterService, settings),
             new FileSettingsHealthIndicatorService()
         );
         var pluginHealthIndicatorServices = pluginsService.filterPlugins(HealthPlugin.class)
diff --git a/server/src/test/java/org/elasticsearch/health/node/ShardsCapacityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/health/node/ShardsCapacityHealthIndicatorServiceTests.java
