Merge remote-tracking branch 'upstream/main' into entitlements/nio-files-1

Author: ldematte

docs/changelog/122458.yaml

@@ -0,0 +1,5 @@
+pr: 122458
+summary: '`DesiredBalanceReconciler` always returns `AllocationStats`'
+area: Allocation
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/cluster/routing/RoutingNode.java

@@ -112,6 +112,9 @@ public String nodeId() {
         return this.nodeId;
     }
 
+    /**
+     * Number of shards assigned to this node. Includes relocating shards. Use {@link #numberOfOwningShards()} to exclude relocating shards.
+     */
     public int size() {
         return shards.size();
     }

server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java

@@ -515,11 +515,6 @@ private void moveShards() {
         }
 
         private DesiredBalanceMetrics.AllocationStats balance() {
-            // Check if rebalancing is disabled.
-            if (allocation.deciders().canRebalance(allocation).type() != Decision.Type.YES) {
-                return DesiredBalanceMetrics.EMPTY_ALLOCATION_STATS;
-            }
-
             int unassignedShards = routingNodes.unassigned().size() + routingNodes.unassigned().ignored().size();
             int totalAllocations = 0;
             int undesiredAllocationsExcludingShuttingDownNodes = 0;
@@ -549,9 +544,15 @@ private DesiredBalanceMetrics.AllocationStats balance() {
                 }
 
                 if (allocation.metadata().nodeShutdowns().contains(shardRouting.currentNodeId()) == false) {
+                    // shard is not on a shutting down node, nor is it on a desired node per the previous check.
                     undesiredAllocationsExcludingShuttingDownNodes++;
                 }
 
+                if (allocation.deciders().canRebalance(allocation).type() != Decision.Type.YES) {
+                    // Rebalancing is disabled, we're just here to collect the AllocationStats to return.
+                    continue;
+                }
+
                 if (allocation.deciders().canRebalance(shardRouting, allocation).type() != Decision.Type.YES) {
                     // rebalancing disabled for this shard
                     continue;

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecider.java

@@ -75,9 +75,9 @@ public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNod
     }
 
     /**
-     * Returns a {@link Decision} whether the cluster can execute
-     * re-balanced operations at all.
-     * {@link Decision#ALWAYS}.
+     * Returns a {@link Decision} on whether the cluster is allowed to rebalance shards to improve relative node shard weights and
+     * performance.
+     * @return {@link Decision#ALWAYS} is returned by default if not overridden.
      */
     public Decision canRebalance(RoutingAllocation allocation) {
         return Decision.ALWAYS;

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java

@@ -82,6 +82,10 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
         );
     }
 
+    /**
+     * Returns whether rebalancing (move shards to improve relative node weights and performance) is allowed right now.
+     * Rebalancing can be disabled via cluster settings, or throttled by cluster settings (e.g. max concurrent shard moves).
+     */
     public Decision canRebalance(RoutingAllocation allocation) {
         return withDeciders(
             allocation,

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java

@@ -150,6 +150,10 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
             + "]"
     );
 
+    /**
+     * Rebalancing may be enabled, disabled, or only allowed after all primaries have started, depending on the cluster setting
+     * {@link #CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING}.
+     */
     @SuppressWarnings("fallthrough")
     @Override
     public Decision canRebalance(RoutingAllocation allocation) {

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java

@@ -61,6 +61,11 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca
         return canRebalance(allocation);
     }
 
+    /**
+     * We allow a limited number of concurrent shard relocations, per the cluster setting
+     * {@link #CLUSTER_ROUTING_ALLOCATION_CLUSTER_CONCURRENT_REBALANCE_SETTING}.
+     * Returns a {@link Decision#THROTTLE} decision if the limit is exceeded, otherwise returns {@link Decision#YES}.
+     */
     @Override
     public Decision canRebalance(RoutingAllocation allocation) {
         int relocatingShards = allocation.routingNodes().getRelocatingShardCount();

server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/EnableAllocationDecider.java

@@ -146,6 +146,11 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
         };
     }
 
+    /**
+     * Rebalancing is limited by the {@link Rebalance} value set on the cluster setting {@link #CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING}.
+     * We might allow movement only of primary shards, or replica shards, or none, or all.
+     * This method only concerns itself with whether {@link Rebalance#NONE} is set: rebalancing is allowed for all other setting values.
+     */
     @Override
     public Decision canRebalance(RoutingAllocation allocation) {
         if (allocation.ignoreDisable()) {
@@ -243,7 +248,7 @@ public String toString() {
     }
 
     /**
-     * Rebalance values or rather their string representation to be used used with
+     * Rebalance values or rather their string representation to be used with
      * {@link EnableAllocationDecider#CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING} /
      * {@link EnableAllocationDecider#INDEX_ROUTING_REBALANCE_ENABLE_SETTING}
      * via cluster / index settings.