Merge branch 'main' into fix/bulk-pipeline

Author: samxbr

build-tools-internal/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=16f2b95838c1ddcf7242b1c39e7bbbb43c842f1f1a1a0dc4959b6d4d68abcac3
-distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.0-all.zip
+distributionSha256Sum=f86344275d1b194688dd330abf9f6f2344cd02872ffee035f2d1ea2fd60cf7f3
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.1-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

build-tools-internal/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-9.2.0
+9.2.1

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=16f2b95838c1ddcf7242b1c39e7bbbb43c842f1f1a1a0dc4959b6d4d68abcac3
-distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.0-all.zip
+distributionSha256Sum=f86344275d1b194688dd330abf9f6f2344cd02872ffee035f2d1ea2fd60cf7f3
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.1-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/DerParser.java

@@ -138,7 +138,12 @@ private int getLength() throws IOException {
         int n = derInputStream.read(bytes);
         if (n < num) throw new IOException("Invalid DER: length too short");
 
-        return new BigInteger(1, bytes).intValue();
+        int len = new BigInteger(1, bytes).intValue();
+        if (len < 0) {
+            throw new IOException("Invalid DER: length larger than max-int");
+        }
+
+        return len;
     }
 
     /**

muted-tests.yml

@@ -60,8 +60,6 @@ tests:
 - class: org.elasticsearch.xpack.ccr.FollowIndexSecurityIT
   method: testCleanShardFollowTaskAfterDeleteFollower
   issue: https://github.com/elastic/elasticsearch/issues/120339
-- class: org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeToCharProcessorTests
-  issue: https://github.com/elastic/elasticsearch/issues/120575
 - class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT
   method: testAuthenticateShouldNotFallThroughInCaseOfFailure
   issue: https://github.com/elastic/elasticsearch/issues/120902

plugins/examples/gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=16f2b95838c1ddcf7242b1c39e7bbbb43c842f1f1a1a0dc4959b6d4d68abcac3
-distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.0-all.zip
+distributionSha256Sum=f86344275d1b194688dd330abf9f6f2344cd02872ffee035f2d1ea2fd60cf7f3
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.1-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

server/src/main/java/org/elasticsearch/cluster/ClusterModule.java

@@ -58,6 +58,7 @@
 import org.elasticsearch.cluster.routing.allocation.decider.DiskThresholdDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
+import org.elasticsearch.cluster.routing.allocation.decider.IndexBalanceAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.IndexVersionAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.elasticsearch.cluster.routing.allocation.decider.NodeReplacementAllocationDecider;
@@ -497,6 +498,7 @@ public static Collection<AllocationDecider> createAllocationDeciders(
         addAllocationDecider(deciders, new ThrottlingAllocationDecider(clusterSettings));
         addAllocationDecider(deciders, new ShardsLimitAllocationDecider(clusterSettings));
         addAllocationDecider(deciders, new AwarenessAllocationDecider(settings, clusterSettings));
+        addAllocationDecider(deciders, new IndexBalanceAllocationDecider(settings, clusterSettings));
 
         clusterPlugins.stream()
             .flatMap(p -> p.createAllocationDeciders(settings, clusterSettings).stream())

server/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java

@@ -250,6 +250,10 @@ private boolean isSingleNodeFilterInternal() {
             || (filters.size() > 1 && opType == OpType.AND && NON_ATTRIBUTE_NAMES.containsAll(filters.keySet()));
     }
 
+    public boolean hasFilters() {
+        return filters.isEmpty() == false;
+    }
+
     /**
      * Generates a human-readable string for the DiscoverNodeFilters.
      * Example: {@code _id:"id1 OR blah",name:"blah OR name2"}

server/src/main/java/org/elasticsearch/cluster/routing/allocation/IndexBalanceConstraintSettings.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.Setting;
+
+/**
+ * Settings definitions for the index shard count allocation decider and associated infrastructure
+ */
+public class IndexBalanceConstraintSettings {
+
+    private static final String SETTING_PREFIX = "cluster.routing.allocation.index_balance_decider.";
+
+    public static final Setting<Boolean> INDEX_BALANCE_DECIDER_ENABLED_SETTING = Setting.boolSetting(
+        SETTING_PREFIX + "enabled",
+        false,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * This setting permits nodes to host more than ideally balanced number of index shards.
+     * Maximum tolerated index shard count = ideal + skew_tolerance
+     * i.e. ideal = 4 shards, skew_tolerance = 1
+     * maximum tolerated index shards = 4 + 1 = 5.
+     */
+    public static final Setting<Integer> INDEX_BALANCE_DECIDER_EXCESS_SHARDS = Setting.intSetting(
+        SETTING_PREFIX + "excess_shards",
+        0,
+        0,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    private volatile boolean deciderEnabled;
+    private volatile int excessShards;
+
+    public IndexBalanceConstraintSettings(ClusterSettings clusterSettings) {
+        clusterSettings.initializeAndWatch(INDEX_BALANCE_DECIDER_ENABLED_SETTING, enabled -> this.deciderEnabled = enabled);
+        clusterSettings.initializeAndWatch(INDEX_BALANCE_DECIDER_EXCESS_SHARDS, value -> this.excessShards = value);
+    }
+
+    public boolean isDeciderEnabled() {
+        return this.deciderEnabled;
+    }
+
+    public int getExcessShards() {
+        return this.excessShards;
+    }
+
+}

server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java

@@ -1581,6 +1581,11 @@ private boolean tryRelocateShard(ModelNode minNode, ModelNode maxNode, ProjectIn
             logger.trace("No shards of [{}] can relocate from [{}] to [{}]", idx, maxNode.getNodeId(), minNode.getNodeId());
             return false;
         }
+
+        // Visible for testing.
+        public RoutingAllocation getAllocation() {
+            return this.allocation;
+        }
     }
 
     public static class ModelNode implements Iterable<ModelIndex> {
@@ -1824,7 +1829,8 @@ public WeightFunction getWeightFunction() {
         }
     }
 
-    record ProjectIndex(ProjectId project, String indexName) {
+    // Visible for testing.
+    public record ProjectIndex(ProjectId project, String indexName) {
         ProjectIndex(RoutingAllocation allocation, ShardRouting shard) {
             this(allocation.metadata().projectFor(shard.index()).id(), shard.getIndexName());
         }