Merge branch 'main' into ivf_hkmeans_struc2

Author: john-wagster

muted-tests.yml

@@ -545,9 +545,6 @@ tests:
 - class: org.elasticsearch.qa.verify_version_constants.VerifyVersionConstantsIT
   method: testLuceneVersionConstant
   issue: https://github.com/elastic/elasticsearch/issues/125638
-- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
-  method: test
-  issue: https://github.com/elastic/elasticsearch/issues/129819
 - class: org.elasticsearch.index.store.FsDirectoryFactoryTests
   method: testPreload
   issue: https://github.com/elastic/elasticsearch/issues/129852
@@ -575,6 +572,11 @@ tests:
 - class: org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests
   method: testMergeEmptyMappingsIntoTemplateWithNonEmptySettings
   issue: https://github.com/elastic/elasticsearch/issues/130050
+- class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/130067
+- class: geoip.GeoIpMultiProjectIT
+  issue: https://github.com/elastic/elasticsearch/issues/130073
 
 # Examples:
 #

qa/mixed-cluster/build.gradle

@@ -23,6 +23,7 @@ apply plugin: 'elasticsearch.rest-resources'
 
 dependencies {
   restTestConfig project(path: ':modules:aggregations', configuration: 'restTests')
+  restTestConfig project(path: ':modules:mapper-extras', configuration: 'restTests')
 }
 
 restResources {

server/src/main/java/org/elasticsearch/cluster/routing/allocation/WriteLoadConstraintSettings.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.cluster.routing.allocation;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.cluster.routing.RerouteService;
+import org.elasticsearch.common.Priority;
+import org.elasticsearch.common.settings.Setting;
+import org.elasticsearch.common.unit.RatioValue;
+import org.elasticsearch.core.TimeValue;
+
+/**
+ * Settings definitions for the write load allocation decider and associated infrastructure
+ */
+public class WriteLoadConstraintSettings {
+
+    private static final String SETTING_PREFIX = "cluster.routing.allocation.write_load_decider.";
+
+    public enum WriteLoadDeciderStatus {
+        /**
+         * The decider is disabled
+         */
+        DISABLED,
+        /**
+         * Only the low-threshold is enabled (write-load will not trigger rebalance)
+         */
+        LOW_ONLY,
+        /**
+         * The decider is enabled
+         */
+        ENABLED
+    }
+
+    public static final Setting<WriteLoadDeciderStatus> WRITE_LOAD_DECIDER_ENABLED_SETTING = Setting.enumSetting(
+        WriteLoadDeciderStatus.class,
+        SETTING_PREFIX + "enabled",
+        WriteLoadDeciderStatus.DISABLED,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * The threshold over which we consider write thread pool utilization "high"
+     */
+    public static final Setting<RatioValue> WRITE_LOAD_DECIDER_HIGH_UTILIZATION_THRESHOLD_SETTING = new Setting<>(
+        SETTING_PREFIX + "high_utilization_threshold",
+        "90%",
+        RatioValue::parseRatioValue,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * The duration for which we need to see "high" utilization before we consider the low threshold exceeded
+     */
+    public static final Setting<TimeValue> WRITE_LOAD_DECIDER_HIGH_UTILIZATION_DURATION_SETTING = Setting.timeSetting(
+        SETTING_PREFIX + "high_utilization_duration",
+        TimeValue.timeValueMinutes(10),
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * When the decider is {@link WriteLoadDeciderStatus#ENABLED}, the write-load monitor will call
+     * {@link RerouteService#reroute(String, Priority, ActionListener)} when we see tasks being delayed by this amount of time
+     * (but no more often than {@link #WRITE_LOAD_DECIDER_REROUTE_INTERVAL_SETTING})
+     */
+    public static final Setting<TimeValue> WRITE_LOAD_DECIDER_QUEUE_LATENCY_THRESHOLD_SETTING = Setting.timeSetting(
+        SETTING_PREFIX + "queue_latency_threshold",
+        TimeValue.timeValueSeconds(30),
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * How often the data node calculates the write-loads for the individual shards
+     */
+    public static final Setting<TimeValue> WRITE_LOAD_DECIDER_SHARD_WRITE_LOAD_POLLING_INTERVAL_SETTING = Setting.timeSetting(
+        SETTING_PREFIX + "shard_write_load_polling_interval",
+        TimeValue.timeValueSeconds(60),
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * The minimum amount of time between successive calls to reroute to address write load hot-spots
+     */
+    public static final Setting<TimeValue> WRITE_LOAD_DECIDER_REROUTE_INTERVAL_SETTING = Setting.timeSetting(
+        SETTING_PREFIX + "reroute_interval",
+        TimeValue.timeValueSeconds(60),
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
+}

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -46,6 +46,7 @@
 import org.elasticsearch.cluster.routing.OperationRouting;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
+import org.elasticsearch.cluster.routing.allocation.WriteLoadConstraintSettings;
 import org.elasticsearch.cluster.routing.allocation.allocator.AllocationBalancingRoundSummaryService;
 import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.elasticsearch.cluster.routing.allocation.allocator.DesiredBalanceComputer;
@@ -643,6 +644,12 @@ public void apply(Settings value, Settings current, Settings previous) {
         DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING,
         IndexingStatsSettings.RECENT_WRITE_LOAD_HALF_LIFE_SETTING,
         SearchStatsSettings.RECENT_READ_LOAD_HALF_LIFE_SETTING,
-        TransportGetAllocationStatsAction.CACHE_TTL_SETTING
+        TransportGetAllocationStatsAction.CACHE_TTL_SETTING,
+        WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_ENABLED_SETTING,
+        WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_HIGH_UTILIZATION_THRESHOLD_SETTING,
+        WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_HIGH_UTILIZATION_DURATION_SETTING,
+        WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_QUEUE_LATENCY_THRESHOLD_SETTING,
+        WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_SHARD_WRITE_LOAD_POLLING_INTERVAL_SETTING,
+        WriteLoadConstraintSettings.WRITE_LOAD_DECIDER_REROUTE_INTERVAL_SETTING
     );
 }

server/src/main/java/org/elasticsearch/index/engine/Engine.java

@@ -2199,7 +2199,7 @@ public void close() throws IOException {
         awaitPendingClose();
     }
 
-    private void awaitPendingClose() {
+    protected final void awaitPendingClose() {
         try {
             closedLatch.await();
         } catch (InterruptedException e) {

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeRestTest.java

@@ -50,6 +50,7 @@ public abstract class GenerativeRestTest extends ESRestTestCase {
         "Plan \\[ProjectExec\\[\\[<no-fields>.* optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/125866
         "optimized incorrectly due to missing references", // https://github.com/elastic/elasticsearch/issues/116781
         "The incoming YAML document exceeds the limit:", // still to investigate, but it seems to be specific to the test framework
+        "Data too large", // Circuit breaker exceptions eg. https://github.com/elastic/elasticsearch/issues/130072
 
         // Awaiting fixes for correctness
         "Expecting the following columns \\[.*\\], got", // https://github.com/elastic/elasticsearch/issues/129000

x-pack/plugin/inference/build.gradle

@@ -405,3 +405,7 @@ tasks.named("thirdPartyAudit").configure {
 tasks.named('yamlRestTest') {
   usesDefaultDistribution("Uses the inference API")
 }
+
+artifacts {
+  restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
+}

x-pack/qa/multi-project/core-rest-tests-with-multiple-projects/src/yamlRestTest/java/org/elasticsearch/multiproject/test/CoreWithMultipleProjectsClientYamlTestSuiteIT.java

@@ -18,7 +18,7 @@
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.junit.ClassRule;
 
-@TimeoutSuite(millis = 30 * TimeUnits.MINUTE)
+@TimeoutSuite(millis = 45 * TimeUnits.MINUTE)
 public class CoreWithMultipleProjectsClientYamlTestSuiteIT extends MultipleProjectsClientYamlSuiteTestCase {
 
     @ClassRule

x-pack/rest-resources-zip/build.gradle

@@ -27,6 +27,7 @@ dependencies {
   platinumTests project(path: ':x-pack:plugin', configuration: 'restXpackTests')
   platinumTests project(path: ':x-pack:plugin:eql:qa:rest', configuration: 'restXpackTests')
   platinumTests project(path: ':x-pack:plugin:ent-search', configuration: 'restXpackTests')
+  platinumTests project(path: ':x-pack:plugin:inference', configuration: 'restXpackTests')
   platinumCompatTests project(path: ':x-pack:plugin', configuration: 'restCompatTests')
   platinumCompatTests project(path: ':x-pack:plugin:eql:qa:rest', configuration: 'restCompatTests')
 }