Merge branch 'main' into bc_upgrades/tolerate_incompatible_versions

Author: ldematte

CONTRIBUTING.md

@@ -203,6 +203,18 @@ Alternative manual steps for IntelliJ.
    3. Navigate to the file `build-conventions/formatterConfig.xml`
    4. Click "OK"
 
+#### Options
+
+When importing to IntelliJ, we offer a few options that can be used to
+configure the behaviour of the import:
+
+| Property                                   | Description                                                                                          | Values (* = default) |
+|--------------------------------------------|------------------------------------------------------------------------------------------------------|----------------------|
+| `org.elasticsearch.idea-configuration-cache` | Should IntelliJ enable the Gradle Configuration cache to speed up builds when generating run configs | *`true`, `false`         |
+| `org.elasticsearch.idea-delegate-to-gradle`  | Should IntelliJ use Gradle for all generated run / test configs or prompt each time                  | `true`, *`false`         |
+
+These options can be set anywhere on the Gradle config path including in `~/.gradle/gradle.properties`
+
 ### REST endpoint conventions
 
 Elasticsearch typically uses singular nouns rather than plurals in URLs.

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -180,6 +180,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   // this path is produced by the extractLibs task above
   String testLibraryPath = TestUtil.getTestLibraryPath("${elasticsearchProject.left()}/libs/native/libraries/build/platform")
   def enableIdeaCC = providers.gradleProperty("org.elasticsearch.idea-configuration-cache").getOrElse("true").toBoolean()
+  def delegateToGradle = providers.gradleProperty("org.elasticsearch.idea-delegate-to-gradle").getOrElse("false").toBoolean()
   idea {
     project {
       vcs = 'Git'
@@ -188,7 +189,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
       settings {
         delegateActions {
           delegateBuildRunToGradle = false
-          testRunner = 'choose_per_test'
+          testRunner = delegateToGradle ? 'gradle' : 'choose_per_test'
         }
         taskTriggers {
           afterSync tasks.named('configureIdeCheckstyle'),

docs/changelog/129464.yaml

@@ -0,0 +1,10 @@
+pr: 129464
+summary: Deprecate `indices.merge.scheduler.use_thread_pool` setting
+area: Engine
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate `indices.merge.scheduler.use_thread_pool` setting
+  area: Ingest
+  details: This deprecates the `indices.merge.scheduler.use_thread_pool` node setting that was introduced in #120869. This setting should not normally be used, unless instructed so by engineering to get around temporary issues with the new threadpool-based merge scheduler.
+  impact: There should be no impact to users since the setting was not released before its deprecation here (and is not documented).

libs/simdvec/src/test/java/org/elasticsearch/simdvec/internal/vectorization/ES91OSQVectorScorerTests.java

@@ -125,12 +125,21 @@ public void testScore() throws Exception {
                             scores2
                         );
                         for (int j = 0; j < ES91OSQVectorsScorer.BULK_SIZE; j++) {
-                            if (scores1[j] > (maxDims * Short.MAX_VALUE)) {
-                                int diff = (int) (scores1[j] - scores2[j]);
-                                assertThat("defaultScores: " + scores1[j] + " bulkScores: " + scores2[j], Math.abs(diff), lessThan(65));
-                            } else if (scores1[j] > (maxDims * Byte.MAX_VALUE)) {
-                                int diff = (int) (scores1[j] - scores2[j]);
-                                assertThat("defaultScores: " + scores1[j] + " bulkScores: " + scores2[j], Math.abs(diff), lessThan(9));
+                            if (scores1[j] == scores2[j]) {
+                                continue;
+                            }
+                            if (scores1[j] > (maxDims * Byte.MAX_VALUE)) {
+                                float diff = Math.abs(scores1[j] - scores2[j]);
+                                assertThat(
+                                    "defaultScores: " + scores1[j] + " bulkScores: " + scores2[j],
+                                    diff / scores1[j],
+                                    lessThan(1e-5f)
+                                );
+                                assertThat(
+                                    "defaultScores: " + scores1[j] + " bulkScores: " + scores2[j],
+                                    diff / scores2[j],
+                                    lessThan(1e-5f)
+                                );
                             } else {
                                 assertEquals(scores1[j], scores2[j], 1e-2f);
                             }

muted-tests.yml

@@ -571,9 +571,21 @@ tests:
 - class: org.elasticsearch.search.query.VectorIT
   method: testFilteredQueryStrategy
   issue: https://github.com/elastic/elasticsearch/issues/129517
-- class: org.elasticsearch.xpack.esql.parser.StatementParserTests
-  method: testInvalidJoinPatterns
-  issue: https://github.com/elastic/elasticsearch/issues/129598
+- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/129453
+- class: org.elasticsearch.server.cli.MachineDependentHeapTests
+  method: testMlOnlyOptions
+  issue: https://github.com/elastic/elasticsearch/issues/129236
+- class: org.elasticsearch.upgrades.RunningSnapshotIT
+  method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=1}
+  issue: https://github.com/elastic/elasticsearch/issues/129644
+- class: org.elasticsearch.upgrades.RunningSnapshotIT
+  method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=2}
+  issue: https://github.com/elastic/elasticsearch/issues/129645
+- class: org.elasticsearch.upgrades.RunningSnapshotIT
+  method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=3}
+  issue: https://github.com/elastic/elasticsearch/issues/129646
 
 # Examples:
 #

qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java

@@ -38,6 +38,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
         .feature(FeatureFlag.DOC_VALUES_SKIPPER)
         .feature(FeatureFlag.USE_LUCENE101_POSTINGS_FORMAT)
+        .feature(FeatureFlag.IVF_FORMAT)
         .build();
 
     public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -197,6 +197,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion NONE_CHUNKING_STRATEGY_8_19 = def(8_841_0_49);
     public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES_ALLOW_LIST_8_19 = def(8_841_0_50);
     public static final TransportVersion SETTINGS_IN_DATA_STREAMS_8_19 = def(8_841_0_51);
+    public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_REMOVE_ERROR_PARSING_8_19 = def(8_841_0_52);
     public static final TransportVersion V_9_0_0 = def(9_000_0_09);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_2 = def(9_000_0_11);
@@ -302,6 +303,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion SECURITY_CLOUD_API_KEY_REALM_AND_TYPE = def(9_099_0_00);
     public static final TransportVersion STATE_PARAM_GET_SNAPSHOT = def(9_100_0_00);
     public static final TransportVersion PROJECT_ID_IN_SNAPSHOTS_DELETIONS_AND_REPO_CLEANUP = def(9_101_0_00);
+    public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_REMOVE_ERROR_PARSING = def(9_102_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/cluster/ClusterState.java

@@ -1005,7 +1005,7 @@ public ClusterState copyAndUpdateMetadata(Consumer<Metadata.Builder> updater) {
     }
 
     public ClusterState copyAndUpdateProject(ProjectId projectId, Consumer<ProjectMetadata.Builder> updater) {
-        return copyAndUpdate(builder -> builder.putProjectMetadata(metadata().getProject(projectId).copyAndUpdate(updater)));
+        return copyAndUpdate(builder -> builder.metadata(metadata.copyAndUpdateProject(projectId, updater)));
     }
 
     @SuppressForbidden(reason = "directly reading ClusterState#clusterFeatures")

server/src/main/java/org/elasticsearch/cluster/ProjectState.java

@@ -73,14 +73,14 @@ public int hashCode() {
     public ClusterState updatedState(Consumer<ProjectMetadata.Builder> projectBuilderConsumer) {
         ProjectMetadata.Builder projectBuilder = ProjectMetadata.builder(metadata());
         projectBuilderConsumer.accept(projectBuilder);
-        return ClusterState.builder(cluster).putProjectMetadata(projectBuilder).build();
+        return updatedState(projectBuilder.build());
     }
 
     /**
      * Build a new {@link ClusterState} with the updated project.
      */
     public ClusterState updatedState(ProjectMetadata updatedProject) {
-        return ClusterState.builder(cluster).putProjectMetadata(updatedProject).build();
+        return ClusterState.builder(cluster).metadata(cluster.metadata().withUpdatedProject(updatedProject)).build();
     }
 
     /**
@@ -100,6 +100,6 @@ public ProjectState updateProject(ProjectMetadata updatedProject) {
                 )
             );
         }
-        return new ProjectState(ClusterState.builder(cluster).putProjectMetadata(updatedProject).build(), project);
+        return new ProjectState(updatedState(updatedProject), project);
     }
 }

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -404,6 +404,44 @@ private Metadata updateSingleProject(Function<ProjectMetadata, ProjectMetadata>
         }
     }
 
+    /**
+     * Updates a single project in the metadata. This offers a more performant way of updating a single project compared to the Builder.
+     */
+    @FixForMultiProject // We should reconsider whether this method is valuable once we update Metadata.Builder to hold constructed projects
+    // instead of project builders.
+    public Metadata withUpdatedProject(ProjectMetadata updatedProject) {
+        final var existingProject = projectMetadata.get(updatedProject.id());
+        if (existingProject == null) {
+            throw new IllegalArgumentException(
+                "Can only update existing project, cannot add a new project [" + updatedProject.id() + "]. Use the builder instead"
+            );
+        }
+        if (updatedProject == existingProject) {
+            return this;
+        }
+        final Map<ProjectId, ProjectMetadata> updatedMap;
+        if (projects().size() == 1) {
+            updatedMap = Map.of(updatedProject.id(), updatedProject);
+        } else {
+            final var hashMap = new HashMap<>(projectMetadata);
+            hashMap.put(updatedProject.id(), updatedProject);
+            updatedMap = Collections.unmodifiableMap(hashMap);
+        }
+        return new Metadata(
+            clusterUUID,
+            clusterUUIDCommitted,
+            version,
+            coordinationMetadata,
+            updatedMap,
+            transientSettings,
+            persistentSettings,
+            settings,
+            hashesOfConsistentSettings,
+            customs,
+            reservedStateMetadata
+        );
+    }
+
     public long version() {
         return this.version;
     }
@@ -1487,6 +1525,18 @@ public Metadata copyAndUpdate(Consumer<Builder> updater) {
         return builder.build();
     }
 
+    public Metadata copyAndUpdateProject(ProjectId projectId, Consumer<ProjectMetadata.Builder> updater) {
+        final var existingProject = projectMetadata.get(projectId);
+        if (existingProject == null) {
+            throw new IllegalArgumentException(
+                "Can only update existing project, cannot add a new project [" + projectId + "]. Use the builder instead"
+            );
+        }
+        final var builder = ProjectMetadata.builder(existingProject);
+        updater.accept(builder);
+        return withUpdatedProject(builder.build());
+    }
+
     public static class Builder {
 
         private String clusterUUID;