Merge branch 'main' into fixup-search-error-trace-it

Author: benchaplin

build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java

@@ -57,13 +57,9 @@ public void apply(Project project) {
             task.into("META-INF", copy -> copy.from(testBuildInfoTask));
         });
 
-        if (project.getRootProject().getName().equals("elasticsearch")) {
-            project.getTasks()
-                .withType(Test.class)
-                .matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
-                .configureEach(test -> {
-                    test.systemProperty("es.entitlement.enableForTests", "true");
-                });
-        }
+        project.getTasks()
+            .withType(Test.class)
+            .matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
+            .configureEach(test -> test.getSystemProperties().putIfAbsent("es.entitlement.enableForTests", "true"));
     }
 }

muted-tests.yml

@@ -476,21 +476,9 @@ tests:
 - class: org.elasticsearch.compute.lucene.read.SortedSetOrdinalsBuilderTests
   method: testReader
   issue: https://github.com/elastic/elasticsearch/issues/131573
-- class: org.elasticsearch.search.SearchWithIndexBlocksIT
-  method: testSearchShardsOnIndicesWithIndexRefreshBlocks
-  issue: https://github.com/elastic/elasticsearch/issues/131662
-- class: org.elasticsearch.search.SearchWithIndexBlocksIT
-  method: testSearchIndicesWithIndexRefreshBlocks
-  issue: https://github.com/elastic/elasticsearch/issues/131693
-- class: org.elasticsearch.search.SearchWithIndexBlocksIT
-  method: testOpenPITOnIndicesWithIndexRefreshBlocks
-  issue: https://github.com/elastic/elasticsearch/issues/131695
 - class: org.elasticsearch.xpack.esql.ccq.MultiClustersIT
   method: testLookupJoinAliasesSkipOld
   issue: https://github.com/elastic/elasticsearch/issues/131697
-- class: org.elasticsearch.search.SearchWithIndexBlocksIT
-  method: testMultiSearchIndicesWithIndexRefreshBlocks
-  issue: https://github.com/elastic/elasticsearch/issues/131698
 - class: org.elasticsearch.indices.cluster.RemoteSearchForceConnectTimeoutIT
   method: testTimeoutSetting
   issue: https://github.com/elastic/elasticsearch/issues/131656
@@ -512,9 +500,6 @@ tests:
 - class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
   method: test {p0=search/600_flattened_ignore_above/flattened ignore_above multi-value field}
   issue: https://github.com/elastic/elasticsearch/issues/131967
-- class: org.elasticsearch.search.routing.SearchReplicaSelectionIT
-  method: testNodeSelection
-  issue: https://github.com/elastic/elasticsearch/issues/132017
 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS1EnrichUnavailableRemotesIT
   method: testEsqlEnrichWithSkipUnavailable
   issue: https://github.com/elastic/elasticsearch/issues/132078
@@ -623,6 +608,13 @@ tests:
 - class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
   method: "testDenseVectorMappingUpdate {initialType=bbq_flat updateType=bbq_disk #2}"
   issue: https://github.com/elastic/elasticsearch/issues/132184
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=bbq_hnsw updateType=bbq_disk}
+  issue: https://github.com/elastic/elasticsearch/issues/132188
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: "testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_disk #2}"
+  issue: https://github.com/elastic/elasticsearch/issues/132189
+
 
 # Examples:
 #

plugins/examples/build.gradle

@@ -20,6 +20,11 @@ subprojects {
     targetCompatibility = 21
   }
 
+  test {
+    // testing with entitlements doesn't work for example plugins ES-12453
+    systemProperty 'es.entitlement.enableForTests', 'false'
+  }
+
   repositories {
     // Only necessary when building plugins against SNAPSHOT versions of Elasticsearch
     if (gradle.includedBuilds.isEmpty()) {

qa/multi-data-path/build.gradle

@@ -0,0 +1,10 @@
+apply plugin: 'elasticsearch.internal-yaml-rest-test'
+
+// This subproject verifies MDP continues to work with entitlements.
+
+restResources {
+  restApi {
+    include '_common', 'capabilities', 'index', 'indices', 'indices.create'
+  }
+}
+

qa/multi-data-path/src/yamlRestTest/java/org/elasticsearch/test/rest/yaml/MDPYamlTestSuiteIT.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.test.rest.yaml;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.nio.file.Files;
+
+public class MDPYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local().setting("path.shared_data", tempSharedDataPath()).build();
+
+    public MDPYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return createParameters();
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    private static String tempSharedDataPath() {
+        try {
+            return Files.createTempDirectory("shared_data").toString();
+        } catch (IOException e) {
+            throw new AssertionError(e);
+        }
+    }
+}

qa/multi-data-path/src/yamlRestTest/resources/rest-api-spec/test/mdp/10_basic.yml

@@ -0,0 +1,31 @@
+---
+"Index using shared data path":
+
+  - requires:
+      test_runner_features: ["warnings"]
+
+  - do:
+      warnings:
+        - "[index.data_path] setting was deprecated in Elasticsearch and will be removed in a future release. See the deprecation documentation for the next major version."
+      indices.create:
+        index: test_index
+        body:
+          settings:
+            data_path: "test_index_data_path"
+
+  - do:
+      index:
+        index:  test_index
+        id:     "1"
+        body:   { foo: bar }
+
+  - match:   { result: created }
+
+  - do:
+      index:
+        index:    test_index
+        id:       "1"
+        body:     { foo: bar }
+        op_type:  index
+
+  - match:   { result: updated }

qa/vector/src/main/java/org/elasticsearch/test/knn/KnnIndexTester.java

@@ -101,7 +101,7 @@ private static String formatIndexPath(CmdLineArgs args) {
     static Codec createCodec(CmdLineArgs args) {
         final KnnVectorsFormat format;
         if (args.indexType() == IndexType.IVF) {
-            format = new IVFVectorsFormat(args.ivfClusterSize());
+            format = new IVFVectorsFormat(args.ivfClusterSize(), IVFVectorsFormat.DEFAULT_CENTROIDS_PER_PARENT_CLUSTER);
         } else {
             if (args.quantizeBits() == 1) {
                 if (args.indexType() == IndexType.FLAT) {

server/src/internalClusterTest/java/org/elasticsearch/search/SearchWithIndexBlocksIT.java

@@ -308,6 +308,9 @@ default void copyBlob(
      * <p>
      * This operation, together with {@link #compareAndSetRegister}, must have linearizable semantics: a collection of such operations must
      * act as if they operate serially, with each operation taking place at some instant in between its invocation and its completion.
+     * <p>
+     * If the listener completes exceptionally then the write operation should be considered as continuing to run and may therefore appear
+     * to occur at some later point in time.
      *
      * @param purpose  The purpose of the operation
      * @param key      key of the value to update
@@ -330,6 +333,9 @@ void compareAndExchangeRegister(
      * <p>
      * This operation, together with {@link #compareAndExchangeRegister}, must have linearizable semantics: a collection of such operations
      * must act as if they operate serially, with each operation taking place at some instant in between its invocation and its completion.
+     * <p>
+     * If the listener completes exceptionally then the write operation should be considered as continuing to run and may therefore appear
+     * to occur at some later point in time.
      *
      * @param purpose  The purpose of the operation
      * @param key      key of the value to update
@@ -361,7 +367,10 @@ default void compareAndSetRegister(
      * This operation has read-after-write consistency with respect to writes performed using {@link #compareAndExchangeRegister} and
      * {@link #compareAndSetRegister}, but does not guarantee full linearizability. In particular, a {@code getRegister} performed during
      * one of these write operations may return either the old or the new value, and a caller may therefore observe the old value
-     * <i>after</i> observing the new value, as long as both such read operations take place before the write operation completes.
+     * <i>after</i> observing the new value, as long as both such read operations take place before the success of the write operation.
+     * <p>
+     * Write operations which complete exceptionally may behave as if they continue to run, thus yielding old or new values for an extended
+     * period of time. If multiple writes fail then {@code getRegister} may return any of the written values.
      *
      * @param purpose The purpose of the operation
      * @param key      key of the value to get