Merge branch 'main' into fixKmeanstests

# Conflicts:
#	server/src/main/java/org/elasticsearch/index/codec/vectors/cluster/KMeansLocal.java

Author: iverase

docs/changelog/130336.yaml

@@ -0,0 +1,5 @@
+pr: 130336
+summary: "[EIS] Rename the elser 2 default model and the default inference endpoint"
+area: Machine Learning
+type: bug
+issues: []

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_data_stream_mappings.yml

@@ -92,3 +92,111 @@ setup:
         index: my-data-stream-1
   - match: { .$oldIndexName.mappings.properties.name: null }
   - match: { .$newIndexName.mappings.properties.name.type: "keyword" }
+
+---
+"Test mappings component templates only":
+  - requires:
+      cluster_features: [ "logs_stream" ]
+      reason: requires setting 'logs_stream' to get or set data stream settings
+
+  - do:
+      cluster.put_component_template:
+        name: mappings-template
+        body:
+          template:
+            mappings:
+              properties:
+                field1:
+                  type: keyword
+                field2:
+                  type: keyword
+
+  - do:
+      allowed_warnings:
+        - "index template [my-component-only-template] has index patterns [my-component-only-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-component-only-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-component-only-template
+        body:
+          index_patterns: [ my-component-only-data-stream-* ]
+          data_stream: { }
+          composed_of:
+            - mappings-template
+
+  - do:
+      indices.create_data_stream:
+        name: my-component-only-data-stream-1
+
+  - do:
+      cluster.health:
+        index: "my-component-only-data-stream-1"
+        wait_for_status: green
+
+  - do:
+      indices.get_data_stream:
+        name: my-component-only-data-stream-1
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.mappings: {} }
+  - match: { data_streams.0.effective_mappings: null }
+
+  - do:
+      indices.put_data_stream_mappings:
+        name: my-component-only-data-stream-1
+        body:
+          properties:
+            field1:
+              type: text
+            field3:
+              type: text
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.applied_to_data_stream: true }
+  - match: { data_streams.0.mappings.properties.field1.type: "text" }
+  - match: { data_streams.0.mappings.properties.field2: null }
+  - match: { data_streams.0.mappings.properties.field3.type: "text" }
+  - match: { data_streams.0.effective_mappings.properties.field1.type: "text" }
+  - match: { data_streams.0.effective_mappings.properties.field2: null }
+  - match: { data_streams.0.effective_mappings.properties.field3.type: "text" }
+
+  - do:
+      indices.rollover:
+        alias: "my-component-only-data-stream-1"
+
+  - do:
+      cluster.health:
+        index: "my-component-only-data-stream-1"
+        wait_for_status: green
+
+  - do:
+      indices.get_data_stream_mappings:
+        name: my-component-only-data-stream-1
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - length: { data_streams.0.effective_mappings.properties: 2 }
+  - match: { data_streams.0.mappings.properties.field1.type: "text" }
+  - match: { data_streams.0.effective_mappings.properties.field3.type: "text" }
+
+  - do:
+      indices.get_data_stream:
+        name: my-component-only-data-stream-1
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.mappings.properties.field1.type: "text" }
+  - match: { data_streams.0.effective_mappings: null }
+  - set: { data_streams.0.indices.0.index_name: oldIndexName }
+  - set: { data_streams.0.indices.1.index_name: newIndexName }
+
+  - do:
+      indices.get_mapping:
+        index: my-component-only-data-stream-1
+  - match: { .$oldIndexName.mappings.properties.field1.type: "keyword" }
+  - match: { .$oldIndexName.mappings.properties.field2.type: "keyword" }
+  - match: { .$oldIndexName.mappings.properties.field3: null }
+  - match: { .$newIndexName.mappings.properties.field1.type: "text" }
+  - match: { .$newIndexName.mappings.properties.field2.type: "keyword" }
+  - match: { .$newIndexName.mappings.properties.field3.type: "text" }
+
+  - do:
+      indices.put_data_stream_mappings:
+        name: my-component-only-data-stream-1
+        body: {}
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.applied_to_data_stream: true }
+  - match: { data_streams.0.mappings null }
+  - match: { data_streams.0.effective_mappings: null }

muted-tests.yml

@@ -500,9 +500,6 @@ tests:
 - class: org.elasticsearch.xpack.security.PermissionsIT
   method: testWhenUserLimitedByOnlyAliasOfIndexCanWriteToIndexWhichWasRolledoverByILMPolicy
   issue: https://github.com/elastic/elasticsearch/issues/129481
-- class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceTests
-  method: testIORateIsAdjustedForAllRunningMergeTasks
-  issue: https://github.com/elastic/elasticsearch/issues/129531
 - class: org.elasticsearch.upgrades.IndexingIT
   method: testIndexing
   issue: https://github.com/elastic/elasticsearch/issues/129533

qa/vector/src/main/java/org/elasticsearch/test/knn/CmdLineArgs.java

@@ -42,6 +42,7 @@ record CmdLineArgs(
     int hnswM,
     int hnswEfConstruction,
     int searchThreads,
+    int numSearchers,
     int indexThreads,
     boolean reindex,
     boolean forceMerge,
@@ -64,6 +65,7 @@ record CmdLineArgs(
     static final ParseField OVER_SAMPLING_FACTOR_FIELD = new ParseField("over_sampling_factor");
     static final ParseField HNSW_M_FIELD = new ParseField("hnsw_m");
     static final ParseField HNSW_EF_CONSTRUCTION_FIELD = new ParseField("hnsw_ef_construction");
+    static final ParseField NUM_SEARCHERS_FIELD = new ParseField("num_searchers");
     static final ParseField SEARCH_THREADS_FIELD = new ParseField("search_threads");
     static final ParseField INDEX_THREADS_FIELD = new ParseField("index_threads");
     static final ParseField REINDEX_FIELD = new ParseField("reindex");
@@ -95,6 +97,7 @@ static CmdLineArgs fromXContent(XContentParser parser) throws IOException {
         PARSER.declareInt(Builder::setHnswM, HNSW_M_FIELD);
         PARSER.declareInt(Builder::setHnswEfConstruction, HNSW_EF_CONSTRUCTION_FIELD);
         PARSER.declareInt(Builder::setSearchThreads, SEARCH_THREADS_FIELD);
+        PARSER.declareInt(Builder::setNumSearchers, NUM_SEARCHERS_FIELD);
         PARSER.declareInt(Builder::setIndexThreads, INDEX_THREADS_FIELD);
         PARSER.declareBoolean(Builder::setReindex, REINDEX_FIELD);
         PARSER.declareBoolean(Builder::setForceMerge, FORCE_MERGE_FIELD);
@@ -125,6 +128,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(HNSW_M_FIELD.getPreferredName(), hnswM);
         builder.field(HNSW_EF_CONSTRUCTION_FIELD.getPreferredName(), hnswEfConstruction);
         builder.field(SEARCH_THREADS_FIELD.getPreferredName(), searchThreads);
+        builder.field(NUM_SEARCHERS_FIELD.getPreferredName(), numSearchers);
         builder.field(INDEX_THREADS_FIELD.getPreferredName(), indexThreads);
         builder.field(REINDEX_FIELD.getPreferredName(), reindex);
         builder.field(FORCE_MERGE_FIELD.getPreferredName(), forceMerge);
@@ -154,6 +158,7 @@ static class Builder {
         private int hnswM = 16;
         private int hnswEfConstruction = 200;
         private int searchThreads = 1;
+        private int numSearchers = 1;
         private int indexThreads = 1;
         private boolean reindex = false;
         private boolean forceMerge = false;
@@ -228,6 +233,11 @@ public Builder setSearchThreads(int searchThreads) {
             return this;
         }
 
+        public Builder setNumSearchers(int numSearchers) {
+            this.numSearchers = numSearchers;
+            return this;
+        }
+
         public Builder setIndexThreads(int indexThreads) {
             this.indexThreads = indexThreads;
             return this;
@@ -291,6 +301,7 @@ public CmdLineArgs build() {
                 hnswM,
                 hnswEfConstruction,
                 searchThreads,
+                numSearchers,
                 indexThreads,
                 reindex,
                 forceMerge,

qa/vector/src/main/java/org/elasticsearch/test/knn/KnnSearcher.java

@@ -55,6 +55,7 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.IntBuffer;
@@ -71,7 +72,9 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
+import java.util.function.IntConsumer;
 
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
 import static org.elasticsearch.test.knn.KnnIndexTester.logger;
@@ -96,6 +99,7 @@ class KnnSearcher {
     private final VectorEncoding vectorEncoding;
     private final float overSamplingFactor;
     private final int searchThreads;
+    private final int numSearchers;
 
     KnnSearcher(Path indexPath, CmdLineArgs cmdLineArgs, int nProbe) {
         this.docPath = cmdLineArgs.docVectors();
@@ -115,6 +119,7 @@ class KnnSearcher {
         this.nProbe = nProbe;
         this.indexType = cmdLineArgs.indexType();
         this.searchThreads = cmdLineArgs.searchThreads();
+        this.numSearchers = cmdLineArgs.numSearchers();
     }
 
     void runSearch(KnnIndexTester.Results finalResults, boolean earlyTermination) throws IOException {
@@ -124,7 +129,10 @@ void runSearch(KnnIndexTester.Results finalResults, boolean earlyTermination) th
         int offsetByteSize = 0;
         try (
             FileChannel input = FileChannel.open(queryPath);
-            ExecutorService executorService = Executors.newFixedThreadPool(searchThreads, r -> new Thread(r, "KnnSearcher-Thread"))
+            ExecutorService executorService = Executors.newFixedThreadPool(searchThreads, r -> new Thread(r, "KnnSearcher-Thread"));
+            ExecutorService numSearchersExecutor = numSearchers > 1
+                ? Executors.newFixedThreadPool(numSearchers, r -> new Thread(r, "KnnSearcher-Caller"))
+                : null
         ) {
             long queryPathSizeInBytes = input.size();
             logger.info(
@@ -163,29 +171,87 @@ void runSearch(KnnIndexTester.Results finalResults, boolean earlyTermination) th
                         }
                     }
                     targetReader.reset();
+                    final IntConsumer[] queryConsumers = new IntConsumer[numSearchers];
+                    if (vectorEncoding.equals(VectorEncoding.BYTE)) {
+                        byte[][] queries = new byte[numQueryVectors][dim];
+                        for (int i = 0; i < numQueryVectors; i++) {
+                            targetReader.next(queries[i]);
+                        }
+                        for (int s = 0; s < numSearchers; s++) {
+                            queryConsumers[s] = i -> {
+                                try {
+                                    results[i] = doVectorQuery(queries[i], searcher, earlyTermination);
+                                } catch (IOException e) {
+                                    throw new UncheckedIOException(e);
+                                }
+                            };
+                        }
+                    } else {
+                        float[][] queries = new float[numQueryVectors][dim];
+                        for (int i = 0; i < numQueryVectors; i++) {
+                            targetReader.next(queries[i]);
+                        }
+                        for (int s = 0; s < numSearchers; s++) {
+                            queryConsumers[s] = i -> {
+                                try {
+                                    results[i] = doVectorQuery(queries[i], searcher, earlyTermination);
+                                } catch (IOException e) {
+                                    throw new UncheckedIOException(e);
+                                }
+                            };
+                        }
+                    }
+                    int[][] querySplits = new int[numSearchers][];
+                    int queriesPerSearcher = numQueryVectors / numSearchers;
+                    for (int s = 0; s < numSearchers; s++) {
+                        int start = s * queriesPerSearcher;
+                        int end = (s == numSearchers - 1) ? numQueryVectors : (s + 1) * queriesPerSearcher;
+                        querySplits[s] = new int[end - start];
+                        for (int i = start; i < end; i++) {
+                            querySplits[s][i - start] = i;
+                        }
+                    }
+                    targetReader.reset();
                     startNS = System.nanoTime();
                     KnnIndexTester.ThreadDetails startThreadDetails = new KnnIndexTester.ThreadDetails();
-                    for (int i = 0; i < numQueryVectors; i++) {
-                        if (vectorEncoding.equals(VectorEncoding.BYTE)) {
-                            targetReader.next(targetBytes);
-                            results[i] = doVectorQuery(targetBytes, searcher, earlyTermination);
-                        } else {
-                            targetReader.next(target);
-                            results[i] = doVectorQuery(target, searcher, earlyTermination);
+                    if (numSearchersExecutor != null) {
+                        // use multiple searchers
+                        var futures = new ArrayList<Future<Void>>();
+                        for (int s = 0; s < numSearchers; s++) {
+                            int[] split = querySplits[s];
+                            IntConsumer queryConsumer = queryConsumers[s];
+                            futures.add(numSearchersExecutor.submit(() -> {
+                                for (int j : split) {
+                                    queryConsumer.accept(j);
+                                }
+                                return null;
+                            }));
+                        }
+                        for (Future<Void> future : futures) {
+                            try {
+                                future.get();
+                            } catch (Exception e) {
+                                throw new RuntimeException("Error executing searcher thread", e);
+                            }
+                        }
+                    } else {
+                        // use a single searcher
+                        for (int i = 0; i < numQueryVectors; i++) {
+                            queryConsumers[0].accept(i);
                         }
                     }
                     KnnIndexTester.ThreadDetails endThreadDetails = new KnnIndexTester.ThreadDetails();
                     elapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNS);
                     long startCPUTimeNS = 0;
                     long endCPUTimeNS = 0;
                     for (int i = 0; i < startThreadDetails.threadInfos.length; i++) {
-                        if (startThreadDetails.threadInfos[i].getThreadName().startsWith("KnnSearcher-Thread")) {
+                        if (startThreadDetails.threadInfos[i].getThreadName().startsWith("KnnSearcher")) {
                             startCPUTimeNS += startThreadDetails.cpuTimesNS[i];
                         }
                     }
 
                     for (int i = 0; i < endThreadDetails.threadInfos.length; i++) {
-                        if (endThreadDetails.threadInfos[i].getThreadName().startsWith("KnnSearcher-Thread")) {
+                        if (endThreadDetails.threadInfos[i].getThreadName().startsWith("KnnSearcher")) {
                             endCPUTimeNS += endThreadDetails.cpuTimesNS[i];
                         }
                     }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/40_knn_search.yml

@@ -605,6 +605,40 @@ setup:
   - match: { hits.hits.0._score: $knn_score0 }
   - match: { hits.hits.1._score: $knn_score1 }
   - match: { hits.hits.2._score: $knn_score2 }
+
+---
+"Dimensions are dynamically set":
+  - do:
+      indices.create:
+        index: test_index
+        body:
+          mappings:
+            properties:
+              embedding:
+                type: dense_vector
+
+  - do:
+      index:
+        index: test_index
+        id: "0"
+        body:
+          embedding: [ 0.5, 111.3, -13.0, 14.8, -156.0 ]
+
+  - do:
+      indices.get_mapping:
+        index: test_index
+
+  - match: { test_index.mappings.properties.embedding.type: dense_vector }
+  - match: { test_index.mappings.properties.embedding.dims: 5 }
+
+  - do:
+      catch: bad_request
+      index:
+        index: test_index
+        id: "0"
+        body:
+          embedding: [ 0.5, 111.3 ]
+
 ---
 "Updating dim to null is not allowed":
   - requires: