Merge branch 'main' into ES-12055_track_snapshot_stats_as_metrics

Author: nicktindall

docs/changelog/130336.yaml

@@ -0,0 +1,5 @@
+pr: 130336
+summary: "[EIS] Rename the elser 2 default model and the default inference endpoint"
+area: Machine Learning
+type: bug
+issues: []

docs/reference/elasticsearch/mapping-reference/sparse-vector.md

@@ -26,7 +26,7 @@ PUT my-index
 
 ## Token pruning
 ```{applies_to}
-stack: preview 9.1
+stack: ga 9.1
 ```
 
 With any new indices created, token pruning will be turned on by default with appropriate defaults. You can control this behaviour using the optional `index_options` parameters for the field:
@@ -63,23 +63,23 @@ The following parameters are accepted by `sparse_vector` fields:
     * Exclude the field from [_source](/reference/elasticsearch/rest-apis/retrieve-selected-fields.md#source-filtering).
     * Use [synthetic `_source`](/reference/elasticsearch/mapping-reference/mapping-source-field.md#synthetic-source).
 
-index_options {applies_to}`stack: preview 9.1`
+index_options {applies_to}`stack: ga 9.1`
 :   (Optional, object) You can set index options for your  `sparse_vector` field to determine if you should prune tokens, and the parameter configurations for the token pruning. If pruning options are not set in your [`sparse_vector` query](/reference/query-languages/query-dsl/query-dsl-sparse-vector-query.md), Elasticsearch will use the default options configured for the field, if any.
 
 Parameters for `index_options` are:
 
-`prune` {applies_to}`stack: preview 9.1`
+`prune` {applies_to}`stack: ga 9.1`
 :   (Optional, boolean) Whether to perform pruning, omitting the non-significant tokens from the query to improve query performance. If `prune` is true but the `pruning_config` is not specified, pruning will occur but default values will be used. Default: true.
 
-`pruning_config` {applies_to}`stack: preview 9.1`
+`pruning_config` {applies_to}`stack: ga 9.1`
 :   (Optional, object) Optional pruning configuration. If enabled, this will omit non-significant tokens from the query in order to improve query performance. This is only used if `prune` is set to `true`. If `prune` is set to `true` but `pruning_config` is not specified, default values will be used. If `prune` is set to false but `pruning_config` is specified, an exception will occur.
 
     Parameters for `pruning_config` include:
 
-    `tokens_freq_ratio_threshold` {applies_to}`stack: preview 9.1`
+    `tokens_freq_ratio_threshold` {applies_to}`stack: ga 9.1`
     :   (Optional, integer) Tokens whose frequency is more than `tokens_freq_ratio_threshold` times the average frequency of all tokens in the specified field are considered outliers and pruned. This value must between 1 and 100. Default: `5`.
 
-    `tokens_weight_threshold` {applies_to}`stack: preview 9.1`
+    `tokens_weight_threshold` {applies_to}`stack: ga 9.1`
     :   (Optional, float) Tokens whose weight is less than `tokens_weight_threshold` are considered insignificant and pruned. This value must be between 0 and 1. Default: `0.4`.
 
     ::::{note}

docs/reference/query-languages/esql/esql-lookup-join.md

@@ -200,6 +200,7 @@ The following are the current limitations with `LOOKUP JOIN`:
 * Indices in [`lookup` mode](/reference/elasticsearch/index-settings/index-modules.md#index-mode-setting) are always single-sharded.
 * Cross cluster search is unsupported initially. Both source and lookup indices must be local.
 * Currently, only matching on equality is supported.
-* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards, aliases, datemath, and datastreams are not supported.
+* `LOOKUP JOIN` can only use a single match field and a single index. Wildcards are not supported.
+  * Aliases, datemath, and datastreams are supported, as long as the index pattern matches a single concrete index {applies_to}`stack: ga 9.1.0`.
 * The name of the match field in `LOOKUP JOIN lu_idx ON match_field` must match an existing field in the query. This may require `RENAME`s or `EVAL`s to achieve.
 * The query will circuit break if there are too many matching documents in the lookup index, or if the documents are too large. More precisely, `LOOKUP JOIN` works in batches of, normally, about 10,000 rows; a large amount of heap space is needed if the matching documents from the lookup index for a batch are multiple megabytes or larger. This is roughly the same as for `ENRICH`.

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_data_stream_mappings.yml

@@ -92,3 +92,111 @@ setup:
         index: my-data-stream-1
   - match: { .$oldIndexName.mappings.properties.name: null }
   - match: { .$newIndexName.mappings.properties.name.type: "keyword" }
+
+---
+"Test mappings component templates only":
+  - requires:
+      cluster_features: [ "logs_stream" ]
+      reason: requires setting 'logs_stream' to get or set data stream settings
+
+  - do:
+      cluster.put_component_template:
+        name: mappings-template
+        body:
+          template:
+            mappings:
+              properties:
+                field1:
+                  type: keyword
+                field2:
+                  type: keyword
+
+  - do:
+      allowed_warnings:
+        - "index template [my-component-only-template] has index patterns [my-component-only-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-component-only-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-component-only-template
+        body:
+          index_patterns: [ my-component-only-data-stream-* ]
+          data_stream: { }
+          composed_of:
+            - mappings-template
+
+  - do:
+      indices.create_data_stream:
+        name: my-component-only-data-stream-1
+
+  - do:
+      cluster.health:
+        index: "my-component-only-data-stream-1"
+        wait_for_status: green
+
+  - do:
+      indices.get_data_stream:
+        name: my-component-only-data-stream-1
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.mappings: {} }
+  - match: { data_streams.0.effective_mappings: null }
+
+  - do:
+      indices.put_data_stream_mappings:
+        name: my-component-only-data-stream-1
+        body:
+          properties:
+            field1:
+              type: text
+            field3:
+              type: text
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.applied_to_data_stream: true }
+  - match: { data_streams.0.mappings.properties.field1.type: "text" }
+  - match: { data_streams.0.mappings.properties.field2: null }
+  - match: { data_streams.0.mappings.properties.field3.type: "text" }
+  - match: { data_streams.0.effective_mappings.properties.field1.type: "text" }
+  - match: { data_streams.0.effective_mappings.properties.field2: null }
+  - match: { data_streams.0.effective_mappings.properties.field3.type: "text" }
+
+  - do:
+      indices.rollover:
+        alias: "my-component-only-data-stream-1"
+
+  - do:
+      cluster.health:
+        index: "my-component-only-data-stream-1"
+        wait_for_status: green
+
+  - do:
+      indices.get_data_stream_mappings:
+        name: my-component-only-data-stream-1
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - length: { data_streams.0.effective_mappings.properties: 2 }
+  - match: { data_streams.0.mappings.properties.field1.type: "text" }
+  - match: { data_streams.0.effective_mappings.properties.field3.type: "text" }
+
+  - do:
+      indices.get_data_stream:
+        name: my-component-only-data-stream-1
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.mappings.properties.field1.type: "text" }
+  - match: { data_streams.0.effective_mappings: null }
+  - set: { data_streams.0.indices.0.index_name: oldIndexName }
+  - set: { data_streams.0.indices.1.index_name: newIndexName }
+
+  - do:
+      indices.get_mapping:
+        index: my-component-only-data-stream-1
+  - match: { .$oldIndexName.mappings.properties.field1.type: "keyword" }
+  - match: { .$oldIndexName.mappings.properties.field2.type: "keyword" }
+  - match: { .$oldIndexName.mappings.properties.field3: null }
+  - match: { .$newIndexName.mappings.properties.field1.type: "text" }
+  - match: { .$newIndexName.mappings.properties.field2.type: "keyword" }
+  - match: { .$newIndexName.mappings.properties.field3.type: "text" }
+
+  - do:
+      indices.put_data_stream_mappings:
+        name: my-component-only-data-stream-1
+        body: {}
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.applied_to_data_stream: true }
+  - match: { data_streams.0.mappings null }
+  - match: { data_streams.0.effective_mappings: null }

muted-tests.yml

@@ -224,9 +224,6 @@ tests:
 - class: org.elasticsearch.packaging.test.BootstrapCheckTests
   method: test20RunWithBootstrapChecks
   issue: https://github.com/elastic/elasticsearch/issues/124940
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
-  method: testStopQueryLocal
-  issue: https://github.com/elastic/elasticsearch/issues/121672
 - class: org.elasticsearch.packaging.test.BootstrapCheckTests
   method: test10Install
   issue: https://github.com/elastic/elasticsearch/issues/124957
@@ -500,9 +497,6 @@ tests:
 - class: org.elasticsearch.xpack.security.PermissionsIT
   method: testWhenUserLimitedByOnlyAliasOfIndexCanWriteToIndexWhichWasRolledoverByILMPolicy
   issue: https://github.com/elastic/elasticsearch/issues/129481
-- class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceTests
-  method: testIORateIsAdjustedForAllRunningMergeTasks
-  issue: https://github.com/elastic/elasticsearch/issues/129531
 - class: org.elasticsearch.upgrades.IndexingIT
   method: testIndexing
   issue: https://github.com/elastic/elasticsearch/issues/129533
@@ -605,6 +599,33 @@ tests:
 - class: org.elasticsearch.repositories.blobstore.BlobStoreCorruptionIT
   method: testCorruptionDetection
   issue: https://github.com/elastic/elasticsearch/issues/130536
+- class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
+  method: test {yaml=indices.resolve_index/10_basic_resolve_index/Resolve index with hidden and closed indices}
+  issue: https://github.com/elastic/elasticsearch/issues/130568
+- class: org.elasticsearch.reindex.ReindexRestClientSslTests
+  method: testClientPassesClientCertificate
+  issue: https://github.com/elastic/elasticsearch/issues/130584
+- class: org.elasticsearch.reindex.ReindexRestClientSslTests
+  method: testClientFailsWithUntrustedCertificate
+  issue: https://github.com/elastic/elasticsearch/issues/130585
+- class: org.elasticsearch.reindex.ReindexRestClientSslTests
+  method: testClientSucceedsWithCertificateAuthorities
+  issue: https://github.com/elastic/elasticsearch/issues/130586
+- class: org.elasticsearch.reindex.ReindexFromRemoteBuildRestClientTests
+  method: testHeaders
+  issue: https://github.com/elastic/elasticsearch/issues/130587
+- class: org.elasticsearch.reindex.ReindexFromRemoteBuildRestClientTests
+  method: testBuildRestClient
+  issue: https://github.com/elastic/elasticsearch/issues/130588
+- class: org.elasticsearch.http.netty4.Netty4HttpServerTransportTests
+  method: testRespondAfterServiceCloseWithClientCancel
+  issue: https://github.com/elastic/elasticsearch/issues/130590
+- class: org.elasticsearch.http.netty4.Netty4HttpServerTransportTests
+  method: testRespondAfterServiceCloseWithServerCancel
+  issue: https://github.com/elastic/elasticsearch/issues/130591
+- class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
+  method: test
+  issue: https://github.com/elastic/elasticsearch/issues/130067
 
 # Examples:
 #

qa/vector/src/main/java/org/elasticsearch/test/knn/CmdLineArgs.java

@@ -42,6 +42,7 @@ record CmdLineArgs(
     int hnswM,
     int hnswEfConstruction,
     int searchThreads,
+    int numSearchers,
     int indexThreads,
     boolean reindex,
     boolean forceMerge,
@@ -64,6 +65,7 @@ record CmdLineArgs(
     static final ParseField OVER_SAMPLING_FACTOR_FIELD = new ParseField("over_sampling_factor");
     static final ParseField HNSW_M_FIELD = new ParseField("hnsw_m");
     static final ParseField HNSW_EF_CONSTRUCTION_FIELD = new ParseField("hnsw_ef_construction");
+    static final ParseField NUM_SEARCHERS_FIELD = new ParseField("num_searchers");
     static final ParseField SEARCH_THREADS_FIELD = new ParseField("search_threads");
     static final ParseField INDEX_THREADS_FIELD = new ParseField("index_threads");
     static final ParseField REINDEX_FIELD = new ParseField("reindex");
@@ -95,6 +97,7 @@ static CmdLineArgs fromXContent(XContentParser parser) throws IOException {
         PARSER.declareInt(Builder::setHnswM, HNSW_M_FIELD);
         PARSER.declareInt(Builder::setHnswEfConstruction, HNSW_EF_CONSTRUCTION_FIELD);
         PARSER.declareInt(Builder::setSearchThreads, SEARCH_THREADS_FIELD);
+        PARSER.declareInt(Builder::setNumSearchers, NUM_SEARCHERS_FIELD);
         PARSER.declareInt(Builder::setIndexThreads, INDEX_THREADS_FIELD);
         PARSER.declareBoolean(Builder::setReindex, REINDEX_FIELD);
         PARSER.declareBoolean(Builder::setForceMerge, FORCE_MERGE_FIELD);
@@ -125,6 +128,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(HNSW_M_FIELD.getPreferredName(), hnswM);
         builder.field(HNSW_EF_CONSTRUCTION_FIELD.getPreferredName(), hnswEfConstruction);
         builder.field(SEARCH_THREADS_FIELD.getPreferredName(), searchThreads);
+        builder.field(NUM_SEARCHERS_FIELD.getPreferredName(), numSearchers);
         builder.field(INDEX_THREADS_FIELD.getPreferredName(), indexThreads);
         builder.field(REINDEX_FIELD.getPreferredName(), reindex);
         builder.field(FORCE_MERGE_FIELD.getPreferredName(), forceMerge);
@@ -154,6 +158,7 @@ static class Builder {
         private int hnswM = 16;
         private int hnswEfConstruction = 200;
         private int searchThreads = 1;
+        private int numSearchers = 1;
         private int indexThreads = 1;
         private boolean reindex = false;
         private boolean forceMerge = false;
@@ -228,6 +233,11 @@ public Builder setSearchThreads(int searchThreads) {
             return this;
         }
 
+        public Builder setNumSearchers(int numSearchers) {
+            this.numSearchers = numSearchers;
+            return this;
+        }
+
         public Builder setIndexThreads(int indexThreads) {
             this.indexThreads = indexThreads;
             return this;
@@ -291,6 +301,7 @@ public CmdLineArgs build() {
                 hnswM,
                 hnswEfConstruction,
                 searchThreads,
+                numSearchers,
                 indexThreads,
                 reindex,
                 forceMerge,