Merge branch 'main' into t-digest-field-mapper

Author: not-napoleon

build-tools-internal/version.properties

@@ -2,7 +2,7 @@ elasticsearch     = 9.3.0
 lucene            = 10.3.1
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 25+36@bd75d5f9689641da8e1daabeccb5528b
+bundled_jdk = 25.0.1+8@2fbf10d8c78e40bd87641c434705079d
 # optional dependencies
 spatial4j         = 0.7
 jts               = 1.15.0

docs/changelog/136065.yaml

@@ -0,0 +1,5 @@
+pr: 136065
+summary: Nightly maintenance for anomaly detection results indices to keep to manageable size.
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/136889.yaml

@@ -0,0 +1,6 @@
+pr: 136889
+summary: Remove early phase failure in batched
+area: Search
+type: bug
+issues:
+ - 134151

docs/changelog/137530.yaml

@@ -0,0 +1,5 @@
+pr: 137530
+summary: Allows field caps to be cross project
+area: Search
+type: enhancement
+issues: []

docs/changelog/137640.yaml

@@ -0,0 +1,5 @@
+pr: 137640
+summary: Update bundled JDK to Java 25.0.1+8
+area: Packaging
+type: upgrade
+issues: []

docs/changelog/137671.yaml

@@ -0,0 +1,5 @@
+pr: 137671
+summary: "[LTR] Fix feature display order when using explain"
+area: Search
+type: bug
+issues: []

docs/reference/elasticsearch/configuration-reference/machine-learning-settings.md

@@ -86,6 +86,9 @@ $$$xpack.ml.max_open_jobs$$$
 `xpack.ml.nightly_maintenance_requests_per_second`
 :   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The rate at which the nightly maintenance task deletes expired model snapshots and results. The setting is a proxy to the [`requests_per_second`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-delete-by-query) parameter used in the delete by query requests and controls throttling. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than `0.0` or equal to `-1.0`, where `-1.0` means a default value is used. Defaults to `-1.0`
 
+`xpack.ml.results_index_rollover_max_size`
+:   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum size the anomaly detection results indices can reach before being rolled over by the nightly maintenance task. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Valid values must be greater than or equal to `-1B`. A value of `-1B` means the indices will never be rolled over. A value of `0B` means the indices will always be rolled over, regardless of size. Defaults to `50GB`.
+
 `xpack.ml.node_concurrent_job_allocations`
 :   ([Dynamic](docs-content://deploy-manage/stack-settings.md#dynamic-cluster-setting)) The maximum number of jobs that can concurrently be in the `opening` state on each node. Typically, jobs spend a small amount of time in this state before they move to `open` state. Jobs that must restore large models when they are opening spend more time in the `opening` state. When the {{operator-feature}} is enabled, this setting can be updated only by operator users. Defaults to `2`.
 

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -48,7 +48,7 @@ the field mappings.
 
 :::::::{tab-set}
 
-::::::{tab-item} Using the default ELSER on EIS endpoint on Serverless
+::::::{tab-item} Default ELSER on EIS endpoint on {{serverless-short}}
 
 ```{applies_to}
 serverless: ga
@@ -72,7 +72,7 @@ PUT my-index-000001
 
 ::::::
 
-::::::{tab-item} Using the preconfigured ELSER on EIS endpoint in Cloud
+::::::{tab-item} Preconfigured ELSER on EIS endpoint in Cloud
 
 ```{applies_to}
 stack: ga 9.2
@@ -98,7 +98,7 @@ PUT my-index-000001
 
 ::::::
 
-::::::{tab-item} Using the default ELSER endpoint
+::::::{tab-item} Default ELSER endpoint
 
 If you use the preconfigured `.elser-2-elasticsearch` endpoint, you can set up `semantic_text` with the following API request:
 
@@ -544,9 +544,14 @@ stack: ga 9.2
 serverless: ga
 ```
 
+:::{important}
+Starting with {{es}} 9.2, the recommended method for retrieving embeddings has changed from that used in previous versions.
+For instructions on retrieving embeddings in versions earlier than 9.2, refer to [Returning semantic field embeddings using `fields`](#return-embeddings-fields).
+:::
+
 By default, the embeddings generated for `semantic_text` fields are stored internally and **not included in `_source`** when retrieving documents.
 
-To include the full inference fields, including their embeddings, in `_source`, set the `_source.exclude_vectors` option to `false`.
+To include the full {{infer}} fields, including their embeddings, in `_source`, set the `_source.exclude_vectors` option to `false`.
 This works with the
 [Get](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get),
 [Search](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-search),
@@ -565,18 +570,23 @@ POST my-index/_search
   }
 }
 ```
-% TEST[skip:Requires inference endpoint]
+% TEST[skip:Requires {{infer}} endpoint]
 
 The embeddings will appear under `_inference_fields` in `_source`.
 
 **Use cases**
+
 Including embeddings in `_source` is useful when you want to:
 
 * Reindex documents into another index **with the same `inference_id`** without re-running inference.
 * Export or migrate documents while preserving their embeddings.
 * Inspect or debug the raw embeddings generated for your content.
 
 ### Example: Reindex while preserving embeddings
+```{applies_to}
+stack: ga 9.2
+serverless: ga
+```
 
 ```console
 POST _reindex
@@ -592,7 +602,7 @@ POST _reindex
   }
 }
 ```
-% TEST[skip:Requires inference endpoint]
+% TEST[skip:Requires {{infer}} endpoint]
 
 1. Sends the source documents with their stored embeddings to the destination index.
 
@@ -602,16 +612,110 @@ the documents will **fail the reindex task**.
 Matching `inference_id` values are required to reuse the existing embeddings.
 ::::
 
-This allows documents to be re-indexed without triggering inference again, **as long as the target `semantic_text` field uses the same `inference_id` as the source**.
+This allows documents to be re-indexed without triggering {{infer}} again, **as long as the target `semantic_text` field uses the same `inference_id` as the source**.
 
-::::{note}
-**For versions prior to 9.2.0**
+### Example: Troubleshooting semantic_text fields [troubleshooting-semantic-text-fields]
+```{applies_to}
+stack: ga 9.2
+serverless: ga
+```
+
+To verify that your embeddings look correct, you can retrieve the {{infer}} data that `semantic_text` normally hides from search results.
 
-Older versions do not support the `exclude_vectors` option to retrieve the embeddings of the semantic text fields.
-To return the `_inference_fields`, use the `fields` option in a search request instead:
+To retrieve the stored embeddings in {{es}} 9.2 and later, set the `exclude_vectors` parameter to `false` in the `_source` field. This ensures that the vector data, which is excluded by default, is included in the search response.
 
 ```console
 POST test-index/_search
+{
+  "_source": {
+    "exclude_vectors": false
+  },
+  "query": {
+    "match": {
+      "my_semantic_field": "Which country is Paris in?"
+    }
+  }
+}
+```
+% TEST[skip:Requires {{infer}} endpoint]
+
+This will return verbose chunked embeddings content that is used to perform
+semantic search for `semantic_text` fields:
+
+```console-response
+{
+  "took": 18,
+  "timed_out": false,
+  "_shards": {
+    "total": 1,
+    "successful": 1,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": { "value": 1, "relation": "eq" },
+    "max_score": 16.532316,
+    "hits": [
+      {
+        "_index": "test-index",
+        "_id": "1",
+        "_score": 16.532316,
+        "_source": {
+          "my_semantic_field": "Paris is the capital of France.",
+          "_inference_fields": {
+            "my_semantic_field": {
+              "inference": {
+                "inference_id": ".elser-2-elasticsearch", <1>
+                "model_settings": { <2>
+                  "service": "elasticsearch",
+                  "task_type": "sparse_embedding"
+                },
+                "chunks": {
+                  "my_semantic_field": [
+                    {
+                      "start_offset": 0,
+                      "end_offset": 31,
+                      "embeddings": { <3>
+                        "airport": 0.12011719,
+                        "brussels": 0.032836914,
+                        "capital": 2.1328125,
+                        "capitals": 0.6386719,
+                        "capitol": 1.2890625,
+                        "cities": 0.78125,
+                        "city": 1.265625,
+                        "continent": 0.26953125,
+                        "country": 0.59765625,
+                        ...
+                      }
+                    }
+                  ]
+                }
+              }
+            }
+          }
+        }
+      }
+    ]
+  }
+}
+```
+% TEST[skip:Requires {{infer}} endpoint]
+1. The {{infer}} endpoint used to generate embeddings.
+2. Lists details about the model used to generate embeddings, such as the service name and task type.
+3. The embeddings generated for this chunk.
+
+## Returning semantic field embeddings using `fields` [return-embeddings-fields]
+
+:::{important}
+This method for returning semantic field embeddings is recommended only for {{es}} versions earlier than 9.2.
+For version 9.2 and later, use the [`exclude_vectors`](#troubleshooting-semantic-text-fields) parameter instead.
+:::
+
+To retrieve stored embeddings, use the `fields` parameter with `_inference_fields`. This lets you include the vector data that is not shown by default in the response.
+The `fields` parameter only works with the `_search` endpoint.  
+
+```console
+POST my-index/_search
 {
   "query": {
     "match": {
@@ -623,11 +727,7 @@ POST test-index/_search
   ]
 }
 ```
-% TEST[skip:Requires inference endpoint]
-
-This returns the chunked embeddings used for semantic search under `_inference_fields` in `_source`.
-Note that the `fields` option is **not** available for the Reindex API.
-::::
+% TEST[skip:Requires {{infer}} endpoint]
 
 ## Customizing `semantic_text` indexing [custom-indexing]
 
@@ -741,30 +841,6 @@ You can query `semantic_text` fields using the following query types:
 
 - [Semantic query](/reference/query-languages/query-dsl/query-dsl-semantic-query.md): We don't recommend this legacy query type for _new_ projects, because the alternatives in this list enable more flexibility and customization. The `semantic` query remains available to support existing implementations.
 
-
-## Troubleshooting semantic_text fields [troubleshooting-semantic-text-fields]
-
-If you want to verify that your embeddings look correct, you can view the
-inference data that `semantic_text` typically hides using `fields`.
-
-```console
-POST test-index/_search
-{
-  "query": {
-    "match": {
-      "my_semantic_field": "Which country is Paris in?"
-    }
-  },
-  "fields": [
-    "_inference_fields"
-  ]
-}
-```
-% TEST[skip:Requires inference endpoint]
-
-This will return verbose chunked embeddings content that is used to perform
-semantic search for `semantic_text` fields.
-
 ### Document count discrepancy in `_cat/indices`
 
 When an index contains a `semantic_text` field, the `docs.count` value returned by the [`_cat/indices`](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cat-indices) API may be higher than the number of documents you indexed. 

modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java

@@ -224,6 +224,12 @@ static IndexMode resolveMode(
                 indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT));
             }
         }
+        if (indexMode == null) {
+            String rawMode = settings.get(IndexSettings.MODE.getKey());
+            if (rawMode != null) {
+                indexMode = Enum.valueOf(IndexMode.class, rawMode.toUpperCase(Locale.ROOT));
+            }
+        }
         return indexMode;
     }
 

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/210_conditional_processor.yml

@@ -221,3 +221,44 @@ teardown:
         index: test
         id: "2"
       catch: missing
+
+---
+"Test conditionals support params and statements":
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body: >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "set" : {
+                  "if" : {
+                    "source": "def hit = params.success_codes.containsKey(ctx.code); return hit != null && hit == true;",
+                    "params": {
+                      "success_codes" : {
+                        "10": true,
+                        "20": true
+                      }
+                    }
+                  },
+                  "field" : "result",
+                  "value" : "success"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "my_pipeline"
+        body: { code: "20" }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match: { _source.result: "success" }