Merge branch 'main' into ml-fix-update-string-format

Author: jonathan-buttner

docs/build.gradle

@@ -85,9 +85,6 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'indices.lifecycle.history_index_enabled', 'false'
     keystorePassword 'keystore-password'
-    if (buildParams.snapshotBuild == false) {
-      requiresFeature 'es.failure_store_feature_flag_enabled', new Version(8, 12, 0)
-    }
   }
 
   // debug ccr test failures:
@@ -126,7 +123,6 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
 
 
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
-  requiresFeature 'es.failure_store_feature_flag_enabled', Version.fromString("8.12.0")
 
   // TODO Rene: clean up this kind of cross project file references
   extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("src/main/resources/oidc/op-jwks.json")

docs/changelog/126035.yaml

@@ -0,0 +1,5 @@
+pr: 126035
+summary: Fix top level knn search with scroll
+area: Vector Search
+type: bug
+issues: []

docs/changelog/126629.yaml

@@ -0,0 +1,89 @@
+pr: 126973
+summary: Add ability to redirect ingestion failures on data streams to a failure store
+area: Data streams
+type: feature
+issues: []
+highlight:
+  title: Add ability to redirect ingestion failures on data streams to a failure store
+  body: |-
+    Documents that encountered ingest pipeline failures or mapping conflicts
+    would previously be returned to the client as errors in the bulk and
+    index operations. Many client applications are not equipped to respond
+    to these failures. This leads to the failed documents often being
+    dropped by the client which cannot hold the broken documents
+    indefinitely. In many end user workloads, these failed documents
+    represent events that could be critical signals for observability or
+    security use cases.
+
+    To help mitigate this problem, data streams can now maintain a "failure
+    store" which is used to accept and hold documents that fail to be
+    ingested due to preventable configuration errors. The data stream's
+    failure store operates like a separate set of backing indices with their
+    own mappings and access patterns that allow Elasticsearch to accept
+    documents that would otherwise be rejected due to unhandled ingest
+    pipeline exceptions or mapping conflicts.
+
+    Users can enable redirection of ingest failures to the failure store on
+    new data streams by specifying it in the new `data_stream_options` field
+    inside of a component or index template:
+
+    [source,yaml]
+    ----
+    PUT _index_template/my-template
+    {
+      "index_patterns": ["logs-test-*"],
+      "data_stream": {},
+      "template": {
+        "data_stream_options": {
+          "failure_store": {
+            "enabled": true
+          }
+        }
+      }
+    }'
+    ----
+
+    Existing data streams can be configured with the new data stream
+    `_options` endpoint:
+
+    [source,yaml]
+    ----
+    PUT _data_stream/logs-test-apache/_options
+    {
+      "failure_store": {
+        "enabled": "true"
+      }
+    }
+    ----
+
+    When redirection is enabled, any ingestion related failures will be
+    captured in the failure store if the cluster is able to, along with the
+    timestamp that the failure occurred, details about the error
+    encountered, and the document that could not be ingested. Since failure
+    stores are a kind of Elasticsearch index, we can search the data stream
+    for the failures that it has collected. The failures are not shown by
+    default as they are stored in different indices than the normal data
+    stream data. In order to retrieve the failures, we use the `_search` API
+    along with a new bit of index pattern syntax, the `::` selector.
+
+    [source,yaml]
+    ----
+    POST logs-test-apache::failures/_search
+    ----
+
+    This index syntax informs the search operation to target the indices in
+    its failure store instead of its backing indices. It can be mixed in a
+    number of ways with other index patterns to include their failure store
+    indices in the search operation:
+
+    [source,yaml]
+    ----
+    POST logs-*::failures/_search
+    POST logs-*,logs-*::failures/_search
+    POST *::failures/_search
+    POST _query
+    {
+      "query": "FROM my_data_stream*::failures"
+    }
+    ----
+  notable: true

docs/changelog/126973.yaml

@@ -6,7 +6,7 @@
 * [`ST_X`](../../functions-operators/spatial-functions.md#esql-st_x)
 * [`ST_Y`](../../functions-operators/spatial-functions.md#esql-st_y)
 * [preview] [`ST_ENVELOPE`](../../functions-operators/spatial-functions.md#esql-st_envelope)
-* [preview] [`ST_XMAX`](../../functions-operators/spatial-functions.md#esql-st_xmax)
-* [preview] [`ST_XMIN`](../../functions-operators/spatial-functions.md#esql-st_xmin)
-* [preview] [`ST_YMAX`](../../functions-operators/spatial-functions.md#esql-st_ymax)
-* [preview] [`ST_YMIN`](../../functions-operators/spatial-functions.md#esql-st_ymin)
+  * [preview] [`ST_XMAX`](../../functions-operators/spatial-functions.md#esql-st_xmax)
+  * [preview] [`ST_XMIN`](../../functions-operators/spatial-functions.md#esql-st_xmin)
+  * [preview] [`ST_YMAX`](../../functions-operators/spatial-functions.md#esql-st_ymax)
+  * [preview] [`ST_YMIN`](../../functions-operators/spatial-functions.md#esql-st_ymin)

docs/reference/query-languages/esql/_snippets/functions/layout/st_xmax.md

@@ -26,6 +26,7 @@
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.startsWith;
 
+@ESTestCase.WithoutSecurityManager
 public class EntitlementInitializationTests extends ESTestCase {
 
     private static PathLookup TEST_PATH_LOOKUP;