Merge branch 'main' into esql-inference-runner-refactoring

Author: afoucret

build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java

@@ -57,13 +57,9 @@ public void apply(Project project) {
             task.into("META-INF", copy -> copy.from(testBuildInfoTask));
         });
 
-        if (project.getRootProject().getName().equals("elasticsearch")) {
-            project.getTasks()
-                .withType(Test.class)
-                .matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
-                .configureEach(test -> {
-                    test.systemProperty("es.entitlement.enableForTests", "true");
-                });
-        }
+        project.getTasks()
+            .withType(Test.class)
+            .matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
+            .configureEach(test -> test.getSystemProperties().putIfAbsent("es.entitlement.enableForTests", "true"));
     }
 }

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -11,7 +11,7 @@ tags:
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.5"
+  BASE_TAG: "9.6"
 # Docker image labels
 labels:
   org.opencontainers.image.title: "elasticsearch"

docs/changelog/129662.yaml

@@ -0,0 +1,6 @@
+pr: 129662
+summary: "[Security] Add entity store and asset criticality index privileges to built\
+  \ in Editor, Viewer and Kibana System roles"
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/131261.yaml

@@ -0,0 +1,13 @@
+pr: 131261
+summary: Enable Failure Store for new logs-*-* data streams
+area: Data streams
+type: feature
+issues:
+ - 131105
+highlight:
+  title: Enable Failure Store for new logs data streams
+  body: |-
+    The [Failure Store](docs-content://manage-data/data-store/data-streams/failure-store.md) is now enabled by default for new logs data streams matching the pattern `logs-*-*`. This means that such data streams will now store invalid documents in a
+    dedicated failure index instead of rejecting them, allowing better visibility and control over data quality issues without loosing data. This can be [enabled manually](docs-content://manage-data/data-store/data-streams/failure-store.md#set-up-failure-store-existing) for existing data streams. 
+    Note: With the failure store enabled, the http response code clients receive when indexing invalid documents will change from `400 Bad Request` to `201 Created`, with an additional response attribute `"failure_store" : "used"`. 
+  notable: true

docs/changelog/131937.yaml

@@ -0,0 +1,5 @@
+pr: 131937
+summary: Fix race condition in `RemoteClusterService.collectNodes()`
+area: Distributed
+type: bug
+issues: []

docs/changelog/132101.yaml

@@ -0,0 +1,6 @@
+pr: 132101
+summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
+  given
+area: Ingest Node
+type: bug
+issues: []

docs/reference/query-languages/esql/_snippets/commands/layout/completion.md

@@ -9,10 +9,26 @@ The `COMPLETION` command allows you to send prompts and context to a Large Langu
 
 **Syntax**
 
+::::{tab-set}
+
+:::{tab-item} 9.2.0+
+
 ```esql
-COMPLETION [column =] prompt WITH inference_id
+COMPLETION [column =] prompt WITH { "inference_id" : "my_inference_endpoint" }
 ```
 
+:::
+
+:::{tab-item} 9.1.x only
+
+```esql
+COMPLETION [column =] prompt WITH my_inference_endpoint
+```
+
+:::
+
+::::
+
 **Parameters**
 
 `column`
@@ -24,7 +40,7 @@ COMPLETION [column =] prompt WITH inference_id
 :   The input text or expression used to prompt the LLM.
     This can be a string literal or a reference to a column containing text.
 
-`inference_id`
+`my_inference_endpoint`
 :   The ID of the [inference endpoint](docs-content://explore-analyze/elastic-inference/inference-api.md) to use for the task.
     The inference endpoint must be configured with the `completion` task type.
 
@@ -75,7 +91,7 @@ How you increase the timeout depends on your deployment type:
 If you don't want to increase the timeout limit, try the following:
 
 * Reduce data volume with `LIMIT` or more selective filters before the `COMPLETION` command
-* Split complex operations into multiple simpler queries 
+* Split complex operations into multiple simpler queries
 * Configure your HTTP client's response timeout (Refer to [HTTP client configuration](/reference/elasticsearch/configuration-reference/networking-settings.md#_http_client_configuration))
 
 
@@ -85,7 +101,7 @@ Use the default column name (results stored in `completion` column):
 
 ```esql
 ROW question = "What is Elasticsearch?"
-| COMPLETION question WITH test_completion_model
+| COMPLETION question WITH { "inference_id" : "my_inference_endpoint" }
 | KEEP question, completion
 ```
 
@@ -97,7 +113,7 @@ Specify the output column (results stored in `answer` column):
 
 ```esql
 ROW question = "What is Elasticsearch?"
-| COMPLETION answer = question WITH test_completion_model
+| COMPLETION answer = question WITH { "inference_id" : "my_inference_endpoint" }
 | KEEP question, answer
 ```
 
@@ -117,7 +133,7 @@ FROM movies
    "Synopsis: ", synopsis, "\n",
    "Actors: ", MV_CONCAT(actors, ", "), "\n",
   )
-| COMPLETION summary = prompt WITH test_completion_model
+| COMPLETION summary = prompt WITH { "inference_id" : "my_inference_endpoint" }
 | KEEP title, summary, rating
 ```
 

gradle/verification-metadata.xml

@@ -2150,24 +2150,24 @@
             <sha256 value="5c8551990307a032336d98ddaed549a39a689f07d4d4c6b950601bf22b3d6a1b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.arrow" name="arrow-format" version="16.1.0">
-         <artifact name="arrow-format-16.1.0.jar">
-            <sha256 value="ad97e0fc72e193b1de3cbce4818d1ff16e81673fd523d001e8d2774bde40ee6c" origin="Generated by Gradle"/>
+      <component group="org.apache.arrow" name="arrow-format" version="18.3.0">
+         <artifact name="arrow-format-18.3.0.jar">
+            <sha256 value="728297d7757d192053ce071cc59e76c8a03ff4f5e430177fa97e96b29dd2de1b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.arrow" name="arrow-memory-core" version="16.1.0">
-         <artifact name="arrow-memory-core-16.1.0.jar">
-            <sha256 value="da7af1a1a899bd5a1b6c71284243b9f3c0e1098f0cb10cd7be4b8b455ced79dd" origin="Generated by Gradle"/>
+      <component group="org.apache.arrow" name="arrow-memory-core" version="18.3.0">
+         <artifact name="arrow-memory-core-18.3.0.jar">
+            <sha256 value="32ed0719bf2ba42becc3a88c95722851cf1bb88b37d06c95654a1152bed6ef2e" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.arrow" name="arrow-memory-unsafe" version="16.1.0">
-         <artifact name="arrow-memory-unsafe-16.1.0.jar">
-            <sha256 value="6534eded25f2c30593416a294c1047f0b017baa9906d98f6f3270737b076c745" origin="Generated by Gradle"/>
+      <component group="org.apache.arrow" name="arrow-memory-unsafe" version="18.3.0">
+         <artifact name="arrow-memory-unsafe-18.3.0.jar">
+            <sha256 value="e9652f322d7be306c400ea0d775b3114a11b48d2afe23e8b8566f13c469d5483" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="org.apache.arrow" name="arrow-vector" version="16.1.0">
-         <artifact name="arrow-vector-16.1.0.jar">
-            <sha256 value="c5837b3aa24dfd93759f57bc5759b9a8fbb5bf3912d55994d70cabb904436aab" origin="Generated by Gradle"/>
+      <component group="org.apache.arrow" name="arrow-vector" version="18.3.0">
+         <artifact name="arrow-vector-18.3.0.jar">
+            <sha256 value="b37eda92daccaffc12abf5ed425db1d0bdb3edea150a6ca856f1cf4292442299" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="org.apache.avro" name="avro" version="1.7.4">

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java

@@ -16,6 +16,7 @@
 
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -740,6 +741,73 @@ public void testIgnoreDynamicBeyondLimit() throws Exception {
         assertThat(ignored.stream().filter(i -> i.startsWith("field") == false).toList(), empty());
     }
 
+    @SuppressWarnings("unchecked")
+    public void testFailureStoreWithInvalidFieldType() throws Exception {
+        String dataStreamName = "logs-app-with-failure-store";
+        createDataStream(client, dataStreamName);
+
+        indexDoc(client, dataStreamName, """
+            {
+              "@timestamp": "2023-11-30T12:00:00Z",
+              "message": "This is a valid message"
+            }
+            """);
+
+        // invalid document (message as an object instead of string)
+        indexDoc(client, dataStreamName, """
+            {
+              "@timestamp": "2023-11-30T12:01:00Z",
+              "message": {
+                "nested": "This should fail because message should be a string"
+              }
+            }
+            """);
+
+        refreshAllIndices();
+
+        Request dsInfoRequest = new Request("GET", "/_data_stream/" + dataStreamName);
+        Map<String, Object> dsInfoResponse = entityAsMap(client.performRequest(dsInfoRequest));
+        List<Map<String, Object>> dataStreams = (List<Map<String, Object>>) dsInfoResponse.get("data_streams");
+        Map<String, Object> dataStream = dataStreams.getFirst();
+        Map<String, Object> failureStoreInfo = (Map<String, Object>) dataStream.get("failure_store");
+        assertNotNull(failureStoreInfo);
+        assertThat(failureStoreInfo.get("enabled"), is(true));
+        List<Map<String, Object>> failureIndices = (List<Map<String, Object>>) failureStoreInfo.get("indices");
+
+        assertThat(failureIndices, not(empty()));
+        String failureIndex = (String) failureIndices.getFirst().get("index_name");
+        assertThat(failureIndex, matchesRegex("\\.fs-" + dataStreamName + "-.*"));
+
+        // query the failure store index
+        Request failureStoreQuery = new Request("GET", "/" + failureIndex + "/_search");
+        failureStoreQuery.setJsonEntity("""
+            {
+              "query": {
+                "match_all": {}
+              }
+            }
+            """);
+        Map<String, Object> failureStoreResponse = entityAsMap(client.performRequest(failureStoreQuery));
+        Map<String, Object> hits = (Map<String, Object>) failureStoreResponse.get("hits");
+        List<Map<String, Object>> hitsList = (List<Map<String, Object>>) hits.get("hits");
+
+        // Verify the failed document is in the failure store
+        assertThat(hitsList.size(), is(1));
+        Map<String, Object> failedDoc = (Map<String, Object>) hitsList.getFirst().get("_source");
+        Map<String, Object> document = (Map<String, Object>) failedDoc.get("document");
+        assertNotNull(document);
+        Map<String, Object> source = (Map<String, Object>) document.get("source");
+        assertNotNull(source);
+        Map<String, Object> message = (Map<String, Object>) source.get("message");
+        assertNotNull(message);
+        assertThat(message.get("nested"), equalTo("This should fail because message should be a string"));
+        Map<String, Object> error = (Map<String, Object>) failedDoc.get("error");
+        assertNotNull(error);
+        assertEquals("document_parsing_exception", error.get("type"));
+        String errorMessage = (String) error.get("message");
+        assertThat(errorMessage, containsString("failed to parse field [message] of type [match_only_text] in document with id"));
+    }
+
     @Override
     protected String indexTemplateName() {
         return "logs";

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java

@@ -28,13 +28,20 @@ public class DataStreamFeatures implements FeatureSpecification {
 
     public static final NodeFeature LOGS_STREAM_FEATURE = new NodeFeature("logs_stream");
 
+    public static final NodeFeature FAILURE_STORE_IN_LOG_DATA_STREAMS = new NodeFeature("logs_data_streams.failure_store.enabled");
+
     @Override
     public Set<NodeFeature> getFeatures() {
         return Set.of(DataStream.DATA_STREAM_FAILURE_STORE_FEATURE);
     }
 
     @Override
     public Set<NodeFeature> getTestFeatures() {
-        return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, LOGS_STREAM_FEATURE);
+        return Set.of(
+            DATA_STREAM_FAILURE_STORE_TSDB_FIX,
+            DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX,
+            LOGS_STREAM_FEATURE,
+            FAILURE_STORE_IN_LOG_DATA_STREAMS
+        );
     }
 }