Merge branch 'main' into security-solution-entity-store-asset-criticality-roles

Author: opauloh

build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java

@@ -57,13 +57,9 @@ public void apply(Project project) {
             task.into("META-INF", copy -> copy.from(testBuildInfoTask));
         });
 
-        if (project.getRootProject().getName().equals("elasticsearch")) {
-            project.getTasks()
-                .withType(Test.class)
-                .matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
-                .configureEach(test -> {
-                    test.systemProperty("es.entitlement.enableForTests", "true");
-                });
-        }
+        project.getTasks()
+            .withType(Test.class)
+            .matching(test -> List.of("test", "internalClusterTest").contains(test.getName()))
+            .configureEach(test -> test.getSystemProperties().putIfAbsent("es.entitlement.enableForTests", "true"));
     }
 }

distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -24,7 +24,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:69f3df4cc5fd08b194a1a44dda2ff9f6665ac2b59063a2b2aedb8948ebd87f97 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ea864ffab2b828ec157b2480c8b8dbd6f27da682c59a93d4b75a46163922b237 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -103,7 +103,7 @@ WORKDIR /usr/share/elasticsearch/config
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:69f3df4cc5fd08b194a1a44dda2ff9f6665ac2b59063a2b2aedb8948ebd87f97
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:ea864ffab2b828ec157b2480c8b8dbd6f27da682c59a93d4b75a46163922b237
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile

@@ -24,7 +24,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:1c4caa90ee9cf26c9143e44074f50ba9bb17636823bde4397751a5e0d846bfd6 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:9ded4d2364e7f263cada56b0b9ca3ef643e8dac958a79df3d18c2a9f0a33fbc7 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -79,7 +79,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:1c4caa90ee9cf26c9143e44074f50ba9bb17636823bde4397751a5e0d846bfd6
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:9ded4d2364e7f263cada56b0b9ca3ef643e8dac958a79df3d18c2a9f0a33fbc7
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -11,7 +11,7 @@ tags:
 # Build args passed to Dockerfile ARGs
 args:
   BASE_IMAGE: "redhat/ubi/ubi9"
-  BASE_TAG: "9.5"
+  BASE_TAG: "9.6"
 # Docker image labels
 labels:
   org.opencontainers.image.title: "elasticsearch"

docs/changelog/113949.yaml

@@ -0,0 +1,7 @@
+pr: 113949
+summary: Support kNN filter on nested metadata
+area: Vector Search
+type: enhancement
+issues:
+ - 128803
+ - 106994

docs/changelog/131517.yaml

@@ -0,0 +1,5 @@
+pr: 131517
+summary: Refresh potential lost connections at query start for field caps
+area: Search
+type: enhancement
+issues: []

docs/changelog/131937.yaml

@@ -0,0 +1,5 @@
+pr: 131937
+summary: Fix race condition in `RemoteClusterService.collectNodes()`
+area: Distributed
+type: bug
+issues: []

docs/changelog/132101.yaml

@@ -0,0 +1,6 @@
+pr: 132101
+summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
+  given
+area: Ingest Node
+type: bug
+issues: []

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -37,6 +37,14 @@ the embedding generation, indexing, and query to use.
 [quantized](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization)
 to `bbq_hnsw` automatically.
 
+## Default and custom endpoints
+
+You can use either preconfigured endpoints in your `semantic_text` fields which
+are ideal for most use cases or create custom endpoints and reference them in
+the field mappings.
+
+### Using the default ELSER endpoint
+
 If you use the preconfigured `.elser-2-elasticsearch` endpoint, you can set up
 `semantic_text` with the following API request:
 
@@ -53,6 +61,8 @@ PUT my-index-000001
 }
 ```
 
+### Using a custom endpoint
+
 To use a custom {{infer}} endpoint instead of the default
 `.elser-2-elasticsearch`, you
 must [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put)
@@ -96,6 +106,35 @@ PUT my-index-000003
 }
 ```
 
+### Using ELSER on EIS
+
+```{applies_to}
+stack: preview 9.1
+serverless: preview
+```
+
+If you use the preconfigured `.elser-2-elastic` endpoint that utilizes the ELSER model as a service through the Elastic Inference Service ([ELSER on EIS](docs-content://explore-analyze/elastic-inference/eis.md#elser-on-eis)), you can
+set up `semantic_text` with the following API request:
+
+```console
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "inference_field": {
+        "type": "semantic_text",
+        "inference_id": ".elser-2-elastic"
+      }
+    }
+  }
+}
+```
+
+::::{note}
+While we do encourage experimentation, we do not recommend implementing production use cases on top of this feature while it is in Technical Preview.
+
+::::
+
 ## Parameters for `semantic_text` fields [semantic-text-params]
 
 `inference_id`

docs/reference/query-languages/esql/_snippets/commands/layout/completion.md

@@ -9,10 +9,26 @@ The `COMPLETION` command allows you to send prompts and context to a Large Langu
 
 **Syntax**
 
+::::{tab-set}
+
+:::{tab-item} 9.2.0+
+
 ```esql
-COMPLETION [column =] prompt WITH inference_id
+COMPLETION [column =] prompt WITH { "inference_id" : "my_inference_endpoint" }
 ```
 
+:::
+
+:::{tab-item} 9.1.x only
+
+```esql
+COMPLETION [column =] prompt WITH my_inference_endpoint
+```
+
+:::
+
+::::
+
 **Parameters**
 
 `column`
@@ -24,7 +40,7 @@ COMPLETION [column =] prompt WITH inference_id
 :   The input text or expression used to prompt the LLM.
     This can be a string literal or a reference to a column containing text.
 
-`inference_id`
+`my_inference_endpoint`
 :   The ID of the [inference endpoint](docs-content://explore-analyze/elastic-inference/inference-api.md) to use for the task.
     The inference endpoint must be configured with the `completion` task type.
 
@@ -46,16 +62,46 @@ including:
 **Requirements**
 
 To use this command, you must deploy your LLM model in Elasticsearch as
-an [≈inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the
+an [inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the
 task type `completion`.
 
+#### Handling timeouts
+
+`COMPLETION` commands may time out when processing large datasets or complex prompts. The default timeout is 10 minutes, but you can increase this limit if necessary.
+
+How you increase the timeout depends on your deployment type:
+
+::::{tab-set}
+:::{tab-item} {{ech}}
+* You can adjust {{es}} settings in the [Elastic Cloud Console](docs-content://deploy-manage/deploy/elastic-cloud/edit-stack-settings.md)
+* You can also adjust the `search.default_search_timeout` cluster setting using [Kibana's Advanced settings](kibana://reference/advanced-settings.md#kibana-search-settings)
+:::
+
+:::{tab-item} Self-managed
+* You can configure at the cluster level by setting `search.default_search_timeout` in `elasticsearch.yml` or updating via [Cluster Settings API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-put-settings)
+* You can also adjust the `search:timeout` setting using [Kibana's Advanced settings](kibana://reference/advanced-settings.md#kibana-search-settings)
+* Alternatively, you can add timeout parameters to individual queries
+:::
+
+:::{tab-item} {{serverless-full}}
+* Requires a manual override from Elastic Support because you cannot modify timeout settings directly
+:::
+::::
+
+If you don't want to increase the timeout limit, try the following:
+
+* Reduce data volume with `LIMIT` or more selective filters before the `COMPLETION` command
+* Split complex operations into multiple simpler queries
+* Configure your HTTP client's response timeout (Refer to [HTTP client configuration](/reference/elasticsearch/configuration-reference/networking-settings.md#_http_client_configuration))
+
+
 **Examples**
 
 Use the default column name (results stored in `completion` column):
 
 ```esql
 ROW question = "What is Elasticsearch?"
-| COMPLETION question WITH test_completion_model
+| COMPLETION question WITH { "inference_id" : "my_inference_endpoint" }
 | KEEP question, completion
 ```
 
@@ -67,7 +113,7 @@ Specify the output column (results stored in `answer` column):
 
 ```esql
 ROW question = "What is Elasticsearch?"
-| COMPLETION answer = question WITH test_completion_model
+| COMPLETION answer = question WITH { "inference_id" : "my_inference_endpoint" }
 | KEEP question, answer
 ```
 
@@ -87,7 +133,7 @@ FROM movies
    "Synopsis: ", synopsis, "\n",
    "Actors: ", MV_CONCAT(actors, ", "), "\n",
   )
-| COMPLETION summary = prompt WITH test_completion_model
+| COMPLETION summary = prompt WITH { "inference_id" : "my_inference_endpoint" }
 | KEEP title, summary, rating
 ```