Merge branch 'main' into 2025/10/24/new-id-format

Author: tlrx

docs/changelog/137023.yaml

@@ -0,0 +1,5 @@
+pr: 137023
+summary: Support choosing the downsampling method in data stream lifecycle
+area: "Data streams"
+type: enhancement
+issues: []

docs/changelog/137331.yaml

@@ -0,0 +1,5 @@
+pr: 137331
+summary: Add ES93BloomFilterStoredFieldsFormat for efficient field existence checks
+area: TSDB
+type: enhancement
+issues: []

docs/changelog/137442.yaml

@@ -0,0 +1,5 @@
+pr: 137442
+summary: Handle ._original stored fields with fls
+area: "Authorization"
+type: bug
+issues: []

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -25,9 +25,12 @@ type make it simpler to perform semantic search on your data. The
 with [match](/reference/query-languages/query-dsl/query-dsl-match-query.md), [sparse_vector](/reference/query-languages/query-dsl/query-dsl-sparse-vector-query.md)
 or [knn](/reference/query-languages/query-dsl/query-dsl-knn-query.md) queries.
 
-If you don’t specify an inference endpoint, the `inference_id` field defaults to
-`.elser-2-elasticsearch`, a preconfigured endpoint for the elasticsearch
-service.
+{applies_to}`serverless: ga` If you don’t specify an {{infer}} endpoint, the `inference_id` field defaults to
+`.elser-v2-elastic`, a preconfigured endpoint for the `elasticsearch` service.
+This endpoint uses the [Elastic {{infer-cap}} Service (EIS)](docs-content://explore-analyze/elastic-inference/eis.md#elser-on-eis).
+
+{applies_to}`stack: ga 9.0` If you don’t specify an {{infer}} endpoint, the `inference_id` field defaults to
+`.elser-2-elasticsearch`, a preconfigured endpoint for the `elasticsearch` service.
 
 Using `semantic_text`, you won’t need to specify how to generate embeddings for
 your data, or how to index it. The {{infer}} endpoint automatically determines
@@ -43,10 +46,15 @@ You can use either preconfigured endpoints in your `semantic_text` fields which
 are ideal for most use cases or create custom endpoints and reference them in
 the field mappings.
 
-### Using the default ELSER endpoint
+:::::::{tab-set}
+
+::::::{tab-item} Using the default ELSER on EIS endpoint on Serverless
+
+```{applies_to}
+serverless: ga
+```
 
-If you use the preconfigured `.elser-2-elasticsearch` endpoint, you can set up
-`semantic_text` with the following API request:
+If you use the default `.elser-v2-elastic` endpoint that runs on EIS, you can set up `semantic_text` with the following API request:
 
 ```console
 PUT my-index-000001
@@ -62,10 +70,58 @@ PUT my-index-000001
 ```
 % TEST[skip:Requires inference endpoint]
 
+::::::
+
+::::::{tab-item} Using the preconfigured ELSER on EIS endpoint in Cloud
+
+```{applies_to}
+stack: ga 9.2
+deployment:
+  self: unavailable
+```
+
+If you use the preconfigured `.elser-v2-elastic` endpoint that runs on EIS, you can set up `semantic_text` with the following API request:
+
+```console
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "inference_field": {
+        "type": "semantic_text",
+        "inference_id": ".elser-v2-elastic"
+      }
+    }
+  }
+}
+```
+
+::::::
+
+::::::{tab-item} Using the default ELSER endpoint
+
+If you use the preconfigured `.elser-2-elasticsearch` endpoint, you can set up `semantic_text` with the following API request:
+
+```console
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "inference_field": {
+        "type": "semantic_text"
+      }
+    }
+  }
+}
+```
+
+::::::
+
+:::::::
+
 ### Using a custom endpoint
 
-To use a custom {{infer}} endpoint instead of the default
-`.elser-2-elasticsearch`, you
+To use a custom {{infer}} endpoint instead of the default, you
 must [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put)
 and specify its `inference_id` when setting up the `semantic_text` field type.
 
@@ -110,14 +166,48 @@ PUT my-index-000003
 % TEST[skip:Requires inference endpoint]
 
 ### Using ELSER on EIS
+
 ```{applies_to}
-stack: preview 9.1
-serverless: preview
+stack: preview 9.1 ga 9.2
+deployment:
+  self: unavailable
+serverless: ga
 ```
 
 If you use the preconfigured `.elser-2-elastic` endpoint that utilizes the ELSER model as a service through the Elastic Inference Service ([ELSER on EIS](docs-content://explore-analyze/elastic-inference/eis.md#elser-on-eis)), you can
 set up `semantic_text` with the following API request:
 
+:::::::{tab-set}
+
+::::::{tab-item} Using ELSER on EIS on Serverless
+
+```{applies_to}
+serverless: ga
+```
+
+```console
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "inference_field": {
+        "type": "semantic_text"
+      }
+    }
+  }
+}
+```
+
+::::::
+
+::::::{tab-item} Using ELSER on EIS in Cloud
+
+```{applies_to}
+stack: ga 9.2
+deployment:
+  self: unavailable
+```
+
 ```console
 PUT my-index-000001
 {
@@ -133,10 +223,9 @@ PUT my-index-000001
 ```
 % TEST[skip:Requires inference endpoint]
 
-::::{note}
-While we do encourage experimentation, we do not recommend implementing production use cases on top of this feature while it is in Technical Preview.
+::::::
 
-::::
+:::::::
 
 ## Parameters for `semantic_text` fields [semantic-text-params]
 

docs/reference/query-languages/esql/_snippets/commands/layout/fork.md

@@ -17,6 +17,8 @@ FORK ( <processing_commands> ) ( <processing_commands> ) ... ( <processing_comma
 The `FORK` processing command creates multiple execution branches to operate
 on the same input data and combines the results in a single output table. A discriminator column (`_fork`) is added to identify which branch each row came from.
 
+Together with the [`FUSE`](/reference/query-languages/esql/commands/fuse.md) command, `FORK` enables hybrid search to combine and score results from multiple queries. To learn more about using {{esql}} for search, refer to [ES|QL for search](docs-content://solutions/search/esql-for-search.md).
+
 **Branch identification:**
 - The `_fork` column identifies each branch with values like `fork1`, `fork2`, `fork3`
 - Values correspond to the order branches are defined

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -524,7 +524,10 @@ Set<Index> maybeExecuteDownsampling(ProjectState projectState, DataStream dataSt
                 // - has matching downsample rounds
                 // - is read-only
                 // So let's wait for an in-progress downsampling operation to succeed or trigger the last matching round
-                affectedIndices.addAll(waitForInProgressOrTriggerDownsampling(dataStream, backingIndexMeta, downsamplingRounds, project));
+                var downsamplingMethod = dataStream.getDataLifecycle().downsamplingMethod();
+                affectedIndices.addAll(
+                    waitForInProgressOrTriggerDownsampling(dataStream, backingIndexMeta, downsamplingRounds, downsamplingMethod, project)
+                );
             }
         }
 
@@ -541,6 +544,7 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
         DataStream dataStream,
         IndexMetadata backingIndex,
         List<DataStreamLifecycle.DownsamplingRound> downsamplingRounds,
+        DownsampleConfig.SamplingMethod downsamplingMethod,
         ProjectMetadata project
     ) {
         assert dataStream.getIndices().contains(backingIndex.getIndex())
@@ -556,7 +560,7 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
             String downsampleIndexName = DownsampleConfig.generateDownsampleIndexName(
                 DOWNSAMPLED_INDEX_PREFIX,
                 backingIndex,
-                round.config().getFixedInterval()
+                round.fixedInterval()
             );
             IndexMetadata targetDownsampleIndexMeta = project.index(downsampleIndexName);
             boolean targetDownsampleIndexExists = targetDownsampleIndexMeta != null;
@@ -568,7 +572,8 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
                     INDEX_DOWNSAMPLE_STATUS.get(targetDownsampleIndexMeta.getSettings()),
                     round,
                     lastRound,
-                    index,
+                    downsamplingMethod,
+                    backingIndex,
                     targetDownsampleIndexMeta.getIndex()
                 );
                 if (downsamplingNotComplete.isEmpty() == false) {
@@ -580,7 +585,7 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
                     // no maintenance needed for previously started downsampling actions and we are on the last matching round so it's time
                     // to kick off downsampling
                     affectedIndices.add(index);
-                    downsampleIndexOnce(round, project.id(), indexName, downsampleIndexName);
+                    downsampleIndexOnce(round, downsamplingMethod, project.id(), backingIndex, downsampleIndexName);
                 }
             }
         }
@@ -592,16 +597,30 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
      */
     private void downsampleIndexOnce(
         DataStreamLifecycle.DownsamplingRound round,
+        DownsampleConfig.SamplingMethod requestedDownsamplingMethod,
         ProjectId projectId,
-        String sourceIndex,
+        IndexMetadata sourceIndexMetadata,
         String downsampleIndexName
     ) {
+        // When an index is already downsampled with a method, we require all later downsampling rounds to use the same method.
+        // This is necessary to preserve the relation of the downsampled index to the raw data. For example, if an index is already
+        // downsampled and downsampled it again to 1 hour; we know that a document represents either the aggregated raw data of an hour
+        // or the last value of the raw data within this hour. If we mix the methods, we cannot derive any meaning from them.
+        // Furthermore, data stream lifecycle is configured on the data stream level and not on the individual index level, meaning that
+        // when a user changes downsampling method, some indices would not be able to be downsampled anymore.
+        // For this reason, when we encounter an already downsampled index, we use the source downsampling method which might be different
+        // from the requested one.
+        var sourceIndexSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata(sourceIndexMetadata);
+        String sourceIndex = sourceIndexMetadata.getIndex().getName();
         DownsampleAction.Request request = new DownsampleAction.Request(
             TimeValue.THIRTY_SECONDS /* TODO should this be longer/configurable? */,
             sourceIndex,
             downsampleIndexName,
             null,
-            round.config()
+            new DownsampleConfig(
+                round.fixedInterval(),
+                sourceIndexSamplingMethod == null ? requestedDownsamplingMethod : sourceIndexSamplingMethod
+            )
         );
         transportActionsDeduplicator.executeOnce(
             Tuple.tuple(projectId, request),
@@ -632,11 +651,12 @@ private Set<Index> evaluateDownsampleStatus(
         IndexMetadata.DownsampleTaskStatus downsampleStatus,
         DataStreamLifecycle.DownsamplingRound currentRound,
         DataStreamLifecycle.DownsamplingRound lastRound,
-        Index backingIndex,
+        DownsampleConfig.SamplingMethod downsamplingMethod,
+        IndexMetadata backingIndex,
         Index downsampleIndex
     ) {
         Set<Index> affectedIndices = new HashSet<>();
-        String indexName = backingIndex.getName();
+        String indexName = backingIndex.getIndex().getName();
         String downsampleIndexName = downsampleIndex.getName();
         return switch (downsampleStatus) {
             case UNKNOWN -> {
@@ -683,15 +703,15 @@ private Set<Index> evaluateDownsampleStatus(
                 // NOTE that the downsample request is made through the deduplicator so it will only really be executed if
                 // there isn't one already in-flight. This can happen if a previous request timed-out, failed, or there was a
                 // master failover and data stream lifecycle needed to restart
-                downsampleIndexOnce(currentRound, projectId, indexName, downsampleIndexName);
-                affectedIndices.add(backingIndex);
+                downsampleIndexOnce(currentRound, downsamplingMethod, projectId, backingIndex, downsampleIndexName);
+                affectedIndices.add(backingIndex.getIndex());
                 yield affectedIndices;
             }
             case SUCCESS -> {
                 if (dataStream.getIndices().contains(downsampleIndex) == false) {
                     // at this point the source index is part of the data stream and the downsample index is complete but not
                     // part of the data stream. we need to replace the source index with the downsample index in the data stream
-                    affectedIndices.add(backingIndex);
+                    affectedIndices.add(backingIndex.getIndex());
                     replaceBackingIndexWithDownsampleIndexOnce(projectId, dataStream, indexName, downsampleIndexName);
                 }
                 yield affectedIndices;

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java

@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
@@ -29,6 +30,9 @@
 @ServerlessScope(Scope.PUBLIC)
 public class RestPutDataStreamLifecycleAction extends BaseRestHandler {
 
+    private static final String SUPPORTS_DOWNSAMPLING_METHOD = "dlm.downsampling_method";
+    private static final Set<String> CAPABILITIES = Set.of(SUPPORTS_DOWNSAMPLING_METHOD);
+
     @Override
     public String getName() {
         return "put_data_lifecycles_action";
@@ -44,13 +48,14 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
         try (XContentParser parser = request.contentParser()) {
             PutDataStreamLifecycleAction.Request putLifecycleRequest = PutDataStreamLifecycleAction.Request.parseRequest(
                 parser,
-                (dataRetention, enabled, downsampling) -> new PutDataStreamLifecycleAction.Request(
+                (dataRetention, enabled, downsamplingRounds, downsamplingMethod) -> new PutDataStreamLifecycleAction.Request(
                     getMasterNodeTimeout(request),
                     getAckTimeout(request),
                     Strings.splitStringByCommaToArray(request.param("name")),
                     dataRetention,
                     enabled,
-                    downsampling
+                    downsamplingRounds,
+                    downsamplingMethod
                 )
             );
             putLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(request, putLifecycleRequest.indicesOptions()));
@@ -61,4 +66,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             );
         }
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return CAPABILITIES;
+    }
 }