Create int bfloat16 vector formats

docs/changelog/136632.yaml

@@ -0,0 +1,6 @@
+pr: 136632
+summary: Field caps transport changes to return for each original expression what
+  it was resolved to
+area: Search
+type: enhancement
+issues: []

docs/changelog/137023.yaml

@@ -0,0 +1,5 @@
+pr: 137023
+summary: Support choosing the downsampling method in data stream lifecycle
+area: "Data streams"
+type: enhancement
+issues: []

docs/changelog/137220.yaml

@@ -0,0 +1,5 @@
+pr: 137220
+summary: Skip dataframes when disabled
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/137442.yaml

@@ -0,0 +1,5 @@
+pr: 137442
+summary: Handle ._original stored fields with fls
+area: "Authorization"
+type: bug
+issues: []

libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MSBitToInt4ESNextOSQVectorsScorer.java

@@ -14,7 +14,6 @@
 import jdk.incubator.vector.LongVector;
 import jdk.incubator.vector.ShortVector;
 import jdk.incubator.vector.VectorOperators;
-import jdk.incubator.vector.VectorSpecies;
 
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.store.IndexInput;
@@ -31,23 +30,6 @@
 /** Panamized scorer for quantized vectors stored as a {@link MemorySegment}. */
 final class MSBitToInt4ESNextOSQVectorsScorer extends MemorySegmentESNextOSQVectorsScorer.MemorySegmentScorer {
 
-    private static final int BULK_SIZE = MemorySegmentESNextOSQVectorsScorer.BULK_SIZE;
-    private static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1);
-
-    private static final VectorSpecies<Integer> INT_SPECIES_128 = IntVector.SPECIES_128;
-
-    private static final VectorSpecies<Long> LONG_SPECIES_128 = LongVector.SPECIES_128;
-    private static final VectorSpecies<Long> LONG_SPECIES_256 = LongVector.SPECIES_256;
-
-    private static final VectorSpecies<Byte> BYTE_SPECIES_128 = ByteVector.SPECIES_128;
-    private static final VectorSpecies<Byte> BYTE_SPECIES_256 = ByteVector.SPECIES_256;
-
-    private static final VectorSpecies<Short> SHORT_SPECIES_128 = ShortVector.SPECIES_128;
-    private static final VectorSpecies<Short> SHORT_SPECIES_256 = ShortVector.SPECIES_256;
-
-    private static final VectorSpecies<Float> FLOAT_SPECIES_128 = FloatVector.SPECIES_128;
-    private static final VectorSpecies<Float> FLOAT_SPECIES_256 = FloatVector.SPECIES_256;
-
     MSBitToInt4ESNextOSQVectorsScorer(IndexInput in, int dimensions, int dataLength, MemorySegment memorySegment) {
         super(in, dimensions, dataLength, memorySegment);
     }

libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/MemorySegmentESNextOSQVectorsScorer.java

@@ -25,20 +25,6 @@
 /** Panamized scorer for quantized vectors stored as a {@link MemorySegment}. */
 public final class MemorySegmentESNextOSQVectorsScorer extends ESNextOSQVectorsScorer {
 
-    private static final VectorSpecies<Integer> INT_SPECIES_128 = IntVector.SPECIES_128;
-
-    private static final VectorSpecies<Long> LONG_SPECIES_128 = LongVector.SPECIES_128;
-    private static final VectorSpecies<Long> LONG_SPECIES_256 = LongVector.SPECIES_256;
-
-    private static final VectorSpecies<Byte> BYTE_SPECIES_128 = ByteVector.SPECIES_128;
-    private static final VectorSpecies<Byte> BYTE_SPECIES_256 = ByteVector.SPECIES_256;
-
-    private static final VectorSpecies<Short> SHORT_SPECIES_128 = ShortVector.SPECIES_128;
-    private static final VectorSpecies<Short> SHORT_SPECIES_256 = ShortVector.SPECIES_256;
-
-    private static final VectorSpecies<Float> FLOAT_SPECIES_128 = FloatVector.SPECIES_128;
-    private static final VectorSpecies<Float> FLOAT_SPECIES_256 = FloatVector.SPECIES_256;
-
     private final MemorySegment memorySegment;
     private final MemorySegmentScorer scorer;
 
@@ -54,6 +40,10 @@ public MemorySegmentESNextOSQVectorsScorer(
         this.memorySegment = memorySegment;
         if (queryBits == 4 && indexBits == 1) {
             this.scorer = new MSBitToInt4ESNextOSQVectorsScorer(in, dimensions, dataLength, memorySegment);
+        } else if (queryBits == 4 && indexBits == 4) {
+            this.scorer = new MSInt4SymmetricESNextOSQVectorsScorer(in, dimensions, dataLength, memorySegment);
+        } else if (queryBits == 4 && indexBits == 2) {
+            throw new IllegalArgumentException("Only symmetric 4-bit query and 1-bit index supported");
         } else {
             throw new IllegalArgumentException("Only asymmetric 4-bit query and 1-bit index supported");
         }
@@ -112,7 +102,24 @@ public float scoreBulk(
         );
     }
 
-    abstract static sealed class MemorySegmentScorer permits MSBitToInt4ESNextOSQVectorsScorer {
+    abstract static sealed class MemorySegmentScorer permits MSBitToInt4ESNextOSQVectorsScorer, MSInt4SymmetricESNextOSQVectorsScorer {
+
+        static final int BULK_SIZE = MemorySegmentESNextOSQVectorsScorer.BULK_SIZE;
+        static final float FOUR_BIT_SCALE = 1f / ((1 << 4) - 1);
+        static final VectorSpecies<Integer> INT_SPECIES_128 = IntVector.SPECIES_128;
+
+        static final VectorSpecies<Long> LONG_SPECIES_128 = LongVector.SPECIES_128;
+        static final VectorSpecies<Long> LONG_SPECIES_256 = LongVector.SPECIES_256;
+
+        static final VectorSpecies<Byte> BYTE_SPECIES_128 = ByteVector.SPECIES_128;
+        static final VectorSpecies<Byte> BYTE_SPECIES_256 = ByteVector.SPECIES_256;
+
+        static final VectorSpecies<Short> SHORT_SPECIES_128 = ShortVector.SPECIES_128;
+        static final VectorSpecies<Short> SHORT_SPECIES_256 = ShortVector.SPECIES_256;
+
+        static final VectorSpecies<Float> FLOAT_SPECIES_128 = FloatVector.SPECIES_128;
+        static final VectorSpecies<Float> FLOAT_SPECIES_256 = FloatVector.SPECIES_256;
+
         protected final MemorySegment memorySegment;
         protected final IndexInput in;
         protected final int length;

libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/vectorization/PanamaESVectorizationProvider.java

@@ -40,7 +40,7 @@ public ESNextOSQVectorsScorer newESNextOSQVectorsScorer(IndexInput input, byte q
         if (PanamaESVectorUtilSupport.HAS_FAST_INTEGER_VECTORS
             && input instanceof MemorySegmentAccessInput msai
             && queryBits == 4
-            && indexBits == 1) {
+            && (indexBits == 1 || indexBits == 4)) {
             MemorySegment ms = msai.segmentSliceOrNull(0, input.length());
             if (ms != null) {
                 return new MemorySegmentESNextOSQVectorsScorer(input, queryBits, indexBits, dimension, dataLength, ms);

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
+import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -44,6 +45,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.action.admin.indices.ResizeIndexTestUtils.resizeRequest;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -267,12 +269,10 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception {
         assertThat(updateSettingsResponse.isAcknowledged(), is(true));
 
         String shrunkenTarget = "k8s-shrunken";
-        var shrinkIndexResponse = client().admin()
-            .indices()
-            .prepareResizeIndex(sourceIndex, shrunkenTarget)
-            .setResizeType(ResizeType.SHRINK)
-            .setSettings(indexSettings(2, 0).build())
-            .get();
+        final var shrinkIndexResponse = client().execute(
+            TransportResizeAction.TYPE,
+            resizeRequest(ResizeType.SHRINK, sourceIndex, shrunkenTarget, indexSettings(2, 0))
+        ).actionGet();
         assertThat(shrinkIndexResponse.isAcknowledged(), is(true));
         assertThat(shrinkIndexResponse.index(), equalTo(shrunkenTarget));
 

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -524,7 +524,10 @@ Set<Index> maybeExecuteDownsampling(ProjectState projectState, DataStream dataSt
                 // - has matching downsample rounds
                 // - is read-only
                 // So let's wait for an in-progress downsampling operation to succeed or trigger the last matching round
-                affectedIndices.addAll(waitForInProgressOrTriggerDownsampling(dataStream, backingIndexMeta, downsamplingRounds, project));
+                var downsamplingMethod = dataStream.getDataLifecycle().downsamplingMethod();
+                affectedIndices.addAll(
+                    waitForInProgressOrTriggerDownsampling(dataStream, backingIndexMeta, downsamplingRounds, downsamplingMethod, project)
+                );
             }
         }
 
@@ -541,6 +544,7 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
         DataStream dataStream,
         IndexMetadata backingIndex,
         List<DataStreamLifecycle.DownsamplingRound> downsamplingRounds,
+        DownsampleConfig.SamplingMethod downsamplingMethod,
         ProjectMetadata project
     ) {
         assert dataStream.getIndices().contains(backingIndex.getIndex())
@@ -556,7 +560,7 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
             String downsampleIndexName = DownsampleConfig.generateDownsampleIndexName(
                 DOWNSAMPLED_INDEX_PREFIX,
                 backingIndex,
-                round.config().getFixedInterval()
+                round.fixedInterval()
             );
             IndexMetadata targetDownsampleIndexMeta = project.index(downsampleIndexName);
             boolean targetDownsampleIndexExists = targetDownsampleIndexMeta != null;
@@ -568,7 +572,8 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
                     INDEX_DOWNSAMPLE_STATUS.get(targetDownsampleIndexMeta.getSettings()),
                     round,
                     lastRound,
-                    index,
+                    downsamplingMethod,
+                    backingIndex,
                     targetDownsampleIndexMeta.getIndex()
                 );
                 if (downsamplingNotComplete.isEmpty() == false) {
@@ -580,7 +585,7 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
                     // no maintenance needed for previously started downsampling actions and we are on the last matching round so it's time
                     // to kick off downsampling
                     affectedIndices.add(index);
-                    downsampleIndexOnce(round, project.id(), indexName, downsampleIndexName);
+                    downsampleIndexOnce(round, downsamplingMethod, project.id(), backingIndex, downsampleIndexName);
                 }
             }
         }
@@ -592,16 +597,30 @@ private Set<Index> waitForInProgressOrTriggerDownsampling(
      */
     private void downsampleIndexOnce(
         DataStreamLifecycle.DownsamplingRound round,
+        DownsampleConfig.SamplingMethod requestedDownsamplingMethod,
         ProjectId projectId,
-        String sourceIndex,
+        IndexMetadata sourceIndexMetadata,
         String downsampleIndexName
     ) {
+        // When an index is already downsampled with a method, we require all later downsampling rounds to use the same method.
+        // This is necessary to preserve the relation of the downsampled index to the raw data. For example, if an index is already
+        // downsampled and downsampled it again to 1 hour; we know that a document represents either the aggregated raw data of an hour
+        // or the last value of the raw data within this hour. If we mix the methods, we cannot derive any meaning from them.
+        // Furthermore, data stream lifecycle is configured on the data stream level and not on the individual index level, meaning that
+        // when a user changes downsampling method, some indices would not be able to be downsampled anymore.
+        // For this reason, when we encounter an already downsampled index, we use the source downsampling method which might be different
+        // from the requested one.
+        var sourceIndexSamplingMethod = DownsampleConfig.SamplingMethod.fromIndexMetadata(sourceIndexMetadata);
+        String sourceIndex = sourceIndexMetadata.getIndex().getName();
         DownsampleAction.Request request = new DownsampleAction.Request(
             TimeValue.THIRTY_SECONDS /* TODO should this be longer/configurable? */,
             sourceIndex,
             downsampleIndexName,
             null,
-            round.config()
+            new DownsampleConfig(
+                round.fixedInterval(),
+                sourceIndexSamplingMethod == null ? requestedDownsamplingMethod : sourceIndexSamplingMethod
+            )
         );
         transportActionsDeduplicator.executeOnce(
             Tuple.tuple(projectId, request),
@@ -632,11 +651,12 @@ private Set<Index> evaluateDownsampleStatus(
         IndexMetadata.DownsampleTaskStatus downsampleStatus,
         DataStreamLifecycle.DownsamplingRound currentRound,
         DataStreamLifecycle.DownsamplingRound lastRound,
-        Index backingIndex,
+        DownsampleConfig.SamplingMethod downsamplingMethod,
+        IndexMetadata backingIndex,
         Index downsampleIndex
     ) {
         Set<Index> affectedIndices = new HashSet<>();
-        String indexName = backingIndex.getName();
+        String indexName = backingIndex.getIndex().getName();
         String downsampleIndexName = downsampleIndex.getName();
         return switch (downsampleStatus) {
             case UNKNOWN -> {
@@ -683,15 +703,15 @@ private Set<Index> evaluateDownsampleStatus(
                 // NOTE that the downsample request is made through the deduplicator so it will only really be executed if
                 // there isn't one already in-flight. This can happen if a previous request timed-out, failed, or there was a
                 // master failover and data stream lifecycle needed to restart
-                downsampleIndexOnce(currentRound, projectId, indexName, downsampleIndexName);
-                affectedIndices.add(backingIndex);
+                downsampleIndexOnce(currentRound, downsamplingMethod, projectId, backingIndex, downsampleIndexName);
+                affectedIndices.add(backingIndex.getIndex());
                 yield affectedIndices;
             }
             case SUCCESS -> {
                 if (dataStream.getIndices().contains(downsampleIndex) == false) {
                     // at this point the source index is part of the data stream and the downsample index is complete but not
                     // part of the data stream. we need to replace the source index with the downsample index in the data stream
-                    affectedIndices.add(backingIndex);
+                    affectedIndices.add(backingIndex.getIndex());
                     replaceBackingIndexWithDownsampleIndexOnce(projectId, dataStream, indexName, downsampleIndexName);
                 }
                 yield affectedIndices;

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java

@@ -21,6 +21,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Set;
 
 import static org.elasticsearch.rest.RestRequest.Method.PUT;
 import static org.elasticsearch.rest.RestUtils.getAckTimeout;
@@ -29,6 +30,9 @@
 @ServerlessScope(Scope.PUBLIC)
 public class RestPutDataStreamLifecycleAction extends BaseRestHandler {
 
+    private static final String SUPPORTS_DOWNSAMPLING_METHOD = "dlm.downsampling_method";
+    private static final Set<String> CAPABILITIES = Set.of(SUPPORTS_DOWNSAMPLING_METHOD);
+
     @Override
     public String getName() {
         return "put_data_lifecycles_action";
@@ -44,13 +48,14 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
         try (XContentParser parser = request.contentParser()) {
             PutDataStreamLifecycleAction.Request putLifecycleRequest = PutDataStreamLifecycleAction.Request.parseRequest(
                 parser,
-                (dataRetention, enabled, downsampling) -> new PutDataStreamLifecycleAction.Request(
+                (dataRetention, enabled, downsamplingRounds, downsamplingMethod) -> new PutDataStreamLifecycleAction.Request(
                     getMasterNodeTimeout(request),
                     getAckTimeout(request),
                     Strings.splitStringByCommaToArray(request.param("name")),
                     dataRetention,
                     enabled,
-                    downsampling
+                    downsamplingRounds,
+                    downsamplingMethod
                 )
             );
             putLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(request, putLifecycleRequest.indicesOptions()));
@@ -61,4 +66,9 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             );
         }
     }
+
+    @Override
+    public Set<String> supportedCapabilities() {
+        return CAPABILITIES;
+    }
 }