Merge branch 'main' into esql-inference-command-serialization

afoucret · web-flow · commit ace5d9d97209 · 2025-12-11T19:41:07.000+01:00
diff --git a/docs/reference/query-languages/esql/_snippets/lists/time-series-aggregation-functions.md b/docs/reference/query-languages/esql/_snippets/lists/time-series-aggregation-functions.md
@@ -2,11 +2,11 @@
 * [`AVG_OVER_TIME`](../../functions-operators/time-series-aggregation-functions.md#esql-avg_over_time) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
 * [`COUNT_OVER_TIME`](../../functions-operators/time-series-aggregation-functions.md#esql-count_over_time) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
 * [`COUNT_DISTINCT_OVER_TIME`](../../functions-operators/time-series-aggregation-functions.md#esql-count_distinct_over_time) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
-* [`DELTA`](../../functions-operators/time-series-aggregation-functions.md#esql-rate) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
+* [`DELTA`](../../functions-operators/time-series-aggregation-functions.md#esql-delta) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
 * [`FIRST_OVER_TIME`](../../functions-operators/time-series-aggregation-functions.md#esql-first_over_time) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
-* [`IDELTA`](../../functions-operators/time-series-aggregation-functions.md#esql-rate) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
-* [`INCREASE`](../../functions-operators/time-series-aggregation-functions.md#esql-rate) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
-* [`IRATE`](../../functions-operators/time-series-aggregation-functions.md#esql-rate) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
+* [`IDELTA`](../../functions-operators/time-series-aggregation-functions.md#esql-idelta) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
+* [`INCREASE`](../../functions-operators/time-series-aggregation-functions.md#esql-increase) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
+* [`IRATE`](../../functions-operators/time-series-aggregation-functions.md#esql-irate) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
 * [`LAST_OVER_TIME`](../../functions-operators/time-series-aggregation-functions.md#esql-last_over_time) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
 * [`MAX_OVER_TIME`](../../functions-operators/time-series-aggregation-functions.md#esql-max_over_time) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
 * [`MIN_OVER_TIME`](../../functions-operators/time-series-aggregation-functions.md#esql-min_over_time) {applies_to}`stack: preview 9.2` {applies_to}`serverless: preview`
diff --git a/muted-tests.yml b/muted-tests.yml
@@ -466,6 +466,9 @@ tests:
 - class: org.elasticsearch.xpack.downsample.DownsampleIT
   method: testAggregateMethod
   issue: https://github.com/elastic/elasticsearch/issues/139382
+- class: org.elasticsearch.xpack.security.authc.saml.SamlRedirectTests
+  method: testRedirectUrlWithRelayStateAndSigning
+  issue: https://github.com/elastic/elasticsearch/issues/139389
 
 # Examples:
 #
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromBinaryBlockLoader.java
@@ -17,6 +17,11 @@
 
 import java.io.IOException;
 
+/**
+ * This block loader should be used for fields that are directly encoded as binary values but are always single valued, such as the
+ * histogram fields.  See also {@link BytesRefsFromCustomBinaryBlockLoader} for multivalued binary fields, and
+ * {@link BytesRefsFromOrdsBlockLoader} for ordinals-based binary values
+ */
 public class BytesRefsFromBinaryBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
 
     private final String fieldName;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromCustomBinaryBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromCustomBinaryBlockLoader.java
@@ -18,6 +18,11 @@
 
 import java.io.IOException;
 
+/**
+ * This block loader should be used for "wildcard-style" binary values, which is to say fields we have encoded into a binary
+ * format that supports multivalued via an encoding on our side.  See also {@link BytesRefsFromOrdsBlockLoader} for ordinals
+ * based multivalue aware binary fields, and {@link BytesRefsFromBinaryBlockLoader} for single-valued binary fields.
+ */
 public class BytesRefsFromCustomBinaryBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader {
     private final String fieldName;
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromOrdsBlockLoader.java
@@ -17,7 +17,9 @@
 import java.io.IOException;
 
 /**
- * Loads {@code keyword} style fields that are stored as a lookup table and ordinals.
+ * Loads {@code keyword} style fields that are stored as a lookup table and ordinals.  See also {@link BytesRefsFromCustomBinaryBlockLoader}
+ * for {@code wildcard} style (i.e. non-ordinal encoded multivalued) and {@link BytesRefsFromBinaryBlockLoader} for {@code histogram}
+ * style (i.e. non-ordinal single valued).
  */
 public class BytesRefsFromOrdsBlockLoader extends AbstractBytesRefsFromOrdsBlockLoader {
     public BytesRefsFromOrdsBlockLoader(String fieldName) {
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java
@@ -229,6 +229,15 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(
             .build();
     }
 
+    /**
+     * For a given mapping and input value, compute the value that will be in the block.  Values are generated from the
+     * {@link DocumentGenerator}, and the behavior can be controled by writing a custom {@link DataSourceHandler}.
+     *
+     * @param fieldMapping Generated parameters for this field mapping
+     * @param value Generated input value to convert
+     * @param testContext Context information for the current test run
+     * @return The value that will be added to the block
+     */
     protected abstract Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext);
 
     protected static Object maybeFoldList(List<?> list) {
diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java
@@ -29,6 +29,7 @@
 import org.elasticsearch.index.fielddata.IndexHistogramFieldData;
 import org.elasticsearch.index.fielddata.LeafHistogramFieldData;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
 import org.elasticsearch.index.mapper.DocumentParserContext;
 import org.elasticsearch.index.mapper.DocumentParsingException;
@@ -42,6 +43,7 @@
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
 import org.elasticsearch.index.mapper.ValueFetcher;
+import org.elasticsearch.index.mapper.blockloader.docvalues.BytesRefsFromBinaryBlockLoader;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
 import org.elasticsearch.search.DocValueFormat;
@@ -56,6 +58,7 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
@@ -305,6 +308,11 @@ public Query termQuery(Object value, SearchExecutionContext context) {
                 "[" + CONTENT_TYPE + "] field do not support searching, " + "use dedicated aggregations instead: [" + name() + "]"
             );
         }
+
+        @Override
+        public BlockLoader blockLoader(BlockLoaderContext blContext) {
+            return new BytesRefsFromBinaryBlockLoader(name());
+        }
     }
 
     @Override
@@ -351,17 +359,11 @@ public void parse(DocumentParserContext context) throws IOException {
                 parsedHistogram = HistogramParser.parse(fullPath(), subParser);
             }
 
-            BytesStreamOutput streamOutput = new BytesStreamOutput();
-            for (int i = 0; i < parsedHistogram.values().size(); i++) {
-                long count = parsedHistogram.counts().get(i);
-                assert count >= 0;
-                // we do not add elements with count == 0
-                if (count > 0) {
-                    streamOutput.writeVLong(count);
-                    streamOutput.writeLong(Double.doubleToRawLongBits(parsedHistogram.values().get(i)));
-                }
-            }
-            BytesRef docValue = streamOutput.bytes().toBytesRef();
+            List<Double> values = parsedHistogram.values();
+            List<Long> counts = parsedHistogram.counts();
+
+            BytesRef docValue = encodeBytesRef(values, counts);
+
             Field field = new BinaryDocValuesField(fullPath(), docValue);
             if (context.doc().getByKey(fieldType().name()) != null) {
                 throw new IllegalArgumentException(
@@ -402,6 +404,22 @@ public void parse(DocumentParserContext context) throws IOException {
         context.path().remove();
     }
 
+    static BytesRef encodeBytesRef(List<Double> values, List<Long> counts) throws IOException {
+        BytesStreamOutput streamOutput = new BytesStreamOutput();
+        assert counts.size() == values.size();
+        for (int i = 0; i < values.size(); i++) {
+            long count = counts.get(i);
+            assert count >= 0;
+            // we do not add elements with count == 0
+            if (count > 0) {
+                streamOutput.writeVLong(count);
+                streamOutput.writeLong(Double.doubleToRawLongBits(values.get(i)));
+            }
+        }
+        BytesRef docValue = streamOutput.bytes().toBytesRef();
+        return docValue;
+    }
+
     /** re-usable {@link HistogramValue} implementation */
     static class InternalHistogramValue extends HistogramValue {
         double value;
diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldBlockLoaderTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldBlockLoaderTests.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.analytics.mapper;
+
+import org.elasticsearch.core.Types;
+import org.elasticsearch.datageneration.datasource.DataSourceHandler;
+import org.elasticsearch.datageneration.datasource.DataSourceRequest;
+import org.elasticsearch.datageneration.datasource.DataSourceResponse;
+import org.elasticsearch.index.mapper.BlockLoaderTestCase;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class HistogramFieldBlockLoaderTests extends BlockLoaderTestCase {
+
+    public HistogramFieldBlockLoaderTests(Params params) {
+        super(HistogramFieldMapper.CONTENT_TYPE, List.of(DATA_SOURCE_HANDLER), params);
+    }
+
+    @Override
+    protected Collection<? extends Plugin> getPlugins() {
+        return List.of(new AnalyticsPlugin());
+    }
+
+    private static DataSourceHandler DATA_SOURCE_HANDLER = new DataSourceHandler() {
+
+        @Override
+        public DataSourceResponse.ObjectArrayGenerator handle(DataSourceRequest.ObjectArrayGenerator request) {
+            // histogram does not support multiple values in a document so we can't have object arrays
+            return new DataSourceResponse.ObjectArrayGenerator(Optional::empty);
+        }
+
+        @Override
+        public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
+            if (request.fieldType().equals(HistogramFieldMapper.CONTENT_TYPE) == false) {
+                return null;
+            }
+
+            return new DataSourceResponse.LeafMappingParametersGenerator(() -> {
+                var map = new HashMap<String, Object>();
+                if (ESTestCase.randomBoolean()) {
+                    map.put("ignore_malformed", ESTestCase.randomBoolean());
+                }
+                return map;
+            });
+        }
+
+        @Override
+        public DataSourceResponse.FieldDataGenerator handle(DataSourceRequest.FieldDataGenerator request) {
+            if (request.fieldType().equals(HistogramFieldMapper.CONTENT_TYPE) == false) {
+                return null;
+            }
+            return new DataSourceResponse.FieldDataGenerator(mapping -> {
+                List<Double> values = randomList(randomIntBetween(1, 1000), ESTestCase::randomDouble);
+                values.sort(Double::compareTo);
+                return Map.of(
+                    "values",
+                    values,
+                    "counts",
+                    // Note - we need the three parameter version of random list here to ensure it's always the same length as values
+                    randomList(values.size(), values.size(), ESTestCase::randomNonNegativeLong)
+                );
+            });
+        }
+    };
+
+    @Override
+    public void testBlockLoaderOfMultiField() throws IOException {
+        // Multi fields are not supported
+    }
+
+    @Override
+    protected Object expected(Map<String, Object> fieldMapping, Object value, TestContext testContext) {
+        Map<String, Object> valueAsMap = Types.forciblyCast(value);
+        List<Double> bucketValues = Types.forciblyCast(valueAsMap.get("values"));
+        List<Long> counts = Types.forciblyCast(valueAsMap.get("counts"));
+
+        try {
+            return HistogramFieldMapper.encodeBytesRef(bucketValues, counts);
+        } catch (IOException e) {
+            fail("failed to encode histogram field values");
+        }
+        throw new IllegalStateException("Unreachable");
+    }
+}
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java
@@ -86,6 +86,7 @@ public class XPackLicenseState {
         );
         messages.put(XPackField.REDACT_PROCESSOR, new String[] { "Executing a redact processor in an ingest pipeline will fail." });
         messages.put(XPackField.INFERENCE, new String[] { "The Inference API is disabled" });
+        messages.put(XPackField.GPU_INDEXING, new String[] { "Indexing using a GPU is disabled." });
         EXPIRATION_MESSAGES = Collections.unmodifiableMap(messages);
     }
 
@@ -109,6 +110,7 @@ public class XPackLicenseState {
         messages.put(XPackField.REDACT_PROCESSOR, XPackLicenseState::redactProcessorAcknowledgementMessages);
         messages.put(XPackField.ESQL, XPackLicenseState::esqlAcknowledgementMessages);
         messages.put(XPackField.INFERENCE, XPackLicenseState::inferenceApiAcknowledgementMessages);
+        messages.put(XPackField.GPU_INDEXING, XPackLicenseState::gpuIndexingAcknowledgementMessages);
         ACKNOWLEDGMENT_MESSAGES = Collections.unmodifiableMap(messages);
     }
 
@@ -376,6 +378,22 @@ private static String[] redactProcessorAcknowledgementMessages(OperationMode cur
         return Strings.EMPTY_ARRAY;
     }
 
+    private static String[] gpuIndexingAcknowledgementMessages(OperationMode currentMode, OperationMode newMode) {
+        switch (newMode) {
+            case BASIC:
+            case STANDARD:
+            case GOLD:
+            case PLATINUM:
+                switch (currentMode) {
+                    case TRIAL:
+                    case ENTERPRISE:
+                        return new String[] { "Indexing using a GPU will be disabled" };
+                }
+                break;
+        }
+        return Strings.EMPTY_ARRAY;
+    }
+
     private static boolean isBasic(OperationMode mode) {
         return mode == OperationMode.BASIC;
     }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
@@ -94,6 +94,8 @@ public final class XPackField {
 
     /** Name constant for the redact processor feature. */
     public static final String REDACT_PROCESSOR = "redact_processor";
+    /** Name constant for the GPU indexing feature. */
+    public static final String GPU_INDEXING = "gpu_indexing";
     public static final String ENTERPRISE_GEOIP_DOWNLOADER = "enterprise_geoip_downloader";
     /** Name for Universal Profiling. */
     public static final String UNIVERSAL_PROFILING = "universal_profiling";
diff --git a/x-pack/plugin/gpu/src/main/java/org/elasticsearch/xpack/gpu/GPUPlugin.java b/x-pack/plugin/gpu/src/main/java/org/elasticsearch/xpack/gpu/GPUPlugin.java
@@ -14,23 +14,34 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.FeatureFlag;
+import org.elasticsearch.core.Strings;
 import org.elasticsearch.gpu.GPUSupport;
 import org.elasticsearch.gpu.codec.ES92GpuHnswSQVectorsFormat;
 import org.elasticsearch.gpu.codec.ES92GpuHnswVectorsFormat;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.mapper.vectors.VectorsFormatProvider;
 import org.elasticsearch.license.License;
+import org.elasticsearch.license.LicensedFeature;
+import org.elasticsearch.logging.LogManager;
+import org.elasticsearch.logging.Logger;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.internal.InternalVectorFormatProviderPlugin;
+import org.elasticsearch.xpack.core.XPackField;
 import org.elasticsearch.xpack.core.XPackPlugin;
 
 import java.util.List;
 
 public class GPUPlugin extends Plugin implements InternalVectorFormatProviderPlugin {
 
+    private static final Logger log = LogManager.getLogger(GPUPlugin.class);
+
     public static final FeatureFlag GPU_FORMAT = new FeatureFlag("gpu_vectors_indexing");
 
-    private static final License.OperationMode MINIMUM_ALLOWED_LICENSE = License.OperationMode.ENTERPRISE;
+    public static final LicensedFeature.Momentary GPU_INDEXING_FEATURE = LicensedFeature.momentary(
+        null,
+        XPackField.GPU_INDEXING,
+        License.OperationMode.ENTERPRISE
+    );
 
     private final GpuMode gpuMode;
 
@@ -75,7 +86,7 @@ public List<Setting<?>> getSettings() {
     // Allow tests to override the license state
     protected boolean isGpuIndexingFeatureAllowed() {
         var licenseState = XPackPlugin.getSharedLicenseState();
-        return licenseState != null && licenseState.isAllowedByLicense(MINIMUM_ALLOWED_LICENSE);
+        return licenseState != null && GPU_INDEXING_FEATURE.check(licenseState);
     }
 
     @Override
@@ -115,10 +126,25 @@ public ReferenceDocs referenceDocs() {
     @Override
     public VectorsFormatProvider getVectorsFormatProvider() {
         return (indexSettings, indexOptions, similarity, elementType) -> {
-            if (GPU_FORMAT.isEnabled() && isGpuIndexingFeatureAllowed()) {
-                if ((gpuMode == GpuMode.TRUE || (gpuMode == GpuMode.AUTO && GPUSupport.isSupported()))
-                    && vectorIndexAndElementTypeSupported(indexOptions.getType(), elementType)) {
-                    return getVectorsFormat(indexOptions, similarity);
+            if (GPU_FORMAT.isEnabled()) {
+                if (vectorIndexAndElementTypeSupported(indexOptions.getType(), elementType) == false) {
+                    return null;
+                }
+
+                if (gpuMode == GpuMode.TRUE || (gpuMode == GpuMode.AUTO && GPUSupport.isSupported())) {
+                    assert GPUSupport.isSupported();
+                    if (isGpuIndexingFeatureAllowed()) {
+                        return getVectorsFormat(indexOptions, similarity);
+                    } else {
+                        log.warn(
+                            Strings.format(
+                                "The current configuration supports GPU indexing, but it is not allowed by the current license. "
+                                    + "If this is intentional, it is possible to suppress this message by setting [%s] to FALSE",
+                                VECTORS_INDEXING_USE_GPU_NODE_SETTING.getKey()
+                            )
+                        );
+                        return null;
+                    }
                 }
             }
             return null;

Original file line number	Diff line number	Diff line change
`@@ -466,6 +466,9 @@ tests:`
`466`	`466`	`- class: org.elasticsearch.xpack.downsample.DownsampleIT`
`467`	`467`	`method: testAggregateMethod`
`468`	`468`	`issue: https://github.com/elastic/elasticsearch/issues/139382`
	`469`	`+- class: org.elasticsearch.xpack.security.authc.saml.SamlRedirectTests`
	`470`	`+ method: testRedirectUrlWithRelayStateAndSigning`
	`471`	`+ issue: https://github.com/elastic/elasticsearch/issues/139389`
`469`	`472`
`470`	`473`	`# Examples:`
`471`	`474`	`#`