Merge branch 'main' into ironscales-add_ilm_delete_index

Author: mohitjha-elastic

docs/changelog/137677.yaml

@@ -0,0 +1,5 @@
+pr: 137677
+summary: "[Inference] Implementing the completion task type on EIS"
+area: "Inference"
+type: enhancement
+issues: []

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContent.java

@@ -63,7 +63,10 @@ public static void serialize(XContentBuilder builder, @Nullable ExponentialHisto
         builder.startObject();
 
         builder.field(SCALE_FIELD, histogram.scale());
-        builder.field(SUM_FIELD, histogram.sum());
+
+        if (histogram.sum() != 0.0 || histogram.valueCount() > 0) {
+            builder.field(SUM_FIELD, histogram.sum());
+        }
         if (Double.isNaN(histogram.min()) == false) {
             builder.field(MIN_FIELD, histogram.min());
         }

libs/exponential-histogram/src/test/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramXContentTests.java

@@ -40,7 +40,7 @@ public void testNullHistogram() {
 
     public void testEmptyHistogram() {
         ExponentialHistogram emptyHistogram = ExponentialHistogram.empty();
-        assertThat(toJson(emptyHistogram), equalTo("{\"scale\":" + emptyHistogram.scale() + ",\"sum\":0.0}"));
+        assertThat(toJson(emptyHistogram), equalTo("{\"scale\":" + emptyHistogram.scale() + "}"));
     }
 
     public void testFullHistogram() {

muted-tests.yml

@@ -417,6 +417,12 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.RevertModelSnapshotIT
   method: testRevertModelSnapshot_DeleteInterveningResults
   issue: https://github.com/elastic/elasticsearch/issues/132349
+- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmAuthenticateTests
+  method: testJwkUpdatesByReloadWithFile
+  issue: https://github.com/elastic/elasticsearch/issues/138397
+- class: org.elasticsearch.compute.data.ExponentialHistogramBlockTests
+  method: testComponentAccess
+  issue: https://github.com/elastic/elasticsearch/issues/138399
 
 # Examples:
 #

test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java

@@ -833,12 +833,13 @@ public static TestBlock parseHistogramsToBlock(
                 }
                 CompressedExponentialHistogram result = new CompressedExponentialHistogram();
                 try {
+                    Double sum = (Double) sums.get(i);
                     Double min = (Double) minima.get(i);
                     Double max = (Double) maxima.get(i);
                     result.reset(
                         (Double) zeroThresholds.get(i),
                         (Long) valueCounts.get(i),
-                        (Double) sums.get(i),
+                        sum == null ? 0.0 : sum,
                         min == null ? Double.NaN : min,
                         max == null ? Double.NaN : max,
                         (BytesRef) encodedHistograms.get(i)

x-pack/plugin/downsample/build.gradle

@@ -15,6 +15,7 @@ dependencies {
   compileOnly project(xpackModule('core'))
   testImplementation project(':modules:data-streams')
   testImplementation project(xpackModule('ilm'))
+  compileOnly project(xpackModule('analytics'))
   compileOnly project(xpackModule('mapper-aggregate-metric'))
   testImplementation(testArtifact(project(xpackModule('core'))))
   testImplementation project(xpackModule('ccr'))

x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsampleIT.java

@@ -147,6 +147,18 @@ public void testLastValueMode() throws Exception {
                 "metrics.cpu_usage": {
                   "type": "double",
                   "time_series_metric": "gauge"
+                },
+                "my_labels": {
+                  "properties": {
+                    "my_histogram": {
+                      "type": "histogram"
+                    },
+                    "my_aggregate": {
+                      "type": "aggregate_metric_double",
+                      "metrics": [ "min", "max", "sum", "value_count" ],
+                      "default_metric": "max"
+                    }
+                  }
                 }
               }
             }
@@ -157,12 +169,25 @@ public void testLastValueMode() throws Exception {
         Supplier<XContentBuilder> sourceSupplier = () -> {
             String ts = randomDateForRange(now.minusSeconds(60 * 60).toEpochMilli(), now.plusSeconds(60 * 29).toEpochMilli());
             try {
+                int maxHistogramSize = randomIntBetween(2, 10);
                 return XContentFactory.jsonBuilder()
                     .startObject()
                     .field("@timestamp", ts)
                     .field("attributes.host.name", randomFrom("host1", "host2", "host3"))
                     .field("attributes.os.name", randomFrom("linux", "windows", "macos"))
                     .field("metrics.cpu_usage", randomDouble())
+
+                    .startObject("my_labels.my_histogram")
+                    .array("values", randomHistogramValues(maxHistogramSize))
+                    .array("counts", randomHistogramValueCounts(maxHistogramSize))
+                    .endObject()
+
+                    .startObject("my_labels.my_aggregate")
+                    .field("min", randomFloatBetween(0.0f, 10.0f, true))
+                    .field("max", randomFloatBetween(10.0f, 20.0f, true))
+                    .field("sum", randomFloatBetween(20.0f, 30.0f, true))
+                    .field("value_count", randomIntBetween(1, 10))
+                    .endObject()
                     .endObject();
             } catch (IOException e) {
                 throw new RuntimeException(e);
@@ -536,4 +561,21 @@ private String rolloverAndDownsample(String dataStreamName, DownsampleConfig dow
     private EsqlQueryResponse esqlCommand(String command) throws IOException {
         return client().execute(EsqlQueryAction.INSTANCE, new EsqlQueryRequest().query(command)).actionGet(30, TimeUnit.SECONDS);
     }
+
+    private static double[] randomHistogramValues(int size) {
+        final double[] array = new double[size];
+        double minHistogramValue = randomDoubleBetween(0.0, 0.1, true);
+        for (int i = 0; i < array.length; i++) {
+            array[i] = minHistogramValue += randomDoubleBetween(0.0, 0.5, true);
+        }
+        return array;
+    }
+
+    private static int[] randomHistogramValueCounts(int size) {
+        final int[] array = new int[size];
+        for (int i = 0; i < array.length; i++) {
+            array[i] = randomIntBetween(1, 100);
+        }
+        return array;
+    }
 }

x-pack/plugin/downsample/src/internalClusterTest/java/org/elasticsearch/xpack/downsample/DownsamplingIntegTestCase.java

@@ -44,6 +44,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin;
+import org.elasticsearch.xpack.analytics.AnalyticsPlugin;
 import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 
@@ -88,7 +89,8 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
             LocalStateCompositeXPackPlugin.class,
             Downsample.class,
             AggregateMetricMapperPlugin.class,
-            EsqlPlugin.class
+            EsqlPlugin.class,
+            AnalyticsPlugin.class
         );
     }
 

x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldSerializer.java

@@ -12,6 +12,10 @@
 import java.io.IOException;
 import java.util.Collection;
 
+/**
+ * This serialiser can be used to convert a list of producers to an aggregate metric double field. The producer should produce an
+ * aggregate metric double or a sub metric of one, any other producers will trigger an error.
+ */
 final class AggregateMetricFieldSerializer implements DownsampleFieldSerializer {
     private final Collection<AbstractDownsampleFieldProducer> producers;
     private final String name;
@@ -36,31 +40,33 @@ public void write(XContentBuilder builder) throws IOException {
         builder.startObject(name);
         for (AbstractDownsampleFieldProducer fieldProducer : producers) {
             assert name.equals(fieldProducer.name()) : "producer has a different name";
-            if (fieldProducer.isEmpty() == false) {
-                if (fieldProducer instanceof MetricFieldProducer metricFieldProducer) {
-                    if (metricFieldProducer instanceof MetricFieldProducer.AggregateScalarMetricFieldProducer gaugeProducer) {
-                        builder.field("max", gaugeProducer.max);
-                        builder.field("min", gaugeProducer.min);
-                        builder.field("sum", gaugeProducer.sum.value());
-                        builder.field("value_count", gaugeProducer.count);
-                    } else if (metricFieldProducer instanceof MetricFieldProducer.LastValueScalarMetricFieldProducer lastValueProducer) {
-                        builder.field(lastValueProducer.sampleLabel(), lastValueProducer.lastValue);
-                    } else if (metricFieldProducer instanceof MetricFieldProducer.AggregatePreAggregatedMetricFieldProducer producer) {
-                        switch (producer.metric) {
-                            case max -> builder.field("max", producer.max);
-                            case min -> builder.field("min", producer.min);
-                            case sum -> builder.field("sum", producer.sum.value());
-                            case value_count -> builder.field("value_count", producer.count);
-                        }
-                    } else {
-                        throw new IllegalStateException();
+            if (fieldProducer.isEmpty()) {
+                continue;
+            }
+            switch (fieldProducer) {
+                case MetricFieldProducer.AggregateGaugeMetricFieldProducer producer -> {
+                    builder.field("max", producer.max);
+                    builder.field("min", producer.min);
+                    builder.field("sum", producer.sum.value());
+                    builder.field("value_count", producer.count);
+                }
+                case MetricFieldProducer.AggregateSubMetricFieldProducer producer -> {
+                    switch (producer.metric) {
+                        case max -> builder.field("max", producer.max);
+                        case min -> builder.field("min", producer.min);
+                        case sum -> builder.field("sum", producer.sum.value());
+                        case value_count -> builder.field("value_count", producer.count);
                     }
-                } else if (fieldProducer instanceof LabelFieldProducer labelFieldProducer) {
-                    LabelFieldProducer.Label label = labelFieldProducer.label();
-                    if (label.get() != null) {
-                        builder.field(label.name(), label.get());
+                }
+                case LastValueFieldProducer.AggregateSubMetricFieldProducer lastValueFieldProducer -> {
+                    Object lastValue = lastValueFieldProducer.lastValue();
+                    if (lastValue != null) {
+                        builder.field(lastValueFieldProducer.subMetric(), lastValue);
                     }
                 }
+                default -> throw new IllegalStateException(
+                    "Unexpected field producer class: " + fieldProducer.getClass().getSimpleName() + " for " + name + " field"
+                );
             }
         }
         builder.endObject();

…le/AggregateMetricFieldValueFetcher.java …AggregateSubMetricFieldValueFetcher.javax-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldValueFetcher.java renamed to x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateSubMetricFieldValueFetcher.java x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateMetricFieldValueFetcher.java renamed to x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/AggregateSubMetricFieldValueFetcher.java

@@ -11,16 +11,24 @@
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
+import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper;
 import org.elasticsearch.xpack.aggregatemetric.mapper.AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType;
 
-public final class AggregateMetricFieldValueFetcher extends FieldValueFetcher {
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This {@link FieldValueFetcher} is responsible for fetching the values of aggregate metric double fields.
+ * For performance reasons, we load an aggregate metric double by loading its sub-metrics separately.
+ */
+public final class AggregateSubMetricFieldValueFetcher extends FieldValueFetcher {
 
     private final AggregateMetricDoubleFieldType aggMetricFieldType;
 
     private final AbstractDownsampleFieldProducer fieldProducer;
 
-    AggregateMetricFieldValueFetcher(
+    AggregateSubMetricFieldValueFetcher(
         MappedFieldType fieldType,
         AggregateMetricDoubleFieldType aggMetricFieldType,
         IndexFieldData<?> fieldData,
@@ -31,10 +39,31 @@ public final class AggregateMetricFieldValueFetcher extends FieldValueFetcher {
         this.fieldProducer = createFieldProducer(samplingMethod);
     }
 
+    @Override
     public AbstractDownsampleFieldProducer fieldProducer() {
         return fieldProducer;
     }
 
+    /**
+     * For aggregate_metric_double fields we create separate fetchers for each sub-metric. This is usually a downsample-of-downsample case.
+     */
+    static List<AggregateSubMetricFieldValueFetcher> create(
+        SearchExecutionContext context,
+        AggregateMetricDoubleFieldMapper.AggregateMetricDoubleFieldType aggMetricFieldType,
+        DownsampleConfig.SamplingMethod samplingMethod
+    ) {
+        List<AggregateSubMetricFieldValueFetcher> fetchers = new ArrayList<>();
+        // If the field is an aggregate_metric_double field, we should load all its subfields
+        // This is usually a downsample-of-downsample case
+        for (NumberFieldMapper.NumberFieldType metricSubField : aggMetricFieldType.getMetricFields().values()) {
+            if (context.fieldExistsInIndex(metricSubField.name())) {
+                IndexFieldData<?> fieldData = context.getForField(metricSubField, MappedFieldType.FielddataOperation.SEARCH);
+                fetchers.add(new AggregateSubMetricFieldValueFetcher(metricSubField, aggMetricFieldType, fieldData, samplingMethod));
+            }
+        }
+        return fetchers;
+    }
+
     private AbstractDownsampleFieldProducer createFieldProducer(DownsampleConfig.SamplingMethod samplingMethod) {
         AggregateMetricDoubleFieldMapper.Metric metric = null;
         for (var e : aggMetricFieldType.getMetricFields().entrySet()) {
@@ -50,13 +79,13 @@ private AbstractDownsampleFieldProducer createFieldProducer(DownsampleConfig.Sam
             if (samplingMethod != DownsampleConfig.SamplingMethod.LAST_VALUE) {
                 // If the field is an aggregate_metric_double field, we should use the correct subfields
                 // for each aggregation. This is a downsample-of-downsample case
-                return new MetricFieldProducer.AggregatePreAggregatedMetricFieldProducer(aggMetricFieldType.name(), metric);
+                return new MetricFieldProducer.AggregateSubMetricFieldProducer(aggMetricFieldType.name(), metric);
             } else {
-                return new MetricFieldProducer.LastValueScalarMetricFieldProducer(aggMetricFieldType.name(), metric);
+                return LastValueFieldProducer.createForAggregateSubMetricMetric(aggMetricFieldType.name(), metric);
             }
         } else {
-            // If field is not a metric, we downsample it as a label
-            return new LabelFieldProducer.AggregateMetricFieldProducer.AggregateMetricFieldProducer(aggMetricFieldType.name(), metric);
+            // If a field is not a metric, we downsample it as a label
+            return LastValueFieldProducer.createForAggregateSubMetricLabel(aggMetricFieldType.name(), metric);
         }
     }
 }