[ES|QL] Render aggregate_metric_double (#122660) (#122996)

limotova · web-flow · commit d440d72eb988 · 2025-02-20T06:46:22.000-10:00
This commit allows users to read aggregate_metric_double fields from
indices in ES|QL, with any subset of metrics.
diff --git a/docs/changelog/122660.yaml b/docs/changelog/122660.yaml
@@ -0,0 +1,5 @@
+pr: 122660
+summary: Render `aggregate_metric_double`
+area: "ES|QL"
+type: enhancement
+issues: []
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java
@@ -139,20 +139,26 @@ public BlockLoader.IntBuilder count() {
     }
 
     public enum Metric {
-        MIN(0),
-        MAX(1),
-        SUM(2),
-        COUNT(3);
+        MIN(0, "min"),
+        MAX(1, "max"),
+        SUM(2, "sum"),
+        COUNT(3, "value_count");
 
         private final int index;
+        private final String label;
 
-        Metric(int index) {
+        Metric(int index, String label) {
             this.index = index;
+            this.label = label;
         }
 
         public int getIndex() {
             return index;
         }
+
+        public String getLabel() {
+            return label;
+        }
     }
 
     public record AggregateMetricDoubleLiteral(Double min, Double max, Double sum, Integer count) {
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java
@@ -96,7 +96,11 @@ public int getFirstValueIndex(int position) {
 
     @Override
     public int getValueCount(int position) {
-        return blocks[0].getValueCount(position);
+        int max = 0;
+        for (var block : blocks) {
+            max = Math.max(max, block.getValueCount(position));
+        }
+        return max;
     }
 
     @Override
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -691,7 +691,12 @@ public enum Cap {
         /**
          * Support partial_results
          */
-        SUPPORT_PARTIAL_RESULTS;
+        SUPPORT_PARTIAL_RESULTS,
+
+        /**
+         * Support for rendering aggregate_metric_double type
+         */
+        AGGREGATE_METRIC_DOUBLE_RENDERING(AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG);
 
         private final boolean enabled;
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java
@@ -13,6 +13,7 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.CompositeBlock;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
@@ -25,6 +26,7 @@
 import java.io.IOException;
 
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.aggregateMetricDoubleBlockToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString;
@@ -148,8 +150,14 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa
                     return builder.value(versionToString(val));
                 }
             };
-            // TODO: Add implementation for aggregate_metric_double
-            case NULL, AGGREGATE_METRIC_DOUBLE -> new PositionToXContent(block) {
+            case AGGREGATE_METRIC_DOUBLE -> new PositionToXContent(block) {
+                @Override
+                protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
+                    throws IOException {
+                    return builder.value(aggregateMetricDoubleBlockToString((CompositeBlock) block, valueIndex));
+                }
+            };
+            case NULL -> new PositionToXContent(block) {
                 @Override
                 protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex)
                     throws IOException {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BooleanBlock;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.CompositeBlock;
 import org.elasticsearch.compute.data.DoubleBlock;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
@@ -30,6 +31,7 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.aggregateMetricDoubleBlockToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
 import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString;
@@ -132,7 +134,8 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR
             case GEO_POINT, GEO_SHAPE, CARTESIAN_POINT, CARTESIAN_SHAPE -> spatialToString(
                 ((BytesRefBlock) block).getBytesRef(offset, scratch)
             );
-            case UNSUPPORTED, AGGREGATE_METRIC_DOUBLE -> (String) null;
+            case AGGREGATE_METRIC_DOUBLE -> aggregateMetricDoubleBlockToString((CompositeBlock) block, offset);
+            case UNSUPPORTED -> (String) null;
             case SOURCE -> {
                 BytesRef val = ((BytesRefBlock) block).getBytesRef(offset, scratch);
                 try {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
@@ -24,6 +24,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
 import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
 import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
@@ -67,7 +68,8 @@ public class ToString extends AbstractConvertFunction implements EvaluatorMapper
         Map.entry(GEO_POINT, ToStringFromGeoPointEvaluator.Factory::new),
         Map.entry(CARTESIAN_POINT, ToStringFromCartesianPointEvaluator.Factory::new),
         Map.entry(CARTESIAN_SHAPE, ToStringFromCartesianShapeEvaluator.Factory::new),
-        Map.entry(GEO_SHAPE, ToStringFromGeoShapeEvaluator.Factory::new)
+        Map.entry(GEO_SHAPE, ToStringFromGeoShapeEvaluator.Factory::new),
+        Map.entry(AGGREGATE_METRIC_DOUBLE, ToStringFromAggregateMetricDoubleEvaluator.Factory::new)
     );
 
     @FunctionInfo(
@@ -82,6 +84,7 @@ public ToString(
         @Param(
             name = "field",
             type = {
+                "aggregate_metric_double",
                 "boolean",
                 "cartesian_point",
                 "cartesian_shape",
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromAggregateMetricDoubleEvaluator.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromAggregateMetricDoubleEvaluator.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.compute.data.Block;
+import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.CompositeBlock;
+import org.elasticsearch.compute.data.Vector;
+import org.elasticsearch.compute.operator.DriverContext;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+
+import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.aggregateMetricDoubleBlockToString;
+
+public class ToStringFromAggregateMetricDoubleEvaluator extends AbstractConvertFunction.AbstractEvaluator {
+    public ToStringFromAggregateMetricDoubleEvaluator(EvalOperator.ExpressionEvaluator field, Source source, DriverContext driverContext) {
+        super(driverContext, field, source);
+    }
+
+    @Override
+    protected String name() {
+        return "ToStringFromAggregateMetricDouble";
+    }
+
+    @Override
+    protected Block evalVector(Vector v) {
+        return evalBlock(v.asBlock());
+    }
+
+    private static BytesRef evalValue(CompositeBlock compositeBlock, int index) {
+        return new BytesRef(aggregateMetricDoubleBlockToString(compositeBlock, index));
+    }
+
+    @Override
+    public Block evalBlock(Block b) {
+        CompositeBlock block = (CompositeBlock) b;
+        int positionCount = block.getPositionCount();
+        try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) {
+            for (int p = 0; p < positionCount; p++) {
+                if (block.isNull(p)) {
+                    builder.appendNull();
+                } else {
+                    builder.appendBytesRef(evalValue(block, p));
+                }
+            }
+            return builder.build();
+        }
+    }
+
+    public static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+        private final Source source;
+        private final EvalOperator.ExpressionEvaluator.Factory field;
+
+        public Factory(EvalOperator.ExpressionEvaluator.Factory field, Source source) {
+            this.field = field;
+            this.source = source;
+        }
+
+        @Override
+        public EvalOperator.ExpressionEvaluator get(DriverContext context) {
+            return new ToStringFromAggregateMetricDoubleEvaluator(field.get(context), source, context);
+        }
+    }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
@@ -8,14 +8,21 @@
 package org.elasticsearch.xpack.esql.type;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
 import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.time.DateUtils;
+import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder.Metric;
+import org.elasticsearch.compute.data.CompositeBlock;
+import org.elasticsearch.compute.data.DoubleBlock;
+import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
@@ -671,6 +678,26 @@ public static long booleanToUnsignedLong(boolean number) {
         return number ? ONE_AS_UNSIGNED_LONG : ZERO_AS_UNSIGNED_LONG;
     }
 
+    public static String aggregateMetricDoubleBlockToString(CompositeBlock compositeBlock, int index) {
+        try (XContentBuilder builder = JsonXContent.contentBuilder()) {
+            builder.startObject();
+            for (Metric metric : List.of(Metric.MIN, Metric.MAX, Metric.SUM)) {
+                var block = compositeBlock.getBlock(metric.getIndex());
+                if (block.isNull(index) == false) {
+                    builder.field(metric.getLabel(), ((DoubleBlock) block).getDouble(index));
+                }
+            }
+            var countBlock = compositeBlock.getBlock(Metric.COUNT.getIndex());
+            if (countBlock.isNull(index) == false) {
+                builder.field(Metric.COUNT.getLabel(), ((IntBlock) countBlock).getInt(index));
+            }
+            builder.endObject();
+            return Strings.toString(builder);
+        } catch (IOException e) {
+            throw new IllegalStateException("error rendering aggregate metric double", e);
+        }
+    }
+
     public enum EsqlConverter implements Converter {
 
         STRING_TO_DATE_PERIOD(x -> EsqlDataTypeConverter.parseTemporalAmount(x, DataType.DATE_PERIOD)),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java
@@ -1580,7 +1580,7 @@ public void testRegexOnInt() {
 
     public void testUnsupportedTypesWithToString() {
         // DATE_PERIOD and TIME_DURATION types have been added, but not really patched through the engine; i.e. supported.
-        final String supportedTypes = "boolean or cartesian_point or cartesian_shape or date_nanos or datetime "
+        final String supportedTypes = "aggregate_metric_double or boolean or cartesian_point or cartesian_shape or date_nanos or datetime "
             + "or geo_point or geo_shape or ip or numeric or string or version";
         verifyUnsupported(
             "row period = 1 year | eval to_string(period)",
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml
@@ -143,6 +143,8 @@ setup:
           - '{"@timestamp": "2021-04-28T19:50:24.467Z", "agg_metric": {"max": 10, "min": 3}, "k8s": {"pod": {"uid":"947e4ced-1786-4e53-9e0c-5c447e959507"}}}'
           - '{"index": {}}'
           - '{"@timestamp": "2021-04-28T19:50:44.467Z", "agg_metric": {"max": 17, "min": 2}, "k8s": {"pod": {"uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}}}'
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T19:51:04.467Z", "k8s": {"pod": {"uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9"}}}'
 
   - do:
       indices.create:
@@ -447,6 +449,77 @@ stats on aggregate_metric_double missing min and max:
   - match: {values.0.2: 1.0}
   - match: {values.0.3: 10}
 
+---
+render aggregate_metric_double when missing min and max:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [ ]
+          capabilities: [ aggregate_metric_double_rendering ]
+      reason: "Support for rendering aggregate_metric_doubles"
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test4 | KEEP agg_metric'
+
+  - length: {values: 1}
+  - length: {values.0: 1}
+  - match: {columns.0.name: "agg_metric"}
+  - match: {columns.0.type: "aggregate_metric_double"}
+  - match: {values.0.0: '{"sum":1.0,"value_count":10}'}
+
+
+---
+render aggregate_metric_double when missing value:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [ ]
+          capabilities: [ aggregate_metric_double_rendering ]
+      reason: "Support for rendering aggregate_metric_doubles"
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test3 | WHERE @timestamp == "2021-04-28T19:51:04.467Z" | KEEP agg_metric'
+
+  - length: {values: 1}
+  - length: {values.0: 1}
+  - match: {columns.0.name: "agg_metric"}
+  - match: {columns.0.type: "aggregate_metric_double"}
+  - match: {values.0.0: null}
+
+
+---
+to_string aggregate_metric_double:
+  - requires:
+      test_runner_features: [ capabilities ]
+      capabilities:
+        - method: POST
+          path: /_query
+          parameters: [ ]
+          capabilities: [ aggregate_metric_double_rendering ]
+      reason: "Support for rendering aggregate_metric_doubles"
+  - do:
+      allowed_warnings_regex:
+        - "No limit defined, adding default limit of \\[.*\\]"
+      esql.query:
+        body:
+          query: 'FROM test4 | EVAL agg = to_string(agg_metric) | KEEP agg'
+
+  - length: {values: 1}
+  - length: {values.0: 1}
+  - match: {columns.0.name: "agg"}
+  - match: {columns.0.type: "keyword"}
+  - match: {values.0.0: '{"sum":1.0,"value_count":10}'}
+
 ---
 from index pattern unsupported counter:
   - requires:
@@ -482,7 +555,7 @@ from index pattern unsupported counter:
   - match: {columns.7.type: "keyword"}
   - match: {columns.8.name: "metricset"}
   - match: {columns.8.type: "keyword"}
-  - length: {values: 15}
+  - length: {values: 16}
 
 ---
 from index pattern explicit counter use:
@@ -503,7 +576,7 @@ from index pattern explicit counter use:
           query: 'FROM test* | keep *.tx'
   - match: {columns.0.name: "k8s.pod.network.tx"}
   - match: {columns.0.type: "unsupported"}
-  - length: {values: 15}
+  - length: {values: 16}
 
 ---
 _source:
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml
