From ae1669493f3e11e53c81146210e27de0ca15eda6 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Wed, 15 Jan 2025 19:26:10 -1000 Subject: [PATCH 01/18] [ES|QL] Support some stats on aggregate_double_metric Adds support for min, max, sum, and count --- .../index/mapper/BlockLoader.java | 8 + .../elasticsearch/index/mapper/TestBlock.java | 5 + .../esql/core/plugin/EsqlCorePlugin.java | 1 + .../xpack/esql/core/type/DataType.java | 8 +- .../AggregateDoubleMetricBlockBuilder.java | 89 +++++++++++ .../compute/data/BlockFactory.java | 4 + .../compute/data/CompositeBlock.java | 16 +- .../lucene/ValuesSourceReaderOperator.java | 5 + .../xpack/esql/EsqlTestUtils.java | 5 +- .../xpack/esql/action/EsqlCapabilities.java | 7 +- .../xpack/esql/action/PositionToXContent.java | 2 +- .../xpack/esql/action/ResponseValueUtils.java | 2 +- .../expression/function/aggregate/Count.java | 14 ++ .../expression/function/aggregate/Max.java | 31 +++- .../expression/function/aggregate/Min.java | 31 +++- .../expression/function/aggregate/Sum.java | 40 ++++- .../scalar/ScalarFunctionWritables.java | 2 + .../convert/FromAggregateDoubleMetric.java | 141 ++++++++++++++++++ .../xpack/esql/planner/AggregateMapper.java | 4 + .../esql/planner/LocalExecutionPlanner.java | 2 +- .../xpack/esql/planner/PlannerUtils.java | 2 +- .../esql/action/EsqlQueryResponseTests.java | 16 +- .../xpack/esql/analysis/AnalyzerTests.java | 5 +- .../xpack/esql/analysis/VerifierTests.java | 2 +- .../scalar/conditional/CaseTests.java | 2 +- .../AggregateDoubleMetricFieldMapper.java | 116 +++++++++++++- ...AggregateDoubleMetricFieldMapperTests.java | 12 ++ .../rest-api-spec/test/esql/40_tsdb.yml | 55 ++++++- .../test/esql/40_unsupported_types.yml | 133 ++++++++--------- 29 files changed, 659 insertions(+), 101 deletions(-) create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index 7b4ceb67f04d7..ed40aab44cb5f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -417,6 +417,8 @@ interface BlockFactory { SingletonOrdinalsBuilder singletonOrdinalsBuilder(SortedDocValues ordinals, int count); // TODO support non-singleton ords + + AggregateDoubleMetricBuilder aggregateDoubleMetricBuilder(int count); } /** @@ -501,4 +503,10 @@ interface SingletonOrdinalsBuilder extends Builder { */ SingletonOrdinalsBuilder appendOrd(int value); } + + interface AggregateDoubleMetricBuilder extends Builder { + + AggregateDoubleMetricBuilder append(double min, double max, double sum, int valueCount); + + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java index 2c53fa782db85..8fca80d459de4 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java @@ -147,6 +147,11 @@ public SingletonOrdsBuilder appendOrd(int value) { } return new SingletonOrdsBuilder(); } + + @Override + public BlockLoader.AggregateDoubleMetricBuilder aggregateDoubleMetricBuilder(int count) { + return null; + } }; } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java index 61b480968e974..729188e2981d9 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java @@ -14,4 +14,5 @@ public class EsqlCorePlugin extends Plugin implements ExtensiblePlugin { public static final FeatureFlag SEMANTIC_TEXT_FEATURE_FLAG = new FeatureFlag("esql_semantic_text"); + public static final FeatureFlag AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG = new FeatureFlag("esql_aggregate_metric_double"); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index d86cdb0de038c..0d9dafb1ca265 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -307,7 +307,9 @@ public enum DataType { * loaded from the index and ESQL will load these fields as strings without their attached * chunks or embeddings. */ - SEMANTIC_TEXT(builder().esType("semantic_text").unknownSize()); + SEMANTIC_TEXT(builder().esType("semantic_text").unknownSize()), + + AGGREGATE_METRIC_DOUBLE(builder().esType("aggregate_metric_double").unknownSize()); /** * Types that are actively being built. These types are not returned @@ -316,7 +318,8 @@ public enum DataType { * check that sending them to a function produces a sane error message. */ public static final Map UNDER_CONSTRUCTION = Map.ofEntries( - Map.entry(SEMANTIC_TEXT, EsqlCorePlugin.SEMANTIC_TEXT_FEATURE_FLAG) + Map.entry(SEMANTIC_TEXT, EsqlCorePlugin.SEMANTIC_TEXT_FEATURE_FLAG), + Map.entry(AGGREGATE_METRIC_DOUBLE, EsqlCorePlugin.AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG) ); private final String typeName; @@ -553,6 +556,7 @@ public static boolean isRepresentable(DataType t) { && t != SOURCE && t != HALF_FLOAT && t != PARTIAL_AGG + && t != AGGREGATE_METRIC_DOUBLE && t.isCounter() == false; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java new file mode 100644 index 0000000000000..bff74adf4c150 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.elasticsearch.index.mapper.BlockLoader; + +public class AggregateDoubleMetricBlockBuilder extends AbstractBlockBuilder implements BlockLoader.AggregateDoubleMetricBuilder { + + private final DoubleBlockBuilder minBuilder; + private final DoubleBlockBuilder maxBuilder; + private final DoubleBlockBuilder sumBuilder; + private final IntBlockBuilder countBuilder; + + public AggregateDoubleMetricBlockBuilder(int estimatedSize, BlockFactory blockFactory) { + super(blockFactory); + minBuilder = new DoubleBlockBuilder(estimatedSize, blockFactory); + maxBuilder = new DoubleBlockBuilder(estimatedSize, blockFactory); + sumBuilder = new DoubleBlockBuilder(estimatedSize, blockFactory); + countBuilder = new IntBlockBuilder(estimatedSize, blockFactory); + } + + @Override + protected int valuesLength() { + return minBuilder.valuesLength(); + } + + @Override + protected void growValuesArray(int newSize) { + minBuilder.growValuesArray(newSize); + maxBuilder.growValuesArray(newSize); + sumBuilder.growValuesArray(newSize); + countBuilder.growValuesArray(newSize); + } + + @Override + protected int elementSize() { + return minBuilder.elementSize() + maxBuilder.elementSize() + sumBuilder.elementSize() + countBuilder.elementSize(); + } + + @Override + public Block.Builder copyFrom(Block block, int beginInclusive, int endExclusive) { + CompositeBlock composite = (CompositeBlock) block; + minBuilder.copyFrom(composite.getBlock(Metric.MIN.ordinal()), beginInclusive, endExclusive); + maxBuilder.copyFrom(composite.getBlock(Metric.MAX.ordinal()), beginInclusive, endExclusive); + sumBuilder.copyFrom(composite.getBlock(Metric.SUM.ordinal()), beginInclusive, endExclusive); + countBuilder.copyFrom(composite.getBlock(Metric.COUNT.ordinal()), beginInclusive, endExclusive); + return this; + } + + @Override + public Block.Builder mvOrdering(Block.MvOrdering mvOrdering) { + minBuilder.mvOrdering(mvOrdering); + maxBuilder.mvOrdering(mvOrdering); + sumBuilder.mvOrdering(mvOrdering); + countBuilder.mvOrdering(mvOrdering); + return this; + } + + @Override + public Block build() { + Block[] blocks = new Block[4]; + blocks[Metric.MIN.ordinal()] = minBuilder.build(); + blocks[Metric.MAX.ordinal()] = maxBuilder.build(); + blocks[Metric.SUM.ordinal()] = sumBuilder.build(); + blocks[Metric.COUNT.ordinal()] = countBuilder.build(); + return new CompositeBlock(blocks); + } + + @Override + public BlockLoader.AggregateDoubleMetricBuilder append(double min, double max, double sum, int valueCount) { + minBuilder.appendDouble(min); + maxBuilder.appendDouble(max); + sumBuilder.appendDouble(sum); + countBuilder.appendInt(valueCount); + return this; + } + + public enum Metric { + MIN, + MAX, + SUM, + COUNT; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java index f66ae42106ca2..8cbdb71e89a44 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java @@ -432,6 +432,10 @@ public Block newConstantNullBlock(int positions) { return b; } + public AggregateDoubleMetricBlockBuilder newAggregatedDoubleMetricBlockBuilder(int estimatedSize) { + return new AggregateDoubleMetricBlockBuilder(estimatedSize, this); + } + /** * Returns the maximum number of bytes that a Block should be backed by a primitive array before switching to using BigArrays. */ diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java index b83e2d1efc259..3836680e95751 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java @@ -86,22 +86,30 @@ public int getPositionCount() { @Override public int getTotalValueCount() { - throw new UnsupportedOperationException("Composite block"); + // TODO: this works for aggregate metric double fields, because the four blocks are guarenteed to have the same length / positions. + // So check just for one sub block is sufficient. + return blocks[0].getTotalValueCount(); } @Override public int getFirstValueIndex(int position) { - throw new UnsupportedOperationException("Composite block"); + // TODO: this works for aggregate metric double fields, because the four blocks are guarenteed to have the same length / positions. + // So check just for one sub block is sufficient. + return blocks[0].getFirstValueIndex(position); } @Override public int getValueCount(int position) { - throw new UnsupportedOperationException("Composite block"); + // TODO: this works for aggregate metric double fields, because the four blocks are guarenteed to have the same length / positions. + // So check just for one sub block is sufficient. + return blocks[0].getValueCount(position); } @Override public boolean isNull(int position) { - throw new UnsupportedOperationException("Composite block"); + // TODO: this works for aggregate metric double fields, because the four blocks are guarenteed to have the same length / positions. + // So check just for one sub block is sufficient. + return blocks[0].isNull(position); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java index 8fbb946587470..7c3bde315a23c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java @@ -698,6 +698,11 @@ public BytesRefBlock constantBytes(BytesRef value) { public BlockLoader.SingletonOrdinalsBuilder singletonOrdinalsBuilder(SortedDocValues ordinals, int count) { return new SingletonOrdinalsBuilder(factory, ordinals, count); } + + @Override + public BlockLoader.AggregateDoubleMetricBuilder aggregateDoubleMetricBuilder(int count) { + return factory.newAggregatedDoubleMetricBlockBuilder(count); + } } // TODO tests that mix source loaded fields and doc values in the same block diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 852cefe83f989..7213de9c3f703 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -765,9 +765,8 @@ public static Literal randomLiteral(DataType type) { throw new UncheckedIOException(e); } } - case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( - "can't make random values for [" + type.typeName() + "]" - ); + case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG, AGGREGATE_METRIC_DOUBLE -> + throw new IllegalArgumentException("can't make random values for [" + type.typeName() + "]"); }, type); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index fb5433d7662af..53abff1c810ec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -738,7 +738,12 @@ public enum Cap { /** * Support named argument for function in map format. */ - OPTIONAL_NAMED_ARGUMENT_MAP_FOR_FUNCTION(Build.current().isSnapshot()); + OPTIONAL_NAMED_ARGUMENT_MAP_FOR_FUNCTION(Build.current().isSnapshot()), + + /** + * Support for aggregate_metric_double type + */ + AGGREGATE_METRIC_DOUBLE; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java index 0def56c70dc35..7d54c00328ae0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java @@ -148,7 +148,7 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa return builder.value(versionToString(val)); } }; - case NULL -> new PositionToXContent(block) { + case NULL, AGGREGATE_METRIC_DOUBLE -> new PositionToXContent(block) { @Override protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex) throws IOException { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java index 49fcc167dce0f..710a66fb1d9f4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java @@ -132,7 +132,7 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR case GEO_POINT, GEO_SHAPE, CARTESIAN_POINT, CARTESIAN_SHAPE -> spatialToString( ((BytesRefBlock) block).getBytesRef(offset, scratch) ); - case UNSUPPORTED -> (String) null; + case UNSUPPORTED, AGGREGATE_METRIC_DOUBLE -> (String) null; case SOURCE -> { BytesRef val = ((BytesRefBlock) block).getBytesRef(offset, scratch); try { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index 3a0d616d407a3..ab3d81d9126d7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.CountAggregatorFunction; +import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; @@ -22,6 +23,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateDoubleMetric; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul; @@ -33,6 +35,7 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; public class Count extends AggregateFunction implements ToAggregator, SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Count", Count::new); @@ -71,6 +74,7 @@ public Count( optional = true, name = "field", type = { + "aggregate_metric_double", "boolean", "cartesian_point", "date", @@ -141,6 +145,16 @@ protected TypeResolution resolveType() { public Expression surrogate() { var s = source(); var field = field(); + if (field.dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { + return new Sum( + s, + new FromAggregateDoubleMetric( + source(), + field, + new Literal(s, AggregateDoubleMetricBlockBuilder.Metric.COUNT.ordinal(), INTEGER) + ) + ); + } if (field.foldable()) { if (field instanceof Literal l) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index eb0c8abd1080b..401ebf77cfaff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.aggregation.MaxIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxIpAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxLongAggregatorFunctionSupplier; +import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -27,6 +28,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateDoubleMetric; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMax; import org.elasticsearch.xpack.esql.planner.ToAggregator; @@ -73,7 +75,19 @@ public Max( Source source, @Param( name = "field", - type = { "boolean", "double", "integer", "long", "date", "date_nanos", "ip", "keyword", "text", "long", "version" } + type = { + "aggregate_metric_double", + "boolean", + "double", + "integer", + "long", + "date", + "date_nanos", + "ip", + "keyword", + "text", + "long", + "version" } ) Expression field ) { this(source, field, Literal.TRUE); @@ -111,7 +125,7 @@ public Max replaceChildren(List newChildren) { protected TypeResolution resolveType() { return TypeResolutions.isType( field(), - SUPPLIERS::containsKey, + dt -> SUPPLIERS.containsKey(dt) || dt == DataType.AGGREGATE_METRIC_DOUBLE, sourceText(), DEFAULT, "representable except unsigned_long and spatial types" @@ -120,6 +134,9 @@ protected TypeResolution resolveType() { @Override public DataType dataType() { + if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { + return DataType.DOUBLE; + } return field().dataType().noText(); } @@ -135,6 +152,16 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { @Override public Expression surrogate() { + if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { + return new Max( + source(), + new FromAggregateDoubleMetric( + source(), + field(), + new Literal(source(), AggregateDoubleMetricBlockBuilder.Metric.MAX.ordinal(), DataType.INTEGER) + ) + ); + } return field().foldable() ? new MvMax(source(), field()) : null; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index 472f0b1ff5cd1..c900b2c2aa995 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.aggregation.MinIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinIpAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinLongAggregatorFunctionSupplier; +import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -27,6 +28,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateDoubleMetric; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.planner.ToAggregator; @@ -73,7 +75,19 @@ public Min( Source source, @Param( name = "field", - type = { "boolean", "double", "integer", "long", "date", "date_nanos", "ip", "keyword", "text", "long", "version" } + type = { + "aggregate_metric_double", + "boolean", + "double", + "integer", + "long", + "date", + "date_nanos", + "ip", + "keyword", + "text", + "long", + "version" } ) Expression field ) { this(source, field, Literal.TRUE); @@ -111,7 +125,7 @@ public Min withFilter(Expression filter) { protected TypeResolution resolveType() { return TypeResolutions.isType( field(), - SUPPLIERS::containsKey, + dt -> SUPPLIERS.containsKey(dt) || dt == DataType.AGGREGATE_METRIC_DOUBLE, sourceText(), DEFAULT, "representable except unsigned_long and spatial types" @@ -120,6 +134,9 @@ protected TypeResolution resolveType() { @Override public DataType dataType() { + if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { + return DataType.DOUBLE; + } return field().dataType().noText(); } @@ -135,6 +152,16 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { @Override public Expression surrogate() { + if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { + return new Min( + source(), + new FromAggregateDoubleMetric( + source(), + field(), + new Literal(source(), AggregateDoubleMetricBlockBuilder.Metric.MIN.ordinal(), DataType.INTEGER) + ) + ); + } return field().foldable() ? new MvMin(source(), field()) : null; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java index 37c2abaae1e4e..4fec0af4ac32d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java @@ -12,8 +12,10 @@ import org.elasticsearch.compute.aggregation.SumDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.SumIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.SumLongAggregatorFunctionSupplier; +import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -22,6 +24,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateDoubleMetric; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul; @@ -29,7 +32,11 @@ import java.util.List; import static java.util.Collections.emptyList; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; @@ -53,7 +60,7 @@ public class Sum extends NumericAggregate implements SurrogateExpression { tag = "docsStatsSumNestedExpression" ) } ) - public Sum(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { + public Sum(Source source, @Param(name = "number", type = { "aggregate_metric_double", "double", "integer", "long" }) Expression field) { this(source, field, Literal.TRUE); } @@ -106,10 +113,41 @@ protected AggregatorFunctionSupplier doubleSupplier(List inputChannels) return new SumDoubleAggregatorFunctionSupplier(inputChannels); } + @Override + protected TypeResolution resolveType() { + if (supportsDates()) { + return TypeResolutions.isType( + this, + e -> e == DataType.DATETIME || e == DataType.AGGREGATE_METRIC_DOUBLE || e.isNumeric() && e != DataType.UNSIGNED_LONG, + sourceText(), + DEFAULT, + "datetime", + "aggregate_metric_double or numeric except unsigned_long or counter types" + ); + } + return isType( + field(), + dt -> dt == DataType.AGGREGATE_METRIC_DOUBLE || dt.isNumeric() && dt != DataType.UNSIGNED_LONG, + sourceText(), + DEFAULT, + "aggregate_metric_double or numeric except unsigned_long or counter types" + ); + } + @Override public Expression surrogate() { var s = source(); var field = field(); + if (field.dataType() == AGGREGATE_METRIC_DOUBLE) { + return new Sum( + s, + new FromAggregateDoubleMetric( + source(), + field, + new Literal(s, AggregateDoubleMetricBlockBuilder.Metric.SUM.ordinal(), INTEGER) + ) + ); + } // SUM(const) is equivalent to MV_SUM(const)*COUNT(*). return field.foldable() diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java index 0d3bacbd47605..fd93f5273fd1c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Least; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateDoubleMetric; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateDiff; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateExtract; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateFormat; @@ -67,6 +68,7 @@ public static List getNamedWriteables() { entries.add(Concat.ENTRY); entries.add(E.ENTRY); entries.add(EndsWith.ENTRY); + entries.add(FromAggregateDoubleMetric.ENTRY); entries.add(Greatest.ENTRY); entries.add(Hash.ENTRY); entries.add(Hypot.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java new file mode 100644 index 0000000000000..bd12ccab14510 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.CompositeBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; +import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; + +public class FromAggregateDoubleMetric extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "FromAggregateDoubleMetric", + FromAggregateDoubleMetric::new + ); + + private final Expression field; + private final Expression subfieldIndex; + + @FunctionInfo(returnType = { "long", "double" }, description = "Convert aggregate double metric to a block of a single subfield.") + public FromAggregateDoubleMetric( + Source source, + @Param( + name = "aggregate_metric_double", + type = { "aggregate_metric_double" }, + description = "Aggregate double metric to convert." + ) Expression field, + @Param(name = "subfieldIndex", type = "int", description = "Index of subfield") Expression subfieldIndex + ) { + super(source, List.of(field, subfieldIndex)); + this.field = field; + this.subfieldIndex = subfieldIndex; + } + + private FromAggregateDoubleMetric(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeNamedWriteable(field); + out.writeNamedWriteable(subfieldIndex); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + public DataType dataType() { + if (subfieldIndex.foldable() == false) { + throw new EsqlIllegalArgumentException("Received a non-foldable value for subfield index"); + } + + var subfield = ((Number) subfieldIndex.fold(FoldContext.small())).intValue(); + if (subfield == AggregateDoubleMetricBlockBuilder.Metric.COUNT.ordinal()) { + return INTEGER; + } + return DOUBLE; + } + + @Override + public Expression replaceChildren(List newChildren) { + return new FromAggregateDoubleMetric(source(), newChildren.get(0), newChildren.get(1)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, FromAggregateDoubleMetric::new, field, subfieldIndex); + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + return isType(field, dt -> dt == DataType.AGGREGATE_METRIC_DOUBLE, sourceText(), DEFAULT, "aggregate_metric_double only"); + } + + @Override + public boolean foldable() { + return Expressions.foldable(children()); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + var fieldEvaluator = toEvaluator.apply(field); + return driverContext -> { + final EvalOperator.ExpressionEvaluator eval = fieldEvaluator.get(driverContext); + return new EvalOperator.ExpressionEvaluator() { + @Override + public Block eval(Page page) { + CompositeBlock compositeBlock = (CompositeBlock) eval.eval(page); + try { + Block block = compositeBlock.getBlock(((Number) subfieldIndex.fold(FoldContext.small())).intValue()); + block.incRef(); + return block; + } finally { + compositeBlock.close(); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(eval); + } + }; + }; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index e420cd501cccd..5449aeea2adcb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -212,6 +212,9 @@ private static Stream groupingAndNonGrouping(Tuple, Tuple aggClass) { case CARTESIAN_POINT -> "CartesianPoint"; case GEO_SHAPE -> "GeoShape"; case CARTESIAN_SHAPE -> "CartesianShape"; + case AGGREGATE_METRIC_DOUBLE -> "AggregatedMetricDouble"; case UNSUPPORTED, NULL, UNSIGNED_LONG, SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, SOURCE, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new EsqlIllegalArgumentException( "illegal agg type: " + type.typeName() diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index ecd0284c7cb57..20b104f506b31 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -364,7 +364,7 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE, COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE, SOURCE -> TopNEncoder.DEFAULT_UNSORTABLE; // unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point - case PARTIAL_AGG, UNSUPPORTED -> TopNEncoder.UNSUPPORTED; + case PARTIAL_AGG, UNSUPPORTED, AGGREGATE_METRIC_DOUBLE -> TopNEncoder.UNSUPPORTED; }; } List orders = topNExec.order().stream().map(order -> { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index af5146f7b6926..1c6e22bb9badb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -285,7 +285,7 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field case TSID_DATA_TYPE -> ElementType.BYTES_REF; case GEO_POINT, CARTESIAN_POINT -> fieldExtractPreference == DOC_VALUES ? ElementType.LONG : ElementType.BYTES_REF; case GEO_SHAPE, CARTESIAN_SHAPE -> fieldExtractPreference == EXTRACT_SPATIAL_BOUNDS ? ElementType.INT : ElementType.BYTES_REF; - case PARTIAL_AGG -> ElementType.COMPOSITE; + case PARTIAL_AGG, AGGREGATE_METRIC_DOUBLE -> ElementType.COMPOSITE; case SHORT, BYTE, DATE_PERIOD, TIME_DURATION, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT -> throw EsqlIllegalArgumentException .illegalDataType(dataType); }; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index f4c68f141460b..08662c162ea2a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.test.AbstractChunkedSerializingTestCase; import org.elasticsearch.transport.RemoteClusterAware; @@ -175,7 +176,8 @@ private ColumnInfoImpl randomColumnInfo() { t -> false == DataType.isPrimitiveAndSupported(t) || t == DataType.DATE_PERIOD || t == DataType.TIME_DURATION - || t == DataType.PARTIAL_AGG, + || t == DataType.PARTIAL_AGG + || t == DataType.AGGREGATE_METRIC_DOUBLE, () -> randomFrom(DataType.types()) ).widenSmallNumeric(); return new ColumnInfoImpl(randomAlphaOfLength(10), type.esType()); @@ -214,6 +216,12 @@ private Page randomPage(List columns) { case CARTESIAN_SHAPE -> ((BytesRefBlock.Builder) builder).appendBytesRef( CARTESIAN.asWkb(ShapeTestUtils.randomGeometry(randomBoolean())) ); + case AGGREGATE_METRIC_DOUBLE -> ((BlockLoader.AggregateDoubleMetricBuilder) builder).append( + randomDouble(), + randomDouble(), + randomDouble(), + randomInt() + ); case NULL -> builder.appendNull(); case SOURCE -> { try { @@ -903,6 +911,12 @@ static Page valuesToPage(BlockFactory blockFactory, List columns BytesRef wkb = stringToSpatial(value.toString()); ((BytesRefBlock.Builder) builder).appendBytesRef(wkb); } + case AGGREGATE_METRIC_DOUBLE -> ((BlockLoader.AggregateDoubleMetricBuilder) builder).append( + ((Number) value).doubleValue(), + ((Number) value).doubleValue(), + ((Number) value).doubleValue(), + ((Number) value).intValue() + ); } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index feab6bbbe56a6..85dd8fde61b33 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1961,7 +1961,7 @@ public void testUnsupportedTypesInStats() { found value [x] type [unsigned_long] line 2:96: first argument of [percentile(x, 10)] must be [numeric except unsigned_long],\ found value [x] type [unsigned_long] - line 2:115: argument of [sum(x)] must be [numeric except unsigned_long or counter types],\ + line 2:115: argument of [sum(x)] must be [aggregate_metric_double or numeric except unsigned_long or counter types],\ found value [x] type [unsigned_long]"""); verifyUnsupported(""" @@ -1976,7 +1976,8 @@ public void testUnsupportedTypesInStats() { line 2:29: argument of [median_absolute_deviation(x)] must be [numeric except unsigned_long or counter types],\ found value [x] type [version] line 2:59: first argument of [percentile(x, 10)] must be [numeric except unsigned_long], found value [x] type [version] - line 2:78: argument of [sum(x)] must be [numeric except unsigned_long or counter types], found value [x] type [version]"""); + line 2:78: argument of [sum(x)] must be [aggregate_metric_double or numeric except unsigned_long or counter types],\ + found value [x] type [version]"""); } public void testInOnText() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index e3214411698b0..1289dd7ed2f67 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -755,7 +755,7 @@ public void testUnsignedLongNegation() { public void testSumOnDate() { assertEquals( - "1:19: argument of [sum(hire_date)] must be [numeric except unsigned_long or counter types]," + "1:19: argument of [sum(hire_date)] must be [aggregate_metric_double or numeric except unsigned_long or counter types]," + " found value [hire_date] type [datetime]", error("from test | stats sum(hire_date)") ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index b196bd49f6bb2..2fa82b9f1caa2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -59,7 +59,7 @@ public class CaseTests extends AbstractScalarFunctionTestCase { DataType.NULL ).collect(Collectors.toList()); if (Build.current().isSnapshot()) { - t.addAll(DataType.UNDER_CONSTRUCTION.keySet()); + t.addAll(DataType.UNDER_CONSTRUCTION.keySet().stream().filter(type -> type != DataType.AGGREGATE_METRIC_DOUBLE).toList()); } TYPES = unmodifiableList(t); } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index 6944f91042311..c4331734b471c 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; @@ -27,6 +28,8 @@ import org.elasticsearch.index.fielddata.ScriptDocValues.DoublesSupplier; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.BlockDocValuesReader; +import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; @@ -288,7 +291,7 @@ public AggregateDoubleMetricFieldType(String name) { } public AggregateDoubleMetricFieldType(String name, Map meta, MetricType metricType) { - super(name, true, false, false, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); + super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, meta); this.metricType = metricType; } @@ -508,6 +511,117 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) return SourceValueFetcher.identity(name(), context, format); } + public class AggregateDoubleMetricBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { + NumberFieldMapper.NumberFieldType minFieldType = metricFields.get(Metric.min); + NumberFieldMapper.NumberFieldType maxFieldType = metricFields.get(Metric.max); + NumberFieldMapper.NumberFieldType sumFieldType = metricFields.get(Metric.sum); + NumberFieldMapper.NumberFieldType countFieldType = metricFields.get(Metric.value_count); + + private AggregateDoubleMetricBlockLoader() {} + + static NumericDocValues getNumericDocValues(NumberFieldMapper.NumberFieldType field, LeafReader leafReader) throws IOException { + if (field == null) { + return null; + } + String fieldName = field.name(); + var values = leafReader.getNumericDocValues(fieldName); + if (values != null) { + return values; + } + + var sortedValues = leafReader.getSortedNumericDocValues(fieldName); + return DocValues.unwrapSingleton(sortedValues); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + NumericDocValues minValues = getNumericDocValues(minFieldType, context.reader()); + NumericDocValues maxValues = getNumericDocValues(maxFieldType, context.reader()); + NumericDocValues sumValues = getNumericDocValues(sumFieldType, context.reader()); + NumericDocValues valueCountValues = getNumericDocValues(countFieldType, context.reader()); + + if (minValues == null || maxValues == null || sumValues == null || valueCountValues == null) { + throw new UnsupportedOperationException("Must have all subfields to use aggregate double metric in ESQL"); + } + return new BlockDocValuesReader() { + + private int docID = -1; + + @Override + protected int docId() { + return docID; + } + + @Override + public String toString() { + return "BlockDocValuesReader.AggregatedDoubleMetrics"; + } + + @Override + public Block read(BlockFactory factory, Docs docs) throws IOException { + try (var builder = factory.aggregateDoubleMetricBuilder(docs.count())) { + int lastDoc = -1; + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < lastDoc) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (minValues.advanceExact(doc)) { + boolean found = maxValues.advanceExact(doc); + assert found; + found = sumValues.advanceExact(doc); + assert found; + found = valueCountValues.advanceExact(doc); + assert found; + double min = NumericUtils.sortableLongToDouble(minValues.longValue()); + double max = NumericUtils.sortableLongToDouble(maxValues.longValue()); + double sum = NumericUtils.sortableLongToDouble(sumValues.longValue()); + int count = Math.toIntExact(valueCountValues.longValue()); + builder.append(min, max, sum, count); + } else { + builder.appendNull(); + } + lastDoc = doc; + this.docID = doc; + } + return builder.build(); + } + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + this.docID = docId; + var blockBuilder = (BlockLoader.AggregateDoubleMetricBuilder) builder; + if (minValues.advanceExact(this.docID)) { + boolean found = maxValues.advanceExact(this.docID); + assert found; + found = sumValues.advanceExact(this.docID); + assert found; + found = valueCountValues.advanceExact(this.docID); + assert found; + double min = NumericUtils.sortableLongToDouble(minValues.longValue()); + double max = NumericUtils.sortableLongToDouble(maxValues.longValue()); + double sum = NumericUtils.sortableLongToDouble(sumValues.longValue()); + int count = Math.toIntExact(valueCountValues.longValue()); + blockBuilder.append(min, max, sum, count); + } else { + blockBuilder.appendNull(); + } + } + }; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + return factory.aggregateDoubleMetricBuilder(expectedCount); + } + } + + @Override + public BlockLoader blockLoader(BlockLoaderContext blContext) { + return new AggregateDoubleMetricBlockLoader(); + } + /** * If field is a time series metric field, returns its metric type * @return the metric type or null diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java index 72c2beeed3ba4..0d62e7a9c1fd2 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java @@ -36,6 +36,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Names.IGNORE_MALFORMED; import static org.elasticsearch.xpack.aggregatemetric.mapper.AggregateDoubleMetricFieldMapper.Names.METRICS; @@ -618,4 +619,15 @@ public void testSyntheticSourceKeepArrays() { protected boolean supportsCopyTo() { return false; } + + @Override + protected Function loadBlockExpected() { + return n -> ((Number) n); + } + + @Override + protected Function loadBlockExpected(BlockReaderSupport blockReaderSupport, boolean columnReader) { + assumeTrue("Not supporting", false); + return null; + } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml index b9415bce62ea9..5bdd2baf60506 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_tsdb.yml @@ -80,8 +80,8 @@ setup: time_series_dimension: true agg_metric: type: aggregate_metric_double - metrics: - - max + # TODO: tests with a subset of metrics + metrics: [ min, max, sum, value_count ] default_metric: max k8s: properties: @@ -99,9 +99,9 @@ setup: index: test2 body: - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": "A", "agg_metric": {"max": 10}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": "A", "agg_metric": {"max": 10, "min": -1, "sum": 20, "value_count": 5}}' - '{"index": {}}' - - '{"@timestamp": "2021-04-28T18:50:24.467Z", "dim": "B", "agg_metric": {"max": 20}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "dim": "B", "agg_metric": {"max": 20, "min": 3, "sum": 50, "value_count": 7}}' --- load everything: @@ -201,6 +201,14 @@ cast then sort on counter: --- from doc with aggregate_metric_double: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [aggregate_metric_double] + reason: "Support for aggregate_metric_double" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" @@ -211,7 +219,7 @@ from doc with aggregate_metric_double: - match: {columns.0.name: "@timestamp"} - match: {columns.0.type: "date"} - match: {columns.1.name: "agg_metric"} - - match: {columns.1.type: "unsupported"} + - match: {columns.1.type: "aggregate_metric_double"} - match: {columns.2.name: "dim"} - match: {columns.2.type: "keyword"} - match: {columns.3.name: "k8s.pod.ip"} @@ -222,14 +230,45 @@ from doc with aggregate_metric_double: --- stats on aggregate_metric_double: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [aggregate_metric_double] + reason: "Support for aggregate_metric_double" - do: - catch: /Cannot use field \[agg_metric\] with unsupported type \[aggregate_metric_double\]/ + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test2 | STATS max(agg_metric) BY dim' + query: 'FROM test2 | STATS max(agg_metric), min(agg_metric), sum(agg_metric), count(agg_metric)' + - length: {values: 1} + - length: {values.0: 4} + - match: {columns.0.name: "max(agg_metric)"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "min(agg_metric)"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "sum(agg_metric)"} + - match: {columns.2.type: "double"} + - match: {columns.3.name: "count(agg_metric)"} + - match: {columns.3.type: "long"} + - match: {values.0.0: 20.0} + - match: {values.0.1: -1.0} + - match: {values.0.2: 70.0} + - match: {values.0.3: 12.0} --- from index pattern unsupported counter: + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [aggregate_metric_double] + reason: "Support for aggregate_metric_double" - do: allowed_warnings_regex: - "No limit defined, adding default limit of \\[.*\\]" @@ -240,7 +279,7 @@ from index pattern unsupported counter: - match: {columns.0.name: "@timestamp"} - match: {columns.0.type: "date"} - match: {columns.1.name: "agg_metric"} - - match: {columns.1.type: "unsupported"} + - match: {columns.1.type: "aggregate_metric_double"} - match: {columns.2.name: "dim"} - match: {columns.2.type: "keyword"} - match: {columns.3.name: "k8s.pod.ip"} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index e100f30717aef..8e5a6e6d231d6 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -13,7 +13,7 @@ setup: properties: aggregate_metric_double: type: aggregate_metric_double - metrics: [ min, max ] + metrics: [ min, max, sum, value_count ] default_metric: max binary: type: binary @@ -81,7 +81,7 @@ setup: body: - { "index": { } } - { - "aggregate_metric_double": { "min": 1.0, "max": 3.0 }, + "aggregate_metric_double": { "min": 1.0, "max": 3.0, "sum": 10.1, "value_count": 5 }, "binary": "U29tZSBiaW5hcnkgYmxvYg==", "completion": "foo bar", "date_nanos": "2015-01-01T12:10:30.123456789Z", @@ -119,8 +119,8 @@ unsupported: - method: POST path: /_query parameters: [ ] - capabilities: [ date_nanos_type ] - reason: "support for date nanos type" + capabilities: [ aggregate_metric_double ] + reason: "support for aggregate_metric_double type" - do: allowed_warnings_regex: @@ -131,7 +131,7 @@ unsupported: query: 'from test' - match: { columns.0.name: aggregate_metric_double } - - match: { columns.0.type: unsupported } + - match: { columns.0.type: aggregate_metric_double } - match: { columns.1.name: binary } - match: { columns.1.type: unsupported } - match: { columns.2.name: completion } @@ -227,7 +227,7 @@ unsupported: body: query: 'from test | limit 0' - match: { columns.0.name: aggregate_metric_double } - - match: { columns.0.type: unsupported } + - match: { columns.0.type: aggregate_metric_double } - match: { columns.1.name: binary } - match: { columns.1.type: unsupported } - match: { columns.2.name: completion } @@ -308,8 +308,8 @@ unsupported with sort: - method: POST path: /_query parameters: [ ] - capabilities: [ date_nanos_type ] - reason: "support for date nanos type" + capabilities: [ aggregate_metric_double ] + reason: "support for aggregate_metric_double type" - do: allowed_warnings_regex: @@ -317,97 +317,94 @@ unsupported with sort: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'from test | sort some_doc.bar' + query: 'from test | drop aggregate_metric_double | sort some_doc.bar' - - match: { columns.0.name: aggregate_metric_double } + - match: { columns.0.name: binary } - match: { columns.0.type: unsupported } - - match: { columns.1.name: binary } + - match: { columns.1.name: completion } - match: { columns.1.type: unsupported } - - match: { columns.2.name: completion } - - match: { columns.2.type: unsupported } - - match: { columns.3.name: date_nanos } - - match: { columns.3.type: date_nanos } - - match: { columns.4.name: date_range } + - match: { columns.2.name: date_nanos } + - match: { columns.2.type: date_nanos } + - match: { columns.3.name: date_range } + - match: { columns.3.type: unsupported } + - match: { columns.4.name: dense_vector } - match: { columns.4.type: unsupported } - - match: { columns.5.name: dense_vector } + - match: { columns.5.name: double_range } - match: { columns.5.type: unsupported } - - match: { columns.6.name: double_range } + - match: { columns.6.name: float_range } - match: { columns.6.type: unsupported } - - match: { columns.7.name: float_range } - - match: { columns.7.type: unsupported } - - match: { columns.8.name: geo_point } + - match: { columns.7.name: geo_point } + - match: { columns.7.type: geo_point } + - match: { columns.8.name: geo_point_alias } - match: { columns.8.type: geo_point } - - match: { columns.9.name: geo_point_alias } - - match: { columns.9.type: geo_point } - - match: { columns.10.name: geo_shape } - - match: { columns.10.type: geo_shape } - - match: { columns.11.name: histogram } + - match: { columns.9.name: geo_shape } + - match: { columns.9.type: geo_shape } + - match: { columns.10.name: histogram } + - match: { columns.10.type: unsupported } + - match: { columns.11.name: integer_range } - match: { columns.11.type: unsupported } - - match: { columns.12.name: integer_range } + - match: { columns.12.name: ip_range } - match: { columns.12.type: unsupported } - - match: { columns.13.name: ip_range } + - match: { columns.13.name: long_range } - match: { columns.13.type: unsupported } - - match: { columns.14.name: long_range } - - match: { columns.14.type: unsupported } - - match: { columns.15.name: match_only_text } - - match: { columns.15.type: text } - - match: { columns.16.name: name } - - match: { columns.16.type: keyword } - - match: { columns.17.name: point } - - match: { columns.17.type: cartesian_point } - - match: { columns.18.name: rank_feature } + - match: { columns.14.name: match_only_text } + - match: { columns.14.type: text } + - match: { columns.15.name: name } + - match: { columns.15.type: keyword } + - match: { columns.16.name: point } + - match: { columns.16.type: cartesian_point } + - match: { columns.17.name: rank_feature } + - match: { columns.17.type: unsupported } + - match: { columns.18.name: rank_features } - match: { columns.18.type: unsupported } - - match: { columns.19.name: rank_features } + - match: { columns.19.name: search_as_you_type } - match: { columns.19.type: unsupported } - - match: { columns.20.name: search_as_you_type } + - match: { columns.20.name: search_as_you_type._2gram } - match: { columns.20.type: unsupported } - - match: { columns.21.name: search_as_you_type._2gram } + - match: { columns.21.name: search_as_you_type._3gram } - match: { columns.21.type: unsupported } - - match: { columns.22.name: search_as_you_type._3gram } + - match: { columns.22.name: search_as_you_type._index_prefix } - match: { columns.22.type: unsupported } - - match: { columns.23.name: search_as_you_type._index_prefix } - - match: { columns.23.type: unsupported } - - match: { columns.24.name: shape } - - match: { columns.24.type: cartesian_shape } - - match: { columns.25.name: some_doc.bar } - - match: { columns.25.type: long } - - match: { columns.26.name: some_doc.foo } - - match: { columns.26.type: keyword } - - match: { columns.27.name: text } - - match: { columns.27.type: text } - - match: { columns.28.name: token_count } - - match: { columns.28.type: integer } + - match: { columns.23.name: shape } + - match: { columns.23.type: cartesian_shape } + - match: { columns.24.name: some_doc.bar } + - match: { columns.24.type: long } + - match: { columns.25.name: some_doc.foo } + - match: { columns.25.type: keyword } + - match: { columns.26.name: text } + - match: { columns.26.type: text } + - match: { columns.27.name: token_count } + - match: { columns.27.type: integer } - length: { values: 1 } - match: { values.0.0: null } - match: { values.0.1: null } - - match: { values.0.2: null } - - match: { values.0.3: "2015-01-01T12:10:30.123456789Z" } + - match: { values.0.2: "2015-01-01T12:10:30.123456789Z" } + - match: { values.0.3: null } - match: { values.0.4: null } - match: { values.0.5: null } - match: { values.0.6: null } - - match: { values.0.7: null } + - match: { values.0.7: "POINT (10.0 12.0)" } - match: { values.0.8: "POINT (10.0 12.0)" } - - match: { values.0.9: "POINT (10.0 12.0)" } - - match: { values.0.10: "LINESTRING (-97.154 25.996, -97.159 25.998, -97.181 25.991, -97.187 25.985)" } + - match: { values.0.9: "LINESTRING (-97.154 25.996, -97.159 25.998, -97.181 25.991, -97.187 25.985)" } + - match: { values.0.10: null } - match: { values.0.11: null } - match: { values.0.12: null } - match: { values.0.13: null } - - match: { values.0.14: null } - - match: { values.0.15: "foo bar baz" } - - match: { values.0.16: Alice } - - match: { values.0.17: "POINT (-97.15447 25.9961525)" } + - match: { values.0.14: "foo bar baz" } + - match: { values.0.15: Alice } + - match: { values.0.16: "POINT (-97.15447 25.9961525)" } + - match: { values.0.17: null } - match: { values.0.18: null } - match: { values.0.19: null } - match: { values.0.20: null } - match: { values.0.21: null } - match: { values.0.22: null } - - match: { values.0.23: null } - - match: { values.0.24: "LINESTRING (-377.03653 389.897676, -377.009051 389.889939)" } - - match: { values.0.25: 12 } - - match: { values.0.26: xy } - - match: { values.0.27: "foo bar" } - - match: { values.0.28: 3 } + - match: { values.0.23: "LINESTRING (-377.03653 389.897676, -377.009051 389.889939)" } + - match: { values.0.24: 12 } + - match: { values.0.25: xy } + - match: { values.0.26: "foo bar" } + - match: { values.0.27: 3 } --- nested declared inline: From a5be73b97d30f590adeeeba1b88fb89fce0f9509 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Tue, 21 Jan 2025 17:13:10 -1000 Subject: [PATCH 02/18] addressed most comments minus tests --- .../xpack/esql/core/type/DataType.java | 2 +- .../AggregateDoubleMetricBlockBuilder.java | 75 +++++++++++++------ .../compute/data/CompositeBlock.java | 8 -- .../expression/function/aggregate/Count.java | 10 +-- .../expression/function/aggregate/Max.java | 9 +-- .../expression/function/aggregate/Min.java | 9 +-- .../expression/function/aggregate/Sum.java | 10 +-- .../convert/FromAggregateDoubleMetric.java | 12 +-- .../xpack/esql/planner/AggregateMapper.java | 2 +- .../AggregateDoubleMetricFieldMapper.java | 34 +++------ 10 files changed, 79 insertions(+), 92 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 0d9dafb1ca265..671e2df3650dd 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -309,7 +309,7 @@ public enum DataType { */ SEMANTIC_TEXT(builder().esType("semantic_text").unknownSize()), - AGGREGATE_METRIC_DOUBLE(builder().esType("aggregate_metric_double").unknownSize()); + AGGREGATE_METRIC_DOUBLE(builder().esType("aggregate_metric_double").estimatedSize(Double.BYTES * 3 + Integer.BYTES)); /** * Types that are actively being built. These types are not returned diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java index bff74adf4c150..f22322033239b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java @@ -7,21 +7,36 @@ package org.elasticsearch.compute.data; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.core.Releasables; import org.elasticsearch.index.mapper.BlockLoader; public class AggregateDoubleMetricBlockBuilder extends AbstractBlockBuilder implements BlockLoader.AggregateDoubleMetricBuilder { - private final DoubleBlockBuilder minBuilder; - private final DoubleBlockBuilder maxBuilder; - private final DoubleBlockBuilder sumBuilder; - private final IntBlockBuilder countBuilder; + private DoubleBlockBuilder minBuilder; + private DoubleBlockBuilder maxBuilder; + private DoubleBlockBuilder sumBuilder; + private IntBlockBuilder countBuilder; public AggregateDoubleMetricBlockBuilder(int estimatedSize, BlockFactory blockFactory) { super(blockFactory); - minBuilder = new DoubleBlockBuilder(estimatedSize, blockFactory); - maxBuilder = new DoubleBlockBuilder(estimatedSize, blockFactory); - sumBuilder = new DoubleBlockBuilder(estimatedSize, blockFactory); - countBuilder = new IntBlockBuilder(estimatedSize, blockFactory); + minBuilder = null; + maxBuilder = null; + sumBuilder = null; + countBuilder = null; + try { + minBuilder = new DoubleBlockBuilder(estimatedSize, blockFactory); + maxBuilder = new DoubleBlockBuilder(estimatedSize, blockFactory); + sumBuilder = new DoubleBlockBuilder(estimatedSize, blockFactory); + countBuilder = new IntBlockBuilder(estimatedSize, blockFactory); + } finally { + if (countBuilder == null) { + Releasables.closeWhileHandlingException(minBuilder); + Releasables.closeWhileHandlingException(maxBuilder); + Releasables.closeWhileHandlingException(sumBuilder); + Releasables.closeWhileHandlingException(countBuilder); + } + } } @Override @@ -45,10 +60,10 @@ protected int elementSize() { @Override public Block.Builder copyFrom(Block block, int beginInclusive, int endExclusive) { CompositeBlock composite = (CompositeBlock) block; - minBuilder.copyFrom(composite.getBlock(Metric.MIN.ordinal()), beginInclusive, endExclusive); - maxBuilder.copyFrom(composite.getBlock(Metric.MAX.ordinal()), beginInclusive, endExclusive); - sumBuilder.copyFrom(composite.getBlock(Metric.SUM.ordinal()), beginInclusive, endExclusive); - countBuilder.copyFrom(composite.getBlock(Metric.COUNT.ordinal()), beginInclusive, endExclusive); + minBuilder.copyFrom(composite.getBlock(Metric.MIN.getIndex()), beginInclusive, endExclusive); + maxBuilder.copyFrom(composite.getBlock(Metric.MAX.getIndex()), beginInclusive, endExclusive); + sumBuilder.copyFrom(composite.getBlock(Metric.SUM.getIndex()), beginInclusive, endExclusive); + countBuilder.copyFrom(composite.getBlock(Metric.COUNT.getIndex()), beginInclusive, endExclusive); return this; } @@ -64,11 +79,19 @@ public Block.Builder mvOrdering(Block.MvOrdering mvOrdering) { @Override public Block build() { Block[] blocks = new Block[4]; - blocks[Metric.MIN.ordinal()] = minBuilder.build(); - blocks[Metric.MAX.ordinal()] = maxBuilder.build(); - blocks[Metric.SUM.ordinal()] = sumBuilder.build(); - blocks[Metric.COUNT.ordinal()] = countBuilder.build(); - return new CompositeBlock(blocks); + try { + finish(); + blocks[Metric.MIN.getIndex()] = minBuilder.build(); + blocks[Metric.MAX.getIndex()] = maxBuilder.build(); + blocks[Metric.SUM.getIndex()] = sumBuilder.build(); + blocks[Metric.COUNT.getIndex()] = countBuilder.build(); + return new CompositeBlock(blocks); + } catch (CircuitBreakingException e) { + for (Block block : blocks) { + block.close(); + } + throw e; + } } @Override @@ -81,9 +104,19 @@ public BlockLoader.AggregateDoubleMetricBuilder append(double min, double max, d } public enum Metric { - MIN, - MAX, - SUM, - COUNT; + MIN(0), + MAX(1), + SUM(2), + COUNT(3); + + private final int index; + + Metric(int index) { + this.index = index; + } + + public int getIndex() { + return index; + } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java index 3836680e95751..acbb1b81ae3ce 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java @@ -86,29 +86,21 @@ public int getPositionCount() { @Override public int getTotalValueCount() { - // TODO: this works for aggregate metric double fields, because the four blocks are guarenteed to have the same length / positions. - // So check just for one sub block is sufficient. return blocks[0].getTotalValueCount(); } @Override public int getFirstValueIndex(int position) { - // TODO: this works for aggregate metric double fields, because the four blocks are guarenteed to have the same length / positions. - // So check just for one sub block is sufficient. return blocks[0].getFirstValueIndex(position); } @Override public int getValueCount(int position) { - // TODO: this works for aggregate metric double fields, because the four blocks are guarenteed to have the same length / positions. - // So check just for one sub block is sufficient. return blocks[0].getValueCount(position); } @Override public boolean isNull(int position) { - // TODO: this works for aggregate metric double fields, because the four blocks are guarenteed to have the same length / positions. - // So check just for one sub block is sufficient. return blocks[0].isNull(position); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index ab3d81d9126d7..d536e22a3b842 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -35,7 +35,6 @@ import static java.util.Collections.emptyList; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; public class Count extends AggregateFunction implements ToAggregator, SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Count", Count::new); @@ -146,14 +145,7 @@ public Expression surrogate() { var s = source(); var field = field(); if (field.dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { - return new Sum( - s, - new FromAggregateDoubleMetric( - source(), - field, - new Literal(s, AggregateDoubleMetricBlockBuilder.Metric.COUNT.ordinal(), INTEGER) - ) - ); + return new Sum(s, new FromAggregateDoubleMetric(source(), field, AggregateDoubleMetricBlockBuilder.Metric.COUNT)); } if (field.foldable()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index 401ebf77cfaff..50b885eb0dac1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -153,14 +153,7 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { @Override public Expression surrogate() { if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { - return new Max( - source(), - new FromAggregateDoubleMetric( - source(), - field(), - new Literal(source(), AggregateDoubleMetricBlockBuilder.Metric.MAX.ordinal(), DataType.INTEGER) - ) - ); + return new Max(source(), new FromAggregateDoubleMetric(source(), field(), AggregateDoubleMetricBlockBuilder.Metric.MAX)); } return field().foldable() ? new MvMax(source(), field()) : null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index c900b2c2aa995..7296b18552082 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -153,14 +153,7 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { @Override public Expression surrogate() { if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { - return new Min( - source(), - new FromAggregateDoubleMetric( - source(), - field(), - new Literal(source(), AggregateDoubleMetricBlockBuilder.Metric.MIN.ordinal(), DataType.INTEGER) - ) - ); + return new Min(source(), new FromAggregateDoubleMetric(source(), field(), AggregateDoubleMetricBlockBuilder.Metric.MIN)); } return field().foldable() ? new MvMin(source(), field()) : null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java index 4fec0af4ac32d..4dce5caa9da22 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java @@ -36,7 +36,6 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; -import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; @@ -139,14 +138,7 @@ public Expression surrogate() { var s = source(); var field = field(); if (field.dataType() == AGGREGATE_METRIC_DOUBLE) { - return new Sum( - s, - new FromAggregateDoubleMetric( - source(), - field, - new Literal(s, AggregateDoubleMetricBlockBuilder.Metric.SUM.ordinal(), INTEGER) - ) - ); + return new Sum(s, new FromAggregateDoubleMetric(source(), field, AggregateDoubleMetricBlockBuilder.Metric.SUM)); } // SUM(const) is equivalent to MV_SUM(const)*COUNT(*). diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java index bd12ccab14510..7e30efc3ef8e7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -61,6 +62,10 @@ public FromAggregateDoubleMetric( this.subfieldIndex = subfieldIndex; } + public FromAggregateDoubleMetric(Source source, Expression field, AggregateDoubleMetricBlockBuilder.Metric metric) { + this(source, field, new Literal(source, metric.getIndex(), INTEGER)); + } + private FromAggregateDoubleMetric(StreamInput in) throws IOException { this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); } @@ -84,7 +89,7 @@ public DataType dataType() { } var subfield = ((Number) subfieldIndex.fold(FoldContext.small())).intValue(); - if (subfield == AggregateDoubleMetricBlockBuilder.Metric.COUNT.ordinal()) { + if (subfield == AggregateDoubleMetricBlockBuilder.Metric.COUNT.getIndex()) { return INTEGER; } return DOUBLE; @@ -121,13 +126,10 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua return new EvalOperator.ExpressionEvaluator() { @Override public Block eval(Page page) { - CompositeBlock compositeBlock = (CompositeBlock) eval.eval(page); - try { + try (CompositeBlock compositeBlock = (CompositeBlock) eval.eval(page)) { Block block = compositeBlock.getBlock(((Number) subfieldIndex.fold(FoldContext.small())).intValue()); block.incRef(); return block; - } finally { - compositeBlock.close(); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index 5449aeea2adcb..fcb5b7166accf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -213,7 +213,7 @@ private static Stream groupingAndNonGrouping(Tuple, Tuple Date: Fri, 24 Jan 2025 19:56:34 -1000 Subject: [PATCH 03/18] FromAggregateDoubleMetricTests --- .../index/mapper/BlockLoader.java | 2 +- x-pack/plugin/build.gradle | 5 ++ .../AggregateDoubleMetricBlockBuilder.java | 61 ++++++++++++--- .../data/AggregateMetricDoubleLiteral.java | 38 ++++++++++ .../compute/data/BlockUtils.java | 11 +++ .../compute/data/ElementType.java | 4 +- .../expression/function/aggregate/Count.java | 2 +- .../expression/function/aggregate/Max.java | 2 +- .../expression/function/aggregate/Min.java | 2 +- .../expression/function/aggregate/Sum.java | 2 +- .../convert/FromAggregateDoubleMetric.java | 63 ++++++++++----- .../function/scalar/nulls/Coalesce.java | 4 +- .../FromAggregateDoubleMetricTests.java | 76 +++++++++++++++++++ 13 files changed, 236 insertions(+), 36 deletions(-) create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index ed40aab44cb5f..69bca6d0df55d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -506,7 +506,7 @@ interface SingletonOrdinalsBuilder extends Builder { interface AggregateDoubleMetricBuilder extends Builder { - AggregateDoubleMetricBuilder append(double min, double max, double sum, int valueCount); + AggregateDoubleMetricBuilder append(Double min, Double max, Double sum, int valueCount); } } diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 5ab0112d822ce..c03f7afbb7d75 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -96,5 +96,10 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields") task.skipTest("esql/180_match_operator/match with functions", "Error message changed") task.skipTest("esql/40_unsupported_types/semantic_text declared in mapping", "The semantic text field format changed") + task.skipTest("esql/40_tsdb/from doc with aggregate_metric_double", "TODO: support for subset of metric fields") + task.skipTest("esql/40_tsdb/stats on aggregate_metric_double", "TODO: support for subset of metric fields") + task.skipTest("esql/40_tsdb/from index pattern unsupported counter", "TODO: support for subset of metric fields") + task.skipTest("esql/40_unsupported_types/unsupported", "TODO: support for subset of metric fields") + task.skipTest("esql/40_unsupported_types/unsupported with sort", "TODO: support for subset of metric fields") }) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java index f22322033239b..ea52424b21df2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java @@ -59,11 +59,35 @@ protected int elementSize() { @Override public Block.Builder copyFrom(Block block, int beginInclusive, int endExclusive) { - CompositeBlock composite = (CompositeBlock) block; - minBuilder.copyFrom(composite.getBlock(Metric.MIN.getIndex()), beginInclusive, endExclusive); - maxBuilder.copyFrom(composite.getBlock(Metric.MAX.getIndex()), beginInclusive, endExclusive); - sumBuilder.copyFrom(composite.getBlock(Metric.SUM.getIndex()), beginInclusive, endExclusive); - countBuilder.copyFrom(composite.getBlock(Metric.COUNT.getIndex()), beginInclusive, endExclusive); + Block minBlock; + Block maxBlock; + Block sumBlock; + Block countBlock; + if (block.areAllValuesNull()) { + minBlock = block; + maxBlock = block; + sumBlock = block; + countBlock = block; + } else { + CompositeBlock composite = (CompositeBlock) block; + minBlock = composite.getBlock(Metric.MIN.getIndex()); + maxBlock = composite.getBlock(Metric.MAX.getIndex()); + sumBlock = composite.getBlock(Metric.SUM.getIndex()); + countBlock = composite.getBlock(Metric.COUNT.getIndex()); + } + minBuilder.copyFrom(minBlock, beginInclusive, endExclusive); + maxBuilder.copyFrom(maxBlock, beginInclusive, endExclusive); + sumBuilder.copyFrom(sumBlock, beginInclusive, endExclusive); + countBuilder.copyFrom(countBlock, beginInclusive, endExclusive); + return this; + } + + @Override + public AbstractBlockBuilder appendNull() { + minBuilder.appendNull(); + maxBuilder.appendNull(); + sumBuilder.appendNull(); + countBuilder.appendNull(); return this; } @@ -88,17 +112,34 @@ public Block build() { return new CompositeBlock(blocks); } catch (CircuitBreakingException e) { for (Block block : blocks) { - block.close(); + Releasables.closeExpectNoException(block); } throw e; } } @Override - public BlockLoader.AggregateDoubleMetricBuilder append(double min, double max, double sum, int valueCount) { - minBuilder.appendDouble(min); - maxBuilder.appendDouble(max); - sumBuilder.appendDouble(sum); + protected void extraClose() { + Releasables.closeExpectNoException(minBuilder, maxBuilder, sumBuilder, countBuilder); + } + + @Override + public BlockLoader.AggregateDoubleMetricBuilder append(Double min, Double max, Double sum, int valueCount) { + if (min == null) { + minBuilder.appendNull(); + } else { + minBuilder.appendDouble(min); + } + if (max == null) { + maxBuilder.appendNull(); + } else { + maxBuilder.appendDouble(max); + } + if (sum == null) { + sumBuilder.appendNull(); + } else { + sumBuilder.appendDouble(sum); + } countBuilder.appendInt(valueCount); return this; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java new file mode 100644 index 0000000000000..4cc7e7594d30d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +public class AggregateMetricDoubleLiteral { + private final Double min; + private final Double max; + private final Double sum; + private final Integer count; + + public AggregateMetricDoubleLiteral(Double min, Double max, Double sum, Integer count) { + this.min = min.isNaN() ? null : min; + this.max = max.isNaN() ? null : max; + this.sum = sum.isNaN() ? null : sum; + this.count = count; + } + + public Double getMax() { + return max; + } + + public Double getMin() { + return min; + } + + public Double getSum() { + return sum; + } + + public Integer getCount() { + return count; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index 3df389135e9d3..69d82272b70b2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -233,6 +233,17 @@ private static Block constantBlock(BlockFactory blockFactory, ElementType type, case BYTES_REF -> blockFactory.newConstantBytesRefBlockWith(toBytesRef(val), size); case DOUBLE -> blockFactory.newConstantDoubleBlockWith((double) val, size); case BOOLEAN -> blockFactory.newConstantBooleanBlockWith((boolean) val, size); + case COMPOSITE -> { + var builder = blockFactory.newAggregatedDoubleMetricBlockBuilder(size); + var aggregate_metric_double = (AggregateMetricDoubleLiteral) val; + builder.append( + aggregate_metric_double.getMin(), + aggregate_metric_double.getMax(), + aggregate_metric_double.getSum(), + aggregate_metric_double.getCount() + ); + yield builder.build(); + } default -> throw new UnsupportedOperationException("unsupported element type [" + type + "]"); }; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java index f38c6d70991f9..6a210d57194ca 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java @@ -33,7 +33,7 @@ public enum ElementType { /** * Composite blocks which contain array of sub-blocks. */ - COMPOSITE("Composite", (blockFactory, estimatedSize) -> { throw new UnsupportedOperationException("can't build composite blocks"); }), + COMPOSITE("Composite", BlockFactory::newAggregatedDoubleMetricBlockBuilder), /** * Intermediate blocks which don't support retrieving elements. @@ -73,6 +73,8 @@ public static ElementType fromJava(Class type) { elementType = BYTES_REF; } else if (type == Boolean.class) { elementType = BOOLEAN; + } else if (type == AggregateMetricDoubleLiteral.class) { + elementType = COMPOSITE; } else if (type == null || type == Void.class) { elementType = NULL; } else { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index d536e22a3b842..13ffe993921f6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -145,7 +145,7 @@ public Expression surrogate() { var s = source(); var field = field(); if (field.dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { - return new Sum(s, new FromAggregateDoubleMetric(source(), field, AggregateDoubleMetricBlockBuilder.Metric.COUNT)); + return new Sum(s, FromAggregateDoubleMetric.withMetric(source(), field, AggregateDoubleMetricBlockBuilder.Metric.COUNT)); } if (field.foldable()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index 50b885eb0dac1..a858b654c3211 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -153,7 +153,7 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { @Override public Expression surrogate() { if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { - return new Max(source(), new FromAggregateDoubleMetric(source(), field(), AggregateDoubleMetricBlockBuilder.Metric.MAX)); + return new Max(source(), FromAggregateDoubleMetric.withMetric(source(), field(), AggregateDoubleMetricBlockBuilder.Metric.MAX)); } return field().foldable() ? new MvMax(source(), field()) : null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index 7296b18552082..b23cbf85222d1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -153,7 +153,7 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { @Override public Expression surrogate() { if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { - return new Min(source(), new FromAggregateDoubleMetric(source(), field(), AggregateDoubleMetricBlockBuilder.Metric.MIN)); + return new Min(source(), FromAggregateDoubleMetric.withMetric(source(), field(), AggregateDoubleMetricBlockBuilder.Metric.MIN)); } return field().foldable() ? new MvMin(source(), field()) : null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java index 4dce5caa9da22..28da31fcb4140 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java @@ -138,7 +138,7 @@ public Expression surrogate() { var s = source(); var field = field(); if (field.dataType() == AGGREGATE_METRIC_DOUBLE) { - return new Sum(s, new FromAggregateDoubleMetric(source(), field, AggregateDoubleMetricBlockBuilder.Metric.SUM)); + return new Sum(s, FromAggregateDoubleMetric.withMetric(source(), field, AggregateDoubleMetricBlockBuilder.Metric.SUM)); } // SUM(const) is equivalent to MV_SUM(const)*COUNT(*). diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java index 7e30efc3ef8e7..e8bebcaec8855 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.CompositeBlock; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -36,6 +37,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; +import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; public class FromAggregateDoubleMetric extends EsqlScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -62,8 +64,8 @@ public FromAggregateDoubleMetric( this.subfieldIndex = subfieldIndex; } - public FromAggregateDoubleMetric(Source source, Expression field, AggregateDoubleMetricBlockBuilder.Metric metric) { - this(source, field, new Literal(source, metric.getIndex(), INTEGER)); + public static FromAggregateDoubleMetric withMetric(Source source, Expression field, AggregateDoubleMetricBlockBuilder.Metric metric) { + return new FromAggregateDoubleMetric(source, field, new Literal(source, metric.getIndex(), INTEGER)); } private FromAggregateDoubleMetric(StreamInput in) throws IOException { @@ -87,8 +89,11 @@ public DataType dataType() { if (subfieldIndex.foldable() == false) { throw new EsqlIllegalArgumentException("Received a non-foldable value for subfield index"); } - - var subfield = ((Number) subfieldIndex.fold(FoldContext.small())).intValue(); + var folded = subfieldIndex.fold(FoldContext.small()); + if (folded == null) { + return NULL; + } + var subfield = ((Number) folded).intValue(); if (subfield == AggregateDoubleMetricBlockBuilder.Metric.COUNT.getIndex()) { return INTEGER; } @@ -121,23 +126,43 @@ public boolean foldable() { @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(field); - return driverContext -> { - final EvalOperator.ExpressionEvaluator eval = fieldEvaluator.get(driverContext); - return new EvalOperator.ExpressionEvaluator() { - @Override - public Block eval(Page page) { - try (CompositeBlock compositeBlock = (CompositeBlock) eval.eval(page)) { - Block block = compositeBlock.getBlock(((Number) subfieldIndex.fold(FoldContext.small())).intValue()); - block.incRef(); - return block; + return new EvalOperator.ExpressionEvaluator.Factory() { + + @Override + public String toString() { + return "FromAggregateDoubleMetricEvaluator[" + "field=" + fieldEvaluator + ",subfieldIndex=" + subfieldIndex + "]"; + } + + @Override + public EvalOperator.ExpressionEvaluator get(DriverContext context) { + final EvalOperator.ExpressionEvaluator eval = fieldEvaluator.get(context); + + return new EvalOperator.ExpressionEvaluator() { + @Override + public Block eval(Page page) { + Block block = eval.eval(page); + if (block.areAllValuesNull()) { + return block; + } + try (CompositeBlock compositeBlock = (CompositeBlock) block) { + Block resultBlock = compositeBlock.getBlock(((Number) subfieldIndex.fold(FoldContext.small())).intValue()); + resultBlock.incRef(); + return resultBlock; + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(eval); + } + + @Override + public String toString() { + return "FromAggregateDoubleMetricEvaluator[field=" + eval + ",subfieldIndex=" + subfieldIndex + "]"; } - } + }; - @Override - public void close() { - Releasables.closeExpectNoException(eval); - } - }; + } }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java index 611c7a456864a..a426a14b0a319 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java @@ -210,7 +210,9 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { CoalesceBytesRefEvaluator.toEvaluator(toEvaluator, children()); case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; case UNSUPPORTED, SHORT, BYTE, DATE_PERIOD, OBJECT, DOC_DATA_TYPE, SOURCE, TIME_DURATION, FLOAT, HALF_FLOAT, TSID_DATA_TYPE, - SCALED_FLOAT, PARTIAL_AGG -> throw new UnsupportedOperationException(dataType() + " can't be coalesced"); + SCALED_FLOAT, PARTIAL_AGG, AGGREGATE_METRIC_DOUBLE -> throw new UnsupportedOperationException( + dataType() + " can't be coalesced" + ); }; } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java new file mode 100644 index 0000000000000..1c1138269dcce --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.compute.data.AggregateMetricDoubleLiteral; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; +import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matchers; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +@FunctionName("from_aggregate_double_metric") +public class FromAggregateDoubleMetricTests extends AbstractScalarFunctionTestCase { + public FromAggregateDoubleMetricTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @Override + protected Expression build(Source source, List args) { + assumeTrue("Test sometimes wraps literals as fields", args.get(1).foldable()); + return new FromAggregateDoubleMetric(source, args.get(0), args.get(1)); + } + + @ParametersFactory + public static Iterable parameters() { + List suppliers = new ArrayList<>(); + DataType dataType = DataType.AGGREGATE_METRIC_DOUBLE; + suppliers.add(new TestCaseSupplier(List.of(dataType, DataType.INTEGER), () -> { + var agg_metric = new AggregateMetricDoubleLiteral( + randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), + randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), + randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), + randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE) + ); + int index = randomIntBetween(0, 3); + Double expectedValue = index == 0 ? agg_metric.getMin() + : index == 1 ? agg_metric.getMax() + : index == 2 ? agg_metric.getSum() + : (Double) agg_metric.getCount().doubleValue(); + + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(agg_metric, dataType, "agg_metric"), + new TestCaseSupplier.TypedData(index, DataType.INTEGER, "subfield_index").forceLiteral() + ), + "FromAggregateDoubleMetricEvaluator[field=Attribute[channel=0],subfieldIndex=" + index + "]", + index == 3 ? DataType.INTEGER : DataType.DOUBLE, + index == 3 ? Matchers.equalTo(agg_metric.getCount()) + : expectedValue == null ? Matchers.nullValue() + : Matchers.closeTo(expectedValue, Math.abs(expectedValue * 0.00001)) + ); + })); + + return parameterSuppliersFromTypedData( + anyNullIsNull( + suppliers, + (nullPosition, nullValueDataType, original) -> nullPosition == 1 ? DataType.NULL : original.expectedType(), + (nullPosition, nullData, original) -> nullData.isForceLiteral() ? Matchers.equalTo("LiteralsEvaluator[lit=null]") : original + ) + ); + } +} From b3b27aa6d9b44ce8c63f8e9d3e8b8fd242478fe4 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Mon, 27 Jan 2025 13:14:38 -1000 Subject: [PATCH 04/18] bring back unsupported exceptions --- .../AggregateDoubleMetricBlockBuilder.java | 27 ++++++++----------- .../compute/data/CompositeBlock.java | 13 ++++++--- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java index ea52424b21df2..0d62de6e88dab 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java @@ -7,7 +7,6 @@ package org.elasticsearch.compute.data; -import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.core.Releasables; import org.elasticsearch.index.mapper.BlockLoader; @@ -31,30 +30,24 @@ public AggregateDoubleMetricBlockBuilder(int estimatedSize, BlockFactory blockFa countBuilder = new IntBlockBuilder(estimatedSize, blockFactory); } finally { if (countBuilder == null) { - Releasables.closeWhileHandlingException(minBuilder); - Releasables.closeWhileHandlingException(maxBuilder); - Releasables.closeWhileHandlingException(sumBuilder); - Releasables.closeWhileHandlingException(countBuilder); + Releasables.closeWhileHandlingException(minBuilder, maxBuilder, sumBuilder, countBuilder); } } } @Override protected int valuesLength() { - return minBuilder.valuesLength(); + throw new UnsupportedOperationException("Not available on aggregate_metric_double"); } @Override protected void growValuesArray(int newSize) { - minBuilder.growValuesArray(newSize); - maxBuilder.growValuesArray(newSize); - sumBuilder.growValuesArray(newSize); - countBuilder.growValuesArray(newSize); + throw new UnsupportedOperationException("Not available on aggregate_metric_double"); } @Override protected int elementSize() { - return minBuilder.elementSize() + maxBuilder.elementSize() + sumBuilder.elementSize() + countBuilder.elementSize(); + throw new UnsupportedOperationException("Not available on aggregate_metric_double"); } @Override @@ -103,18 +96,20 @@ public Block.Builder mvOrdering(Block.MvOrdering mvOrdering) { @Override public Block build() { Block[] blocks = new Block[4]; + boolean success = false; try { finish(); blocks[Metric.MIN.getIndex()] = minBuilder.build(); blocks[Metric.MAX.getIndex()] = maxBuilder.build(); blocks[Metric.SUM.getIndex()] = sumBuilder.build(); blocks[Metric.COUNT.getIndex()] = countBuilder.build(); - return new CompositeBlock(blocks); - } catch (CircuitBreakingException e) { - for (Block block : blocks) { - Releasables.closeExpectNoException(block); + CompositeBlock block = new CompositeBlock(blocks); + success = true; + return block; + } finally { + if (success == false) { + Releasables.closeExpectNoException(blocks); } - throw e; } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java index acbb1b81ae3ce..4b9c8dbf99011 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java @@ -86,22 +86,27 @@ public int getPositionCount() { @Override public int getTotalValueCount() { - return blocks[0].getTotalValueCount(); + throw new UnsupportedOperationException("Composite block"); } @Override public int getFirstValueIndex(int position) { - return blocks[0].getFirstValueIndex(position); + throw new UnsupportedOperationException("Composite block"); } @Override public int getValueCount(int position) { - return blocks[0].getValueCount(position); + throw new UnsupportedOperationException("Composite block"); } @Override public boolean isNull(int position) { - return blocks[0].isNull(position); + for (Block block : blocks) { + if (block.isNull(position) == false) { + return false; + } + } + return true; } @Override From 71a05b24014041a6ecfd9b4eeb7a37a6865de191 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Mon, 27 Jan 2025 14:16:51 -1000 Subject: [PATCH 05/18] change valueCount to Integer and revert other getValueCount() --- .../java/org/elasticsearch/index/mapper/BlockLoader.java | 2 +- .../compute/data/AggregateDoubleMetricBlockBuilder.java | 8 ++++++-- .../org/elasticsearch/compute/data/CompositeBlock.java | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index 69bca6d0df55d..921eeb61864cb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -506,7 +506,7 @@ interface SingletonOrdinalsBuilder extends Builder { interface AggregateDoubleMetricBuilder extends Builder { - AggregateDoubleMetricBuilder append(Double min, Double max, Double sum, int valueCount); + AggregateDoubleMetricBuilder append(Double min, Double max, Double sum, Integer valueCount); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java index 0d62de6e88dab..ebb9b89953ef2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java @@ -119,7 +119,7 @@ protected void extraClose() { } @Override - public BlockLoader.AggregateDoubleMetricBuilder append(Double min, Double max, Double sum, int valueCount) { + public BlockLoader.AggregateDoubleMetricBuilder append(Double min, Double max, Double sum, Integer valueCount) { if (min == null) { minBuilder.appendNull(); } else { @@ -135,7 +135,11 @@ public BlockLoader.AggregateDoubleMetricBuilder append(Double min, Double max, D } else { sumBuilder.appendDouble(sum); } - countBuilder.appendInt(valueCount); + if (valueCount == null) { + countBuilder.appendNull(); + } else { + countBuilder.appendInt(valueCount); + } return this; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java index 4b9c8dbf99011..3c4658fa2ba00 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java @@ -96,7 +96,7 @@ public int getFirstValueIndex(int position) { @Override public int getValueCount(int position) { - throw new UnsupportedOperationException("Composite block"); + return blocks[0].getValueCount(position); } @Override From 648e0af1f37373516aff088bcac05e95c345d2c8 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Mon, 27 Jan 2025 15:17:11 -1000 Subject: [PATCH 06/18] and revert getFirstValueIndex() --- .../java/org/elasticsearch/compute/data/CompositeBlock.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java index 3c4658fa2ba00..6dfe4c9229e76 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/CompositeBlock.java @@ -91,7 +91,7 @@ public int getTotalValueCount() { @Override public int getFirstValueIndex(int position) { - throw new UnsupportedOperationException("Composite block"); + return blocks[0].getFirstValueIndex(position); } @Override From 83612ec78220bf85731e42ec6eca56efc5020455 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Mon, 27 Jan 2025 17:28:59 -1000 Subject: [PATCH 07/18] test all metrics in aggregate double metric test --- .../FromAggregateDoubleMetricTests.java | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java index 1c1138269dcce..700fc7e293e77 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java @@ -10,6 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; import org.elasticsearch.compute.data.AggregateMetricDoubleLiteral; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -39,31 +40,33 @@ protected Expression build(Source source, List args) { public static Iterable parameters() { List suppliers = new ArrayList<>(); DataType dataType = DataType.AGGREGATE_METRIC_DOUBLE; - suppliers.add(new TestCaseSupplier(List.of(dataType, DataType.INTEGER), () -> { - var agg_metric = new AggregateMetricDoubleLiteral( - randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), - randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), - randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), - randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE) - ); - int index = randomIntBetween(0, 3); - Double expectedValue = index == 0 ? agg_metric.getMin() - : index == 1 ? agg_metric.getMax() - : index == 2 ? agg_metric.getSum() - : (Double) agg_metric.getCount().doubleValue(); + for (int i = 0; i < 4; i++) { + int index = i; + suppliers.add(new TestCaseSupplier(List.of(dataType, DataType.INTEGER), () -> { + var agg_metric = new AggregateMetricDoubleLiteral( + randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), + randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), + randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), + randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE) + ); + Double expectedValue = index == AggregateDoubleMetricBlockBuilder.Metric.MIN.getIndex() ? agg_metric.getMin() + : index == AggregateDoubleMetricBlockBuilder.Metric.MAX.getIndex() ? agg_metric.getMax() + : index == AggregateDoubleMetricBlockBuilder.Metric.SUM.getIndex() ? agg_metric.getSum() + : (Double) agg_metric.getCount().doubleValue(); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(agg_metric, dataType, "agg_metric"), - new TestCaseSupplier.TypedData(index, DataType.INTEGER, "subfield_index").forceLiteral() - ), - "FromAggregateDoubleMetricEvaluator[field=Attribute[channel=0],subfieldIndex=" + index + "]", - index == 3 ? DataType.INTEGER : DataType.DOUBLE, - index == 3 ? Matchers.equalTo(agg_metric.getCount()) - : expectedValue == null ? Matchers.nullValue() - : Matchers.closeTo(expectedValue, Math.abs(expectedValue * 0.00001)) - ); - })); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(agg_metric, dataType, "agg_metric"), + new TestCaseSupplier.TypedData(index, DataType.INTEGER, "subfield_index").forceLiteral() + ), + "FromAggregateDoubleMetricEvaluator[field=Attribute[channel=0],subfieldIndex=" + index + "]", + index == AggregateDoubleMetricBlockBuilder.Metric.COUNT.getIndex() ? DataType.INTEGER : DataType.DOUBLE, + index == AggregateDoubleMetricBlockBuilder.Metric.COUNT.getIndex() ? Matchers.equalTo(agg_metric.getCount()) + : expectedValue == null ? Matchers.nullValue() + : Matchers.closeTo(expectedValue, Math.abs(expectedValue * 0.00001)) + ); + })); + } return parameterSuppliersFromTypedData( anyNullIsNull( From 38659b94b1ecf0a6b5ab66df76cc2431cb4c8327 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Mon, 27 Jan 2025 18:06:29 -1000 Subject: [PATCH 08/18] refactor things touched in this PR to all be aggregate metric double (in that order) --- .../index/mapper/BlockLoader.java | 6 ++--- .../elasticsearch/index/mapper/TestBlock.java | 2 +- ...=> AggregateMetricDoubleBlockBuilder.java} | 6 ++--- .../compute/data/BlockFactory.java | 4 ++-- .../compute/data/BlockUtils.java | 2 +- .../compute/data/ElementType.java | 2 +- .../lucene/ValuesSourceReaderOperator.java | 4 ++-- .../expression/function/aggregate/Count.java | 6 ++--- .../expression/function/aggregate/Max.java | 6 ++--- .../expression/function/aggregate/Min.java | 6 ++--- .../expression/function/aggregate/Sum.java | 6 ++--- .../scalar/ScalarFunctionWritables.java | 4 ++-- ...ic.java => FromAggregateMetricDouble.java} | 20 ++++++++-------- .../xpack/esql/planner/AggregateMapper.java | 4 ++-- .../esql/action/EsqlQueryResponseTests.java | 4 ++-- ...va => FromAggregateMetricDoubleTests.java} | 24 +++++++++---------- .../AggregateDoubleMetricFieldMapper.java | 14 +++++------ 17 files changed, 60 insertions(+), 60 deletions(-) rename x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/{AggregateDoubleMetricBlockBuilder.java => AggregateMetricDoubleBlockBuilder.java} (95%) rename x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/{FromAggregateDoubleMetric.java => FromAggregateMetricDouble.java} (90%) rename x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/{FromAggregateDoubleMetricTests.java => FromAggregateMetricDoubleTests.java} (69%) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index 921eeb61864cb..a6ed5261611ac 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -418,7 +418,7 @@ interface BlockFactory { // TODO support non-singleton ords - AggregateDoubleMetricBuilder aggregateDoubleMetricBuilder(int count); + AggregateMetricDoubleBuilder aggregateMetricDoubleBuilder(int count); } /** @@ -504,9 +504,9 @@ interface SingletonOrdinalsBuilder extends Builder { SingletonOrdinalsBuilder appendOrd(int value); } - interface AggregateDoubleMetricBuilder extends Builder { + interface AggregateMetricDoubleBuilder extends Builder { - AggregateDoubleMetricBuilder append(Double min, Double max, Double sum, Integer valueCount); + AggregateMetricDoubleBuilder append(Double min, Double max, Double sum, Integer valueCount); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java index 8fca80d459de4..14beb979b96cf 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java @@ -149,7 +149,7 @@ public SingletonOrdsBuilder appendOrd(int value) { } @Override - public BlockLoader.AggregateDoubleMetricBuilder aggregateDoubleMetricBuilder(int count) { + public BlockLoader.AggregateMetricDoubleBuilder aggregateMetricDoubleBuilder(int count) { return null; } }; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java similarity index 95% rename from x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java rename to x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java index ebb9b89953ef2..1586c836f5e63 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateDoubleMetricBlockBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java @@ -10,14 +10,14 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.index.mapper.BlockLoader; -public class AggregateDoubleMetricBlockBuilder extends AbstractBlockBuilder implements BlockLoader.AggregateDoubleMetricBuilder { +public class AggregateMetricDoubleBlockBuilder extends AbstractBlockBuilder implements BlockLoader.AggregateMetricDoubleBuilder { private DoubleBlockBuilder minBuilder; private DoubleBlockBuilder maxBuilder; private DoubleBlockBuilder sumBuilder; private IntBlockBuilder countBuilder; - public AggregateDoubleMetricBlockBuilder(int estimatedSize, BlockFactory blockFactory) { + public AggregateMetricDoubleBlockBuilder(int estimatedSize, BlockFactory blockFactory) { super(blockFactory); minBuilder = null; maxBuilder = null; @@ -119,7 +119,7 @@ protected void extraClose() { } @Override - public BlockLoader.AggregateDoubleMetricBuilder append(Double min, Double max, Double sum, Integer valueCount) { + public BlockLoader.AggregateMetricDoubleBuilder append(Double min, Double max, Double sum, Integer valueCount) { if (min == null) { minBuilder.appendNull(); } else { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java index 8cbdb71e89a44..c4f0be7533a00 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java @@ -432,8 +432,8 @@ public Block newConstantNullBlock(int positions) { return b; } - public AggregateDoubleMetricBlockBuilder newAggregatedDoubleMetricBlockBuilder(int estimatedSize) { - return new AggregateDoubleMetricBlockBuilder(estimatedSize, this); + public AggregateMetricDoubleBlockBuilder newAggregateMetricDoubleBlockBuilder(int estimatedSize) { + return new AggregateMetricDoubleBlockBuilder(estimatedSize, this); } /** diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index 69d82272b70b2..99518614b8888 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -234,7 +234,7 @@ private static Block constantBlock(BlockFactory blockFactory, ElementType type, case DOUBLE -> blockFactory.newConstantDoubleBlockWith((double) val, size); case BOOLEAN -> blockFactory.newConstantBooleanBlockWith((boolean) val, size); case COMPOSITE -> { - var builder = blockFactory.newAggregatedDoubleMetricBlockBuilder(size); + var builder = blockFactory.newAggregateMetricDoubleBlockBuilder(size); var aggregate_metric_double = (AggregateMetricDoubleLiteral) val; builder.append( aggregate_metric_double.getMin(), diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java index 6a210d57194ca..dc11cdd107464 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java @@ -33,7 +33,7 @@ public enum ElementType { /** * Composite blocks which contain array of sub-blocks. */ - COMPOSITE("Composite", BlockFactory::newAggregatedDoubleMetricBlockBuilder), + COMPOSITE("Composite", BlockFactory::newAggregateMetricDoubleBlockBuilder), /** * Intermediate blocks which don't support retrieving elements. diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java index 7c3bde315a23c..841789e8ada3c 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java @@ -700,8 +700,8 @@ public BlockLoader.SingletonOrdinalsBuilder singletonOrdinalsBuilder(SortedDocVa } @Override - public BlockLoader.AggregateDoubleMetricBuilder aggregateDoubleMetricBuilder(int count) { - return factory.newAggregatedDoubleMetricBlockBuilder(count); + public BlockLoader.AggregateMetricDoubleBuilder aggregateMetricDoubleBuilder(int count) { + return factory.newAggregateMetricDoubleBlockBuilder(count); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index 13ffe993921f6..5ce43c7b3872d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -11,7 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.CountAggregatorFunction; -import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; +import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; @@ -23,7 +23,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; -import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateDoubleMetric; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateMetricDouble; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul; @@ -145,7 +145,7 @@ public Expression surrogate() { var s = source(); var field = field(); if (field.dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { - return new Sum(s, FromAggregateDoubleMetric.withMetric(source(), field, AggregateDoubleMetricBlockBuilder.Metric.COUNT)); + return new Sum(s, FromAggregateMetricDouble.withMetric(source(), field, AggregateMetricDoubleBlockBuilder.Metric.COUNT)); } if (field.foldable()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index a858b654c3211..6a8ce792ec8c1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -16,7 +16,7 @@ import org.elasticsearch.compute.aggregation.MaxIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxIpAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MaxLongAggregatorFunctionSupplier; -import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; +import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -28,7 +28,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; -import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateDoubleMetric; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateMetricDouble; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMax; import org.elasticsearch.xpack.esql.planner.ToAggregator; @@ -153,7 +153,7 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { @Override public Expression surrogate() { if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { - return new Max(source(), FromAggregateDoubleMetric.withMetric(source(), field(), AggregateDoubleMetricBlockBuilder.Metric.MAX)); + return new Max(source(), FromAggregateMetricDouble.withMetric(source(), field(), AggregateMetricDoubleBlockBuilder.Metric.MAX)); } return field().foldable() ? new MvMax(source(), field()) : null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index b23cbf85222d1..f2ae1292e47e8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -16,7 +16,7 @@ import org.elasticsearch.compute.aggregation.MinIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinIpAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.MinLongAggregatorFunctionSupplier; -import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; +import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -28,7 +28,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; -import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateDoubleMetric; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateMetricDouble; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; import org.elasticsearch.xpack.esql.planner.ToAggregator; @@ -153,7 +153,7 @@ public final AggregatorFunctionSupplier supplier(List inputChannels) { @Override public Expression surrogate() { if (field().dataType() == DataType.AGGREGATE_METRIC_DOUBLE) { - return new Min(source(), FromAggregateDoubleMetric.withMetric(source(), field(), AggregateDoubleMetricBlockBuilder.Metric.MIN)); + return new Min(source(), FromAggregateMetricDouble.withMetric(source(), field(), AggregateMetricDoubleBlockBuilder.Metric.MIN)); } return field().foldable() ? new MvMin(source(), field()) : null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java index 28da31fcb4140..1c69edb9f0da9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sum.java @@ -12,7 +12,7 @@ import org.elasticsearch.compute.aggregation.SumDoubleAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.SumIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.SumLongAggregatorFunctionSupplier; -import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; +import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -24,7 +24,7 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; -import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateDoubleMetric; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateMetricDouble; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul; @@ -138,7 +138,7 @@ public Expression surrogate() { var s = source(); var field = field(); if (field.dataType() == AGGREGATE_METRIC_DOUBLE) { - return new Sum(s, FromAggregateDoubleMetric.withMetric(source(), field, AggregateDoubleMetricBlockBuilder.Metric.SUM)); + return new Sum(s, FromAggregateMetricDouble.withMetric(source(), field, AggregateMetricDoubleBlockBuilder.Metric.SUM)); } // SUM(const) is equivalent to MV_SUM(const)*COUNT(*). diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java index fd93f5273fd1c..90152d546097c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/ScalarFunctionWritables.java @@ -12,7 +12,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Least; -import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateDoubleMetric; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromAggregateMetricDouble; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateDiff; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateExtract; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateFormat; @@ -68,7 +68,7 @@ public static List getNamedWriteables() { entries.add(Concat.ENTRY); entries.add(E.ENTRY); entries.add(EndsWith.ENTRY); - entries.add(FromAggregateDoubleMetric.ENTRY); + entries.add(FromAggregateMetricDouble.ENTRY); entries.add(Greatest.ENTRY); entries.add(Hash.ENTRY); entries.add(Hypot.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java similarity index 90% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java index e8bebcaec8855..cfa8624800bae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetric.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java @@ -10,7 +10,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; +import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.CompositeBlock; import org.elasticsearch.compute.data.Page; @@ -39,18 +39,18 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; -public class FromAggregateDoubleMetric extends EsqlScalarFunction { +public class FromAggregateMetricDouble extends EsqlScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "FromAggregateDoubleMetric", - FromAggregateDoubleMetric::new + FromAggregateMetricDouble::new ); private final Expression field; private final Expression subfieldIndex; @FunctionInfo(returnType = { "long", "double" }, description = "Convert aggregate double metric to a block of a single subfield.") - public FromAggregateDoubleMetric( + public FromAggregateMetricDouble( Source source, @Param( name = "aggregate_metric_double", @@ -64,11 +64,11 @@ public FromAggregateDoubleMetric( this.subfieldIndex = subfieldIndex; } - public static FromAggregateDoubleMetric withMetric(Source source, Expression field, AggregateDoubleMetricBlockBuilder.Metric metric) { - return new FromAggregateDoubleMetric(source, field, new Literal(source, metric.getIndex(), INTEGER)); + public static FromAggregateMetricDouble withMetric(Source source, Expression field, AggregateMetricDoubleBlockBuilder.Metric metric) { + return new FromAggregateMetricDouble(source, field, new Literal(source, metric.getIndex(), INTEGER)); } - private FromAggregateDoubleMetric(StreamInput in) throws IOException { + private FromAggregateMetricDouble(StreamInput in) throws IOException { this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(Expression.class), in.readNamedWriteable(Expression.class)); } @@ -94,7 +94,7 @@ public DataType dataType() { return NULL; } var subfield = ((Number) folded).intValue(); - if (subfield == AggregateDoubleMetricBlockBuilder.Metric.COUNT.getIndex()) { + if (subfield == AggregateMetricDoubleBlockBuilder.Metric.COUNT.getIndex()) { return INTEGER; } return DOUBLE; @@ -102,12 +102,12 @@ public DataType dataType() { @Override public Expression replaceChildren(List newChildren) { - return new FromAggregateDoubleMetric(source(), newChildren.get(0), newChildren.get(1)); + return new FromAggregateMetricDouble(source(), newChildren.get(0), newChildren.get(1)); } @Override protected NodeInfo info() { - return NodeInfo.create(this, FromAggregateDoubleMetric::new, field, subfieldIndex); + return NodeInfo.create(this, FromAggregateMetricDouble::new, field, subfieldIndex); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index fcb5b7166accf..a66a302354df2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -212,7 +212,7 @@ private static Stream groupingAndNonGrouping(Tuple, Tuple aggClass) { case CARTESIAN_POINT -> "CartesianPoint"; case GEO_SHAPE -> "GeoShape"; case CARTESIAN_SHAPE -> "CartesianShape"; - case AGGREGATE_METRIC_DOUBLE -> "AggregatedMetricDouble"; + case AGGREGATE_METRIC_DOUBLE -> "AggregateMetricDouble"; case UNSUPPORTED, NULL, UNSIGNED_LONG, SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, SOURCE, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new EsqlIllegalArgumentException( "illegal agg type: " + type.typeName() diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 0bd28f456bfc8..bb03a45ef0ab0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -216,7 +216,7 @@ private Page randomPage(List columns) { case CARTESIAN_SHAPE -> ((BytesRefBlock.Builder) builder).appendBytesRef( CARTESIAN.asWkb(ShapeTestUtils.randomGeometry(randomBoolean())) ); - case AGGREGATE_METRIC_DOUBLE -> ((BlockLoader.AggregateDoubleMetricBuilder) builder).append( + case AGGREGATE_METRIC_DOUBLE -> ((BlockLoader.AggregateMetricDoubleBuilder) builder).append( randomDouble(), randomDouble(), randomDouble(), @@ -947,7 +947,7 @@ static Page valuesToPage(BlockFactory blockFactory, List columns BytesRef wkb = stringToSpatial(value.toString()); ((BytesRefBlock.Builder) builder).appendBytesRef(wkb); } - case AGGREGATE_METRIC_DOUBLE -> ((BlockLoader.AggregateDoubleMetricBuilder) builder).append( + case AGGREGATE_METRIC_DOUBLE -> ((BlockLoader.AggregateMetricDoubleBuilder) builder).append( ((Number) value).doubleValue(), ((Number) value).doubleValue(), ((Number) value).doubleValue(), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java similarity index 69% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java index 700fc7e293e77..bcd6d5b04667c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateDoubleMetricTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java @@ -10,7 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.elasticsearch.compute.data.AggregateDoubleMetricBlockBuilder; +import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; import org.elasticsearch.compute.data.AggregateMetricDoubleLiteral; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -24,16 +24,16 @@ import java.util.List; import java.util.function.Supplier; -@FunctionName("from_aggregate_double_metric") -public class FromAggregateDoubleMetricTests extends AbstractScalarFunctionTestCase { - public FromAggregateDoubleMetricTests(@Name("TestCase") Supplier testCaseSupplier) { +@FunctionName("from_aggregate_metric_double") +public class FromAggregateMetricDoubleTests extends AbstractScalarFunctionTestCase { + public FromAggregateMetricDoubleTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } @Override protected Expression build(Source source, List args) { assumeTrue("Test sometimes wraps literals as fields", args.get(1).foldable()); - return new FromAggregateDoubleMetric(source, args.get(0), args.get(1)); + return new FromAggregateMetricDouble(source, args.get(0), args.get(1)); } @ParametersFactory @@ -49,10 +49,10 @@ public static Iterable parameters() { randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE) ); - Double expectedValue = index == AggregateDoubleMetricBlockBuilder.Metric.MIN.getIndex() ? agg_metric.getMin() - : index == AggregateDoubleMetricBlockBuilder.Metric.MAX.getIndex() ? agg_metric.getMax() - : index == AggregateDoubleMetricBlockBuilder.Metric.SUM.getIndex() ? agg_metric.getSum() - : (Double) agg_metric.getCount().doubleValue(); + Double expectedValue = index == AggregateMetricDoubleBlockBuilder.Metric.MIN.getIndex() ? agg_metric.getMin() + : index == AggregateMetricDoubleBlockBuilder.Metric.MAX.getIndex() ? agg_metric.getMax() + : index == AggregateMetricDoubleBlockBuilder.Metric.SUM.getIndex() ? agg_metric.getSum() + : (Double) agg_metric.getCount().doubleValue(); return new TestCaseSupplier.TestCase( List.of( @@ -60,9 +60,9 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(index, DataType.INTEGER, "subfield_index").forceLiteral() ), "FromAggregateDoubleMetricEvaluator[field=Attribute[channel=0],subfieldIndex=" + index + "]", - index == AggregateDoubleMetricBlockBuilder.Metric.COUNT.getIndex() ? DataType.INTEGER : DataType.DOUBLE, - index == AggregateDoubleMetricBlockBuilder.Metric.COUNT.getIndex() ? Matchers.equalTo(agg_metric.getCount()) - : expectedValue == null ? Matchers.nullValue() + index == AggregateMetricDoubleBlockBuilder.Metric.COUNT.getIndex() ? DataType.INTEGER : DataType.DOUBLE, + index == AggregateMetricDoubleBlockBuilder.Metric.COUNT.getIndex() ? Matchers.equalTo(agg_metric.getCount()) + : expectedValue == null ? Matchers.nullValue() : Matchers.closeTo(expectedValue, Math.abs(expectedValue * 0.00001)) ); })); diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index d11192b198a7e..765de5d16d43a 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -511,13 +511,13 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) return SourceValueFetcher.identity(name(), context, format); } - public class AggregateDoubleMetricBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { + public class AggregateMetricDoubleBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { NumberFieldMapper.NumberFieldType minFieldType = metricFields.get(Metric.min); NumberFieldMapper.NumberFieldType maxFieldType = metricFields.get(Metric.max); NumberFieldMapper.NumberFieldType sumFieldType = metricFields.get(Metric.sum); NumberFieldMapper.NumberFieldType countFieldType = metricFields.get(Metric.value_count); - private AggregateDoubleMetricBlockLoader() {} + private AggregateMetricDoubleBlockLoader() {} static NumericDocValues getNumericDocValues(NumberFieldMapper.NumberFieldType field, LeafReader leafReader) throws IOException { if (field == null) { @@ -559,7 +559,7 @@ public String toString() { @Override public Block read(BlockFactory factory, Docs docs) throws IOException { - try (var builder = factory.aggregateDoubleMetricBuilder(docs.count())) { + try (var builder = factory.aggregateMetricDoubleBuilder(docs.count())) { int lastDoc = -1; for (int i = 0; i < docs.count(); i++) { int doc = docs.get(i); @@ -576,12 +576,12 @@ public Block read(BlockFactory factory, Docs docs) throws IOException { @Override public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { - var blockBuilder = (BlockLoader.AggregateDoubleMetricBuilder) builder; + var blockBuilder = (AggregateMetricDoubleBuilder) builder; this.docID = docId; read(docId, blockBuilder); } - private void read(int docId, BlockLoader.AggregateDoubleMetricBuilder builder) throws IOException { + private void read(int docId, AggregateMetricDoubleBuilder builder) throws IOException { if (minValues.advanceExact(docId)) { boolean found = maxValues.advanceExact(docId); assert found; @@ -603,13 +603,13 @@ private void read(int docId, BlockLoader.AggregateDoubleMetricBuilder builder) t @Override public Builder builder(BlockFactory factory, int expectedCount) { - return factory.aggregateDoubleMetricBuilder(expectedCount); + return factory.aggregateMetricDoubleBuilder(expectedCount); } } @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { - return new AggregateDoubleMetricBlockLoader(); + return new AggregateMetricDoubleBlockLoader(); } /** From bfe84c1d8e6e14a2ef41071384f4e3cfbf74aefe Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 28 Jan 2025 04:14:17 +0000 Subject: [PATCH 09/18] [CI] Auto commit changes from spotless --- .../scalar/convert/FromAggregateMetricDoubleTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java index bcd6d5b04667c..4167bad90b171 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java @@ -50,9 +50,9 @@ public static Iterable parameters() { randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE) ); Double expectedValue = index == AggregateMetricDoubleBlockBuilder.Metric.MIN.getIndex() ? agg_metric.getMin() - : index == AggregateMetricDoubleBlockBuilder.Metric.MAX.getIndex() ? agg_metric.getMax() - : index == AggregateMetricDoubleBlockBuilder.Metric.SUM.getIndex() ? agg_metric.getSum() - : (Double) agg_metric.getCount().doubleValue(); + : index == AggregateMetricDoubleBlockBuilder.Metric.MAX.getIndex() ? agg_metric.getMax() + : index == AggregateMetricDoubleBlockBuilder.Metric.SUM.getIndex() ? agg_metric.getSum() + : (Double) agg_metric.getCount().doubleValue(); return new TestCaseSupplier.TestCase( List.of( @@ -62,7 +62,7 @@ public static Iterable parameters() { "FromAggregateDoubleMetricEvaluator[field=Attribute[channel=0],subfieldIndex=" + index + "]", index == AggregateMetricDoubleBlockBuilder.Metric.COUNT.getIndex() ? DataType.INTEGER : DataType.DOUBLE, index == AggregateMetricDoubleBlockBuilder.Metric.COUNT.getIndex() ? Matchers.equalTo(agg_metric.getCount()) - : expectedValue == null ? Matchers.nullValue() + : expectedValue == null ? Matchers.nullValue() : Matchers.closeTo(expectedValue, Math.abs(expectedValue * 0.00001)) ); })); From 022eebe0b49f1b2afa986a7683a3c2b6baed913f Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Mon, 27 Jan 2025 18:17:29 -1000 Subject: [PATCH 10/18] renaming stragglers --- .../function/scalar/convert/FromAggregateMetricDouble.java | 6 +++--- .../scalar/convert/FromAggregateMetricDoubleTests.java | 2 +- .../mapper/AggregateDoubleMetricFieldMapper.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java index cfa8624800bae..fd9e1796c531d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java @@ -42,7 +42,7 @@ public class FromAggregateMetricDouble extends EsqlScalarFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, - "FromAggregateDoubleMetric", + "FromAggregateMetricDouble", FromAggregateMetricDouble::new ); @@ -130,7 +130,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua @Override public String toString() { - return "FromAggregateDoubleMetricEvaluator[" + "field=" + fieldEvaluator + ",subfieldIndex=" + subfieldIndex + "]"; + return "FromAggregateMetricDoubleEvaluator[" + "field=" + fieldEvaluator + ",subfieldIndex=" + subfieldIndex + "]"; } @Override @@ -158,7 +158,7 @@ public void close() { @Override public String toString() { - return "FromAggregateDoubleMetricEvaluator[field=" + eval + ",subfieldIndex=" + subfieldIndex + "]"; + return "FromAggregateMetricDoubleEvaluator[field=" + eval + ",subfieldIndex=" + subfieldIndex + "]"; } }; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java index 4167bad90b171..98e1a9e687c9e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java @@ -59,7 +59,7 @@ public static Iterable parameters() { new TestCaseSupplier.TypedData(agg_metric, dataType, "agg_metric"), new TestCaseSupplier.TypedData(index, DataType.INTEGER, "subfield_index").forceLiteral() ), - "FromAggregateDoubleMetricEvaluator[field=Attribute[channel=0],subfieldIndex=" + index + "]", + "FromAggregateMetricDoubleEvaluator[field=Attribute[channel=0],subfieldIndex=" + index + "]", index == AggregateMetricDoubleBlockBuilder.Metric.COUNT.getIndex() ? DataType.INTEGER : DataType.DOUBLE, index == AggregateMetricDoubleBlockBuilder.Metric.COUNT.getIndex() ? Matchers.equalTo(agg_metric.getCount()) : expectedValue == null ? Matchers.nullValue() diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index 765de5d16d43a..d61109d5c2f9d 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -554,7 +554,7 @@ protected int docId() { @Override public String toString() { - return "BlockDocValuesReader.AggregatedDoubleMetrics"; + return "BlockDocValuesReader.AggregateMetricDouble"; } @Override From 769e924bea9ac864aad602eeefef994ece1c9cde Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Mon, 27 Jan 2025 19:34:01 -1000 Subject: [PATCH 11/18] Update docs/changelog/120343.yaml --- docs/changelog/120343.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/120343.yaml diff --git a/docs/changelog/120343.yaml b/docs/changelog/120343.yaml new file mode 100644 index 0000000000000..6c527325bfc5a --- /dev/null +++ b/docs/changelog/120343.yaml @@ -0,0 +1,6 @@ +pr: 120343 +summary: Support some stats on `aggregate_metric_double` +area: "ES|QL, TSDB" +type: enhancement +issues: + - 110649 From 6dd6e36a6c4f71bea5bedee84972b9457131927a Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Mon, 27 Jan 2025 19:42:36 -1000 Subject: [PATCH 12/18] fix changelog --- docs/changelog/120343.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog/120343.yaml b/docs/changelog/120343.yaml index 6c527325bfc5a..f33bd215877c7 100644 --- a/docs/changelog/120343.yaml +++ b/docs/changelog/120343.yaml @@ -1,6 +1,6 @@ pr: 120343 -summary: Support some stats on `aggregate_metric_double` -area: "ES|QL, TSDB" +summary: Support some stats on aggregate_metric_double +area: "ES|QL" type: enhancement issues: - 110649 From af0e24274d4b6643698a66cce1bca0e70e0df8dc Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Tue, 28 Jan 2025 08:11:12 -1000 Subject: [PATCH 13/18] Add downsampled yaml test --- .../rest-api-spec/test/esql/46_downsample.yml | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml new file mode 100644 index 0000000000000..18096c16479aa --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml @@ -0,0 +1,111 @@ +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + index: + mode: time_series + routing_path: [ metricset, k8s.pod.uid ] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + k8s: + properties: + pod: + properties: + uid: + type: keyword + time_series_dimension: true + name: + type: keyword + created_at: + type: date_nanos + running: + type: boolean + number_of_containers: + type: integer + ip: + type: ip + tags: + type: keyword + values: + type: integer + network: + properties: + tx: + type: long + time_series_metric: gauge + rx: + type: long + time_series_metric: gauge + - do: + bulk: + refresh: true + index: test + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001810, "rx": 802133}, "created_at": "2021-04-28T19:34:00.000Z", "running": false, "number_of_containers": 2, "tags": ["backend", "prod"], "values": [2, 3, 6]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:24.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.26", "network": {"tx": 2005177, "rx": 801479}, "created_at": "2021-04-28T19:35:00.000Z", "running": true, "number_of_containers": 2, "tags": ["backend", "prod", "us-west1"], "values": [1, 1, 3]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T20:50:44.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.41", "network": {"tx": 2006223, "rx": 802337}, "created_at": "2021-04-28T19:36:00.000Z", "running": true, "number_of_containers": 2, "tags": ["backend", "prod", "us-west2"], "values": [4, 1, 2]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T20:51:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.22", "network": {"tx": 2012916, "rx": 803685}, "created_at": "2021-04-28T19:37:00.000Z", "running": true, "number_of_containers": 2, "tags": ["backend", "prod"], "values": [2, 3, 1]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.33", "network": {"tx": 1434521, "rx": 530575}, "created_at": "2021-04-28T19:42:00.000Z", "running": false, "number_of_containers": 1, "tags": ["backend", "test"], "values": [2, 3, 4]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:23.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.56", "network": {"tx": 1434577, "rx": 530600}, "created_at": "2021-04-28T19:43:00.000Z", "running": false, "number_of_containers": 1, "tags": ["backend", "test", "us-west2"], "values": [2, 1, 1]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T19:50:53.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.37", "network": {"tx": 1434587, "rx": 530604}, "created_at": "2021-04-28T19:44:00.000Z", "running": true, "number_of_containers": 1, "tags": ["backend", "test", "us-west1"], "values": [4, 5, 2]}}}' + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T19:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.120", "network": {"tx": 1434595, "rx": 530605}, "created_at": "2021-04-28T19:45:00.000Z", "running": true, "number_of_containers": 1, "tags": ["backend", "test", "us-west1"], "values": [3, 2, 1]}}}' + + - do: + indices.put_settings: + index: test + body: + index.blocks.write: true + +--- +"Query stats on downsampled index": + - do: + indices.downsample: + index: test + target_index: test-downsample + body: > + { + "fixed_interval": "1h" + } + - is_true: acknowledged + + - do: + esql.query: + body: + query: "FROM test-downsample | + STATS max(k8s.pod.network.rx), min(k8s.pod.network.rx), sum(k8s.pod.network.rx), count(k8s.pod.network.rx) + | LIMIT 100" + + - length: {values: 1} + - length: {values.0: 4} + - match: {columns.0.name: "max(k8s.pod.network.rx)"} + - match: {columns.0.type: "double"} + - match: {columns.1.name: "min(k8s.pod.network.rx)"} + - match: {columns.1.type: "double"} + - match: {columns.2.name: "sum(k8s.pod.network.rx)"} + - match: {columns.2.type: "double"} + - match: {columns.3.name: "count(k8s.pod.network.rx)"} + - match: {columns.3.type: "long"} + - match: {values.0.0: 803685.0} + - match: {values.0.1: 530575.0} + - match: {values.0.2: 5332018.0} + - match: {values.0.3: 8} + From 1838a820bfb6c6d3ea2c2b3fec93e17998e75f27 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Tue, 28 Jan 2025 12:03:25 -1000 Subject: [PATCH 14/18] address some comments --- .../data/AggregateMetricDoubleLiteral.java | 16 ++++++++++++++++ .../xpack/esql/action/PositionToXContent.java | 1 + .../convert/FromAggregateMetricDouble.java | 5 ++++- .../rest-api-spec/test/esql/46_downsample.yml | 8 ++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java index 4cc7e7594d30d..c0f51965882b7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java @@ -7,6 +7,8 @@ package org.elasticsearch.compute.data; +import java.util.Objects; + public class AggregateMetricDoubleLiteral { private final Double min; private final Double max; @@ -35,4 +37,18 @@ public Double getSum() { public Integer getCount() { return count; } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != this.getClass()) { + return false; + } + var aggMetric = (AggregateMetricDoubleLiteral) obj; + return min.equals(aggMetric.min) && max.equals(aggMetric.max) && sum.equals(aggMetric.sum) && count.equals(aggMetric.count); + } + + @Override + public int hashCode() { + return Objects.hash(max, min, sum, count); + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java index 7d54c00328ae0..a065d0bd5e3a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java @@ -148,6 +148,7 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa return builder.value(versionToString(val)); } }; + // TODO: Add implementation for aggregate_metric_double case NULL, AGGREGATE_METRIC_DOUBLE -> new PositionToXContent(block) { @Override protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Params params, int valueIndex) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java index fd9e1796c531d..f1bde9f57b671 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDouble.java @@ -144,10 +144,13 @@ public Block eval(Page page) { if (block.areAllValuesNull()) { return block; } - try (CompositeBlock compositeBlock = (CompositeBlock) block) { + try { + CompositeBlock compositeBlock = (CompositeBlock) block; Block resultBlock = compositeBlock.getBlock(((Number) subfieldIndex.fold(FoldContext.small())).intValue()); resultBlock.incRef(); return resultBlock; + } finally { + block.close(); } } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml index 18096c16479aa..5a0b8b281e88f 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/46_downsample.yml @@ -77,6 +77,14 @@ setup: --- "Query stats on downsampled index": + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [aggregate_metric_double] + reason: "Support for aggregate_metric_double" - do: indices.downsample: index: test From 335fe3b3c885dc083e39a60fc5b642924e167b84 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Tue, 28 Jan 2025 14:24:32 -1000 Subject: [PATCH 15/18] change Double etc to double --- .../index/mapper/BlockLoader.java | 8 +- .../AggregateMetricDoubleBlockBuilder.java | 39 ++++------ .../compute/data/BlockUtils.java | 33 +++++--- .../esql/action/EsqlQueryResponseTests.java | 26 ++++--- .../AggregateDoubleMetricFieldMapper.java | 77 ++++++++++++++----- 5 files changed, 119 insertions(+), 64 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index a6ed5261611ac..451da5bfdbaf0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -506,7 +506,13 @@ interface SingletonOrdinalsBuilder extends Builder { interface AggregateMetricDoubleBuilder extends Builder { - AggregateMetricDoubleBuilder append(Double min, Double max, Double sum, Integer valueCount); + DoubleBuilder min(); + + DoubleBuilder max(); + + DoubleBuilder sum(); + + IntBuilder count(); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java index 1586c836f5e63..18d76c1573e69 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java @@ -119,28 +119,23 @@ protected void extraClose() { } @Override - public BlockLoader.AggregateMetricDoubleBuilder append(Double min, Double max, Double sum, Integer valueCount) { - if (min == null) { - minBuilder.appendNull(); - } else { - minBuilder.appendDouble(min); - } - if (max == null) { - maxBuilder.appendNull(); - } else { - maxBuilder.appendDouble(max); - } - if (sum == null) { - sumBuilder.appendNull(); - } else { - sumBuilder.appendDouble(sum); - } - if (valueCount == null) { - countBuilder.appendNull(); - } else { - countBuilder.appendInt(valueCount); - } - return this; + public BlockLoader.DoubleBuilder min() { + return minBuilder; + } + + @Override + public BlockLoader.DoubleBuilder max() { + return maxBuilder; + } + + @Override + public BlockLoader.DoubleBuilder sum() { + return sumBuilder; + } + + @Override + public BlockLoader.IntBuilder count() { + return countBuilder; } public enum Metric { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index 99518614b8888..8284f4d9ba158 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -234,15 +234,30 @@ private static Block constantBlock(BlockFactory blockFactory, ElementType type, case DOUBLE -> blockFactory.newConstantDoubleBlockWith((double) val, size); case BOOLEAN -> blockFactory.newConstantBooleanBlockWith((boolean) val, size); case COMPOSITE -> { - var builder = blockFactory.newAggregateMetricDoubleBlockBuilder(size); - var aggregate_metric_double = (AggregateMetricDoubleLiteral) val; - builder.append( - aggregate_metric_double.getMin(), - aggregate_metric_double.getMax(), - aggregate_metric_double.getSum(), - aggregate_metric_double.getCount() - ); - yield builder.build(); + try (AggregateMetricDoubleBlockBuilder builder = blockFactory.newAggregateMetricDoubleBlockBuilder(size)) { + var aggregate_metric_double = (AggregateMetricDoubleLiteral) val; + if (aggregate_metric_double.getMin() != null) { + builder.min().appendDouble(aggregate_metric_double.getMin()); + } else { + builder.min().appendNull(); + } + if (aggregate_metric_double.getMax() != null) { + builder.max().appendDouble(aggregate_metric_double.getMax()); + } else { + builder.max().appendNull(); + } + if (aggregate_metric_double.getSum() != null) { + builder.sum().appendDouble(aggregate_metric_double.getSum()); + } else { + builder.sum().appendNull(); + } + if (aggregate_metric_double.getCount() != null) { + builder.count().appendInt(aggregate_metric_double.getCount()); + } else { + builder.count().appendNull(); + } + yield builder.build(); + } } default -> throw new UnsupportedOperationException("unsupported element type [" + type + "]"); }; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java index 3e425d4919152..4fdb4a7bf042b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponseTests.java @@ -216,12 +216,13 @@ private Page randomPage(List columns) { case CARTESIAN_SHAPE -> ((BytesRefBlock.Builder) builder).appendBytesRef( CARTESIAN.asWkb(ShapeTestUtils.randomGeometry(randomBoolean())) ); - case AGGREGATE_METRIC_DOUBLE -> ((BlockLoader.AggregateMetricDoubleBuilder) builder).append( - randomDouble(), - randomDouble(), - randomDouble(), - randomInt() - ); + case AGGREGATE_METRIC_DOUBLE -> { + BlockLoader.AggregateMetricDoubleBuilder aggBuilder = (BlockLoader.AggregateMetricDoubleBuilder) builder; + aggBuilder.min().appendDouble(randomDouble()); + aggBuilder.max().appendDouble(randomDouble()); + aggBuilder.sum().appendDouble(randomDouble()); + aggBuilder.count().appendInt(randomInt()); + } case NULL -> builder.appendNull(); case SOURCE -> { try { @@ -947,12 +948,13 @@ static Page valuesToPage(BlockFactory blockFactory, List columns BytesRef wkb = stringToSpatial(value.toString()); ((BytesRefBlock.Builder) builder).appendBytesRef(wkb); } - case AGGREGATE_METRIC_DOUBLE -> ((BlockLoader.AggregateMetricDoubleBuilder) builder).append( - ((Number) value).doubleValue(), - ((Number) value).doubleValue(), - ((Number) value).doubleValue(), - ((Number) value).intValue() - ); + case AGGREGATE_METRIC_DOUBLE -> { + BlockLoader.AggregateMetricDoubleBuilder aggBuilder = (BlockLoader.AggregateMetricDoubleBuilder) builder; + aggBuilder.min().appendDouble(((Number) value).doubleValue()); + aggBuilder.max().appendDouble(((Number) value).doubleValue()); + aggBuilder.sum().appendDouble(((Number) value).doubleValue()); + aggBuilder.count().appendInt(((Number) value).intValue()); + } } } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index d61109d5c2f9d..a58f8dae8cc73 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -560,17 +560,49 @@ public String toString() { @Override public Block read(BlockFactory factory, Docs docs) throws IOException { try (var builder = factory.aggregateMetricDoubleBuilder(docs.count())) { - int lastDoc = -1; - for (int i = 0; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < lastDoc) { - throw new IllegalStateException("docs within same block must be in order"); - } - read(doc, builder); + copyDoubleValuesToBuilder(docs, builder.min(), minValues); + copyDoubleValuesToBuilder(docs, builder.max(), maxValues); + copyDoubleValuesToBuilder(docs, builder.sum(), sumValues); + copyIntValuesToBuilder(docs, builder.count(), valueCountValues); + return builder.build(); + } + } + + private void copyDoubleValuesToBuilder(Docs docs, BlockLoader.DoubleBuilder builder, NumericDocValues values) + throws IOException { + int lastDoc = -1; + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < lastDoc) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (values.advanceExact(doc)) { + double value = NumericUtils.sortableLongToDouble(values.longValue()); lastDoc = doc; this.docID = doc; + builder.appendDouble(value); + } else { + builder.appendNull(); + } + } + } + + private void copyIntValuesToBuilder(Docs docs, BlockLoader.IntBuilder builder, NumericDocValues values) + throws IOException { + int lastDoc = -1; + for (int i = 0; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < lastDoc) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (values.advanceExact(doc)) { + int value = Math.toIntExact(values.longValue()); + lastDoc = doc; + this.docID = doc; + builder.appendInt(value); + } else { + builder.appendNull(); } - return builder.build(); } } @@ -583,19 +615,24 @@ public void read(int docId, StoredFields storedFields, Builder builder) throws I private void read(int docId, AggregateMetricDoubleBuilder builder) throws IOException { if (minValues.advanceExact(docId)) { - boolean found = maxValues.advanceExact(docId); - assert found; - found = sumValues.advanceExact(docId); - assert found; - found = valueCountValues.advanceExact(docId); - assert found; - double min = NumericUtils.sortableLongToDouble(minValues.longValue()); - double max = NumericUtils.sortableLongToDouble(maxValues.longValue()); - double sum = NumericUtils.sortableLongToDouble(sumValues.longValue()); - int count = Math.toIntExact(valueCountValues.longValue()); - builder.append(min, max, sum, count); + builder.min().appendDouble(NumericUtils.sortableLongToDouble(minValues.longValue())); + } else { + builder.min().appendNull(); + } + if (maxValues.advanceExact(docId)) { + builder.max().appendDouble(NumericUtils.sortableLongToDouble(maxValues.longValue())); + } else { + builder.max().appendNull(); + } + if (sumValues.advanceExact(docId)) { + builder.sum().appendDouble(NumericUtils.sortableLongToDouble(sumValues.longValue())); + } else { + builder.sum().appendNull(); + } + if (valueCountValues.advanceExact(docId)) { + builder.count().appendInt(Math.toIntExact(valueCountValues.longValue())); } else { - builder.appendNull(); + builder.count().appendNull(); } } }; From c417b3c8cd6e902c3b079397e25daa3c56bba524 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Tue, 28 Jan 2025 15:34:01 -1000 Subject: [PATCH 16/18] add instanceof check --- .../compute/data/BlockUtils.java | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index 8284f4d9ba158..b1be28d3e72e2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -234,30 +234,34 @@ private static Block constantBlock(BlockFactory blockFactory, ElementType type, case DOUBLE -> blockFactory.newConstantDoubleBlockWith((double) val, size); case BOOLEAN -> blockFactory.newConstantBooleanBlockWith((boolean) val, size); case COMPOSITE -> { - try (AggregateMetricDoubleBlockBuilder builder = blockFactory.newAggregateMetricDoubleBlockBuilder(size)) { - var aggregate_metric_double = (AggregateMetricDoubleLiteral) val; - if (aggregate_metric_double.getMin() != null) { - builder.min().appendDouble(aggregate_metric_double.getMin()); - } else { - builder.min().appendNull(); - } - if (aggregate_metric_double.getMax() != null) { - builder.max().appendDouble(aggregate_metric_double.getMax()); - } else { - builder.max().appendNull(); - } - if (aggregate_metric_double.getSum() != null) { - builder.sum().appendDouble(aggregate_metric_double.getSum()); - } else { - builder.sum().appendNull(); - } - if (aggregate_metric_double.getCount() != null) { - builder.count().appendInt(aggregate_metric_double.getCount()); - } else { - builder.count().appendNull(); + if (val instanceof AggregateMetricDoubleLiteral aggregateMetricDoubleLiteral) { + try (AggregateMetricDoubleBlockBuilder builder = blockFactory.newAggregateMetricDoubleBlockBuilder(size)) { + if (aggregateMetricDoubleLiteral.getMin() != null) { + builder.min().appendDouble(aggregateMetricDoubleLiteral.getMin()); + } else { + builder.min().appendNull(); + } + if (aggregateMetricDoubleLiteral.getMax() != null) { + builder.max().appendDouble(aggregateMetricDoubleLiteral.getMax()); + } else { + builder.max().appendNull(); + } + if (aggregateMetricDoubleLiteral.getSum() != null) { + builder.sum().appendDouble(aggregateMetricDoubleLiteral.getSum()); + } else { + builder.sum().appendNull(); + } + if (aggregateMetricDoubleLiteral.getCount() != null) { + builder.count().appendInt(aggregateMetricDoubleLiteral.getCount()); + } else { + builder.count().appendNull(); + } + yield builder.build(); } - yield builder.build(); } + throw new UnsupportedOperationException( + "Composite block but received value that wasn't AggregateMetricDoubleLiteral [" + val + "]" + ); } default -> throw new UnsupportedOperationException("unsupported element type [" + type + "]"); }; From 827eaf38aef5897aab8c30d1bcffcf1e8a9a5c23 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Wed, 29 Jan 2025 09:48:50 -1000 Subject: [PATCH 17/18] change AggMetDoubLit to record --- .../AggregateMetricDoubleBlockBuilder.java | 8 +++ .../data/AggregateMetricDoubleLiteral.java | 54 ------------------- .../compute/data/BlockUtils.java | 18 +++---- .../compute/data/ElementType.java | 2 +- .../FromAggregateMetricDoubleTests.java | 13 +++-- 5 files changed, 24 insertions(+), 71 deletions(-) delete mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java index 18d76c1573e69..d5eecc3e6ed70 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleBlockBuilder.java @@ -154,4 +154,12 @@ public int getIndex() { return index; } } + + public record AggregateMetricDoubleLiteral(Double min, Double max, Double sum, Integer count) { + public AggregateMetricDoubleLiteral { + min = min.isNaN() ? null : min; + max = max.isNaN() ? null : max; + sum = sum.isNaN() ? null : sum; + } + } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java deleted file mode 100644 index c0f51965882b7..0000000000000 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/AggregateMetricDoubleLiteral.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.compute.data; - -import java.util.Objects; - -public class AggregateMetricDoubleLiteral { - private final Double min; - private final Double max; - private final Double sum; - private final Integer count; - - public AggregateMetricDoubleLiteral(Double min, Double max, Double sum, Integer count) { - this.min = min.isNaN() ? null : min; - this.max = max.isNaN() ? null : max; - this.sum = sum.isNaN() ? null : sum; - this.count = count; - } - - public Double getMax() { - return max; - } - - public Double getMin() { - return min; - } - - public Double getSum() { - return sum; - } - - public Integer getCount() { - return count; - } - - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != this.getClass()) { - return false; - } - var aggMetric = (AggregateMetricDoubleLiteral) obj; - return min.equals(aggMetric.min) && max.equals(aggMetric.max) && sum.equals(aggMetric.sum) && count.equals(aggMetric.count); - } - - @Override - public int hashCode() { - return Objects.hash(max, min, sum, count); - } -} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index b1be28d3e72e2..c02736ee5d06d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -234,25 +234,25 @@ private static Block constantBlock(BlockFactory blockFactory, ElementType type, case DOUBLE -> blockFactory.newConstantDoubleBlockWith((double) val, size); case BOOLEAN -> blockFactory.newConstantBooleanBlockWith((boolean) val, size); case COMPOSITE -> { - if (val instanceof AggregateMetricDoubleLiteral aggregateMetricDoubleLiteral) { + if (val instanceof AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral(Double min, Double max, Double sum, Integer count)) { try (AggregateMetricDoubleBlockBuilder builder = blockFactory.newAggregateMetricDoubleBlockBuilder(size)) { - if (aggregateMetricDoubleLiteral.getMin() != null) { - builder.min().appendDouble(aggregateMetricDoubleLiteral.getMin()); + if (min != null) { + builder.min().appendDouble(min); } else { builder.min().appendNull(); } - if (aggregateMetricDoubleLiteral.getMax() != null) { - builder.max().appendDouble(aggregateMetricDoubleLiteral.getMax()); + if (max != null) { + builder.max().appendDouble(max); } else { builder.max().appendNull(); } - if (aggregateMetricDoubleLiteral.getSum() != null) { - builder.sum().appendDouble(aggregateMetricDoubleLiteral.getSum()); + if (sum != null) { + builder.sum().appendDouble(sum); } else { builder.sum().appendNull(); } - if (aggregateMetricDoubleLiteral.getCount() != null) { - builder.count().appendInt(aggregateMetricDoubleLiteral.getCount()); + if (count != null) { + builder.count().appendInt(count); } else { builder.count().appendNull(); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java index dc11cdd107464..cdf6711e14058 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java @@ -73,7 +73,7 @@ public static ElementType fromJava(Class type) { elementType = BYTES_REF; } else if (type == Boolean.class) { elementType = BOOLEAN; - } else if (type == AggregateMetricDoubleLiteral.class) { + } else if (type == AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral.class) { elementType = COMPOSITE; } else if (type == null || type == Void.class) { elementType = NULL; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java index 98e1a9e687c9e..94d9bd5f64cbd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FromAggregateMetricDoubleTests.java @@ -11,7 +11,6 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; -import org.elasticsearch.compute.data.AggregateMetricDoubleLiteral; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -43,16 +42,16 @@ public static Iterable parameters() { for (int i = 0; i < 4; i++) { int index = i; suppliers.add(new TestCaseSupplier(List.of(dataType, DataType.INTEGER), () -> { - var agg_metric = new AggregateMetricDoubleLiteral( + var agg_metric = new AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral( randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), randomDoubleBetween(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), randomIntBetween(Integer.MIN_VALUE, Integer.MAX_VALUE) ); - Double expectedValue = index == AggregateMetricDoubleBlockBuilder.Metric.MIN.getIndex() ? agg_metric.getMin() - : index == AggregateMetricDoubleBlockBuilder.Metric.MAX.getIndex() ? agg_metric.getMax() - : index == AggregateMetricDoubleBlockBuilder.Metric.SUM.getIndex() ? agg_metric.getSum() - : (Double) agg_metric.getCount().doubleValue(); + Double expectedValue = index == AggregateMetricDoubleBlockBuilder.Metric.MIN.getIndex() ? agg_metric.min() + : index == AggregateMetricDoubleBlockBuilder.Metric.MAX.getIndex() ? agg_metric.max() + : index == AggregateMetricDoubleBlockBuilder.Metric.SUM.getIndex() ? agg_metric.sum() + : (Double) agg_metric.count().doubleValue(); return new TestCaseSupplier.TestCase( List.of( @@ -61,7 +60,7 @@ public static Iterable parameters() { ), "FromAggregateMetricDoubleEvaluator[field=Attribute[channel=0],subfieldIndex=" + index + "]", index == AggregateMetricDoubleBlockBuilder.Metric.COUNT.getIndex() ? DataType.INTEGER : DataType.DOUBLE, - index == AggregateMetricDoubleBlockBuilder.Metric.COUNT.getIndex() ? Matchers.equalTo(agg_metric.getCount()) + index == AggregateMetricDoubleBlockBuilder.Metric.COUNT.getIndex() ? Matchers.equalTo(agg_metric.count()) : expectedValue == null ? Matchers.nullValue() : Matchers.closeTo(expectedValue, Math.abs(expectedValue * 0.00001)) ); From fbdb24dc4ee727575298daeca813446d5aa98929 Mon Sep 17 00:00:00 2001 From: Larisa Motova Date: Wed, 29 Jan 2025 10:56:26 -1000 Subject: [PATCH 18/18] move function to make checkstyle work --- .../compute/data/BlockFactory.java | 29 +++++++++++++++++++ .../compute/data/BlockUtils.java | 27 ++--------------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java index c4f0be7533a00..55053f509591d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java @@ -436,6 +436,35 @@ public AggregateMetricDoubleBlockBuilder newAggregateMetricDoubleBlockBuilder(in return new AggregateMetricDoubleBlockBuilder(estimatedSize, this); } + public final Block newConstantAggregateMetricDoubleBlock( + AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral value, + int positions + ) { + try (AggregateMetricDoubleBlockBuilder builder = newAggregateMetricDoubleBlockBuilder(positions)) { + if (value.min() != null) { + builder.min().appendDouble(value.min()); + } else { + builder.min().appendNull(); + } + if (value.max() != null) { + builder.max().appendDouble(value.max()); + } else { + builder.max().appendNull(); + } + if (value.sum() != null) { + builder.sum().appendDouble(value.sum()); + } else { + builder.sum().appendNull(); + } + if (value.count() != null) { + builder.count().appendInt(value.count()); + } else { + builder.count().appendNull(); + } + return builder.build(); + } + } + /** * Returns the maximum number of bytes that a Block should be backed by a primitive array before switching to using BigArrays. */ diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index c02736ee5d06d..8773a3b9785e0 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -9,6 +9,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Randomness; +import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -234,30 +235,8 @@ private static Block constantBlock(BlockFactory blockFactory, ElementType type, case DOUBLE -> blockFactory.newConstantDoubleBlockWith((double) val, size); case BOOLEAN -> blockFactory.newConstantBooleanBlockWith((boolean) val, size); case COMPOSITE -> { - if (val instanceof AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral(Double min, Double max, Double sum, Integer count)) { - try (AggregateMetricDoubleBlockBuilder builder = blockFactory.newAggregateMetricDoubleBlockBuilder(size)) { - if (min != null) { - builder.min().appendDouble(min); - } else { - builder.min().appendNull(); - } - if (max != null) { - builder.max().appendDouble(max); - } else { - builder.max().appendNull(); - } - if (sum != null) { - builder.sum().appendDouble(sum); - } else { - builder.sum().appendNull(); - } - if (count != null) { - builder.count().appendInt(count); - } else { - builder.count().appendNull(); - } - yield builder.build(); - } + if (val instanceof AggregateMetricDoubleLiteral aggregateMetricDoubleLiteral) { + yield blockFactory.newConstantAggregateMetricDoubleBlock(aggregateMetricDoubleLiteral, size); } throw new UnsupportedOperationException( "Composite block but received value that wasn't AggregateMetricDoubleLiteral [" + val + "]"