From e105a42f76d4cbb26d70372651a72c3dd05522a2 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Fri, 31 Jan 2025 10:29:02 +0100 Subject: [PATCH 01/15] initial --- .../src/main/resources/observability.csv-spec | 23 ++++ .../xpack/esql/action/EsqlCapabilities.java | 5 + .../function/EsqlFunctionRegistry.java | 3 + .../aggregate/AggregateWritables.java | 1 + .../function/aggregate/FirstValue.java | 100 ++++++++++++++++++ .../AbstractExpressionSerializationTests.java | 10 ++ .../FirstValueSerializationTests.java | 38 +++++++ 7 files changed, 180 insertions(+) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec create mode 100644 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValueSerializationTests.java diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec new file mode 100644 index 0000000000000..aff770cc29c59 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec @@ -0,0 +1,23 @@ +first +required_capability: fn_first_last +FROM sample_data +| STATS message = first_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) +| SORT @timestamp +; + +@timestamp:long | message:keyword +2023-10-23T12:00:00.000Z | Connected to 10.1.0.3 +2023-10-23T13:00:00.000Z | Disconnected +; + +last +required_capability: fn_first_last +FROM sample_data +| STATS message = last_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) +| SORT @timestamp +; + +@timestamp:long | message:keyword +2023-10-23T12:00:00.000Z | Connected to 10.1.0.2 +2023-10-23T13:00:00.000Z | Connected to 10.1.0.1 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 25518220e308b..3e2ffb1fa33be 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -199,6 +199,11 @@ public enum Cap { */ FN_ROUND_UL_FIXES, + /** + * Support for function {@code FIRST} and {@code LAST}. + */ + FN_FIRST_LAST, + /** * All functions that take TEXT should never emit TEXT, only KEYWORD. #114334 */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index a614a473ebe41..2c975a258d154 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Avg; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.aggregate.CountDistinct; +import org.elasticsearch.xpack.esql.expression.function.aggregate.FirstValue; import org.elasticsearch.xpack.esql.expression.function.aggregate.Max; import org.elasticsearch.xpack.esql.expression.function.aggregate.Median; import org.elasticsearch.xpack.esql.expression.function.aggregate.MedianAbsoluteDeviation; @@ -293,6 +294,8 @@ private static FunctionDefinition[][] functions() { def(Avg.class, uni(Avg::new), "avg"), def(Count.class, uni(Count::new), "count"), def(CountDistinct.class, bi(CountDistinct::new), "count_distinct"), + def(FirstValue.class, bi(FirstValue::new), "first_value"), + // def(Last.class, uni(Last::new), "last"), def(Max.class, uni(Max::new), "max"), def(Median.class, uni(Median::new), "median"), def(MedianAbsoluteDeviation.class, uni(MedianAbsoluteDeviation::new), "median_absolute_deviation"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java index db1d2a9e6f254..322da944a3217 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java @@ -18,6 +18,7 @@ public static List getNamedWriteables() { Avg.ENTRY, Count.ENTRY, CountDistinct.ENTRY, + FirstValue.ENTRY, Max.ENTRY, Median.ENTRY, MedianAbsoluteDeviation.ENTRY, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java new file mode 100644 index 0000000000000..d0b53b37b45e6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.SurrogateExpression; +import org.elasticsearch.xpack.esql.expression.function.OptionalArgument; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.planner.ToAggregator; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; + +public class FirstValue extends AggregateFunction implements OptionalArgument, ToAggregator, SurrogateExpression { + + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "First", FirstValue::new); + + // TODO @FunctionInfo + public FirstValue(Source source, Expression field, Expression by) { + this(source, field, Literal.TRUE, by != null ? List.of(by) : List.of()); + } + + private FirstValue(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + in.readNamedWriteable(Expression.class), + in.readNamedWriteable(Expression.class), + in.readNamedWriteableCollectionAsList(Expression.class) + ); + } + + private FirstValue(Source source, Expression field, Expression filter, List params) { + super(source, field, filter, params); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, FirstValue::new, field(), filter(), parameters()); + } + + @Override + public Expression replaceChildren(List newChildren) { + return new FirstValue(source(), newChildren.get(0), newChildren.get(1), newChildren.subList(2, newChildren.size())); + } + + @Override + public AggregateFunction withFilter(Expression filter) { + return new FirstValue(source(), field(), filter, parameters()); + } + + @Override + public DataType dataType() { + return field().dataType().noText(); + } + + @Override + protected TypeResolution resolveType() { + return TypeResolutions.isType( + field(), + dt -> dt == DataType.KEYWORD, // TODO implement for all types + sourceText(), + DEFAULT, + "representable except unsigned_long and spatial types" + ); + } + + @Override + public Expression surrogate() { + return null; + } + + @Override + public AggregatorFunctionSupplier supplier(List inputChannels) { + return null; + } + + public Expression by() { + return parameters().isEmpty() ? null : parameters().getFirst(); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java index 050293e58c19d..46078b6de8d20 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java @@ -23,6 +23,16 @@ public static Expression mutateExpression(Expression expression) { return randomValueOtherThan(expression, AbstractExpressionSerializationTests::randomChild); } + public static Expression mutateNullableExpression(Expression expression) { + if (expression == null) { + return randomChild(); + } else if (randomBoolean()) { + return null; + } else { + return randomValueOtherThan(expression, AbstractExpressionSerializationTests::randomChild); + } + } + @Override protected final NamedWriteableRegistry getNamedWriteableRegistry() { return new NamedWriteableRegistry(ExpressionWritables.getNamedWriteables()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValueSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValueSerializationTests.java new file mode 100644 index 0000000000000..fc7d1cd0645e7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValueSerializationTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class FirstValueSerializationTests extends AbstractExpressionSerializationTests { + + @Override + protected FirstValue createTestInstance() { + return new FirstValue(randomSource(), randomChild(), randomBoolean() ? randomChild() : null); + } + + @Override + protected FirstValue mutateInstance(FirstValue instance) throws IOException { + var source = instance.source(); + var field = instance.field(); + var precision = instance.by(); + if (randomBoolean()) { + field = mutateExpression(field); + } else { + precision = mutateNullableExpression(precision); + } + return new FirstValue(source, field, precision); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} From 903340786243430c8bebae24485610d1d8454485 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Fri, 31 Jan 2025 13:56:15 +0100 Subject: [PATCH 02/15] upd --- .../FirstValueBytesRefAggregatorFunction.java | 176 ++++++++++++++ ...lueBytesRefAggregatorFunctionSupplier.java | 39 +++ ...lueBytesRefGroupingAggregatorFunction.java | 223 ++++++++++++++++++ .../FirstValueLongAggregatorFunction.java | 181 ++++++++++++++ ...stValueLongAggregatorFunctionSupplier.java | 38 +++ ...stValueLongGroupingAggregatorFunction.java | 221 +++++++++++++++++ .../FirstValueBytesRefAggregator.java | 147 ++++++++++++ .../aggregation/FirstValueLongAggregator.java | 25 ++ .../src/main/resources/observability.csv-spec | 6 +- .../function/aggregate/FirstValue.java | 28 ++- .../xpack/esql/planner/AggregateMapper.java | 4 + 11 files changed, 1083 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java new file mode 100644 index 0000000000000..e42efcabc9744 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java @@ -0,0 +1,176 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link FirstValueBytesRefAggregator}. + * This class is generated. Edit {@code AggregatorImplementer} instead. + */ +public final class FirstValueBytesRefAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("first", ElementType.BYTES_REF), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final FirstValueBytesRefAggregator.SingleState state; + + private final List channels; + + public FirstValueBytesRefAggregatorFunction(DriverContext driverContext, List channels, + FirstValueBytesRefAggregator.SingleState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static FirstValueBytesRefAggregatorFunction create(DriverContext driverContext, + List channels) { + return new FirstValueBytesRefAggregatorFunction(driverContext, channels, FirstValueBytesRefAggregator.initSingle(driverContext)); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page, BooleanVector mask) { + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { + // No masking + BytesRefBlock block = page.getBlock(channels.get(0)); + BytesRefVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + return; + } + // Some positions masked away, others kept + BytesRefBlock block = page.getBlock(channels.get(0)); + BytesRefVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector, mask); + } else { + addRawBlock(block, mask); + } + } + + private void addRawVector(BytesRefVector vector) { + BytesRef scratch = new BytesRef(); + for (int i = 0; i < vector.getPositionCount(); i++) { + FirstValueBytesRefAggregator.combine(state, vector.getBytesRef(i, scratch)); + } + } + + private void addRawVector(BytesRefVector vector, BooleanVector mask) { + BytesRef scratch = new BytesRef(); + for (int i = 0; i < vector.getPositionCount(); i++) { + if (mask.getBoolean(i) == false) { + continue; + } + FirstValueBytesRefAggregator.combine(state, vector.getBytesRef(i, scratch)); + } + } + + private void addRawBlock(BytesRefBlock block) { + BytesRef scratch = new BytesRef(); + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + FirstValueBytesRefAggregator.combine(state, block.getBytesRef(i, scratch)); + } + } + } + + private void addRawBlock(BytesRefBlock block, BooleanVector mask) { + BytesRef scratch = new BytesRef(); + for (int p = 0; p < block.getPositionCount(); p++) { + if (mask.getBoolean(p) == false) { + continue; + } + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + FirstValueBytesRefAggregator.combine(state, block.getBytesRef(i, scratch)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block firstUncast = page.getBlock(channels.get(0)); + if (firstUncast.areAllValuesNull()) { + return; + } + BytesRefVector first = ((BytesRefBlock) firstUncast).asVector(); + assert first.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + FirstValueBytesRefAggregator.combineIntermediate(state, first.getBytesRef(0, scratch), seen.getBoolean(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = FirstValueBytesRefAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..bcd10a3f1ec6f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunctionSupplier.java @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link FirstValueBytesRefAggregator}. + * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. + */ +public final class FirstValueBytesRefAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public FirstValueBytesRefAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public FirstValueBytesRefAggregatorFunction aggregator(DriverContext driverContext) { + return FirstValueBytesRefAggregatorFunction.create(driverContext, channels); + } + + @Override + public FirstValueBytesRefGroupingAggregatorFunction groupingAggregator( + DriverContext driverContext) { + return FirstValueBytesRefGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "first_value of bytes"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..713b572cda2c5 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java @@ -0,0 +1,223 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link FirstValueBytesRefAggregator}. + * This class is generated. Edit {@code GroupingAggregatorImplementer} instead. + */ +public final class FirstValueBytesRefGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("first", ElementType.BYTES_REF), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final FirstValueBytesRefAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public FirstValueBytesRefGroupingAggregatorFunction(List channels, + FirstValueBytesRefAggregator.GroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static FirstValueBytesRefGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new FirstValueBytesRefGroupingAggregatorFunction(channels, FirstValueBytesRefAggregator.initGrouping(driverContext), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + BytesRefBlock valuesBlock = page.getBlock(channels.get(0)); + BytesRefVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void close() { + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void close() { + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + FirstValueBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + FirstValueBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = groups.getInt(g); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + FirstValueBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values) { + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = groups.getInt(g); + FirstValueBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + } + + @Override + public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { + state.enableGroupIdTracking(seenGroupIds); + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block firstUncast = page.getBlock(channels.get(0)); + if (firstUncast.areAllValuesNull()) { + return; + } + BytesRefVector first = ((BytesRefBlock) firstUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert first.getPositionCount() == seen.getPositionCount(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + FirstValueBytesRefAggregator.combineIntermediate(state, groupId, first.getBytesRef(groupPosition + positionOffset, scratch), seen.getBoolean(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + FirstValueBytesRefAggregator.GroupingState inState = ((FirstValueBytesRefGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + FirstValueBytesRefAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = FirstValueBytesRefAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java new file mode 100644 index 0000000000000..4ffe73e1724ab --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java @@ -0,0 +1,181 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link FirstValueLongAggregator}. + * This class is generated. Edit {@code AggregatorImplementer} instead. + */ +public final class FirstValueLongAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("first", ElementType.LONG), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final LongState state; + + private final List channels; + + public FirstValueLongAggregatorFunction(DriverContext driverContext, List channels, + LongState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static FirstValueLongAggregatorFunction create(DriverContext driverContext, + List channels) { + return new FirstValueLongAggregatorFunction(driverContext, channels, new LongState(FirstValueLongAggregator.init())); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page, BooleanVector mask) { + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { + // No masking + LongBlock block = page.getBlock(channels.get(0)); + LongVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + return; + } + // Some positions masked away, others kept + LongBlock block = page.getBlock(channels.get(0)); + LongVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector, mask); + } else { + addRawBlock(block, mask); + } + } + + private void addRawVector(LongVector vector) { + state.seen(true); + for (int i = 0; i < vector.getPositionCount(); i++) { + state.longValue(FirstValueLongAggregator.combine(state.longValue(), vector.getLong(i))); + } + } + + private void addRawVector(LongVector vector, BooleanVector mask) { + state.seen(true); + for (int i = 0; i < vector.getPositionCount(); i++) { + if (mask.getBoolean(i) == false) { + continue; + } + state.longValue(FirstValueLongAggregator.combine(state.longValue(), vector.getLong(i))); + } + } + + private void addRawBlock(LongBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + state.seen(true); + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + state.longValue(FirstValueLongAggregator.combine(state.longValue(), block.getLong(i))); + } + } + } + + private void addRawBlock(LongBlock block, BooleanVector mask) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (mask.getBoolean(p) == false) { + continue; + } + if (block.isNull(p)) { + continue; + } + state.seen(true); + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + state.longValue(FirstValueLongAggregator.combine(state.longValue(), block.getLong(i))); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block firstUncast = page.getBlock(channels.get(0)); + if (firstUncast.areAllValuesNull()) { + return; + } + LongVector first = ((LongBlock) firstUncast).asVector(); + assert first.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + if (seen.getBoolean(0)) { + state.longValue(FirstValueLongAggregator.combine(state.longValue(), first.getLong(0))); + state.seen(true); + } + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + if (state.seen() == false) { + blocks[offset] = driverContext.blockFactory().newConstantNullBlock(1); + return; + } + blocks[offset] = driverContext.blockFactory().newConstantLongBlockWith(state.longValue(), 1); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..26d86ee5158ff --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link FirstValueLongAggregator}. + * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. + */ +public final class FirstValueLongAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public FirstValueLongAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public FirstValueLongAggregatorFunction aggregator(DriverContext driverContext) { + return FirstValueLongAggregatorFunction.create(driverContext, channels); + } + + @Override + public FirstValueLongGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return FirstValueLongGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "first_value of longs"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..0ce898a204624 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java @@ -0,0 +1,221 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link FirstValueLongAggregator}. + * This class is generated. Edit {@code GroupingAggregatorImplementer} instead. + */ +public final class FirstValueLongGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("first", ElementType.LONG), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final LongArrayState state; + + private final List channels; + + private final DriverContext driverContext; + + public FirstValueLongGroupingAggregatorFunction(List channels, LongArrayState state, + DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static FirstValueLongGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new FirstValueLongGroupingAggregatorFunction(channels, new LongArrayState(driverContext.bigArrays(), FirstValueLongAggregator.init()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + LongBlock valuesBlock = page.getBlock(channels.get(0)); + LongVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void close() { + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void close() { + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, LongBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), values.getLong(v))); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, LongVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), values.getLong(groupPosition + positionOffset))); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, LongBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = groups.getInt(g); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), values.getLong(v))); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, LongVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = groups.getInt(g); + state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), values.getLong(groupPosition + positionOffset))); + } + } + } + + @Override + public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { + state.enableGroupIdTracking(seenGroupIds); + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block firstUncast = page.getBlock(channels.get(0)); + if (firstUncast.areAllValuesNull()) { + return; + } + LongVector first = ((LongBlock) firstUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert first.getPositionCount() == seen.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + if (seen.getBoolean(groupPosition + positionOffset)) { + state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), first.getLong(groupPosition + positionOffset))); + } + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + LongArrayState inState = ((FirstValueLongGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + if (inState.hasValue(position)) { + state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), inState.get(position))); + } + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = state.toValuesBlock(selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java new file mode 100644 index 0000000000000..9404683e9db55 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; + +@Aggregator({ @IntermediateState(name = "first", type = "BYTES_REF"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@GroupingAggregator +public class FirstValueBytesRefAggregator { + + public static FirstValueBytesRefAggregator.SingleState initSingle(DriverContext driverContext) { + return new FirstValueBytesRefAggregator.SingleState(driverContext.breaker()); + } + + public static void combine(FirstValueBytesRefAggregator.SingleState state, BytesRef value) { + state.add(value); + } + + public static void combineIntermediate(FirstValueBytesRefAggregator.SingleState state, BytesRef value, boolean seen) { + if (seen) { + combine(state, value); + } + } + + public static Block evaluateFinal(FirstValueBytesRefAggregator.SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext); + } + + public static FirstValueBytesRefAggregator.GroupingState initGrouping(DriverContext driverContext) { + return new FirstValueBytesRefAggregator.GroupingState(driverContext.bigArrays(), driverContext.breaker()); + } + + public static void combine(FirstValueBytesRefAggregator.GroupingState state, int groupId, BytesRef value) { + state.add(groupId, value); + } + + public static void combineIntermediate(FirstValueBytesRefAggregator.GroupingState state, int groupId, BytesRef value, boolean seen) { + if (seen) { + state.add(groupId, value); + } + } + + public static void combineStates( + FirstValueBytesRefAggregator.GroupingState state, + int groupId, + FirstValueBytesRefAggregator.GroupingState otherState, + int otherGroupId + ) { + state.combine(groupId, otherState, otherGroupId); + } + + public static Block evaluateFinal(FirstValueBytesRefAggregator.GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(selected, driverContext); + } + + public static class GroupingState implements Releasable { + private final BytesRefArrayState internalState; + + private GroupingState(BigArrays bigArrays, CircuitBreaker breaker) { + this.internalState = new BytesRefArrayState(bigArrays, breaker, "max_bytes_ref_grouping_aggregator"); + } + + public void add(int groupId, BytesRef value) { + if (internalState.hasValue(groupId) == false) { + internalState.set(groupId, value); + } + } + + public void combine(int groupId, FirstValueBytesRefAggregator.GroupingState otherState, int otherGroupId) { + if (otherState.internalState.hasValue(otherGroupId)) { + add(groupId, otherState.internalState.get(otherGroupId)); + } + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + internalState.toIntermediate(blocks, offset, selected, driverContext); + } + + Block toBlock(IntVector selected, DriverContext driverContext) { + return internalState.toValuesBlock(selected, driverContext); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + internalState.enableGroupIdTracking(seen); + } + + @Override + public void close() { + Releasables.close(internalState); + } + } + + public static class SingleState implements Releasable { + private final BreakingBytesRefBuilder internalState; + private boolean seen; + + private SingleState(CircuitBreaker breaker) { + this.internalState = new BreakingBytesRefBuilder(breaker, "max_bytes_ref_aggregator"); + this.seen = false; + } + + public void add(BytesRef value) { + if (seen == false) { + seen = true; + + internalState.grow(value.length); + internalState.setLength(value.length); + + System.arraycopy(value.bytes, value.offset, internalState.bytes(), 0, value.length); + } + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + } + + Block toBlock(DriverContext driverContext) { + if (seen == false) { + return driverContext.blockFactory().newConstantNullBlock(1); + } + + return driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1); + } + + @Override + public void close() { + Releasables.close(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java new file mode 100644 index 0000000000000..03605ee4c5194 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +@Aggregator({ @IntermediateState(name = "first", type = "LONG"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@GroupingAggregator +class FirstValueLongAggregator { + + public static long init() { + return Long.MIN_VALUE; + } + + public static long combine(long current, long v) { + return current;// TODO compare by other fields + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec index aff770cc29c59..004ad8005d947 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec @@ -3,9 +3,10 @@ required_capability: fn_first_last FROM sample_data | STATS message = first_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) | SORT @timestamp +| KEEp @timestamp, message ; -@timestamp:long | message:keyword +@timestamp:date | message:keyword 2023-10-23T12:00:00.000Z | Connected to 10.1.0.3 2023-10-23T13:00:00.000Z | Disconnected ; @@ -15,9 +16,10 @@ required_capability: fn_first_last FROM sample_data | STATS message = last_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) | SORT @timestamp +| KEEp @timestamp, message ; -@timestamp:long | message:keyword +@timestamp:date | message:keyword 2023-10-23T12:00:00.000Z | Connected to 10.1.0.2 2023-10-23T13:00:00.000Z | Connected to 10.1.0.1 ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java index d0b53b37b45e6..335e6ccb29489 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java @@ -10,6 +10,9 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.FirstValueBytesRefAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.FirstValueLongAggregatorFunctionSupplier; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -23,6 +26,8 @@ import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; @@ -30,6 +35,14 @@ public class FirstValue extends AggregateFunction implements OptionalArgument, T public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "First", FirstValue::new); + private static final Map, AggregatorFunctionSupplier>> SUPPLIERS = Map.ofEntries( + Map.entry(DataType.LONG, FirstValueLongAggregatorFunctionSupplier::new), + Map.entry(DataType.KEYWORD, FirstValueBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.TEXT, FirstValueBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.SEMANTIC_TEXT, FirstValueBytesRefAggregatorFunctionSupplier::new), + Map.entry(DataType.VERSION, FirstValueBytesRefAggregatorFunctionSupplier::new) + ); + // TODO @FunctionInfo public FirstValue(Source source, Expression field, Expression by) { this(source, field, Literal.TRUE, by != null ? List.of(by) : List.of()); @@ -68,6 +81,8 @@ public AggregateFunction withFilter(Expression filter) { return new FirstValue(source(), field(), filter, parameters()); } + // TODO ensure BY is always timestamp/long + @Override public DataType dataType() { return field().dataType().noText(); @@ -77,7 +92,7 @@ public DataType dataType() { protected TypeResolution resolveType() { return TypeResolutions.isType( field(), - dt -> dt == DataType.KEYWORD, // TODO implement for all types + SUPPLIERS::containsKey, sourceText(), DEFAULT, "representable except unsigned_long and spatial types" @@ -86,12 +101,19 @@ protected TypeResolution resolveType() { @Override public Expression surrogate() { - return null; + // TODO can this be optimized even further? + return field().foldable() ? field() : null; } @Override public AggregatorFunctionSupplier supplier(List inputChannels) { - return null; + var type = field().dataType(); + var supplier = SUPPLIERS.get(type); + if (supplier == null) { + // If the type checking did its job, this should never happen + throw EsqlIllegalArgumentException.illegalDataType(type); + } + return supplier.apply(inputChannels); } public Expression by() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index a66a302354df2..b6599b15f3bae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.aggregate.CountDistinct; +import org.elasticsearch.xpack.esql.expression.function.aggregate.FirstValue; import org.elasticsearch.xpack.esql.expression.function.aggregate.FromPartial; import org.elasticsearch.xpack.esql.expression.function.aggregate.Max; import org.elasticsearch.xpack.esql.expression.function.aggregate.MedianAbsoluteDeviation; @@ -73,6 +74,7 @@ final class AggregateMapper { private static final List> AGG_FUNCTIONS = List.of( Count.class, CountDistinct.class, + FirstValue.class, Max.class, MedianAbsoluteDeviation.class, Min.class, @@ -193,6 +195,8 @@ private static Stream, Tuple>> typeAndNames(Class types = List.of(""); // no type } else if (CountDistinct.class.isAssignableFrom(clazz)) { types = Stream.concat(NUMERIC.stream(), Stream.of("Boolean", "BytesRef")).toList(); + } else if (FirstValue.class.isAssignableFrom(clazz)) { + types = List.of("Long", "BytesRef"); } else { assert false : "unknown aggregate type " + clazz; throw new IllegalArgumentException("unknown aggregate type " + clazz); From 9fe3db586017d52d956f23ed63e42254a6f941de Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 3 Feb 2025 12:44:36 +0100 Subject: [PATCH 03/15] upd test --- .../src/main/resources/observability.csv-spec | 45 ++++++++++++++----- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec index 004ad8005d947..439cbbb0d12df 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec @@ -1,25 +1,48 @@ -first +first_long_no_grouping required_capability: fn_first_last FROM sample_data -| STATS message = first_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) +| STATS event_duration = first_value(event_duration) +; + +event_duration:long +3450233 +; + + +first_no_grouping +required_capability: fn_first_last +FROM sample_data +| STATS message = first_value(message) +; + +message:long +Connected to 10.1.0.3 +; + + +first_long_grouped +required_capability: fn_first_last +FROM sample_data +| STATS event_duration = first_value(event_duration) BY @timestamp = BUCKET(@timestamp, 1 hour) | SORT @timestamp -| KEEp @timestamp, message +| KEEP @timestamp, event_duration ; -@timestamp:date | message:keyword -2023-10-23T12:00:00.000Z | Connected to 10.1.0.3 -2023-10-23T13:00:00.000Z | Disconnected +@timestamp:date | event_duration:long +2023-10-23T12:00:00.000Z | 3450233 +2023-10-23T13:00:00.000Z | 1232382 ; -last + +first_grouped required_capability: fn_first_last FROM sample_data -| STATS message = last_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) +| STATS message = first_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) | SORT @timestamp -| KEEp @timestamp, message +| KEEP @timestamp, message ; @timestamp:date | message:keyword -2023-10-23T12:00:00.000Z | Connected to 10.1.0.2 -2023-10-23T13:00:00.000Z | Connected to 10.1.0.1 +2023-10-23T12:00:00.000Z | Connected to 10.1.0.3 +2023-10-23T13:00:00.000Z | Disconnected ; From cb49afef1833a22f66d981c4f2038c0b8373902c Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 3 Feb 2025 14:40:36 +0100 Subject: [PATCH 04/15] signatures --- .../FirstValueLongAggregatorFunction.java | 51 ++++------ ...stValueLongGroupingAggregatorFunction.java | 49 ++++------ .../aggregation/FirstValueLongAggregator.java | 97 ++++++++++++++++++- 3 files changed, 130 insertions(+), 67 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java index 4ffe73e1724ab..d4f50ae1c3a23 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java @@ -9,9 +9,11 @@ import java.lang.String; import java.lang.StringBuilder; import java.util.List; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.LongVector; @@ -24,17 +26,16 @@ */ public final class FirstValueLongAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("first", ElementType.LONG), - new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + new IntermediateStateDesc("agg", ElementType.BYTES_REF) ); private final DriverContext driverContext; - private final LongState state; + private final FirstValueLongAggregator.FirstValueLongSingleState state; private final List channels; public FirstValueLongAggregatorFunction(DriverContext driverContext, List channels, - LongState state) { + FirstValueLongAggregator.FirstValueLongSingleState state) { this.driverContext = driverContext; this.channels = channels; this.state = state; @@ -42,7 +43,7 @@ public FirstValueLongAggregatorFunction(DriverContext driverContext, List channels) { - return new FirstValueLongAggregatorFunction(driverContext, channels, new LongState(FirstValueLongAggregator.init())); + return new FirstValueLongAggregatorFunction(driverContext, channels, FirstValueLongAggregator.initSingle()); } public static List intermediateStateDesc() { @@ -82,19 +83,17 @@ public void addRawInput(Page page, BooleanVector mask) { } private void addRawVector(LongVector vector) { - state.seen(true); for (int i = 0; i < vector.getPositionCount(); i++) { - state.longValue(FirstValueLongAggregator.combine(state.longValue(), vector.getLong(i))); + FirstValueLongAggregator.combine(state, vector.getLong(i)); } } private void addRawVector(LongVector vector, BooleanVector mask) { - state.seen(true); for (int i = 0; i < vector.getPositionCount(); i++) { if (mask.getBoolean(i) == false) { continue; } - state.longValue(FirstValueLongAggregator.combine(state.longValue(), vector.getLong(i))); + FirstValueLongAggregator.combine(state, vector.getLong(i)); } } @@ -103,11 +102,10 @@ private void addRawBlock(LongBlock block) { if (block.isNull(p)) { continue; } - state.seen(true); int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - state.longValue(FirstValueLongAggregator.combine(state.longValue(), block.getLong(i))); + FirstValueLongAggregator.combine(state, block.getLong(i)); } } } @@ -120,11 +118,10 @@ private void addRawBlock(LongBlock block, BooleanVector mask) { if (block.isNull(p)) { continue; } - state.seen(true); int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - state.longValue(FirstValueLongAggregator.combine(state.longValue(), block.getLong(i))); + FirstValueLongAggregator.combine(state, block.getLong(i)); } } } @@ -133,22 +130,14 @@ private void addRawBlock(LongBlock block, BooleanVector mask) { public void addIntermediateInput(Page page) { assert channels.size() == intermediateBlockCount(); assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); - Block firstUncast = page.getBlock(channels.get(0)); - if (firstUncast.areAllValuesNull()) { + Block aggUncast = page.getBlock(channels.get(0)); + if (aggUncast.areAllValuesNull()) { return; } - LongVector first = ((LongBlock) firstUncast).asVector(); - assert first.getPositionCount() == 1; - Block seenUncast = page.getBlock(channels.get(1)); - if (seenUncast.areAllValuesNull()) { - return; - } - BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); - assert seen.getPositionCount() == 1; - if (seen.getBoolean(0)) { - state.longValue(FirstValueLongAggregator.combine(state.longValue(), first.getLong(0))); - state.seen(true); - } + BytesRefVector agg = ((BytesRefBlock) aggUncast).asVector(); + assert agg.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + FirstValueLongAggregator.combineIntermediate(state, agg.getBytesRef(0, scratch)); } @Override @@ -158,11 +147,7 @@ public void evaluateIntermediate(Block[] blocks, int offset, DriverContext drive @Override public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { - if (state.seen() == false) { - blocks[offset] = driverContext.blockFactory().newConstantNullBlock(1); - return; - } - blocks[offset] = driverContext.blockFactory().newConstantLongBlockWith(state.longValue(), 1); + blocks[offset] = FirstValueLongAggregator.evaluateFinal(state, driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java index 0ce898a204624..f9f7fbe8e0e39 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java @@ -9,9 +9,10 @@ import java.lang.String; import java.lang.StringBuilder; import java.util.List; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BooleanBlock; -import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; @@ -26,17 +27,16 @@ */ public final class FirstValueLongGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("first", ElementType.LONG), - new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + new IntermediateStateDesc("agg", ElementType.BYTES_REF) ); - private final LongArrayState state; + private final FirstValueLongAggregator.FirstValueLongGroupingState state; private final List channels; private final DriverContext driverContext; - public FirstValueLongGroupingAggregatorFunction(List channels, LongArrayState state, - DriverContext driverContext) { + public FirstValueLongGroupingAggregatorFunction(List channels, + FirstValueLongAggregator.FirstValueLongGroupingState state, DriverContext driverContext) { this.channels = channels; this.state = state; this.driverContext = driverContext; @@ -44,7 +44,7 @@ public FirstValueLongGroupingAggregatorFunction(List channels, LongArra public static FirstValueLongGroupingAggregatorFunction create(List channels, DriverContext driverContext) { - return new FirstValueLongGroupingAggregatorFunction(channels, new LongArrayState(driverContext.bigArrays(), FirstValueLongAggregator.init()), driverContext); + return new FirstValueLongGroupingAggregatorFunction(channels, FirstValueLongAggregator.initGrouping(), driverContext); } public static List intermediateStateDesc() { @@ -107,7 +107,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values) int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), values.getLong(v))); + FirstValueLongAggregator.combine(state, groupId, values.getLong(v)); } } } @@ -115,7 +115,7 @@ private void addRawInput(int positionOffset, IntVector groups, LongBlock values) private void addRawInput(int positionOffset, IntVector groups, LongVector values) { for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), values.getLong(groupPosition + positionOffset))); + FirstValueLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset)); } } @@ -134,7 +134,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongBlock values) int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), values.getLong(v))); + FirstValueLongAggregator.combine(state, groupId, values.getLong(v)); } } } @@ -149,7 +149,7 @@ private void addRawInput(int positionOffset, IntBlock groups, LongVector values) int groupEnd = groupStart + groups.getValueCount(groupPosition); for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); - state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), values.getLong(groupPosition + positionOffset))); + FirstValueLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset)); } } } @@ -163,22 +163,15 @@ public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { state.enableGroupIdTracking(new SeenGroupIds.Empty()); assert channels.size() == intermediateBlockCount(); - Block firstUncast = page.getBlock(channels.get(0)); - if (firstUncast.areAllValuesNull()) { - return; - } - LongVector first = ((LongBlock) firstUncast).asVector(); - Block seenUncast = page.getBlock(channels.get(1)); - if (seenUncast.areAllValuesNull()) { + Block aggUncast = page.getBlock(channels.get(0)); + if (aggUncast.areAllValuesNull()) { return; } - BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); - assert first.getPositionCount() == seen.getPositionCount(); + BytesRefVector agg = ((BytesRefBlock) aggUncast).asVector(); + BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - if (seen.getBoolean(groupPosition + positionOffset)) { - state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), first.getLong(groupPosition + positionOffset))); - } + FirstValueLongAggregator.combineIntermediate(state, groupId, agg.getBytesRef(groupPosition + positionOffset, scratch)); } } @@ -187,11 +180,9 @@ public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction inpu if (input.getClass() != getClass()) { throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); } - LongArrayState inState = ((FirstValueLongGroupingAggregatorFunction) input).state; + FirstValueLongAggregator.FirstValueLongGroupingState inState = ((FirstValueLongGroupingAggregatorFunction) input).state; state.enableGroupIdTracking(new SeenGroupIds.Empty()); - if (inState.hasValue(position)) { - state.set(groupId, FirstValueLongAggregator.combine(state.getOrDefault(groupId), inState.get(position))); - } + FirstValueLongAggregator.combineStates(state, groupId, inState, position); } @Override @@ -202,7 +193,7 @@ public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) @Override public void evaluateFinal(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { - blocks[offset] = state.toValuesBlock(selected, driverContext); + blocks[offset] = FirstValueLongAggregator.evaluateFinal(state, selected, driverContext); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java index 03605ee4c5194..fcdf19b194dfb 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java @@ -7,19 +7,106 @@ package org.elasticsearch.compute.aggregation; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.compute.ann.Aggregator; import org.elasticsearch.compute.ann.GroupingAggregator; import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; -@Aggregator({ @IntermediateState(name = "first", type = "LONG"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@Aggregator({ + @IntermediateState(name = "agg", type = "BYTES_REF") +}) @GroupingAggregator class FirstValueLongAggregator { - public static long init() { - return Long.MIN_VALUE; + // single + + public static FirstValueLongSingleState initSingle() { + return new FirstValueLongSingleState(); + } + + public static void combine(FirstValueLongSingleState current, long v) { + current.add(v, /*TODO get timestamp value*/ 0L); + } + + public static void combineIntermediate(FirstValueLongSingleState current, BytesRef state) { + + } + + public static Block evaluateFinal(FirstValueLongSingleState state, DriverContext driverContext) { + return driverContext.blockFactory().newConstantLongBlockWith(state.value, 1); + } + + + // grouping + + public static FirstValueLongGroupingState initGrouping() { + return new FirstValueLongGroupingState(); + } + + public static void combine(FirstValueLongGroupingState state, int groupId, long v) { + + } + + public static void combineIntermediate(FirstValueLongGroupingState current, int groupId, BytesRef inValue) { + + } + + public static void combineStates(FirstValueLongGroupingState state, int groupId, FirstValueLongGroupingState otherState, int otherGroupId) { + } + + public static Block evaluateFinal(FirstValueLongGroupingState state, IntVector selected, DriverContext driverContext) { + return driverContext.blockFactory().newConstantLongBlockWith(state.value, 1); } - public static long combine(long current, long v) { - return current;// TODO compare by other fields + + public static class FirstValueLongSingleState implements AggregatorState { + + private long value = 0; + private long byTimestamp = Long.MAX_VALUE; + + public void add(long value, long byTimestamp) { + if (byTimestamp > this.byTimestamp) { + this.value = value; + this.byTimestamp = byTimestamp; + } + } + + + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + + } + + @Override + public void close() { + + } + } + + public static class FirstValueLongGroupingState implements GroupingAggregatorState { + + private long value = 0; + private long byTimestamp = Long.MAX_VALUE; + + public void add(long value, long byTimestamp) { + if (byTimestamp > this.byTimestamp) { + this.value = value; + this.byTimestamp = byTimestamp; + } + } + + void enableGroupIdTracking(SeenGroupIds seen) { + } + + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + + } + + @Override + public void close() {} } } From e504952046fc158f20300fee1ffdec6d2482f3a8 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Tue, 4 Feb 2025 16:48:47 +0100 Subject: [PATCH 05/15] attempt to implement aggregator --- .../FirstValueBytesRefAggregatorFunction.java | 27 ++- ...lueBytesRefGroupingAggregatorFunction.java | 28 ++- .../FirstValueLongAggregatorFunction.java | 166 -------------- ...stValueLongAggregatorFunctionSupplier.java | 38 ---- ...stValueLongGroupingAggregatorFunction.java | 212 ------------------ .../FirstValueBytesRefAggregator.java | 171 ++++++++------ .../aggregation/FirstValueLongAggregator.java | 112 --------- .../src/main/resources/observability.csv-spec | 2 +- .../function/aggregate/FirstValue.java | 2 - .../xpack/esql/planner/AggregateMapper.java | 2 +- 10 files changed, 140 insertions(+), 620 deletions(-) delete mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java delete mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java delete mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java delete mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java index e42efcabc9744..d0d030d27e42b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java @@ -16,6 +16,8 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.BytesRefVector; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; @@ -25,17 +27,18 @@ */ public final class FirstValueBytesRefAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("first", ElementType.BYTES_REF), + new IntermediateStateDesc("value", ElementType.BYTES_REF), + new IntermediateStateDesc("by", ElementType.LONG), new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); private final DriverContext driverContext; - private final FirstValueBytesRefAggregator.SingleState state; + private final FirstValueBytesRefAggregator.FirstValueLongSingleState state; private final List channels; public FirstValueBytesRefAggregatorFunction(DriverContext driverContext, List channels, - FirstValueBytesRefAggregator.SingleState state) { + FirstValueBytesRefAggregator.FirstValueLongSingleState state) { this.driverContext = driverContext; this.channels = channels; this.state = state; @@ -134,20 +137,26 @@ private void addRawBlock(BytesRefBlock block, BooleanVector mask) { public void addIntermediateInput(Page page) { assert channels.size() == intermediateBlockCount(); assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); - Block firstUncast = page.getBlock(channels.get(0)); - if (firstUncast.areAllValuesNull()) { + Block valueUncast = page.getBlock(channels.get(0)); + if (valueUncast.areAllValuesNull()) { return; } - BytesRefVector first = ((BytesRefBlock) firstUncast).asVector(); - assert first.getPositionCount() == 1; - Block seenUncast = page.getBlock(channels.get(1)); + BytesRefVector value = ((BytesRefBlock) valueUncast).asVector(); + assert value.getPositionCount() == 1; + Block byUncast = page.getBlock(channels.get(1)); + if (byUncast.areAllValuesNull()) { + return; + } + LongVector by = ((LongBlock) byUncast).asVector(); + assert by.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(2)); if (seenUncast.areAllValuesNull()) { return; } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; BytesRef scratch = new BytesRef(); - FirstValueBytesRefAggregator.combineIntermediate(state, first.getBytesRef(0, scratch), seen.getBoolean(0)); + FirstValueBytesRefAggregator.combineIntermediate(state, value.getBytesRef(0, scratch), by.getLong(0), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java index 713b572cda2c5..b554a2044cd61 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java @@ -18,6 +18,8 @@ import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; @@ -27,17 +29,18 @@ */ public final class FirstValueBytesRefGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("first", ElementType.BYTES_REF), + new IntermediateStateDesc("value", ElementType.BYTES_REF), + new IntermediateStateDesc("by", ElementType.LONG), new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); - private final FirstValueBytesRefAggregator.GroupingState state; + private final FirstValueBytesRefAggregator.FirstValueLongGroupingState state; private final List channels; private final DriverContext driverContext; public FirstValueBytesRefGroupingAggregatorFunction(List channels, - FirstValueBytesRefAggregator.GroupingState state, DriverContext driverContext) { + FirstValueBytesRefAggregator.FirstValueLongGroupingState state, DriverContext driverContext) { this.channels = channels; this.state = state; this.driverContext = driverContext; @@ -168,21 +171,26 @@ public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { state.enableGroupIdTracking(new SeenGroupIds.Empty()); assert channels.size() == intermediateBlockCount(); - Block firstUncast = page.getBlock(channels.get(0)); - if (firstUncast.areAllValuesNull()) { + Block valueUncast = page.getBlock(channels.get(0)); + if (valueUncast.areAllValuesNull()) { return; } - BytesRefVector first = ((BytesRefBlock) firstUncast).asVector(); - Block seenUncast = page.getBlock(channels.get(1)); + BytesRefVector value = ((BytesRefBlock) valueUncast).asVector(); + Block byUncast = page.getBlock(channels.get(1)); + if (byUncast.areAllValuesNull()) { + return; + } + LongVector by = ((LongBlock) byUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(2)); if (seenUncast.areAllValuesNull()) { return; } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); - assert first.getPositionCount() == seen.getPositionCount(); + assert value.getPositionCount() == by.getPositionCount() && value.getPositionCount() == seen.getPositionCount(); BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - FirstValueBytesRefAggregator.combineIntermediate(state, groupId, first.getBytesRef(groupPosition + positionOffset, scratch), seen.getBoolean(groupPosition + positionOffset)); + FirstValueBytesRefAggregator.combineIntermediate(state, groupId, value.getBytesRef(groupPosition + positionOffset, scratch), by.getLong(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset)); } } @@ -191,7 +199,7 @@ public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction inpu if (input.getClass() != getClass()) { throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); } - FirstValueBytesRefAggregator.GroupingState inState = ((FirstValueBytesRefGroupingAggregatorFunction) input).state; + FirstValueBytesRefAggregator.FirstValueLongGroupingState inState = ((FirstValueBytesRefGroupingAggregatorFunction) input).state; state.enableGroupIdTracking(new SeenGroupIds.Empty()); FirstValueBytesRefAggregator.combineStates(state, groupId, inState, position); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java deleted file mode 100644 index d4f50ae1c3a23..0000000000000 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License -// 2.0; you may not use this file except in compliance with the Elastic License -// 2.0. -package org.elasticsearch.compute.aggregation; - -import java.lang.Integer; -import java.lang.Override; -import java.lang.String; -import java.lang.StringBuilder; -import java.util.List; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BooleanVector; -import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.BytesRefVector; -import org.elasticsearch.compute.data.ElementType; -import org.elasticsearch.compute.data.LongBlock; -import org.elasticsearch.compute.data.LongVector; -import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.DriverContext; - -/** - * {@link AggregatorFunction} implementation for {@link FirstValueLongAggregator}. - * This class is generated. Edit {@code AggregatorImplementer} instead. - */ -public final class FirstValueLongAggregatorFunction implements AggregatorFunction { - private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("agg", ElementType.BYTES_REF) ); - - private final DriverContext driverContext; - - private final FirstValueLongAggregator.FirstValueLongSingleState state; - - private final List channels; - - public FirstValueLongAggregatorFunction(DriverContext driverContext, List channels, - FirstValueLongAggregator.FirstValueLongSingleState state) { - this.driverContext = driverContext; - this.channels = channels; - this.state = state; - } - - public static FirstValueLongAggregatorFunction create(DriverContext driverContext, - List channels) { - return new FirstValueLongAggregatorFunction(driverContext, channels, FirstValueLongAggregator.initSingle()); - } - - public static List intermediateStateDesc() { - return INTERMEDIATE_STATE_DESC; - } - - @Override - public int intermediateBlockCount() { - return INTERMEDIATE_STATE_DESC.size(); - } - - @Override - public void addRawInput(Page page, BooleanVector mask) { - if (mask.allFalse()) { - // Entire page masked away - return; - } - if (mask.allTrue()) { - // No masking - LongBlock block = page.getBlock(channels.get(0)); - LongVector vector = block.asVector(); - if (vector != null) { - addRawVector(vector); - } else { - addRawBlock(block); - } - return; - } - // Some positions masked away, others kept - LongBlock block = page.getBlock(channels.get(0)); - LongVector vector = block.asVector(); - if (vector != null) { - addRawVector(vector, mask); - } else { - addRawBlock(block, mask); - } - } - - private void addRawVector(LongVector vector) { - for (int i = 0; i < vector.getPositionCount(); i++) { - FirstValueLongAggregator.combine(state, vector.getLong(i)); - } - } - - private void addRawVector(LongVector vector, BooleanVector mask) { - for (int i = 0; i < vector.getPositionCount(); i++) { - if (mask.getBoolean(i) == false) { - continue; - } - FirstValueLongAggregator.combine(state, vector.getLong(i)); - } - } - - private void addRawBlock(LongBlock block) { - for (int p = 0; p < block.getPositionCount(); p++) { - if (block.isNull(p)) { - continue; - } - int start = block.getFirstValueIndex(p); - int end = start + block.getValueCount(p); - for (int i = start; i < end; i++) { - FirstValueLongAggregator.combine(state, block.getLong(i)); - } - } - } - - private void addRawBlock(LongBlock block, BooleanVector mask) { - for (int p = 0; p < block.getPositionCount(); p++) { - if (mask.getBoolean(p) == false) { - continue; - } - if (block.isNull(p)) { - continue; - } - int start = block.getFirstValueIndex(p); - int end = start + block.getValueCount(p); - for (int i = start; i < end; i++) { - FirstValueLongAggregator.combine(state, block.getLong(i)); - } - } - } - - @Override - public void addIntermediateInput(Page page) { - assert channels.size() == intermediateBlockCount(); - assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); - Block aggUncast = page.getBlock(channels.get(0)); - if (aggUncast.areAllValuesNull()) { - return; - } - BytesRefVector agg = ((BytesRefBlock) aggUncast).asVector(); - assert agg.getPositionCount() == 1; - BytesRef scratch = new BytesRef(); - FirstValueLongAggregator.combineIntermediate(state, agg.getBytesRef(0, scratch)); - } - - @Override - public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { - state.toIntermediate(blocks, offset, driverContext); - } - - @Override - public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { - blocks[offset] = FirstValueLongAggregator.evaluateFinal(state, driverContext); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(getClass().getSimpleName()).append("["); - sb.append("channels=").append(channels); - sb.append("]"); - return sb.toString(); - } - - @Override - public void close() { - state.close(); - } -} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java deleted file mode 100644 index 26d86ee5158ff..0000000000000 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License -// 2.0; you may not use this file except in compliance with the Elastic License -// 2.0. -package org.elasticsearch.compute.aggregation; - -import java.lang.Integer; -import java.lang.Override; -import java.lang.String; -import java.util.List; -import org.elasticsearch.compute.operator.DriverContext; - -/** - * {@link AggregatorFunctionSupplier} implementation for {@link FirstValueLongAggregator}. - * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. - */ -public final class FirstValueLongAggregatorFunctionSupplier implements AggregatorFunctionSupplier { - private final List channels; - - public FirstValueLongAggregatorFunctionSupplier(List channels) { - this.channels = channels; - } - - @Override - public FirstValueLongAggregatorFunction aggregator(DriverContext driverContext) { - return FirstValueLongAggregatorFunction.create(driverContext, channels); - } - - @Override - public FirstValueLongGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { - return FirstValueLongGroupingAggregatorFunction.create(channels, driverContext); - } - - @Override - public String describe() { - return "first_value of longs"; - } -} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java deleted file mode 100644 index f9f7fbe8e0e39..0000000000000 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License -// 2.0; you may not use this file except in compliance with the Elastic License -// 2.0. -package org.elasticsearch.compute.aggregation; - -import java.lang.Integer; -import java.lang.Override; -import java.lang.String; -import java.lang.StringBuilder; -import java.util.List; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BytesRefBlock; -import org.elasticsearch.compute.data.BytesRefVector; -import org.elasticsearch.compute.data.ElementType; -import org.elasticsearch.compute.data.IntBlock; -import org.elasticsearch.compute.data.IntVector; -import org.elasticsearch.compute.data.LongBlock; -import org.elasticsearch.compute.data.LongVector; -import org.elasticsearch.compute.data.Page; -import org.elasticsearch.compute.operator.DriverContext; - -/** - * {@link GroupingAggregatorFunction} implementation for {@link FirstValueLongAggregator}. - * This class is generated. Edit {@code GroupingAggregatorImplementer} instead. - */ -public final class FirstValueLongGroupingAggregatorFunction implements GroupingAggregatorFunction { - private static final List INTERMEDIATE_STATE_DESC = List.of( - new IntermediateStateDesc("agg", ElementType.BYTES_REF) ); - - private final FirstValueLongAggregator.FirstValueLongGroupingState state; - - private final List channels; - - private final DriverContext driverContext; - - public FirstValueLongGroupingAggregatorFunction(List channels, - FirstValueLongAggregator.FirstValueLongGroupingState state, DriverContext driverContext) { - this.channels = channels; - this.state = state; - this.driverContext = driverContext; - } - - public static FirstValueLongGroupingAggregatorFunction create(List channels, - DriverContext driverContext) { - return new FirstValueLongGroupingAggregatorFunction(channels, FirstValueLongAggregator.initGrouping(), driverContext); - } - - public static List intermediateStateDesc() { - return INTERMEDIATE_STATE_DESC; - } - - @Override - public int intermediateBlockCount() { - return INTERMEDIATE_STATE_DESC.size(); - } - - @Override - public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, - Page page) { - LongBlock valuesBlock = page.getBlock(channels.get(0)); - LongVector valuesVector = valuesBlock.asVector(); - if (valuesVector == null) { - if (valuesBlock.mayHaveNulls()) { - state.enableGroupIdTracking(seenGroupIds); - } - return new GroupingAggregatorFunction.AddInput() { - @Override - public void add(int positionOffset, IntBlock groupIds) { - addRawInput(positionOffset, groupIds, valuesBlock); - } - - @Override - public void add(int positionOffset, IntVector groupIds) { - addRawInput(positionOffset, groupIds, valuesBlock); - } - - @Override - public void close() { - } - }; - } - return new GroupingAggregatorFunction.AddInput() { - @Override - public void add(int positionOffset, IntBlock groupIds) { - addRawInput(positionOffset, groupIds, valuesVector); - } - - @Override - public void add(int positionOffset, IntVector groupIds) { - addRawInput(positionOffset, groupIds, valuesVector); - } - - @Override - public void close() { - } - }; - } - - private void addRawInput(int positionOffset, IntVector groups, LongBlock values) { - for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { - int groupId = groups.getInt(groupPosition); - if (values.isNull(groupPosition + positionOffset)) { - continue; - } - int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); - int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); - for (int v = valuesStart; v < valuesEnd; v++) { - FirstValueLongAggregator.combine(state, groupId, values.getLong(v)); - } - } - } - - private void addRawInput(int positionOffset, IntVector groups, LongVector values) { - for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { - int groupId = groups.getInt(groupPosition); - FirstValueLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset)); - } - } - - private void addRawInput(int positionOffset, IntBlock groups, LongBlock values) { - for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { - if (groups.isNull(groupPosition)) { - continue; - } - int groupStart = groups.getFirstValueIndex(groupPosition); - int groupEnd = groupStart + groups.getValueCount(groupPosition); - for (int g = groupStart; g < groupEnd; g++) { - int groupId = groups.getInt(g); - if (values.isNull(groupPosition + positionOffset)) { - continue; - } - int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); - int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); - for (int v = valuesStart; v < valuesEnd; v++) { - FirstValueLongAggregator.combine(state, groupId, values.getLong(v)); - } - } - } - } - - private void addRawInput(int positionOffset, IntBlock groups, LongVector values) { - for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { - if (groups.isNull(groupPosition)) { - continue; - } - int groupStart = groups.getFirstValueIndex(groupPosition); - int groupEnd = groupStart + groups.getValueCount(groupPosition); - for (int g = groupStart; g < groupEnd; g++) { - int groupId = groups.getInt(g); - FirstValueLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset)); - } - } - } - - @Override - public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { - state.enableGroupIdTracking(seenGroupIds); - } - - @Override - public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { - state.enableGroupIdTracking(new SeenGroupIds.Empty()); - assert channels.size() == intermediateBlockCount(); - Block aggUncast = page.getBlock(channels.get(0)); - if (aggUncast.areAllValuesNull()) { - return; - } - BytesRefVector agg = ((BytesRefBlock) aggUncast).asVector(); - BytesRef scratch = new BytesRef(); - for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { - int groupId = groups.getInt(groupPosition); - FirstValueLongAggregator.combineIntermediate(state, groupId, agg.getBytesRef(groupPosition + positionOffset, scratch)); - } - } - - @Override - public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { - if (input.getClass() != getClass()) { - throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); - } - FirstValueLongAggregator.FirstValueLongGroupingState inState = ((FirstValueLongGroupingAggregatorFunction) input).state; - state.enableGroupIdTracking(new SeenGroupIds.Empty()); - FirstValueLongAggregator.combineStates(state, groupId, inState, position); - } - - @Override - public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { - state.toIntermediate(blocks, offset, selected, driverContext); - } - - @Override - public void evaluateFinal(Block[] blocks, int offset, IntVector selected, - DriverContext driverContext) { - blocks[offset] = FirstValueLongAggregator.evaluateFinal(state, selected, driverContext); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(getClass().getSimpleName()).append("["); - sb.append("channels=").append(channels); - sb.append("]"); - return sb.toString(); - } - - @Override - public void close() { - state.close(); - } -} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java index 9404683e9db55..df29cab1d5cda 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java @@ -17,131 +17,164 @@ import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; import org.elasticsearch.compute.operator.DriverContext; -import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; -@Aggregator({ @IntermediateState(name = "first", type = "BYTES_REF"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@Aggregator({ + @IntermediateState(name = "value", type = "BYTES_REF"), + @IntermediateState(name = "by", type = "LONG"), + @IntermediateState(name = "seen", type = "BOOLEAN") +}) @GroupingAggregator public class FirstValueBytesRefAggregator { - public static FirstValueBytesRefAggregator.SingleState initSingle(DriverContext driverContext) { - return new FirstValueBytesRefAggregator.SingleState(driverContext.breaker()); + // single + + public static FirstValueLongSingleState initSingle(DriverContext driverContext) { + return new FirstValueLongSingleState(driverContext.breaker()); } - public static void combine(FirstValueBytesRefAggregator.SingleState state, BytesRef value) { - state.add(value); + public static void combine(FirstValueLongSingleState state, BytesRef value) { + state.add(value, /*TODO get timestamp value*/ 0L); } - public static void combineIntermediate(FirstValueBytesRefAggregator.SingleState state, BytesRef value, boolean seen) { - if (seen) { - combine(state, value); - } + public static void combineIntermediate(FirstValueLongSingleState current, BytesRef value, long by, boolean seen) { + current.combine(value, by, seen); } - public static Block evaluateFinal(FirstValueBytesRefAggregator.SingleState state, DriverContext driverContext) { - return state.toBlock(driverContext); + public static Block evaluateFinal(FirstValueLongSingleState state, DriverContext driverContext) { + return state.toFinal(driverContext); } - public static FirstValueBytesRefAggregator.GroupingState initGrouping(DriverContext driverContext) { - return new FirstValueBytesRefAggregator.GroupingState(driverContext.bigArrays(), driverContext.breaker()); + // grouping + + public static FirstValueLongGroupingState initGrouping(DriverContext driverContext) { + return new FirstValueLongGroupingState(driverContext.bigArrays(), driverContext.breaker()); } - public static void combine(FirstValueBytesRefAggregator.GroupingState state, int groupId, BytesRef value) { - state.add(groupId, value); + public static void combine(FirstValueLongGroupingState state, int groupId, BytesRef value) { + state.add(groupId, value, /*TODO get timestamp value*/ 0L); } - public static void combineIntermediate(FirstValueBytesRefAggregator.GroupingState state, int groupId, BytesRef value, boolean seen) { + public static void combineIntermediate(FirstValueLongGroupingState current, int groupId, BytesRef value, long by, boolean seen) { if (seen) { - state.add(groupId, value); + current.add(groupId, value, by); } } - public static void combineStates( - FirstValueBytesRefAggregator.GroupingState state, - int groupId, - FirstValueBytesRefAggregator.GroupingState otherState, - int otherGroupId - ) { - state.combine(groupId, otherState, otherGroupId); + public static void combineStates(FirstValueLongGroupingState state, int groupId, FirstValueLongGroupingState otherState, int otherGroupId) { + if (otherState.byState.hasValue(otherGroupId)) { + state.add(groupId, otherState.valueState.get(otherGroupId), otherState.byState.get(otherGroupId)); + } } - public static Block evaluateFinal(FirstValueBytesRefAggregator.GroupingState state, IntVector selected, DriverContext driverContext) { - return state.toBlock(selected, driverContext); + public static Block evaluateFinal(FirstValueLongGroupingState state, IntVector selected, DriverContext driverContext) { + return state.toFinal(driverContext, selected); } - public static class GroupingState implements Releasable { - private final BytesRefArrayState internalState; + public static class FirstValueLongSingleState implements AggregatorState { - private GroupingState(BigArrays bigArrays, CircuitBreaker breaker) { - this.internalState = new BytesRefArrayState(bigArrays, breaker, "max_bytes_ref_grouping_aggregator"); - } + private final BreakingBytesRefBuilder value; + private long by = Long.MIN_VALUE; + private boolean seen = false; - public void add(int groupId, BytesRef value) { - if (internalState.hasValue(groupId) == false) { - internalState.set(groupId, value); - } + public FirstValueLongSingleState(CircuitBreaker breaker) { + this.value = new BreakingBytesRefBuilder(breaker, "first_value_bytes_ref_aggregator"); } - public void combine(int groupId, FirstValueBytesRefAggregator.GroupingState otherState, int otherGroupId) { - if (otherState.internalState.hasValue(otherGroupId)) { - add(groupId, otherState.internalState.get(otherGroupId)); + public void add(BytesRef value, long by) { + if (seen == false || by < this.by) { + this.seen = true; + this.value.grow(value.length); + this.value.setLength(value.length); + System.arraycopy(value.bytes, value.offset, this.value.bytes(), 0, value.length); + this.by = by; } } - void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { - internalState.toIntermediate(blocks, offset, selected, driverContext); + public void combine(BytesRef value, long by, boolean seen) { + if (this.seen == false || (seen && by < this.by)) { + this.seen = true; + this.value.grow(value.length); + this.value.setLength(value.length); + System.arraycopy(value.bytes, value.offset, this.value.bytes(), 0, value.length); + this.by = by; + } } - Block toBlock(IntVector selected, DriverContext driverContext) { - return internalState.toValuesBlock(selected, driverContext); + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(value.bytesRefView(), 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantLongBlockWith(by, 1); + blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); } - void enableGroupIdTracking(SeenGroupIds seen) { - internalState.enableGroupIdTracking(seen); + public Block toFinal(DriverContext driverContext) { + return seen + ? driverContext.blockFactory().newConstantBytesRefBlockWith(value.bytesRefView(), 1) + : driverContext.blockFactory().newConstantNullBlock(1); } @Override public void close() { - Releasables.close(internalState); + Releasables.close(value); } } - public static class SingleState implements Releasable { - private final BreakingBytesRefBuilder internalState; - private boolean seen; - - private SingleState(CircuitBreaker breaker) { - this.internalState = new BreakingBytesRefBuilder(breaker, "max_bytes_ref_aggregator"); - this.seen = false; - } + public static class FirstValueLongGroupingState implements GroupingAggregatorState { - public void add(BytesRef value) { - if (seen == false) { - seen = true; + private final BytesRefArrayState valueState; + private final LongArrayState byState; - internalState.grow(value.length); - internalState.setLength(value.length); + public FirstValueLongGroupingState(BigArrays bigArrays, CircuitBreaker breaker) { + this.valueState = new BytesRefArrayState(bigArrays, breaker, "first_value_bytes_ref_grouping_aggregator"); + this.byState = new LongArrayState(bigArrays, Long.MIN_VALUE); + } - System.arraycopy(value.bytes, value.offset, internalState.bytes(), 0, value.length); + public void add(int groupId, BytesRef value, long byTimestamp) { + if (byState.hasValue(groupId) == false || byTimestamp < byState.getOrDefault(groupId)) { + valueState.set(groupId, value); + byState.set(groupId, byTimestamp); } } - void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { - blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1); - blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + void enableGroupIdTracking(SeenGroupIds seen) { + valueState.enableGroupIdTracking(seen); + byState.enableGroupIdTracking(seen); } - Block toBlock(DriverContext driverContext) { - if (seen == false) { - return driverContext.blockFactory().newConstantNullBlock(1); - } + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + valueState.toIntermediate(blocks, offset, selected, driverContext); + byState.toIntermediate(blocks, offset + 1, selected, driverContext); + } - return driverContext.blockFactory().newConstantBytesRefBlockWith(internalState.bytesRefView(), 1); + public Block toFinal(DriverContext driverContext, IntVector selected) { + if (byState.trackingGroupIds()) { + try (var builder = driverContext.blockFactory().newBytesRefBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (byState.hasValue(group)) { + builder.appendBytesRef(valueState.get(group)); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } else { + try (var builder = driverContext.blockFactory().newBytesRefVectorBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + builder.appendBytesRef(valueState.get(group)); + } + return builder.build().asBlock(); + } + } } @Override public void close() { - Releasables.close(internalState); + Releasables.close(valueState, byState); } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java deleted file mode 100644 index fcdf19b194dfb..0000000000000 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.compute.aggregation; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.compute.ann.Aggregator; -import org.elasticsearch.compute.ann.GroupingAggregator; -import org.elasticsearch.compute.ann.IntermediateState; -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.IntVector; -import org.elasticsearch.compute.operator.DriverContext; - -@Aggregator({ - @IntermediateState(name = "agg", type = "BYTES_REF") -}) -@GroupingAggregator -class FirstValueLongAggregator { - - // single - - public static FirstValueLongSingleState initSingle() { - return new FirstValueLongSingleState(); - } - - public static void combine(FirstValueLongSingleState current, long v) { - current.add(v, /*TODO get timestamp value*/ 0L); - } - - public static void combineIntermediate(FirstValueLongSingleState current, BytesRef state) { - - } - - public static Block evaluateFinal(FirstValueLongSingleState state, DriverContext driverContext) { - return driverContext.blockFactory().newConstantLongBlockWith(state.value, 1); - } - - - // grouping - - public static FirstValueLongGroupingState initGrouping() { - return new FirstValueLongGroupingState(); - } - - public static void combine(FirstValueLongGroupingState state, int groupId, long v) { - - } - - public static void combineIntermediate(FirstValueLongGroupingState current, int groupId, BytesRef inValue) { - - } - - public static void combineStates(FirstValueLongGroupingState state, int groupId, FirstValueLongGroupingState otherState, int otherGroupId) { - } - - public static Block evaluateFinal(FirstValueLongGroupingState state, IntVector selected, DriverContext driverContext) { - return driverContext.blockFactory().newConstantLongBlockWith(state.value, 1); - } - - - public static class FirstValueLongSingleState implements AggregatorState { - - private long value = 0; - private long byTimestamp = Long.MAX_VALUE; - - public void add(long value, long byTimestamp) { - if (byTimestamp > this.byTimestamp) { - this.value = value; - this.byTimestamp = byTimestamp; - } - } - - - @Override - public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { - - } - - @Override - public void close() { - - } - } - - public static class FirstValueLongGroupingState implements GroupingAggregatorState { - - private long value = 0; - private long byTimestamp = Long.MAX_VALUE; - - public void add(long value, long byTimestamp) { - if (byTimestamp > this.byTimestamp) { - this.value = value; - this.byTimestamp = byTimestamp; - } - } - - void enableGroupIdTracking(SeenGroupIds seen) { - } - - @Override - public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { - - } - - @Override - public void close() {} - } -} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec index 439cbbb0d12df..c6e45cd39eaa3 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec @@ -15,7 +15,7 @@ FROM sample_data | STATS message = first_value(message) ; -message:long +message:keyword Connected to 10.1.0.3 ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java index 335e6ccb29489..24d11fe808434 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.FirstValueBytesRefAggregatorFunctionSupplier; -import org.elasticsearch.compute.aggregation.FirstValueLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -36,7 +35,6 @@ public class FirstValue extends AggregateFunction implements OptionalArgument, T public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "First", FirstValue::new); private static final Map, AggregatorFunctionSupplier>> SUPPLIERS = Map.ofEntries( - Map.entry(DataType.LONG, FirstValueLongAggregatorFunctionSupplier::new), Map.entry(DataType.KEYWORD, FirstValueBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.TEXT, FirstValueBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.SEMANTIC_TEXT, FirstValueBytesRefAggregatorFunctionSupplier::new), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index b6599b15f3bae..e2254ae970cd9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -196,7 +196,7 @@ private static Stream, Tuple>> typeAndNames(Class } else if (CountDistinct.class.isAssignableFrom(clazz)) { types = Stream.concat(NUMERIC.stream(), Stream.of("Boolean", "BytesRef")).toList(); } else if (FirstValue.class.isAssignableFrom(clazz)) { - types = List.of("Long", "BytesRef"); + types = List.of("BytesRef"); } else { assert false : "unknown aggregate type " + clazz; throw new IllegalArgumentException("unknown aggregate type " + clazz); From c5781ea91fd28a4419dad523c19341233b9b8a23 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Tue, 4 Feb 2025 16:55:59 +0100 Subject: [PATCH 06/15] ignore tests for long --- .../qa/testFixtures/src/main/resources/observability.csv-spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec index c6e45cd39eaa3..6eed76937cc92 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec @@ -1,4 +1,4 @@ -first_long_no_grouping +first_long_no_grouping-Ignore required_capability: fn_first_last FROM sample_data | STATS event_duration = first_value(event_duration) @@ -20,7 +20,7 @@ Connected to 10.1.0.3 ; -first_long_grouped +first_long_grouped-Ignore required_capability: fn_first_last FROM sample_data | STATS event_duration = first_value(event_duration) BY @timestamp = BUCKET(@timestamp, 1 hour) From 30dc9497aee148f3de4a2cf1fe24dc7d19a71d46 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 10 Feb 2025 14:12:00 +0100 Subject: [PATCH 07/15] supply timestamp --- .../elasticsearch/compute/ann/Aggregator.java | 4 + .../compute/gen/AggregatorImplementer.java | 121 +++++++++++++----- .../compute/gen/AggregatorProcessor.java | 11 +- .../gen/GroupingAggregatorImplementer.java | 105 +++++++++------ .../FirstValueBytesRefAggregatorFunction.java | 34 +++-- ...lueBytesRefGroupingAggregatorFunction.java | 35 +++-- .../FirstValueBytesRefAggregator.java | 31 +++-- .../function/aggregate/FirstValue.java | 10 +- 8 files changed, 244 insertions(+), 107 deletions(-) diff --git a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java index 794baf1759204..8096153459003 100644 --- a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java +++ b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/Aggregator.java @@ -58,4 +58,8 @@ */ Class[] warnExceptions() default {}; + /** + * If {@code true} then the @timestamp LongVector will be appended to the input blocks of the aggregation function. + */ + boolean includeTimestamps() default false; } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index a07796882d46e..8a02f8bc4c69a 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -54,6 +54,8 @@ import static org.elasticsearch.compute.gen.Types.INTERMEDIATE_STATE_DESC; import static org.elasticsearch.compute.gen.Types.LIST_AGG_FUNC_DESC; import static org.elasticsearch.compute.gen.Types.LIST_INTEGER; +import static org.elasticsearch.compute.gen.Types.LONG_BLOCK; +import static org.elasticsearch.compute.gen.Types.LONG_VECTOR; import static org.elasticsearch.compute.gen.Types.PAGE; import static org.elasticsearch.compute.gen.Types.WARNINGS; import static org.elasticsearch.compute.gen.Types.blockType; @@ -73,9 +75,10 @@ public class AggregatorImplementer { private final List warnExceptions; private final ExecutableElement init; private final ExecutableElement combine; + private final List createParameters; private final ClassName implementation; private final List intermediateState; - private final List createParameters; + private final boolean includeTimestampVector; private final AggregationState aggState; private final AggregationParameter aggParam; @@ -84,7 +87,8 @@ public AggregatorImplementer( Elements elements, TypeElement declarationType, IntermediateState[] interStateAnno, - List warnExceptions + List warnExceptions, + boolean includeTimestampVector ) { this.declarationType = declarationType; this.warnExceptions = warnExceptions; @@ -102,10 +106,10 @@ public AggregatorImplementer( declarationType, aggState.declaredType().isPrimitive() ? requireType(aggState.declaredType()) : requireVoidType(), requireName("combine"), - requireArgs(requireType(aggState.declaredType()), requireAnyType("")) + combineArgs(aggState, includeTimestampVector) ); // TODO support multiple parameters - this.aggParam = AggregationParameter.create(combine.getParameters().get(1).asType()); + this.aggParam = AggregationParameter.create(combine.getParameters().getLast().asType()); this.createParameters = init.getParameters() .stream() @@ -117,7 +121,20 @@ public AggregatorImplementer( elements.getPackageOf(declarationType).toString(), (declarationType.getSimpleName() + "AggregatorFunction").replace("AggregatorAggregator", "Aggregator") ); - intermediateState = Arrays.stream(interStateAnno).map(IntermediateStateDesc::newIntermediateStateDesc).toList(); + this.intermediateState = Arrays.stream(interStateAnno).map(IntermediateStateDesc::newIntermediateStateDesc).toList(); + this.includeTimestampVector = includeTimestampVector; + } + + private static Methods.ArgumentMatcher combineArgs(AggregationState aggState, boolean includeTimestampVector) { + if (includeTimestampVector) { + return requireArgs( + requireType(aggState.declaredType()), + requireType(TypeName.LONG), // @timestamp + requireAnyType("") + ); + } else { + return requireArgs(requireType(aggState.declaredType()), requireAnyType("")); + } } ClassName implementation() { @@ -295,10 +312,18 @@ private MethodSpec addRawInput() { builder.addComment("No masking"); builder.addStatement("$T block = page.getBlock(channels.get(0))", blockType(aggParam.type())); builder.addStatement("$T vector = block.asVector()", vectorType(aggParam.type())); + if (includeTimestampVector) { + builder.addStatement("$T timestampsBlock = page.getBlock(channels.get(1))", LONG_BLOCK); + builder.addStatement("$T timestampsVector = timestampsBlock.asVector()", LONG_VECTOR); + + builder.beginControlFlow("if (timestampsVector == null) "); + builder.addStatement("throw new IllegalStateException($S)", "expected @timestamp vector; but got a block"); + builder.endControlFlow(); + } builder.beginControlFlow("if (vector != null)"); - builder.addStatement("addRawVector(vector)"); + builder.addStatement(includeTimestampVector ? "addRawVector(vector, timestampsVector)" : "addRawVector(vector)"); builder.nextControlFlow("else"); - builder.addStatement("addRawBlock(block)"); + builder.addStatement(includeTimestampVector ? "addRawBlock(block, timestampsVector)" : "addRawBlock(block)"); builder.endControlFlow(); builder.addStatement("return"); } @@ -307,10 +332,18 @@ private MethodSpec addRawInput() { builder.addComment("Some positions masked away, others kept"); builder.addStatement("$T block = page.getBlock(channels.get(0))", blockType(aggParam.type())); builder.addStatement("$T vector = block.asVector()", vectorType(aggParam.type())); + if (includeTimestampVector) { + builder.addStatement("$T timestampsBlock = page.getBlock(channels.get(1))", LONG_BLOCK); + builder.addStatement("$T timestampsVector = timestampsBlock.asVector()", LONG_VECTOR); + + builder.beginControlFlow("if (timestampsVector == null) "); + builder.addStatement("throw new IllegalStateException($S)", "expected @timestamp vector; but got a block"); + builder.endControlFlow(); + } builder.beginControlFlow("if (vector != null)"); - builder.addStatement("addRawVector(vector, mask)"); + builder.addStatement(includeTimestampVector ? "addRawVector(vector, timestampsVector, mask)" : "addRawVector(vector, mask)"); builder.nextControlFlow("else"); - builder.addStatement("addRawBlock(block, mask)"); + builder.addStatement(includeTimestampVector ? "addRawBlock(block, timestampsVector, mask)" : "addRawBlock(block, mask)"); builder.endControlFlow(); return builder.build(); } @@ -318,6 +351,9 @@ private MethodSpec addRawInput() { private MethodSpec addRawVector(boolean masked) { MethodSpec.Builder builder = MethodSpec.methodBuilder("addRawVector"); builder.addModifiers(Modifier.PRIVATE).addParameter(vectorType(aggParam.type()), "vector"); + if (includeTimestampVector) { + builder.addParameter(LONG_VECTOR, "timestamps"); + } if (masked) { builder.addParameter(BOOLEAN_VECTOR, "mask"); } @@ -348,6 +384,9 @@ private MethodSpec addRawVector(boolean masked) { private MethodSpec addRawBlock(boolean masked) { MethodSpec.Builder builder = MethodSpec.methodBuilder("addRawBlock"); builder.addModifiers(Modifier.PRIVATE).addParameter(blockType(aggParam.type()), "block"); + if (includeTimestampVector) { + builder.addParameter(LONG_VECTOR, "timestamps"); + } if (masked) { builder.addParameter(BOOLEAN_VECTOR, "mask"); } @@ -401,33 +440,57 @@ private void combineRawInput(MethodSpec.Builder builder, String blockVariable) { }); } - private void combineRawInputForPrimitive(TypeName returnType, MethodSpec.Builder builder, String blockVariable) { - builder.addStatement( - "state.$TValue($T.combine(state.$TValue(), $L.get$L(i)))", - returnType, - declarationType, - returnType, - blockVariable, - capitalize(combine.getParameters().get(1).asType().toString()) - ); + private void combineRawInputForBytesRef(MethodSpec.Builder builder, String blockVariable) { + // scratch is a BytesRef var that must have been defined before the iteration starts + if (includeTimestampVector) { + builder.addStatement("$T.combine(state, timestamps.getLong(i), $L.getBytesRef(i, scratch))", declarationType, blockVariable); + } else { + builder.addStatement("$T.combine(state, $L.getBytesRef(i, scratch))", declarationType, blockVariable); + } } - private void combineRawInputForArray(MethodSpec.Builder builder, String arrayVariable) { - warningsBlock(builder, () -> builder.addStatement("$T.combine(state, $L)", declarationType, arrayVariable)); + private void combineRawInputForPrimitive(TypeName returnType, MethodSpec.Builder builder, String blockVariable) { + if (includeTimestampVector) { + builder.addStatement( + "state.$TValue($T.combine(state.$TValue(), timestamps.getLong(i), $L.get$L(i)))", + returnType, + declarationType, + returnType, + blockVariable, + capitalize(combine.getParameters().get(1).asType().toString()) + ); + } else { + builder.addStatement( + "state.$TValue($T.combine(state.$TValue(), $L.get$L(i)))", + returnType, + declarationType, + returnType, + blockVariable, + capitalize(combine.getParameters().get(1).asType().toString()) + ); + } } private void combineRawInputForVoid(MethodSpec.Builder builder, String blockVariable) { - builder.addStatement( - "$T.combine(state, $L.get$L(i))", - declarationType, - blockVariable, - capitalize(combine.getParameters().get(1).asType().toString()) - ); + if (includeTimestampVector) { + builder.addStatement( + "$T.combine(state, timestamps.getLong(i), $L.get$L(i))", + declarationType, + blockVariable, + capitalize(combine.getParameters().get(1).asType().toString()) + ); + } else { + builder.addStatement( + "$T.combine(state, $L.get$L(i))", + declarationType, + blockVariable, + capitalize(combine.getParameters().get(1).asType().toString()) + ); + } } - private void combineRawInputForBytesRef(MethodSpec.Builder builder, String blockVariable) { - // scratch is a BytesRef var that must have been defined before the iteration starts - builder.addStatement("$T.combine(state, $L.getBytesRef(i, scratch))", declarationType, blockVariable); + private void combineRawInputForArray(MethodSpec.Builder builder, String arrayVariable) { + warningsBlock(builder, () -> builder.addStatement("$T.combine(state, $L)", declarationType, arrayVariable)); } private void warningsBlock(MethodSpec.Builder builder, Runnable block) { diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java index 863db86eb934a..3ad2343ad1658 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorProcessor.java @@ -87,7 +87,13 @@ public boolean process(Set set, RoundEnvironment roundEnv ); if (aggClass.getAnnotation(Aggregator.class) != null) { IntermediateState[] intermediateState = aggClass.getAnnotation(Aggregator.class).value(); - implementer = new AggregatorImplementer(env.getElementUtils(), aggClass, intermediateState, warnExceptionsTypes); + implementer = new AggregatorImplementer( + env.getElementUtils(), + aggClass, + intermediateState, + warnExceptionsTypes, + aggClass.getAnnotation(Aggregator.class).includeTimestamps() + ); write(aggClass, "aggregator", implementer.sourceFile(), env); } GroupingAggregatorImplementer groupingAggregatorImplementer = null; @@ -96,13 +102,12 @@ public boolean process(Set set, RoundEnvironment roundEnv if (intermediateState.length == 0 && aggClass.getAnnotation(Aggregator.class) != null) { intermediateState = aggClass.getAnnotation(Aggregator.class).value(); } - boolean includeTimestamps = aggClass.getAnnotation(GroupingAggregator.class).includeTimestamps(); groupingAggregatorImplementer = new GroupingAggregatorImplementer( env.getElementUtils(), aggClass, intermediateState, warnExceptionsTypes, - includeTimestamps + aggClass.getAnnotation(GroupingAggregator.class).includeTimestamps() ); write(aggClass, "grouping aggregator", groupingAggregatorImplementer.sourceFile(), env); } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java index 180589fc64373..fc377f22bbbc6 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java @@ -112,7 +112,8 @@ public GroupingAggregatorImplementer( requireName("combine"), combineArgs(aggState, includeTimestampVector) ); - this.aggParam = AggregationParameter.create(combine.getParameters().get(combine.getParameters().size() - 1).asType()); + // TODO support multiple parameters + this.aggParam = AggregationParameter.create(combine.getParameters().getLast().asType()); this.createParameters = init.getParameters() .stream() @@ -125,7 +126,7 @@ public GroupingAggregatorImplementer( (declarationType.getSimpleName() + "GroupingAggregatorFunction").replace("AggregatorGroupingAggregator", "GroupingAggregator") ); - intermediateState = Arrays.stream(interStateAnno) + this.intermediateState = Arrays.stream(interStateAnno) .map(AggregatorImplementer.IntermediateStateDesc::newIntermediateStateDesc) .toList(); this.includeTimestampVector = includeTimestampVector; @@ -370,8 +371,7 @@ private TypeSpec addInput(Consumer addBlock) { private MethodSpec addRawInputLoop(TypeName groupsType, TypeName valuesType) { boolean groupsIsBlock = groupsType.toString().endsWith("Block"); boolean valuesIsBlock = valuesType.toString().endsWith("Block"); - String methodName = "addRawInput"; - MethodSpec.Builder builder = MethodSpec.methodBuilder(methodName); + MethodSpec.Builder builder = MethodSpec.methodBuilder("addRawInput"); builder.addModifiers(Modifier.PRIVATE); builder.addParameter(TypeName.INT, "positionOffset").addParameter(groupsType, "groups").addParameter(valuesType, "values"); if (includeTimestampVector) { @@ -443,8 +443,6 @@ private void combineRawInput(MethodSpec.Builder builder, String blockVariable, S warningsBlock(builder, () -> { if (aggParam.isBytesRef()) { combineRawInputForBytesRef(builder, blockVariable, offsetVariable); - } else if (includeTimestampVector) { - combineRawInputWithTimestamp(builder, offsetVariable); } else if (valueType.isPrimitive() == false) { throw new IllegalArgumentException("second parameter to combine must be a primitive, array or BytesRef: " + valueType); } else if (returnType.isPrimitive()) { @@ -457,48 +455,75 @@ private void combineRawInput(MethodSpec.Builder builder, String blockVariable, S }); } - private void combineRawInputForPrimitive(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { - builder.addStatement( - "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.get$L($L)))", - declarationType, - blockVariable, - capitalize(aggParam.type().toString()), - offsetVariable - ); + private void combineRawInputForBytesRef(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { + // scratch is a BytesRef var that must have been defined before the iteration starts + if (includeTimestampVector) { + if (offsetVariable.contains(" + ")) { + builder.addStatement("var valuePosition = $L", offsetVariable); + offsetVariable = "valuePosition"; + } + builder.addStatement( + "$T.combine(state, groupId, timestamps.getLong($L), $L.getBytesRef($L, scratch))", + declarationType, + offsetVariable, + blockVariable, + offsetVariable + ); + } else { + builder.addStatement("$T.combine(state, groupId, $L.getBytesRef($L, scratch))", declarationType, blockVariable, offsetVariable); + } } - private void combineRawInputForArray(MethodSpec.Builder builder, String arrayVariable) { - warningsBlock(builder, () -> builder.addStatement("$T.combine(state, groupId, $L)", declarationType, arrayVariable)); + private void combineRawInputForPrimitive(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { + if (includeTimestampVector) { + if (offsetVariable.contains(" + ")) { + builder.addStatement("var valuePosition = $L", offsetVariable); + offsetVariable = "valuePosition"; + } + builder.addStatement( + "$T.combine(state, groupId, timestamps.getLong($L), values.get$L($L))", + declarationType, + offsetVariable, + capitalize(aggParam.type().toString()), + offsetVariable + ); + } else { + builder.addStatement( + "state.set(groupId, $T.combine(state.getOrDefault(groupId), $L.get$L($L)))", + declarationType, + blockVariable, + capitalize(aggParam.type().toString()), + offsetVariable + ); + } } private void combineRawInputForVoid(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { - builder.addStatement( - "$T.combine(state, groupId, $L.get$L($L))", - declarationType, - blockVariable, - capitalize(aggParam.type().toString()), - offsetVariable - ); - } - - private void combineRawInputWithTimestamp(MethodSpec.Builder builder, String offsetVariable) { - String blockType = capitalize(aggParam.type().toString()); - if (offsetVariable.contains(" + ")) { - builder.addStatement("var valuePosition = $L", offsetVariable); - offsetVariable = "valuePosition"; + if (includeTimestampVector) { + if (offsetVariable.contains(" + ")) { + builder.addStatement("var valuePosition = $L", offsetVariable); + offsetVariable = "valuePosition"; + } + builder.addStatement( + "$T.combine(state, groupId, timestamps.getLong($L), values.get$L($L))", + declarationType, + offsetVariable, + capitalize(aggParam.type().toString()), + offsetVariable + ); + } else { + builder.addStatement( + "$T.combine(state, groupId, $L.get$L($L))", + declarationType, + blockVariable, + capitalize(aggParam.type().toString()), + offsetVariable + ); } - builder.addStatement( - "$T.combine(state, groupId, timestamps.getLong($L), values.get$L($L))", - declarationType, - offsetVariable, - blockType, - offsetVariable - ); } - private void combineRawInputForBytesRef(MethodSpec.Builder builder, String blockVariable, String offsetVariable) { - // scratch is a BytesRef var that must have been defined before the iteration starts - builder.addStatement("$T.combine(state, groupId, $L.getBytesRef($L, scratch))", declarationType, blockVariable, offsetVariable); + private void combineRawInputForArray(MethodSpec.Builder builder, String arrayVariable) { + warningsBlock(builder, () -> builder.addStatement("$T.combine(state, groupId, $L)", declarationType, arrayVariable)); } private void warningsBlock(MethodSpec.Builder builder, Runnable block) { diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java index d0d030d27e42b..e326c283beb28 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java @@ -68,41 +68,51 @@ public void addRawInput(Page page, BooleanVector mask) { // No masking BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); + LongBlock timestampsBlock = page.getBlock(channels.get(1)); + LongVector timestampsVector = timestampsBlock.asVector(); + if (timestampsVector == null) { + throw new IllegalStateException("expected @timestamp vector; but got a block"); + } if (vector != null) { - addRawVector(vector); + addRawVector(vector, timestampsVector); } else { - addRawBlock(block); + addRawBlock(block, timestampsVector); } return; } // Some positions masked away, others kept BytesRefBlock block = page.getBlock(channels.get(0)); BytesRefVector vector = block.asVector(); + LongBlock timestampsBlock = page.getBlock(channels.get(1)); + LongVector timestampsVector = timestampsBlock.asVector(); + if (timestampsVector == null) { + throw new IllegalStateException("expected @timestamp vector; but got a block"); + } if (vector != null) { - addRawVector(vector, mask); + addRawVector(vector, timestampsVector, mask); } else { - addRawBlock(block, mask); + addRawBlock(block, timestampsVector, mask); } } - private void addRawVector(BytesRefVector vector) { + private void addRawVector(BytesRefVector vector, LongVector timestamps) { BytesRef scratch = new BytesRef(); for (int i = 0; i < vector.getPositionCount(); i++) { - FirstValueBytesRefAggregator.combine(state, vector.getBytesRef(i, scratch)); + FirstValueBytesRefAggregator.combine(state, timestamps.getLong(i), vector.getBytesRef(i, scratch)); } } - private void addRawVector(BytesRefVector vector, BooleanVector mask) { + private void addRawVector(BytesRefVector vector, LongVector timestamps, BooleanVector mask) { BytesRef scratch = new BytesRef(); for (int i = 0; i < vector.getPositionCount(); i++) { if (mask.getBoolean(i) == false) { continue; } - FirstValueBytesRefAggregator.combine(state, vector.getBytesRef(i, scratch)); + FirstValueBytesRefAggregator.combine(state, timestamps.getLong(i), vector.getBytesRef(i, scratch)); } } - private void addRawBlock(BytesRefBlock block) { + private void addRawBlock(BytesRefBlock block, LongVector timestamps) { BytesRef scratch = new BytesRef(); for (int p = 0; p < block.getPositionCount(); p++) { if (block.isNull(p)) { @@ -111,12 +121,12 @@ private void addRawBlock(BytesRefBlock block) { int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - FirstValueBytesRefAggregator.combine(state, block.getBytesRef(i, scratch)); + FirstValueBytesRefAggregator.combine(state, timestamps.getLong(i), block.getBytesRef(i, scratch)); } } } - private void addRawBlock(BytesRefBlock block, BooleanVector mask) { + private void addRawBlock(BytesRefBlock block, LongVector timestamps, BooleanVector mask) { BytesRef scratch = new BytesRef(); for (int p = 0; p < block.getPositionCount(); p++) { if (mask.getBoolean(p) == false) { @@ -128,7 +138,7 @@ private void addRawBlock(BytesRefBlock block, BooleanVector mask) { int start = block.getFirstValueIndex(p); int end = start + block.getValueCount(p); for (int i = start; i < end; i++) { - FirstValueBytesRefAggregator.combine(state, block.getBytesRef(i, scratch)); + FirstValueBytesRefAggregator.combine(state, timestamps.getLong(i), block.getBytesRef(i, scratch)); } } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java index b554a2044cd61..1dd90140cfd9b 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java @@ -65,6 +65,11 @@ public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenG Page page) { BytesRefBlock valuesBlock = page.getBlock(channels.get(0)); BytesRefVector valuesVector = valuesBlock.asVector(); + LongBlock timestampsBlock = page.getBlock(channels.get(1)); + LongVector timestampsVector = timestampsBlock.asVector(); + if (timestampsVector == null) { + throw new IllegalStateException("expected @timestamp vector; but got a block"); + } if (valuesVector == null) { if (valuesBlock.mayHaveNulls()) { state.enableGroupIdTracking(seenGroupIds); @@ -72,12 +77,12 @@ public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenG return new GroupingAggregatorFunction.AddInput() { @Override public void add(int positionOffset, IntBlock groupIds) { - addRawInput(positionOffset, groupIds, valuesBlock); + addRawInput(positionOffset, groupIds, valuesBlock, timestampsVector); } @Override public void add(int positionOffset, IntVector groupIds) { - addRawInput(positionOffset, groupIds, valuesBlock); + addRawInput(positionOffset, groupIds, valuesBlock, timestampsVector); } @Override @@ -88,12 +93,12 @@ public void close() { return new GroupingAggregatorFunction.AddInput() { @Override public void add(int positionOffset, IntBlock groupIds) { - addRawInput(positionOffset, groupIds, valuesVector); + addRawInput(positionOffset, groupIds, valuesVector, timestampsVector); } @Override public void add(int positionOffset, IntVector groupIds) { - addRawInput(positionOffset, groupIds, valuesVector); + addRawInput(positionOffset, groupIds, valuesVector, timestampsVector); } @Override @@ -102,7 +107,8 @@ public void close() { }; } - private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values) { + private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock values, + LongVector timestamps) { BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); @@ -112,20 +118,23 @@ private void addRawInput(int positionOffset, IntVector groups, BytesRefBlock val int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - FirstValueBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + FirstValueBytesRefAggregator.combine(state, groupId, timestamps.getLong(v), values.getBytesRef(v, scratch)); } } } - private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values) { + private void addRawInput(int positionOffset, IntVector groups, BytesRefVector values, + LongVector timestamps) { BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - FirstValueBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + var valuePosition = groupPosition + positionOffset; + FirstValueBytesRefAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getBytesRef(valuePosition, scratch)); } } - private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values) { + private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock values, + LongVector timestamps) { BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { @@ -141,13 +150,14 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefBlock valu int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); for (int v = valuesStart; v < valuesEnd; v++) { - FirstValueBytesRefAggregator.combine(state, groupId, values.getBytesRef(v, scratch)); + FirstValueBytesRefAggregator.combine(state, groupId, timestamps.getLong(v), values.getBytesRef(v, scratch)); } } } } - private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values) { + private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector values, + LongVector timestamps) { BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { if (groups.isNull(groupPosition)) { @@ -157,7 +167,8 @@ private void addRawInput(int positionOffset, IntBlock groups, BytesRefVector val int groupEnd = groupStart + groups.getValueCount(groupPosition); for (int g = groupStart; g < groupEnd; g++) { int groupId = groups.getInt(g); - FirstValueBytesRefAggregator.combine(state, groupId, values.getBytesRef(groupPosition + positionOffset, scratch)); + var valuePosition = groupPosition + positionOffset; + FirstValueBytesRefAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getBytesRef(valuePosition, scratch)); } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java index df29cab1d5cda..efda0619c864b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java @@ -19,12 +19,14 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.core.Releasables; -@Aggregator({ - @IntermediateState(name = "value", type = "BYTES_REF"), - @IntermediateState(name = "by", type = "LONG"), - @IntermediateState(name = "seen", type = "BOOLEAN") -}) -@GroupingAggregator +@Aggregator( + value = { + @IntermediateState(name = "value", type = "BYTES_REF"), + @IntermediateState(name = "by", type = "LONG"), + @IntermediateState(name = "seen", type = "BOOLEAN") }, + includeTimestamps = true +) +@GroupingAggregator(includeTimestamps = true) public class FirstValueBytesRefAggregator { // single @@ -37,6 +39,10 @@ public static void combine(FirstValueLongSingleState state, BytesRef value) { state.add(value, /*TODO get timestamp value*/ 0L); } + public static void combine(FirstValueLongSingleState state, long timestamp, BytesRef value) { + state.add(value, timestamp); + } + public static void combineIntermediate(FirstValueLongSingleState current, BytesRef value, long by, boolean seen) { current.combine(value, by, seen); } @@ -55,13 +61,22 @@ public static void combine(FirstValueLongGroupingState state, int groupId, Bytes state.add(groupId, value, /*TODO get timestamp value*/ 0L); } - public static void combineIntermediate(FirstValueLongGroupingState current, int groupId, BytesRef value, long by, boolean seen) { + public static void combine(FirstValueLongGroupingState state, int groupId, long timestamp, BytesRef value) { + state.add(groupId, value, timestamp); + } + + public static void combineIntermediate(FirstValueLongGroupingState current, int groupId, BytesRef value, long by, boolean seen) { if (seen) { current.add(groupId, value, by); } } - public static void combineStates(FirstValueLongGroupingState state, int groupId, FirstValueLongGroupingState otherState, int otherGroupId) { + public static void combineStates( + FirstValueLongGroupingState state, + int groupId, + FirstValueLongGroupingState otherState, + int otherGroupId + ) { if (otherState.byState.hasValue(otherGroupId)) { state.add(groupId, otherState.valueState.get(otherGroupId), otherState.byState.get(otherGroupId)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java index 24d11fe808434..3c34f62ca9234 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java @@ -15,11 +15,15 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.SurrogateExpression; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.FunctionType; import org.elasticsearch.xpack.esql.expression.function.OptionalArgument; +import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.planner.ToAggregator; @@ -41,9 +45,9 @@ public class FirstValue extends AggregateFunction implements OptionalArgument, T Map.entry(DataType.VERSION, FirstValueBytesRefAggregatorFunctionSupplier::new) ); - // TODO @FunctionInfo - public FirstValue(Source source, Expression field, Expression by) { - this(source, field, Literal.TRUE, by != null ? List.of(by) : List.of()); + @FunctionInfo(returnType = "keyword", description = "TBD", type = FunctionType.AGGREGATE, examples = {}) + public FirstValue(Source source, @Param(name = "field", type = "keyword", description = "TBD") Expression field, Expression timestamp) { + this(source, field, Literal.TRUE, timestamp != null ? List.of(timestamp) : List.of(new UnresolvedAttribute(source, "@timestamp"))); } private FirstValue(StreamInput in) throws IOException { From 65f381ad7a5025e7fcb467340464ac5742b9ddab Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 10 Feb 2025 14:36:34 +0100 Subject: [PATCH 08/15] upd --- .../FirstValueLongAggregatorFunction.java | 187 ++++++++++++++ ...stValueLongAggregatorFunctionSupplier.java | 38 +++ ...stValueLongGroupingAggregatorFunction.java | 234 ++++++++++++++++++ .../FirstValueBytesRefAggregator.java | 48 ++-- .../aggregation/FirstValueLongAggregator.java | 175 +++++++++++++ .../src/main/resources/observability.csv-spec | 39 +-- .../function/aggregate/FirstValue.java | 2 + 7 files changed, 663 insertions(+), 60 deletions(-) create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java create mode 100644 x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java create mode 100644 x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java new file mode 100644 index 0000000000000..c9882012f64d4 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java @@ -0,0 +1,187 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link FirstValueLongAggregator}. + * This class is generated. Edit {@code AggregatorImplementer} instead. + */ +public final class FirstValueLongAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("value", ElementType.LONG), + new IntermediateStateDesc("by", ElementType.LONG), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final FirstValueLongAggregator.FirstValueLongSingleState state; + + private final List channels; + + public FirstValueLongAggregatorFunction(DriverContext driverContext, List channels, + FirstValueLongAggregator.FirstValueLongSingleState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static FirstValueLongAggregatorFunction create(DriverContext driverContext, + List channels) { + return new FirstValueLongAggregatorFunction(driverContext, channels, FirstValueLongAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page, BooleanVector mask) { + if (mask.allFalse()) { + // Entire page masked away + return; + } + if (mask.allTrue()) { + // No masking + LongBlock block = page.getBlock(channels.get(0)); + LongVector vector = block.asVector(); + LongBlock timestampsBlock = page.getBlock(channels.get(1)); + LongVector timestampsVector = timestampsBlock.asVector(); + if (timestampsVector == null) { + throw new IllegalStateException("expected @timestamp vector; but got a block"); + } + if (vector != null) { + addRawVector(vector, timestampsVector); + } else { + addRawBlock(block, timestampsVector); + } + return; + } + // Some positions masked away, others kept + LongBlock block = page.getBlock(channels.get(0)); + LongVector vector = block.asVector(); + LongBlock timestampsBlock = page.getBlock(channels.get(1)); + LongVector timestampsVector = timestampsBlock.asVector(); + if (timestampsVector == null) { + throw new IllegalStateException("expected @timestamp vector; but got a block"); + } + if (vector != null) { + addRawVector(vector, timestampsVector, mask); + } else { + addRawBlock(block, timestampsVector, mask); + } + } + + private void addRawVector(LongVector vector, LongVector timestamps) { + for (int i = 0; i < vector.getPositionCount(); i++) { + FirstValueLongAggregator.combine(state, timestamps.getLong(i), vector.getLong(i)); + } + } + + private void addRawVector(LongVector vector, LongVector timestamps, BooleanVector mask) { + for (int i = 0; i < vector.getPositionCount(); i++) { + if (mask.getBoolean(i) == false) { + continue; + } + FirstValueLongAggregator.combine(state, timestamps.getLong(i), vector.getLong(i)); + } + } + + private void addRawBlock(LongBlock block, LongVector timestamps) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + FirstValueLongAggregator.combine(state, timestamps.getLong(i), block.getLong(i)); + } + } + } + + private void addRawBlock(LongBlock block, LongVector timestamps, BooleanVector mask) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (mask.getBoolean(p) == false) { + continue; + } + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + FirstValueLongAggregator.combine(state, timestamps.getLong(i), block.getLong(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block valueUncast = page.getBlock(channels.get(0)); + if (valueUncast.areAllValuesNull()) { + return; + } + LongVector value = ((LongBlock) valueUncast).asVector(); + assert value.getPositionCount() == 1; + Block byUncast = page.getBlock(channels.get(1)); + if (byUncast.areAllValuesNull()) { + return; + } + LongVector by = ((LongBlock) byUncast).asVector(); + assert by.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(2)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + FirstValueLongAggregator.combineIntermediate(state, value.getLong(0), by.getLong(0), seen.getBoolean(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = FirstValueLongAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..26d86ee5158ff --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link FirstValueLongAggregator}. + * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. + */ +public final class FirstValueLongAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public FirstValueLongAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public FirstValueLongAggregatorFunction aggregator(DriverContext driverContext) { + return FirstValueLongAggregatorFunction.create(driverContext, channels); + } + + @Override + public FirstValueLongGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return FirstValueLongGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "first_value of longs"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..7a28e11941a94 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java @@ -0,0 +1,234 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link FirstValueLongAggregator}. + * This class is generated. Edit {@code GroupingAggregatorImplementer} instead. + */ +public final class FirstValueLongGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("value", ElementType.LONG), + new IntermediateStateDesc("by", ElementType.LONG), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final FirstValueLongAggregator.FirstValueLongGroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public FirstValueLongGroupingAggregatorFunction(List channels, + FirstValueLongAggregator.FirstValueLongGroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static FirstValueLongGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new FirstValueLongGroupingAggregatorFunction(channels, FirstValueLongAggregator.initGrouping(driverContext), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + LongBlock valuesBlock = page.getBlock(channels.get(0)); + LongVector valuesVector = valuesBlock.asVector(); + LongBlock timestampsBlock = page.getBlock(channels.get(1)); + LongVector timestampsVector = timestampsBlock.asVector(); + if (timestampsVector == null) { + throw new IllegalStateException("expected @timestamp vector; but got a block"); + } + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock, timestampsVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock, timestampsVector); + } + + @Override + public void close() { + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector, timestampsVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector, timestampsVector); + } + + @Override + public void close() { + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, LongBlock values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + FirstValueLongAggregator.combine(state, groupId, timestamps.getLong(v), values.getLong(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, LongVector values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + var valuePosition = groupPosition + positionOffset; + FirstValueLongAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getLong(valuePosition)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, LongBlock values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = groups.getInt(g); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + FirstValueLongAggregator.combine(state, groupId, timestamps.getLong(v), values.getLong(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, LongVector values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = groups.getInt(g); + var valuePosition = groupPosition + positionOffset; + FirstValueLongAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getLong(valuePosition)); + } + } + } + + @Override + public void selectedMayContainUnseenGroups(SeenGroupIds seenGroupIds) { + state.enableGroupIdTracking(seenGroupIds); + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block valueUncast = page.getBlock(channels.get(0)); + if (valueUncast.areAllValuesNull()) { + return; + } + LongVector value = ((LongBlock) valueUncast).asVector(); + Block byUncast = page.getBlock(channels.get(1)); + if (byUncast.areAllValuesNull()) { + return; + } + LongVector by = ((LongBlock) byUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(2)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert value.getPositionCount() == by.getPositionCount() && value.getPositionCount() == seen.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = groups.getInt(groupPosition); + FirstValueLongAggregator.combineIntermediate(state, groupId, value.getLong(groupPosition + positionOffset), by.getLong(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + FirstValueLongAggregator.FirstValueLongGroupingState inState = ((FirstValueLongGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + FirstValueLongAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = FirstValueLongAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java index efda0619c864b..222dfb3a61a2d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java @@ -35,10 +35,6 @@ public static FirstValueLongSingleState initSingle(DriverContext driverContext) return new FirstValueLongSingleState(driverContext.breaker()); } - public static void combine(FirstValueLongSingleState state, BytesRef value) { - state.add(value, /*TODO get timestamp value*/ 0L); - } - public static void combine(FirstValueLongSingleState state, long timestamp, BytesRef value) { state.add(value, timestamp); } @@ -57,10 +53,6 @@ public static FirstValueLongGroupingState initGrouping(DriverContext driverConte return new FirstValueLongGroupingState(driverContext.bigArrays(), driverContext.breaker()); } - public static void combine(FirstValueLongGroupingState state, int groupId, BytesRef value) { - state.add(groupId, value, /*TODO get timestamp value*/ 0L); - } - public static void combine(FirstValueLongGroupingState state, int groupId, long timestamp, BytesRef value) { state.add(groupId, value, timestamp); } @@ -77,8 +69,8 @@ public static void combineStates( FirstValueLongGroupingState otherState, int otherGroupId ) { - if (otherState.byState.hasValue(otherGroupId)) { - state.add(groupId, otherState.valueState.get(otherGroupId), otherState.byState.get(otherGroupId)); + if (otherState.timestampState.hasValue(otherGroupId)) { + state.add(groupId, otherState.valueState.get(otherGroupId), otherState.timestampState.get(otherGroupId)); } } @@ -89,37 +81,37 @@ public static Block evaluateFinal(FirstValueLongGroupingState state, IntVector s public static class FirstValueLongSingleState implements AggregatorState { private final BreakingBytesRefBuilder value; - private long by = Long.MIN_VALUE; + private long timestamp = Long.MIN_VALUE; private boolean seen = false; public FirstValueLongSingleState(CircuitBreaker breaker) { this.value = new BreakingBytesRefBuilder(breaker, "first_value_bytes_ref_aggregator"); } - public void add(BytesRef value, long by) { - if (seen == false || by < this.by) { + public void add(BytesRef value, long timestamp) { + if (seen == false || timestamp < this.timestamp) { this.seen = true; this.value.grow(value.length); this.value.setLength(value.length); System.arraycopy(value.bytes, value.offset, this.value.bytes(), 0, value.length); - this.by = by; + this.timestamp = timestamp; } } - public void combine(BytesRef value, long by, boolean seen) { - if (this.seen == false || (seen && by < this.by)) { + public void combine(BytesRef value, long timestamp, boolean seen) { + if (this.seen == false || (seen && timestamp < this.timestamp)) { this.seen = true; this.value.grow(value.length); this.value.setLength(value.length); System.arraycopy(value.bytes, value.offset, this.value.bytes(), 0, value.length); - this.by = by; + this.timestamp = timestamp; } } @Override public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(value.bytesRefView(), 1); - blocks[offset + 1] = driverContext.blockFactory().newConstantLongBlockWith(by, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantLongBlockWith(timestamp, 1); blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); } @@ -138,37 +130,37 @@ public void close() { public static class FirstValueLongGroupingState implements GroupingAggregatorState { private final BytesRefArrayState valueState; - private final LongArrayState byState; + private final LongArrayState timestampState; public FirstValueLongGroupingState(BigArrays bigArrays, CircuitBreaker breaker) { this.valueState = new BytesRefArrayState(bigArrays, breaker, "first_value_bytes_ref_grouping_aggregator"); - this.byState = new LongArrayState(bigArrays, Long.MIN_VALUE); + this.timestampState = new LongArrayState(bigArrays, Long.MIN_VALUE); } - public void add(int groupId, BytesRef value, long byTimestamp) { - if (byState.hasValue(groupId) == false || byTimestamp < byState.getOrDefault(groupId)) { + public void add(int groupId, BytesRef value, long timestamp) { + if (timestampState.hasValue(groupId) == false || timestamp < timestampState.getOrDefault(groupId)) { valueState.set(groupId, value); - byState.set(groupId, byTimestamp); + timestampState.set(groupId, timestamp); } } void enableGroupIdTracking(SeenGroupIds seen) { valueState.enableGroupIdTracking(seen); - byState.enableGroupIdTracking(seen); + timestampState.enableGroupIdTracking(seen); } @Override public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { valueState.toIntermediate(blocks, offset, selected, driverContext); - byState.toIntermediate(blocks, offset + 1, selected, driverContext); + timestampState.toIntermediate(blocks, offset + 1, selected, driverContext); } public Block toFinal(DriverContext driverContext, IntVector selected) { - if (byState.trackingGroupIds()) { + if (timestampState.trackingGroupIds()) { try (var builder = driverContext.blockFactory().newBytesRefBlockBuilder(selected.getPositionCount())) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); - if (byState.hasValue(group)) { + if (timestampState.hasValue(group)) { builder.appendBytesRef(valueState.get(group)); } else { builder.appendNull(); @@ -189,7 +181,7 @@ public Block toFinal(DriverContext driverContext, IntVector selected) { @Override public void close() { - Releasables.close(valueState, byState); + Releasables.close(valueState, timestampState); } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java new file mode 100644 index 0000000000000..eec2a5815a7eb --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasables; + +@Aggregator( + value = { + @IntermediateState(name = "value", type = "LONG"), + @IntermediateState(name = "by", type = "LONG"), + @IntermediateState(name = "seen", type = "BOOLEAN") }, + includeTimestamps = true +) +@GroupingAggregator(includeTimestamps = true) +public class FirstValueLongAggregator { + + // single + + public static FirstValueLongSingleState initSingle() { + return new FirstValueLongSingleState(); + } + + public static void combine(FirstValueLongSingleState state, long timestamp, long value) { + state.add(value, timestamp); + } + + public static void combineIntermediate(FirstValueLongSingleState current, long value, long by, boolean seen) { + current.combine(value, by, seen); + } + + public static Block evaluateFinal(FirstValueLongSingleState state, DriverContext driverContext) { + return state.toFinal(driverContext); + } + + // grouping + + public static FirstValueLongGroupingState initGrouping(DriverContext driverContext) { + return new FirstValueLongGroupingState(driverContext.bigArrays(), driverContext.breaker()); + } + + public static void combine(FirstValueLongGroupingState state, int groupId, long timestamp, long value) { + state.add(groupId, value, timestamp); + } + + public static void combineIntermediate(FirstValueLongGroupingState current, int groupId, long value, long by, boolean seen) { + if (seen) { + current.add(groupId, value, by); + } + } + + public static void combineStates( + FirstValueLongGroupingState state, + int groupId, + FirstValueLongGroupingState otherState, + int otherGroupId + ) { + if (otherState.timestampState.hasValue(otherGroupId)) { + state.add(groupId, otherState.valueState.get(otherGroupId), otherState.timestampState.get(otherGroupId)); + } + } + + public static Block evaluateFinal(FirstValueLongGroupingState state, IntVector selected, DriverContext driverContext) { + return state.toFinal(driverContext, selected); + } + + public static class FirstValueLongSingleState implements AggregatorState { + + private long value = 0; + private long timestamp = Long.MIN_VALUE; + private boolean seen = false; + + public void add(long value, long timestamp) { + if (seen == false || timestamp < this.timestamp) { + this.seen = true; + this.value = value; + this.timestamp = timestamp; + } + } + + public void combine(long value, long timestamp, boolean seen) { + if (this.seen == false || (seen && timestamp < this.timestamp)) { + this.seen = true; + this.value = value; + this.timestamp = timestamp; + } + } + + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = driverContext.blockFactory().newConstantLongBlockWith(value, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantLongBlockWith(timestamp, 1); + blocks[offset + 2] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + } + + public Block toFinal(DriverContext driverContext) { + return seen + ? driverContext.blockFactory().newConstantLongBlockWith(value, 1) + : driverContext.blockFactory().newConstantNullBlock(1); + } + + @Override + public void close() {} + } + + public static class FirstValueLongGroupingState implements GroupingAggregatorState { + + private final LongArrayState valueState; + private final LongArrayState timestampState; + + public FirstValueLongGroupingState(BigArrays bigArrays, CircuitBreaker breaker) { + this.valueState = new LongArrayState(bigArrays, Long.MIN_VALUE); + this.timestampState = new LongArrayState(bigArrays, Long.MIN_VALUE); + } + + public void add(int groupId, long value, long timestamp) { + if (timestampState.hasValue(groupId) == false || timestamp < timestampState.getOrDefault(groupId)) { + valueState.set(groupId, value); + timestampState.set(groupId, timestamp); + } + } + + void enableGroupIdTracking(SeenGroupIds seen) { + valueState.enableGroupIdTracking(seen); + timestampState.enableGroupIdTracking(seen); + } + + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + valueState.toIntermediate(blocks, offset, selected, driverContext); + timestampState.toIntermediate(blocks, offset + 1, selected, driverContext); + } + + public Block toFinal(DriverContext driverContext, IntVector selected) { + if (timestampState.trackingGroupIds()) { + try (var builder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (timestampState.hasValue(group)) { + builder.appendLong(valueState.get(group)); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } else { + try (var builder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + builder.appendLong(valueState.get(group)); + } + return builder.build(); + } + } + } + + @Override + public void close() { + Releasables.close(valueState, timestampState); + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec index 6eed76937cc92..70039180af2b0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec @@ -1,48 +1,23 @@ -first_long_no_grouping-Ignore -required_capability: fn_first_last -FROM sample_data -| STATS event_duration = first_value(event_duration) -; - -event_duration:long -3450233 -; - - first_no_grouping required_capability: fn_first_last FROM sample_data -| STATS message = first_value(message) -; - -message:keyword -Connected to 10.1.0.3 -; - - -first_long_grouped-Ignore -required_capability: fn_first_last -FROM sample_data -| STATS event_duration = first_value(event_duration) BY @timestamp = BUCKET(@timestamp, 1 hour) -| SORT @timestamp -| KEEP @timestamp, event_duration +| STATS event_duration = first_value(event_duration), message = first_value(message) ; -@timestamp:date | event_duration:long -2023-10-23T12:00:00.000Z | 3450233 -2023-10-23T13:00:00.000Z | 1232382 +event_duration:long | message:keyword +3450233 | Connected to 10.1.0.3 ; first_grouped required_capability: fn_first_last FROM sample_data -| STATS message = first_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) +| STATS event_duration = first_value(event_duration), message = first_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) | SORT @timestamp | KEEP @timestamp, message ; -@timestamp:date | message:keyword -2023-10-23T12:00:00.000Z | Connected to 10.1.0.3 -2023-10-23T13:00:00.000Z | Disconnected +@timestamp:date | event_duration:long | message:keyword +2023-10-23T12:00:00.000Z | 3450233 | Connected to 10.1.0.3 +2023-10-23T13:00:00.000Z | 1232382 | Disconnected ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java index 3c34f62ca9234..8da8913dd3caf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.FirstValueBytesRefAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.FirstValueLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -39,6 +40,7 @@ public class FirstValue extends AggregateFunction implements OptionalArgument, T public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "First", FirstValue::new); private static final Map, AggregatorFunctionSupplier>> SUPPLIERS = Map.ofEntries( + Map.entry(DataType.LONG, FirstValueLongAggregatorFunctionSupplier::new), Map.entry(DataType.KEYWORD, FirstValueBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.TEXT, FirstValueBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.SEMANTIC_TEXT, FirstValueBytesRefAggregatorFunctionSupplier::new), From 287906d816985b5c8ea37dbd30ee67cd8369ae6b Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 10 Feb 2025 15:10:13 +0100 Subject: [PATCH 09/15] upd --- ...lueBytesRefAggregatorFunctionSupplier.java | 18 +++++++++---- ...stValueLongAggregatorFunctionSupplier.java | 19 +++++++++---- .../src/main/resources/observability.csv-spec | 27 ++++++++++++++++++- .../function/aggregate/FirstValue.java | 8 +++--- 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunctionSupplier.java index bcd10a3f1ec6f..70dfa538d02fe 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunctionSupplier.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunctionSupplier.java @@ -15,20 +15,28 @@ * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. */ public final class FirstValueBytesRefAggregatorFunctionSupplier implements AggregatorFunctionSupplier { - private final List channels; + public FirstValueBytesRefAggregatorFunctionSupplier() { + } - public FirstValueBytesRefAggregatorFunctionSupplier(List channels) { - this.channels = channels; + @Override + public List nonGroupingIntermediateStateDesc() { + return FirstValueBytesRefAggregatorFunction.intermediateStateDesc(); + } + + @Override + public List groupingIntermediateStateDesc() { + return FirstValueBytesRefGroupingAggregatorFunction.intermediateStateDesc(); } @Override - public FirstValueBytesRefAggregatorFunction aggregator(DriverContext driverContext) { + public FirstValueBytesRefAggregatorFunction aggregator(DriverContext driverContext, + List channels) { return FirstValueBytesRefAggregatorFunction.create(driverContext, channels); } @Override public FirstValueBytesRefGroupingAggregatorFunction groupingAggregator( - DriverContext driverContext) { + DriverContext driverContext, List channels) { return FirstValueBytesRefGroupingAggregatorFunction.create(channels, driverContext); } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java index 26d86ee5158ff..d8e09b0da7242 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunctionSupplier.java @@ -15,19 +15,28 @@ * This class is generated. Edit {@code AggregatorFunctionSupplierImplementer} instead. */ public final class FirstValueLongAggregatorFunctionSupplier implements AggregatorFunctionSupplier { - private final List channels; + public FirstValueLongAggregatorFunctionSupplier() { + } - public FirstValueLongAggregatorFunctionSupplier(List channels) { - this.channels = channels; + @Override + public List nonGroupingIntermediateStateDesc() { + return FirstValueLongAggregatorFunction.intermediateStateDesc(); + } + + @Override + public List groupingIntermediateStateDesc() { + return FirstValueLongGroupingAggregatorFunction.intermediateStateDesc(); } @Override - public FirstValueLongAggregatorFunction aggregator(DriverContext driverContext) { + public FirstValueLongAggregatorFunction aggregator(DriverContext driverContext, + List channels) { return FirstValueLongAggregatorFunction.create(driverContext, channels); } @Override - public FirstValueLongGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + public FirstValueLongGroupingAggregatorFunction groupingAggregator(DriverContext driverContext, + List channels) { return FirstValueLongGroupingAggregatorFunction.create(channels, driverContext); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec index 70039180af2b0..3cde498ecd142 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec @@ -14,10 +14,35 @@ required_capability: fn_first_last FROM sample_data | STATS event_duration = first_value(event_duration), message = first_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) | SORT @timestamp -| KEEP @timestamp, message +| KEEP @timestamp, event_duration, message ; @timestamp:date | event_duration:long | message:keyword 2023-10-23T12:00:00.000Z | 3450233 | Connected to 10.1.0.3 2023-10-23T13:00:00.000Z | 1232382 | Disconnected ; + + +first_no_grouping_long +required_capability: fn_first_last +FROM sample_data +| STATS event_duration = first_value(event_duration) +; + +event_duration:long +3450233 +; + + +first_grouped_long +required_capability: fn_first_last +FROM sample_data +| STATS event_duration = first_value(event_duration) BY @timestamp = BUCKET(@timestamp, 1 hour) +| SORT @timestamp +| KEEP @timestamp, event_duration +; + +@timestamp:date | event_duration:long +2023-10-23T12:00:00.000Z | 3450233 +2023-10-23T13:00:00.000Z | 1232382 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java index 8da8913dd3caf..60e92b954feb5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java @@ -31,7 +31,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.function.Function; +import java.util.function.Supplier; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; @@ -39,7 +39,7 @@ public class FirstValue extends AggregateFunction implements OptionalArgument, T public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "First", FirstValue::new); - private static final Map, AggregatorFunctionSupplier>> SUPPLIERS = Map.ofEntries( + private static final Map> SUPPLIERS = Map.ofEntries( Map.entry(DataType.LONG, FirstValueLongAggregatorFunctionSupplier::new), Map.entry(DataType.KEYWORD, FirstValueBytesRefAggregatorFunctionSupplier::new), Map.entry(DataType.TEXT, FirstValueBytesRefAggregatorFunctionSupplier::new), @@ -110,14 +110,14 @@ public Expression surrogate() { } @Override - public AggregatorFunctionSupplier supplier(List inputChannels) { + public AggregatorFunctionSupplier supplier() { var type = field().dataType(); var supplier = SUPPLIERS.get(type); if (supplier == null) { // If the type checking did its job, this should never happen throw EsqlIllegalArgumentException.illegalDataType(type); } - return supplier.apply(inputChannels); + return supplier.get(); } public Expression by() { From 13d879e92f040dd20e6d8e039212a3c1e54e83f7 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 10 Feb 2025 17:27:36 +0100 Subject: [PATCH 10/15] upd --- .../FirstValueBytesRefAggregatorFunction.java | 12 +- ...lueBytesRefGroupingAggregatorFunction.java | 12 +- .../FirstValueLongAggregatorFunction.java | 12 +- ...stValueLongGroupingAggregatorFunction.java | 12 +- .../FirstValueBytesRefAggregator.java | 115 +++++++++++++----- .../aggregation/FirstValueLongAggregator.java | 102 ++++++++++++---- .../src/main/resources/observability.csv-spec | 11 -- 7 files changed, 181 insertions(+), 95 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java index e326c283beb28..aac146ecacc6c 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregatorFunction.java @@ -28,7 +28,7 @@ public final class FirstValueBytesRefAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( new IntermediateStateDesc("value", ElementType.BYTES_REF), - new IntermediateStateDesc("by", ElementType.LONG), + new IntermediateStateDesc("timestamp", ElementType.LONG), new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); private final DriverContext driverContext; @@ -153,12 +153,12 @@ public void addIntermediateInput(Page page) { } BytesRefVector value = ((BytesRefBlock) valueUncast).asVector(); assert value.getPositionCount() == 1; - Block byUncast = page.getBlock(channels.get(1)); - if (byUncast.areAllValuesNull()) { + Block timestampUncast = page.getBlock(channels.get(1)); + if (timestampUncast.areAllValuesNull()) { return; } - LongVector by = ((LongBlock) byUncast).asVector(); - assert by.getPositionCount() == 1; + LongVector timestamp = ((LongBlock) timestampUncast).asVector(); + assert timestamp.getPositionCount() == 1; Block seenUncast = page.getBlock(channels.get(2)); if (seenUncast.areAllValuesNull()) { return; @@ -166,7 +166,7 @@ public void addIntermediateInput(Page page) { BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; BytesRef scratch = new BytesRef(); - FirstValueBytesRefAggregator.combineIntermediate(state, value.getBytesRef(0, scratch), by.getLong(0), seen.getBoolean(0)); + FirstValueBytesRefAggregator.combineIntermediate(state, value.getBytesRef(0, scratch), timestamp.getLong(0), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java index 1dd90140cfd9b..aec4a39bfa2ce 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueBytesRefGroupingAggregatorFunction.java @@ -30,7 +30,7 @@ public final class FirstValueBytesRefGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( new IntermediateStateDesc("value", ElementType.BYTES_REF), - new IntermediateStateDesc("by", ElementType.LONG), + new IntermediateStateDesc("timestamp", ElementType.LONG), new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); private final FirstValueBytesRefAggregator.FirstValueLongGroupingState state; @@ -187,21 +187,21 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } BytesRefVector value = ((BytesRefBlock) valueUncast).asVector(); - Block byUncast = page.getBlock(channels.get(1)); - if (byUncast.areAllValuesNull()) { + Block timestampUncast = page.getBlock(channels.get(1)); + if (timestampUncast.areAllValuesNull()) { return; } - LongVector by = ((LongBlock) byUncast).asVector(); + LongVector timestamp = ((LongBlock) timestampUncast).asVector(); Block seenUncast = page.getBlock(channels.get(2)); if (seenUncast.areAllValuesNull()) { return; } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); - assert value.getPositionCount() == by.getPositionCount() && value.getPositionCount() == seen.getPositionCount(); + assert value.getPositionCount() == timestamp.getPositionCount() && value.getPositionCount() == seen.getPositionCount(); BytesRef scratch = new BytesRef(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - FirstValueBytesRefAggregator.combineIntermediate(state, groupId, value.getBytesRef(groupPosition + positionOffset, scratch), by.getLong(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset)); + FirstValueBytesRefAggregator.combineIntermediate(state, groupId, value.getBytesRef(groupPosition + positionOffset, scratch), timestamp.getLong(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset)); } } diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java index c9882012f64d4..162bef97688b8 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongAggregatorFunction.java @@ -25,7 +25,7 @@ public final class FirstValueLongAggregatorFunction implements AggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( new IntermediateStateDesc("value", ElementType.LONG), - new IntermediateStateDesc("by", ElementType.LONG), + new IntermediateStateDesc("timestamp", ElementType.LONG), new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); private final DriverContext driverContext; @@ -146,19 +146,19 @@ public void addIntermediateInput(Page page) { } LongVector value = ((LongBlock) valueUncast).asVector(); assert value.getPositionCount() == 1; - Block byUncast = page.getBlock(channels.get(1)); - if (byUncast.areAllValuesNull()) { + Block timestampUncast = page.getBlock(channels.get(1)); + if (timestampUncast.areAllValuesNull()) { return; } - LongVector by = ((LongBlock) byUncast).asVector(); - assert by.getPositionCount() == 1; + LongVector timestamp = ((LongBlock) timestampUncast).asVector(); + assert timestamp.getPositionCount() == 1; Block seenUncast = page.getBlock(channels.get(2)); if (seenUncast.areAllValuesNull()) { return; } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); assert seen.getPositionCount() == 1; - FirstValueLongAggregator.combineIntermediate(state, value.getLong(0), by.getLong(0), seen.getBoolean(0)); + FirstValueLongAggregator.combineIntermediate(state, value.getLong(0), timestamp.getLong(0), seen.getBoolean(0)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java index 7a28e11941a94..06a79902f1101 100644 --- a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/FirstValueLongGroupingAggregatorFunction.java @@ -27,7 +27,7 @@ public final class FirstValueLongGroupingAggregatorFunction implements GroupingAggregatorFunction { private static final List INTERMEDIATE_STATE_DESC = List.of( new IntermediateStateDesc("value", ElementType.LONG), - new IntermediateStateDesc("by", ElementType.LONG), + new IntermediateStateDesc("timestamp", ElementType.LONG), new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); private final FirstValueLongAggregator.FirstValueLongGroupingState state; @@ -180,20 +180,20 @@ public void addIntermediateInput(int positionOffset, IntVector groups, Page page return; } LongVector value = ((LongBlock) valueUncast).asVector(); - Block byUncast = page.getBlock(channels.get(1)); - if (byUncast.areAllValuesNull()) { + Block timestampUncast = page.getBlock(channels.get(1)); + if (timestampUncast.areAllValuesNull()) { return; } - LongVector by = ((LongBlock) byUncast).asVector(); + LongVector timestamp = ((LongBlock) timestampUncast).asVector(); Block seenUncast = page.getBlock(channels.get(2)); if (seenUncast.areAllValuesNull()) { return; } BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); - assert value.getPositionCount() == by.getPositionCount() && value.getPositionCount() == seen.getPositionCount(); + assert value.getPositionCount() == timestamp.getPositionCount() && value.getPositionCount() == seen.getPositionCount(); for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { int groupId = groups.getInt(groupPosition); - FirstValueLongAggregator.combineIntermediate(state, groupId, value.getLong(groupPosition + positionOffset), by.getLong(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset)); + FirstValueLongAggregator.combineIntermediate(state, groupId, value.getLong(groupPosition + positionOffset), timestamp.getLong(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset)); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java index 222dfb3a61a2d..404c325835953 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java @@ -10,6 +10,9 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.LongArray; +import org.elasticsearch.common.util.ObjectArray; import org.elasticsearch.compute.ann.Aggregator; import org.elasticsearch.compute.ann.GroupingAggregator; import org.elasticsearch.compute.ann.IntermediateState; @@ -22,7 +25,7 @@ @Aggregator( value = { @IntermediateState(name = "value", type = "BYTES_REF"), - @IntermediateState(name = "by", type = "LONG"), + @IntermediateState(name = "timestamp", type = "LONG"), @IntermediateState(name = "seen", type = "BOOLEAN") }, includeTimestamps = true ) @@ -39,8 +42,10 @@ public static void combine(FirstValueLongSingleState state, long timestamp, Byte state.add(value, timestamp); } - public static void combineIntermediate(FirstValueLongSingleState current, BytesRef value, long by, boolean seen) { - current.combine(value, by, seen); + public static void combineIntermediate(FirstValueLongSingleState current, BytesRef value, long timestamp, boolean seen) { + if (seen) { + current.add(value, timestamp); + } } public static Block evaluateFinal(FirstValueLongSingleState state, DriverContext driverContext) { @@ -57,9 +62,9 @@ public static void combine(FirstValueLongGroupingState state, int groupId, long state.add(groupId, value, timestamp); } - public static void combineIntermediate(FirstValueLongGroupingState current, int groupId, BytesRef value, long by, boolean seen) { + public static void combineIntermediate(FirstValueLongGroupingState current, int groupId, BytesRef value, long timestamp, boolean seen) { if (seen) { - current.add(groupId, value, by); + current.add(groupId, value, timestamp); } } @@ -69,8 +74,8 @@ public static void combineStates( FirstValueLongGroupingState otherState, int otherGroupId ) { - if (otherState.timestampState.hasValue(otherGroupId)) { - state.add(groupId, otherState.valueState.get(otherGroupId), otherState.timestampState.get(otherGroupId)); + if (otherState.hasValue(otherGroupId)) { + state.add(groupId, otherState.valueState.get(otherGroupId).bytesRefView(), otherState.timestampState.get(otherGroupId)); } } @@ -81,7 +86,7 @@ public static Block evaluateFinal(FirstValueLongGroupingState state, IntVector s public static class FirstValueLongSingleState implements AggregatorState { private final BreakingBytesRefBuilder value; - private long timestamp = Long.MIN_VALUE; + private long timestamp = Long.MAX_VALUE; private boolean seen = false; public FirstValueLongSingleState(CircuitBreaker breaker) { @@ -98,16 +103,6 @@ public void add(BytesRef value, long timestamp) { } } - public void combine(BytesRef value, long timestamp, boolean seen) { - if (this.seen == false || (seen && timestamp < this.timestamp)) { - this.seen = true; - this.value.grow(value.length); - this.value.setLength(value.length); - System.arraycopy(value.bytes, value.offset, this.value.bytes(), 0, value.length); - this.timestamp = timestamp; - } - } - @Override public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = driverContext.blockFactory().newConstantBytesRefBlockWith(value.bytesRefView(), 1); @@ -129,39 +124,93 @@ public void close() { public static class FirstValueLongGroupingState implements GroupingAggregatorState { - private final BytesRefArrayState valueState; - private final LongArrayState timestampState; + private final BigArrays bigArrays; + private final CircuitBreaker breaker; + + private ObjectArray valueState; + private LongArray timestampState; + private BitArray seen; public FirstValueLongGroupingState(BigArrays bigArrays, CircuitBreaker breaker) { - this.valueState = new BytesRefArrayState(bigArrays, breaker, "first_value_bytes_ref_grouping_aggregator"); - this.timestampState = new LongArrayState(bigArrays, Long.MIN_VALUE); + this.bigArrays = bigArrays; + this.breaker = breaker; + valueState = bigArrays.newObjectArray(0); + timestampState = bigArrays.newLongArray(1, false); + seen = null; } public void add(int groupId, BytesRef value, long timestamp) { - if (timestampState.hasValue(groupId) == false || timestamp < timestampState.getOrDefault(groupId)) { - valueState.set(groupId, value); + if (hasValue(groupId) == false || timestamp < getTimestamp(groupId)) { + ensureCapacity(groupId); + var currentBuilder = valueState.get(groupId); + if (currentBuilder == null) { + currentBuilder = new BreakingBytesRefBuilder(breaker, "values", value.length); + valueState.set(groupId, currentBuilder); + } + currentBuilder.copyBytes(value); timestampState.set(groupId, timestamp); + if (seen != null) { + seen.set(groupId); + } } } void enableGroupIdTracking(SeenGroupIds seen) { - valueState.enableGroupIdTracking(seen); - timestampState.enableGroupIdTracking(seen); + if (this.seen == null) { + this.seen = seen.seenGroupIds(bigArrays); + } + } + + public boolean hasValue(int groupId) { + return groupId < valueState.size() && valueState.get(groupId) != null; + } + + public long getTimestamp(int groupId) { + return groupId < timestampState.size() ? timestampState.get(groupId) : Long.MAX_VALUE; + } + + private void ensureCapacity(int groupId) { + if (groupId >= timestampState.size()) { + valueState = bigArrays.grow(valueState, groupId + 1); + long prevSize = timestampState.size(); + timestampState = bigArrays.grow(timestampState, groupId + 1); + timestampState.fill(prevSize, timestampState.size(), Long.MAX_VALUE); + } } @Override public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { - valueState.toIntermediate(blocks, offset, selected, driverContext); - timestampState.toIntermediate(blocks, offset + 1, selected, driverContext); + assert blocks.length >= offset + 3; + try ( + var valuesBuilder = driverContext.blockFactory().newBytesRefVectorBuilder(selected.getPositionCount()); + var timestampBuilder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + var emptyBytesRef = new BytesRef(); + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group)) { + valuesBuilder.appendBytesRef(valueState.get(group).bytesRefView()); + timestampBuilder.appendLong(timestampState.get(group)); + } else { + valuesBuilder.appendBytesRef(emptyBytesRef); + timestampBuilder.appendNull(); + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + } + blocks[offset] = valuesBuilder.build().asBlock(); + blocks[offset + 1] = timestampBuilder.build(); + blocks[offset + 2] = hasValueBuilder.build().asBlock(); + } } public Block toFinal(DriverContext driverContext, IntVector selected) { - if (timestampState.trackingGroupIds()) { + if (seen != null) { try (var builder = driverContext.blockFactory().newBytesRefBlockBuilder(selected.getPositionCount())) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); - if (timestampState.hasValue(group)) { - builder.appendBytesRef(valueState.get(group)); + if (seen.get(group)) { + builder.appendBytesRef(valueState.get(group).bytesRefView()); } else { builder.appendNull(); } @@ -172,7 +221,7 @@ public Block toFinal(DriverContext driverContext, IntVector selected) { try (var builder = driverContext.blockFactory().newBytesRefVectorBuilder(selected.getPositionCount())) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); - builder.appendBytesRef(valueState.get(group)); + builder.appendBytesRef(valueState.get(group).bytesRefView()); } return builder.build().asBlock(); } @@ -181,7 +230,7 @@ public Block toFinal(DriverContext driverContext, IntVector selected) { @Override public void close() { - Releasables.close(valueState, timestampState); + Releasables.close(valueState, timestampState, seen); } } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java index eec2a5815a7eb..63b90b25e82b6 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java @@ -9,6 +9,8 @@ import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.LongArray; import org.elasticsearch.compute.ann.Aggregator; import org.elasticsearch.compute.ann.GroupingAggregator; import org.elasticsearch.compute.ann.IntermediateState; @@ -20,7 +22,7 @@ @Aggregator( value = { @IntermediateState(name = "value", type = "LONG"), - @IntermediateState(name = "by", type = "LONG"), + @IntermediateState(name = "timestamp", type = "LONG"), @IntermediateState(name = "seen", type = "BOOLEAN") }, includeTimestamps = true ) @@ -37,8 +39,10 @@ public static void combine(FirstValueLongSingleState state, long timestamp, long state.add(value, timestamp); } - public static void combineIntermediate(FirstValueLongSingleState current, long value, long by, boolean seen) { - current.combine(value, by, seen); + public static void combineIntermediate(FirstValueLongSingleState current, long value, long timestamp, boolean seen) { + if (seen) { + current.add(value, timestamp); + } } public static Block evaluateFinal(FirstValueLongSingleState state, DriverContext driverContext) { @@ -55,9 +59,9 @@ public static void combine(FirstValueLongGroupingState state, int groupId, long state.add(groupId, value, timestamp); } - public static void combineIntermediate(FirstValueLongGroupingState current, int groupId, long value, long by, boolean seen) { + public static void combineIntermediate(FirstValueLongGroupingState current, int groupId, long value, long timestamp, boolean seen) { if (seen) { - current.add(groupId, value, by); + current.add(groupId, value, timestamp); } } @@ -67,7 +71,7 @@ public static void combineStates( FirstValueLongGroupingState otherState, int otherGroupId ) { - if (otherState.timestampState.hasValue(otherGroupId)) { + if (otherState.hasValue(otherGroupId)) { state.add(groupId, otherState.valueState.get(otherGroupId), otherState.timestampState.get(otherGroupId)); } } @@ -79,7 +83,7 @@ public static Block evaluateFinal(FirstValueLongGroupingState state, IntVector s public static class FirstValueLongSingleState implements AggregatorState { private long value = 0; - private long timestamp = Long.MIN_VALUE; + private long timestamp = Long.MAX_VALUE; private boolean seen = false; public void add(long value, long timestamp) { @@ -90,14 +94,6 @@ public void add(long value, long timestamp) { } } - public void combine(long value, long timestamp, boolean seen) { - if (this.seen == false || (seen && timestamp < this.timestamp)) { - this.seen = true; - this.value = value; - this.timestamp = timestamp; - } - } - @Override public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { blocks[offset] = driverContext.blockFactory().newConstantLongBlockWith(value, 1); @@ -117,38 +113,90 @@ public void close() {} public static class FirstValueLongGroupingState implements GroupingAggregatorState { - private final LongArrayState valueState; - private final LongArrayState timestampState; + private final BigArrays bigArrays; + private final CircuitBreaker breaker; + + private LongArray valueState; + private LongArray timestampState; + private BitArray seen; public FirstValueLongGroupingState(BigArrays bigArrays, CircuitBreaker breaker) { - this.valueState = new LongArrayState(bigArrays, Long.MIN_VALUE); - this.timestampState = new LongArrayState(bigArrays, Long.MIN_VALUE); + this.bigArrays = bigArrays; + this.breaker = breaker; + valueState = bigArrays.newLongArray(1, false); + timestampState = bigArrays.newLongArray(1, false); + seen = null; } public void add(int groupId, long value, long timestamp) { - if (timestampState.hasValue(groupId) == false || timestamp < timestampState.getOrDefault(groupId)) { + long currentValue = groupId < valueState.size() ? valueState.get(groupId) : Long.MAX_VALUE; + long currentTimestamp = groupId < timestampState.size() ? timestampState.get(groupId) : Long.MAX_VALUE; + if (hasValue(groupId) == false || timestamp < getTimestamp(groupId)) { + ensureCapacity(groupId); valueState.set(groupId, value); timestampState.set(groupId, timestamp); + if (seen != null) { + seen.set(groupId); + } } } void enableGroupIdTracking(SeenGroupIds seen) { - valueState.enableGroupIdTracking(seen); - timestampState.enableGroupIdTracking(seen); + if (this.seen == null) { + this.seen = seen.seenGroupIds(bigArrays); + } + } + + public boolean hasValue(int groupId) { + return groupId < valueState.size() && valueState.get(groupId) != Long.MAX_VALUE; + } + + public long getTimestamp(int groupId) { + return groupId < timestampState.size() ? timestampState.get(groupId) : Long.MAX_VALUE; + } + + private void ensureCapacity(int groupId) { + if (groupId >= timestampState.size()) { + long prevSize1 = valueState.size(); + valueState = bigArrays.grow(valueState, groupId + 1); + valueState.fill(prevSize1, valueState.size(), Long.MAX_VALUE); + long prevSize2 = timestampState.size(); + timestampState = bigArrays.grow(timestampState, groupId + 1); + timestampState.fill(prevSize2, timestampState.size(), Long.MAX_VALUE); + } } @Override public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { - valueState.toIntermediate(blocks, offset, selected, driverContext); - timestampState.toIntermediate(blocks, offset + 1, selected, driverContext); + assert blocks.length >= offset + 3; + try ( + var valuesBuilder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount()); + var timestampBuilder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group)) { + valuesBuilder.appendLong(valueState.get(group)); + timestampBuilder.appendLong(timestampState.get(group)); + } else { + valuesBuilder.appendNull(); + timestampBuilder.appendNull(); + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + } + blocks[offset] = valuesBuilder.build(); + blocks[offset + 1] = timestampBuilder.build(); + blocks[offset + 2] = hasValueBuilder.build().asBlock(); + } } public Block toFinal(DriverContext driverContext, IntVector selected) { - if (timestampState.trackingGroupIds()) { + if (seen != null) { try (var builder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount())) { for (int i = 0; i < selected.getPositionCount(); i++) { int group = selected.getInt(i); - if (timestampState.hasValue(group)) { + if (seen.get(group)) { builder.appendLong(valueState.get(group)); } else { builder.appendNull(); @@ -169,7 +217,7 @@ public Block toFinal(DriverContext driverContext, IntVector selected) { @Override public void close() { - Releasables.close(valueState, timestampState); + Releasables.close(valueState, timestampState, seen); } } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec index 3cde498ecd142..b016a5b9a2ed2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec @@ -23,17 +23,6 @@ FROM sample_data ; -first_no_grouping_long -required_capability: fn_first_last -FROM sample_data -| STATS event_duration = first_value(event_duration) -; - -event_duration:long -3450233 -; - - first_grouped_long required_capability: fn_first_last FROM sample_data From 2847bfbb1117b785d3e056b8801a11aab083acd5 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 10 Feb 2025 17:43:51 +0100 Subject: [PATCH 11/15] fix tests --- .../src/main/resources/observability.csv-spec | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec index b016a5b9a2ed2..65571e4552eb9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec @@ -12,12 +12,12 @@ event_duration:long | message:keyword first_grouped required_capability: fn_first_last FROM sample_data -| STATS event_duration = first_value(event_duration), message = first_value(message) BY @timestamp = BUCKET(@timestamp, 1 hour) -| SORT @timestamp -| KEEP @timestamp, event_duration, message +| STATS event_duration = first_value(event_duration), message = first_value(message) BY hourly = BUCKET(@timestamp, 1 hour) +| SORT hourly +| KEEP hourly, event_duration, message ; -@timestamp:date | event_duration:long | message:keyword +hourly:date | event_duration:long | message:keyword 2023-10-23T12:00:00.000Z | 3450233 | Connected to 10.1.0.3 2023-10-23T13:00:00.000Z | 1232382 | Disconnected ; @@ -26,12 +26,12 @@ FROM sample_data first_grouped_long required_capability: fn_first_last FROM sample_data -| STATS event_duration = first_value(event_duration) BY @timestamp = BUCKET(@timestamp, 1 hour) -| SORT @timestamp -| KEEP @timestamp, event_duration +| STATS event_duration = first_value(event_duration) BY hourly = BUCKET(@timestamp, 1 hour) +| SORT hourly +| KEEP hourly, event_duration ; -@timestamp:date | event_duration:long +hourly:date | event_duration:long 2023-10-23T12:00:00.000Z | 3450233 2023-10-23T13:00:00.000Z | 1232382 ; From d560527d2568a0d0009643d5b689a12bdc57f88e Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 10 Feb 2025 17:50:59 +0100 Subject: [PATCH 12/15] fix unreleased state --- .../aggregation/FirstValueBytesRefAggregator.java | 3 +++ .../src/main/resources/observability.csv-spec | 14 -------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java index 404c325835953..2d1a95559a538 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java @@ -230,6 +230,9 @@ public Block toFinal(DriverContext driverContext, IntVector selected) { @Override public void close() { + for (int i = 0; i < valueState.size(); i++) { + Releasables.closeWhileHandlingException(valueState.get(i)); + } Releasables.close(valueState, timestampState, seen); } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec index 65571e4552eb9..2ac16895952d0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec @@ -21,17 +21,3 @@ hourly:date | event_duration:long | message:keyword 2023-10-23T12:00:00.000Z | 3450233 | Connected to 10.1.0.3 2023-10-23T13:00:00.000Z | 1232382 | Disconnected ; - - -first_grouped_long -required_capability: fn_first_last -FROM sample_data -| STATS event_duration = first_value(event_duration) BY hourly = BUCKET(@timestamp, 1 hour) -| SORT hourly -| KEEP hourly, event_duration -; - -hourly:date | event_duration:long -2023-10-23T12:00:00.000Z | 3450233 -2023-10-23T13:00:00.000Z | 1232382 -; From 904249d2f695a248f3976450e53dfa028ac6b3e1 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 13 Feb 2025 16:02:08 +0100 Subject: [PATCH 13/15] initial unit test --- .../FirstValueBytesRefAggregator.java | 6 +- .../aggregation/FirstValueLongAggregator.java | 6 +- .../function/aggregate/FirstValue.java | 19 ++++- .../function/aggregate/FirstValueTests.java | 73 +++++++++++++++++++ 4 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValueTests.java diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java index 2d1a95559a538..fd734a6df2bfa 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java @@ -134,7 +134,7 @@ public static class FirstValueLongGroupingState implements GroupingAggregatorSta public FirstValueLongGroupingState(BigArrays bigArrays, CircuitBreaker breaker) { this.bigArrays = bigArrays; this.breaker = breaker; - valueState = bigArrays.newObjectArray(0); + valueState = bigArrays.newObjectArray(1); timestampState = bigArrays.newLongArray(1, false); seen = null; } @@ -170,8 +170,10 @@ public long getTimestamp(int groupId) { } private void ensureCapacity(int groupId) { - if (groupId >= timestampState.size()) { + if (groupId >= valueState.size()) { valueState = bigArrays.grow(valueState, groupId + 1); + } + if (groupId >= timestampState.size()) { long prevSize = timestampState.size(); timestampState = bigArrays.grow(timestampState, groupId + 1); timestampState.fill(prevSize, timestampState.size(), Long.MAX_VALUE); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java index 63b90b25e82b6..e3f77bce49b07 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java @@ -129,8 +129,6 @@ public FirstValueLongGroupingState(BigArrays bigArrays, CircuitBreaker breaker) } public void add(int groupId, long value, long timestamp) { - long currentValue = groupId < valueState.size() ? valueState.get(groupId) : Long.MAX_VALUE; - long currentTimestamp = groupId < timestampState.size() ? timestampState.get(groupId) : Long.MAX_VALUE; if (hasValue(groupId) == false || timestamp < getTimestamp(groupId)) { ensureCapacity(groupId); valueState.set(groupId, value); @@ -156,10 +154,12 @@ public long getTimestamp(int groupId) { } private void ensureCapacity(int groupId) { - if (groupId >= timestampState.size()) { + if (groupId >= valueState.size()) { long prevSize1 = valueState.size(); valueState = bigArrays.grow(valueState, groupId + 1); valueState.fill(prevSize1, valueState.size(), Long.MAX_VALUE); + } + if (groupId >= timestampState.size()) { long prevSize2 = timestampState.size(); timestampState = bigArrays.grow(timestampState, groupId + 1); timestampState.fill(prevSize2, timestampState.size(), Long.MAX_VALUE); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java index 60e92b954feb5..c35700d010622 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValue.java @@ -12,6 +12,7 @@ import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.FirstValueBytesRefAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.FirstValueLongAggregatorFunctionSupplier; +import org.elasticsearch.core.Nullable; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; @@ -47,8 +48,22 @@ public class FirstValue extends AggregateFunction implements OptionalArgument, T Map.entry(DataType.VERSION, FirstValueBytesRefAggregatorFunctionSupplier::new) ); - @FunctionInfo(returnType = "keyword", description = "TBD", type = FunctionType.AGGREGATE, examples = {}) - public FirstValue(Source source, @Param(name = "field", type = "keyword", description = "TBD") Expression field, Expression timestamp) { + @FunctionInfo( + returnType = { "long", "keyword" }, + description = "Picks the first value (by the timestamp) of the series.", + type = FunctionType.AGGREGATE, + examples = {} + ) + public FirstValue( + Source source, + @Param(name = "field", type = { "long", "keyword" }) Expression field, + @Nullable @Param( + optional = true, + name = "timestamp", + type = "long", + description = "Timestamp field to determine values order." + ) Expression timestamp + ) { this(source, field, Literal.TRUE, timestamp != null ? List.of(timestamp) : List.of(new UnresolvedAttribute(source, "@timestamp"))); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValueTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValueTests.java new file mode 100644 index 0000000000000..a9b8878c41bb7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/FirstValueTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.AbstractAggregationTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +public class FirstValueTests extends AbstractAggregationTestCase { + public FirstValueTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + var suppliers = new ArrayList(); + + suppliers.add( + new TestCaseSupplier( + "first(long, long): long", + List.of(DataType.LONG, DataType.LONG), + () -> new TestCaseSupplier.TestCase( + List.of( + TestCaseSupplier.TypedData.multiRow(List.of(1000L, 999L), DataType.LONG, "field"), + TestCaseSupplier.TypedData.multiRow(List.of(2025L, 2026L), DataType.LONG, "timestamp") + ), + "FirstValue[field=Attribute[channel=0], timestamp=Attribute[channel=1]]", + DataType.LONG, + equalTo(1000L) + ) + ) + ); + suppliers.add( + new TestCaseSupplier( + "first(keyword, long): keyword", + List.of(DataType.KEYWORD, DataType.LONG), + () -> new TestCaseSupplier.TestCase( + List.of( + TestCaseSupplier.TypedData.multiRow(List.of(new BytesRef("1000"), new BytesRef("999")), DataType.KEYWORD, "field"), + TestCaseSupplier.TypedData.multiRow(List.of(2025L, 2026L), DataType.LONG, "timestamp") + ), + "FirstValue[field=Attribute[channel=0], timestamp=Attribute[channel=1]]", + DataType.KEYWORD, + equalTo(new BytesRef("1000")) + ) + ) + ); + + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(suppliers); + } + + @Override + protected Expression build(Source source, List args) { + return new FirstValue(source, args.get(0), args.get(1)); + } +} From 38ba4b401d461e20381b9ab79ffe902aebba8415 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Thu, 13 Feb 2025 16:21:38 +0100 Subject: [PATCH 14/15] fix merge --- .../compute/aggregation/FirstValueBytesRefAggregator.java | 3 ++- .../compute/aggregation/FirstValueLongAggregator.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java index fd734a6df2bfa..9e2d3f9e6e8a1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueBytesRefAggregator.java @@ -155,7 +155,8 @@ public void add(int groupId, BytesRef value, long timestamp) { } } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { if (this.seen == null) { this.seen = seen.seenGroupIds(bigArrays); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java index e3f77bce49b07..13d7474b99987 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/FirstValueLongAggregator.java @@ -139,7 +139,8 @@ public void add(int groupId, long value, long timestamp) { } } - void enableGroupIdTracking(SeenGroupIds seen) { + @Override + public void enableGroupIdTracking(SeenGroupIds seen) { if (this.seen == null) { this.seen = seen.seenGroupIds(bigArrays); } From 8ee43b0f5ef343c3bba22b237068e1a7b2abd747 Mon Sep 17 00:00:00 2001 From: "ievgen.degtiarenko" Date: Mon, 24 Feb 2025 08:20:56 +0100 Subject: [PATCH 15/15] upd --- .../src/main/resources/first_last.csv-spec | 48 +++++++++++++++++++ .../src/main/resources/observability.csv-spec | 23 --------- 2 files changed, 48 insertions(+), 23 deletions(-) create mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/first_last.csv-spec delete mode 100644 x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/first_last.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/first_last.csv-spec new file mode 100644 index 0000000000000..48b72593545c7 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/first_last.csv-spec @@ -0,0 +1,48 @@ +first_no_grouping +required_capability: fn_first_last +FROM sample_data +| STATS event_duration = first_value(event_duration), message = first_value(message) +; + +event_duration:long | message:keyword +3450233 | Connected to 10.1.0.3 +; + + +first_no_grouping_with_timestamp +required_capability: fn_first_last +FROM sample_data +| STATS event_duration = first_value(event_duration, @timestamp), message = first_value(message, @timestamp) +; + +event_duration:long | message:keyword +3450233 | Connected to 10.1.0.3 +; + + +first_grouped +required_capability: fn_first_last +FROM sample_data +| STATS event_duration = first_value(event_duration), message = first_value(message) BY hourly = BUCKET(@timestamp, 1 hour) +| SORT hourly +| KEEP hourly, event_duration, message +; + +hourly:date | event_duration:long | message:keyword +2023-10-23T12:00:00.000Z | 3450233 | Connected to 10.1.0.3 +2023-10-23T13:00:00.000Z | 1232382 | Disconnected +; + + +first_grouped_with_timestamp +required_capability: fn_first_last +FROM sample_data +| STATS event_duration = first_value(event_duration, @timestamp), message = first_value(message, @timestamp) BY hourly = BUCKET(@timestamp, 1 hour) +| SORT hourly +| KEEP hourly, event_duration, message +; + +hourly:date | event_duration:long | message:keyword +2023-10-23T12:00:00.000Z | 3450233 | Connected to 10.1.0.3 +2023-10-23T13:00:00.000Z | 1232382 | Disconnected +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec deleted file mode 100644 index 2ac16895952d0..0000000000000 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/observability.csv-spec +++ /dev/null @@ -1,23 +0,0 @@ -first_no_grouping -required_capability: fn_first_last -FROM sample_data -| STATS event_duration = first_value(event_duration), message = first_value(message) -; - -event_duration:long | message:keyword -3450233 | Connected to 10.1.0.3 -; - - -first_grouped -required_capability: fn_first_last -FROM sample_data -| STATS event_duration = first_value(event_duration), message = first_value(message) BY hourly = BUCKET(@timestamp, 1 hour) -| SORT hourly -| KEEP hourly, event_duration, message -; - -hourly:date | event_duration:long | message:keyword -2023-10-23T12:00:00.000Z | 3450233 | Connected to 10.1.0.3 -2023-10-23T13:00:00.000Z | 1232382 | Disconnected -;