Skip to content

Commit 1508be5

Browse files
JonasKunzJonasKunz
committed
ESQL: Implement TopN support for exponential histogram
1 parent aebedd6 commit 1508be5

File tree

8 files changed

+242
-8
lines changed

8 files changed

+242
-8
lines changed

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramArrayBlock.java‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -79,19 +79,55 @@ private List<Block> getSubBlocks() {
7979
}
8080

8181
void loadValue(int valueIndex, CompressedExponentialHistogram resultHistogram, BytesRef tempBytesRef) {
82-
BytesRef bytes = encodedHistograms.getBytesRef(encodedHistograms.getFirstValueIndex(valueIndex), tempBytesRef);
83-
double zeroThreshold = zeroThresholds.getDouble(zeroThresholds.getFirstValueIndex(valueIndex));
84-
long valueCount = valueCounts.getLong(valueCounts.getFirstValueIndex(valueIndex));
85-
double sum = sums.getDouble(sums.getFirstValueIndex(valueIndex));
86-
double min = valueCount == 0 ? Double.NaN : minima.getDouble(minima.getFirstValueIndex(valueIndex));
87-
double max = valueCount == 0 ? Double.NaN : maxima.getDouble(maxima.getFirstValueIndex(valueIndex));
82+
BytesRef bytes = getEncodedHistogramBytes(valueIndex, tempBytesRef);
83+
double zeroThreshold = getHistogramZeroThreshold(valueIndex);
84+
long valueCount = getHistogramValueCount(valueIndex);
85+
double sum = getHistogramSum(valueIndex);
86+
double min = getHistogramMin(valueIndex);
87+
double max = getHistogramMax(valueIndex);
8888
try {
8989
resultHistogram.reset(zeroThreshold, valueCount, sum, min, max, bytes);
9090
} catch (IOException e) {
9191
throw new IllegalStateException("error loading histogram", e);
9292
}
9393
}
9494

95+
/**
96+
* Returns the minimum for the histogram at the given value index.
97+
* @param valueIndex the value index, obtained via {@link #getFirstValueIndex(int)} and {@link #getValueCount(int)}
98+
* @return the minimum, or Double.NaN if the histogram is null or has no values
99+
*/
100+
double getHistogramMin(int valueIndex) {
101+
int minimaValIndex = minima.getFirstValueIndex(valueIndex);
102+
return minima.isNull(minimaValIndex) ? Double.NaN : minima.getDouble(minimaValIndex);
103+
}
104+
105+
/**
106+
* Returns the maximum for the histogram at the given value index.
107+
* @param valueIndex the value index, obtained via {@link #getFirstValueIndex(int)} and {@link #getValueCount(int)}
108+
* @return the maximum, or Double.NaN if the histogram is null or has no values
109+
*/
110+
double getHistogramMax(int valueIndex) {
111+
int maximaValIndex = maxima.getFirstValueIndex(valueIndex);
112+
return maxima.isNull(maximaValIndex) ? Double.NaN : maxima.getDouble(maximaValIndex);
113+
}
114+
115+
double getHistogramSum(int valueIndex) {
116+
return sums.getDouble(sums.getFirstValueIndex(valueIndex));
117+
}
118+
119+
long getHistogramValueCount(int valueIndex) {
120+
return valueCounts.getLong(valueCounts.getFirstValueIndex(valueIndex));
121+
}
122+
123+
double getHistogramZeroThreshold(int valueIndex) {
124+
return zeroThresholds.getDouble(zeroThresholds.getFirstValueIndex(valueIndex));
125+
}
126+
127+
BytesRef getEncodedHistogramBytes(int valueIndex, BytesRef tempBytesRef) {
128+
return encodedHistograms.getBytesRef(encodedHistograms.getFirstValueIndex(valueIndex), tempBytesRef);
129+
}
130+
95131
@Override
96132
protected void closeInternal() {
97133
Releasables.close(getSubBlocks());

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramBlockAccessor.java‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,61 @@ public ExponentialHistogram get(int valueIndex) {
3838
assert block.isReleased() == false;
3939
ExponentialHistogramArrayBlock arrayBlock = (ExponentialHistogramArrayBlock) block;
4040
if (reusedHistogram == null) {
41-
tempBytesRef = new BytesRef();
4241
reusedHistogram = new CompressedExponentialHistogram();
4342
}
43+
if (tempBytesRef == null) {
44+
tempBytesRef = new BytesRef();
45+
}
4446
arrayBlock.loadValue(valueIndex, reusedHistogram, tempBytesRef);
4547
return reusedHistogram;
4648
}
4749

50+
public double getSum(int valueIndex) {
51+
assert block.isNull(valueIndex) == false;
52+
assert block.isReleased() == false;
53+
ExponentialHistogramArrayBlock arrayBlock = (ExponentialHistogramArrayBlock) block;
54+
return arrayBlock.getHistogramSum(valueIndex);
55+
}
56+
57+
public long getValueCount(int valueIndex) {
58+
assert block.isNull(valueIndex) == false;
59+
assert block.isReleased() == false;
60+
ExponentialHistogramArrayBlock arrayBlock = (ExponentialHistogramArrayBlock) block;
61+
return arrayBlock.getHistogramValueCount(valueIndex);
62+
}
63+
64+
public double getMin(int valueIndex) {
65+
assert block.isNull(valueIndex) == false;
66+
assert block.isReleased() == false;
67+
ExponentialHistogramArrayBlock arrayBlock = (ExponentialHistogramArrayBlock) block;
68+
return arrayBlock.getHistogramMin(valueIndex);
69+
}
70+
71+
public double getMax(int valueIndex) {
72+
assert block.isNull(valueIndex) == false;
73+
assert block.isReleased() == false;
74+
ExponentialHistogramArrayBlock arrayBlock = (ExponentialHistogramArrayBlock) block;
75+
return arrayBlock.getHistogramMax(valueIndex);
76+
}
77+
78+
public double getZeroThreshold(int valueIndex) {
79+
assert block.isNull(valueIndex) == false;
80+
assert block.isReleased() == false;
81+
ExponentialHistogramArrayBlock arrayBlock = (ExponentialHistogramArrayBlock) block;
82+
return arrayBlock.getHistogramZeroThreshold(valueIndex);
83+
}
84+
85+
public BytesRef getEncodedHistogramBytes(int valueIndex) {
86+
assert block.isNull(valueIndex) == false;
87+
assert block.isReleased() == false;
88+
ExponentialHistogramArrayBlock arrayBlock = (ExponentialHistogramArrayBlock) block;
89+
if (tempBytesRef == null) {
90+
tempBytesRef = new BytesRef();
91+
}
92+
arrayBlock.getEncodedHistogramBytes(valueIndex, tempBytesRef);
93+
return tempBytesRef;
94+
}
95+
96+
97+
4898
}

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ExponentialHistogramBlockBuilder.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
package org.elasticsearch.compute.data;
99

10+
import org.apache.lucene.util.BytesRef;
1011
import org.elasticsearch.common.io.stream.BytesStreamOutput;
1112
import org.elasticsearch.core.Releasables;
1213
import org.elasticsearch.exponentialhistogram.CompressedExponentialHistogram;
@@ -100,6 +101,25 @@ public ExponentialHistogramBlockBuilder append(ExponentialHistogram histogram) {
100101
return this;
101102
}
102103

104+
public ExponentialHistogramBlockBuilder appendEncoded(double sum, long valueCount, double min, double max, double zeroThreshold, BytesRef encodedHistogram) {
105+
sumsBuilder.appendDouble(sum);
106+
valueCountsBuilder.appendLong(valueCount);
107+
zeroThresholdsBuilder.appendDouble(zeroThreshold);
108+
encodedHistogramsBuilder.appendBytesRef(encodedHistogram);
109+
if (valueCount == 0) {
110+
assert Double.isNaN(min) : "min should be NaN for empty histogram";
111+
assert Double.isNaN(max) : "max should be NaN for empty histogram";
112+
minimaBuilder.appendNull();
113+
maximaBuilder.appendNull();
114+
} else {
115+
assert Double.isNaN(min) == false : "min should be non-NaN for non-empty histogram";
116+
assert Double.isNaN(max) == false : "max should be non-NaN for non-empty histogram";
117+
minimaBuilder.appendDouble(min);
118+
maximaBuilder.appendDouble(max);
119+
}
120+
return this;
121+
}
122+
103123
@Override
104124
public ExponentialHistogramBlock build() {
105125
DoubleBlock minima = null;

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ResultBuilder.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ static ResultBuilder resultBuilderFor(
5555
case NULL -> new ResultBuilderForNull(blockFactory);
5656
case DOC -> new ResultBuilderForDoc(blockFactory, (DocVectorEncoder) encoder, positions);
5757
case AGGREGATE_METRIC_DOUBLE -> new ResultBuilderForAggregateMetricDouble(blockFactory, positions);
58+
case EXPONENTIAL_HISTOGRAM -> new ResultBuilderForExponentialHistogram(blockFactory, positions);
5859
default -> {
5960
assert false : "Result builder for [" + elementType + "]";
6061
throw new UnsupportedOperationException("Result builder for [" + elementType + "]");

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ResultBuilderForExponentialHistogram.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.compute.operator.topn;
9+
10+
import org.apache.lucene.util.BytesRef;
11+
import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder;
12+
import org.elasticsearch.compute.data.Block;
13+
import org.elasticsearch.compute.data.BlockFactory;
14+
import org.elasticsearch.compute.data.ExponentialHistogramBlockBuilder;
15+
import org.elasticsearch.index.mapper.BlockLoader;
16+
17+
import java.util.List;
18+
19+
public class ResultBuilderForExponentialHistogram implements ResultBuilder {
20+
21+
private final ExponentialHistogramBlockBuilder builder;
22+
private final BytesRef scratch = new BytesRef();
23+
24+
ResultBuilderForExponentialHistogram(BlockFactory blockFactory, int positions) {
25+
this.builder = blockFactory.newExponentialHistogramBlockBuilder(positions);
26+
}
27+
28+
@Override
29+
public void decodeKey(BytesRef keys) {
30+
throw new AssertionError("AggregateMetricDoubleBlock can't be a key");
31+
}
32+
33+
@Override
34+
public void decodeValue(BytesRef values) {
35+
int count = TopNEncoder.DEFAULT_UNSORTABLE.decodeVInt(values);
36+
if (count == 0) {
37+
builder.appendNull();
38+
return;
39+
}
40+
assert count == 1: "ExponentialHistogramBlock does not support multi values";
41+
long histogramValueCount = TopNEncoder.DEFAULT_UNSORTABLE.decodeLong(values);
42+
double sum = TopNEncoder.DEFAULT_UNSORTABLE.decodeDouble(values);
43+
double min = TopNEncoder.DEFAULT_UNSORTABLE.decodeDouble(values);
44+
double max = TopNEncoder.DEFAULT_UNSORTABLE.decodeDouble(values);
45+
double zeroThreshold = TopNEncoder.DEFAULT_UNSORTABLE.decodeDouble(values);
46+
BytesRef encodedHistogramBytes = TopNEncoder.DEFAULT_UNSORTABLE.decodeBytesRef(values, scratch);
47+
builder.appendEncoded(
48+
sum,
49+
histogramValueCount,
50+
min,
51+
max,
52+
zeroThreshold,
53+
encodedHistogramBytes
54+
);
55+
}
56+
57+
@Override
58+
public Block build() {
59+
return builder.build();
60+
}
61+
62+
@Override
63+
public String toString() {
64+
return "ResultBuilderForExponentialHistogram";
65+
}
66+
67+
@Override
68+
public void close() {
69+
builder.close();
70+
}
71+
}

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractor.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.compute.data.DocBlock;
1515
import org.elasticsearch.compute.data.DoubleBlock;
1616
import org.elasticsearch.compute.data.ElementType;
17+
import org.elasticsearch.compute.data.ExponentialHistogramBlock;
1718
import org.elasticsearch.compute.data.FloatBlock;
1819
import org.elasticsearch.compute.data.IntBlock;
1920
import org.elasticsearch.compute.data.LongBlock;
@@ -53,6 +54,7 @@ static ValueExtractor extractorFor(ElementType elementType, TopNEncoder encoder,
5354
case NULL -> new ValueExtractorForNull();
5455
case DOC -> new ValueExtractorForDoc(encoder, ((DocBlock) block).asVector());
5556
case AGGREGATE_METRIC_DOUBLE -> new ValueExtractorForAggregateMetricDouble(encoder, (AggregateMetricDoubleBlock) block);
57+
case EXPONENTIAL_HISTOGRAM -> new ValueExtractorForExponentialHistogram(encoder, (ExponentialHistogramBlock) block);
5658
default -> {
5759
assert false : "No value extractor for [" + block.elementType() + "]";
5860
throw new UnsupportedOperationException("No value extractor for [" + block.elementType() + "]");

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractorForExponentialHistogram.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.compute.operator.topn;
9+
10+
import org.elasticsearch.compute.data.AggregateMetricDoubleBlock;
11+
import org.elasticsearch.compute.data.DoubleBlock;
12+
import org.elasticsearch.compute.data.ExponentialHistogramBlock;
13+
import org.elasticsearch.compute.data.ExponentialHistogramBlockAccessor;
14+
import org.elasticsearch.compute.data.IntBlock;
15+
import org.elasticsearch.compute.operator.BreakingBytesRefBuilder;
16+
17+
import java.util.List;
18+
19+
public class ValueExtractorForExponentialHistogram implements ValueExtractor {
20+
private final ExponentialHistogramBlock block;
21+
private final ExponentialHistogramBlockAccessor accessor;
22+
23+
ValueExtractorForExponentialHistogram(TopNEncoder encoder, ExponentialHistogramBlock block) {
24+
assert encoder == TopNEncoder.DEFAULT_UNSORTABLE;
25+
this.block = block;
26+
this.accessor = new ExponentialHistogramBlockAccessor(block);
27+
}
28+
29+
@Override
30+
public void writeValue(BreakingBytesRefBuilder values, int position) {
31+
// number of multi-values first for compatibility with ValueExtractorForNull
32+
if (block.isNull(position)) {
33+
TopNEncoder.DEFAULT_UNSORTABLE.encodeVInt(0, values);
34+
} else {
35+
assert block.getValueCount(position) == 1: "Multi-valued ExponentialHistogram not supported in TopN";
36+
TopNEncoder.DEFAULT_UNSORTABLE.encodeVInt(1, values);
37+
int valueIndex = block.getFirstValueIndex(position);
38+
TopNEncoder.DEFAULT_UNSORTABLE.encodeLong(accessor.getValueCount(valueIndex), values);
39+
TopNEncoder.DEFAULT_UNSORTABLE.encodeDouble(accessor.getSum(valueIndex), values);
40+
TopNEncoder.DEFAULT_UNSORTABLE.encodeDouble(accessor.getMin(valueIndex), values);
41+
TopNEncoder.DEFAULT_UNSORTABLE.encodeDouble(accessor.getMax(valueIndex), values);
42+
TopNEncoder.DEFAULT_UNSORTABLE.encodeDouble(accessor.getZeroThreshold(valueIndex), values);
43+
TopNEncoder.DEFAULT_UNSORTABLE.encodeBytesRef(accessor.getEncodedHistogramBytes(valueIndex), values);
44+
}
45+
}
46+
47+
@Override
48+
public String toString() {
49+
return "ValueExtractorForExponentialHistogram";
50+
}
51+
}

‎x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/ExtractorTests.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public static Iterable<Object[]> parameters() {
4444
case UNKNOWN -> {
4545
supportsNull = false;
4646
}
47-
case COMPOSITE, EXPONENTIAL_HISTOGRAM -> {
47+
case COMPOSITE -> {
4848
// TODO: add later
4949
supportsNull = false;
5050
}
@@ -130,6 +130,9 @@ public static Iterable<Object[]> parameters() {
130130
) }
131131
);
132132
}
133+
case EXPONENTIAL_HISTOGRAM ->
134+
// multi values are not supported
135+
cases.add(valueTestCase("single " + e, e, TopNEncoder.DEFAULT_UNSORTABLE, () -> BlockTestUtils.randomValue(e)));
133136
case NULL -> {
134137
}
135138
default -> {

0 commit comments

Comments
 (0)