Skip to content

Commit a4f6782

Browse files
salvatore-campagnasalvatore-campagna
authored
Use compression equal to 1.0 for empty TDigest histograms and a null empty object (#93736)
* Use zero digits for empty histograms and a null empty object When an empty aggregations uses a TDigest object the underlying arrays used by the AvlTree TDigest implementation are eagerly allocated. If the aggregation produces no result, we serialize and deserialize these arrays which might be large if the compression value is large. So, no matter the precision requested by setting the compression value, we always serialize and deserialize arrays whose size depends on the value of the 'compression' parameter. This also happens if the result is empty. Here we use a null value for empty aggregations while building the result TDigest object and, later on, use a static empty TDigest object at reduce time merging it with non-empty results. This static object just uses 1.0 as the compression value, which is enough for empty results. If a larger compression value is required to compute the final global result there will be a non-empty TDigest result using a larger compression value, which will be used at merging time and provide a result with the expected precision. Here we also make sure empty histogram objects are immutable so that in case a reference leaks or is shared in other parts of the code we make sure we are not changing a histogram expected to be empty.
1 parent 82e8e40 commit a4f6782

File tree

10 files changed

+603
-8
lines changed

10 files changed

+603
-8
lines changed

‎server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalHDRPercentiles.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
abstract class AbstractInternalHDRPercentiles extends InternalNumericMetricsAggregation.MultiValue {
3030

3131
private static final DoubleHistogram EMPTY_HISTOGRAM_THREE_DIGITS = new DoubleHistogram(3);
32-
private static final DoubleHistogram EMPTY_HISTOGRAM_ZERO_DIGITS = new DoubleHistogram(0);
32+
private static final DoubleHistogram EMPTY_HISTOGRAM_ZERO_DIGITS = new EmptyDoubleHdrHistogram();
3333

3434
protected final double[] keys;
3535
protected final DoubleHistogram state;

‎server/src/main/java/org/elasticsearch/search/aggregations/metrics/AbstractInternalTDigestPercentiles.java‎

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
package org.elasticsearch.search.aggregations.metrics;
1010

11+
import org.elasticsearch.TransportVersion;
1112
import org.elasticsearch.common.io.stream.StreamInput;
1213
import org.elasticsearch.common.io.stream.StreamOutput;
1314
import org.elasticsearch.search.DocValueFormat;
@@ -24,6 +25,10 @@
2425

2526
abstract class AbstractInternalTDigestPercentiles extends InternalNumericMetricsAggregation.MultiValue {
2627

28+
// NOTE: using compression = 1.0 empty histograms will track just about 5 centroids.
29+
// This reduces the amount of data to serialize and deserialize.
30+
private static final TDigestState EMPTY_HISTOGRAM = new EmptyTDigestState();
31+
2732
protected final double[] keys;
2833
protected final TDigestState state;
2934
final boolean keyed;
@@ -48,15 +53,33 @@ abstract class AbstractInternalTDigestPercentiles extends InternalNumericMetrics
4853
protected AbstractInternalTDigestPercentiles(StreamInput in) throws IOException {
4954
super(in);
5055
keys = in.readDoubleArray();
51-
state = TDigestState.read(in);
56+
if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_8_0)) {
57+
if (in.readBoolean()) {
58+
state = TDigestState.read(in);
59+
} else {
60+
state = null;
61+
}
62+
} else {
63+
state = TDigestState.read(in);
64+
}
5265
keyed = in.readBoolean();
5366
}
5467

5568
@Override
5669
protected void doWriteTo(StreamOutput out) throws IOException {
5770
out.writeNamedWriteable(format);
5871
out.writeDoubleArray(keys);
59-
TDigestState.write(state, out);
72+
if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_8_0)) {
73+
if (this.state != null) {
74+
out.writeBoolean(true);
75+
TDigestState.write(state, out);
76+
} else {
77+
out.writeBoolean(false);
78+
}
79+
} else {
80+
TDigestState state = this.state != null ? this.state : EMPTY_HISTOGRAM;
81+
TDigestState.write(state, out);
82+
}
6083
out.writeBoolean(keyed);
6184
}
6285

@@ -109,14 +132,40 @@ public AbstractInternalTDigestPercentiles reduce(List<InternalAggregation> aggre
109132
TDigestState merged = null;
110133
for (InternalAggregation aggregation : aggregations) {
111134
final AbstractInternalTDigestPercentiles percentiles = (AbstractInternalTDigestPercentiles) aggregation;
135+
if (percentiles.state == null) {
136+
continue;
137+
}
112138
if (merged == null) {
113139
merged = new TDigestState(percentiles.state.compression());
114140
}
115-
merged.add(percentiles.state);
141+
merged = merge(merged, percentiles.state);
142+
}
143+
if (merged == null) {
144+
merged = EMPTY_HISTOGRAM;
116145
}
117146
return createReduced(getName(), keys, merged, keyed, getMetadata());
118147
}
119148

149+
/**
150+
* Merges two {@link TDigestState}s such that we always merge the one with smaller
151+
* compression into the one with larger compression.
152+
* This prevents producing a result that has lower than expected precision.
153+
*
154+
* @param digest1 The first histogram to merge
155+
* @param digest2 The second histogram to merge
156+
* @return One of the input histograms such that the one with larger compression is used as the one for merging
157+
*/
158+
private TDigestState merge(final TDigestState digest1, final TDigestState digest2) {
159+
TDigestState largerCompression = digest1;
160+
TDigestState smallerCompression = digest2;
161+
if (digest2.compression() > digest1.compression()) {
162+
largerCompression = digest2;
163+
smallerCompression = digest1;
164+
}
165+
largerCompression.add(smallerCompression);
166+
return largerCompression;
167+
}
168+
120169
@Override
121170
public InternalAggregation finalizeSampling(SamplingContext samplingContext) {
122171
return this;
@@ -132,6 +181,7 @@ protected abstract AbstractInternalTDigestPercentiles createReduced(
132181

133182
@Override
134183
public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
184+
TDigestState state = this.state != null ? this.state : EMPTY_HISTOGRAM;
135185
if (keyed) {
136186
builder.startObject(CommonFields.VALUES.getPreferredName());
137187
for (int i = 0; i < keys.length; ++i) {

‎server/src/main/java/org/elasticsearch/search/aggregations/metrics/EmptyDoubleHdrHistogram.java‎

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.search.aggregations.metrics;
10+
11+
import org.HdrHistogram.DoubleHistogram;
12+
13+
import java.io.PrintStream;
14+
15+
public final class EmptyDoubleHdrHistogram extends DoubleHistogram {
16+
17+
public EmptyDoubleHdrHistogram() {
18+
super(0);
19+
setAutoResize(false);
20+
}
21+
22+
@Override
23+
public void setAutoResize(boolean ignored) {
24+
// DO NOT CHANGE 'autoResize'
25+
}
26+
27+
@Override
28+
public void recordValue(double value) throws ArrayIndexOutOfBoundsException {
29+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
30+
}
31+
32+
@Override
33+
public void recordValueWithCount(double value, long count) throws ArrayIndexOutOfBoundsException {
34+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
35+
}
36+
37+
@Override
38+
public void recordValueWithExpectedInterval(double value, double expectedIntervalBetweenValueSamples)
39+
throws ArrayIndexOutOfBoundsException {
40+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
41+
}
42+
43+
@Override
44+
public void reset() {
45+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
46+
}
47+
48+
@Override
49+
public DoubleHistogram copy() {
50+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
51+
}
52+
53+
@Override
54+
public DoubleHistogram copyCorrectedForCoordinatedOmission(double expectedIntervalBetweenValueSamples) {
55+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
56+
}
57+
58+
@Override
59+
public void copyInto(DoubleHistogram targetHistogram) {
60+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
61+
}
62+
63+
@Override
64+
public void copyIntoCorrectedForCoordinatedOmission(DoubleHistogram targetHistogram, double expectedIntervalBetweenValueSamples) {
65+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
66+
}
67+
68+
@Override
69+
public void add(DoubleHistogram fromHistogram) throws ArrayIndexOutOfBoundsException {
70+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
71+
}
72+
73+
@Override
74+
public void addWhileCorrectingForCoordinatedOmission(DoubleHistogram fromHistogram, double expectedIntervalBetweenValueSamples) {
75+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
76+
}
77+
78+
@Override
79+
public void subtract(DoubleHistogram otherHistogram) {
80+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
81+
}
82+
83+
@Override
84+
public double lowestEquivalentValue(double value) {
85+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
86+
}
87+
88+
@Override
89+
public double highestEquivalentValue(double value) {
90+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
91+
}
92+
93+
@Override
94+
public double sizeOfEquivalentValueRange(double value) {
95+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
96+
}
97+
98+
@Override
99+
public double medianEquivalentValue(double value) {
100+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
101+
}
102+
103+
@Override
104+
public double nextNonEquivalentValue(double value) {
105+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
106+
}
107+
108+
@Override
109+
public boolean valuesAreEquivalent(double value1, double value2) {
110+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
111+
}
112+
113+
@Override
114+
public void setStartTimeStamp(long timeStampMsec) {
115+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
116+
}
117+
118+
@Override
119+
public long getStartTimeStamp() {
120+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
121+
}
122+
123+
@Override
124+
public long getEndTimeStamp() {
125+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
126+
}
127+
128+
@Override
129+
public void setEndTimeStamp(long timeStampMsec) {
130+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
131+
}
132+
133+
@Override
134+
public void setTag(String tag) {
135+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
136+
}
137+
138+
@Override
139+
public double getMinValue() {
140+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
141+
}
142+
143+
@Override
144+
public double getMaxValue() {
145+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
146+
}
147+
148+
@Override
149+
public double getMinNonZeroValue() {
150+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
151+
}
152+
153+
@Override
154+
public double getMaxValueAsDouble() {
155+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
156+
}
157+
158+
@Override
159+
public double getMean() {
160+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
161+
}
162+
163+
@Override
164+
public double getStdDeviation() {
165+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
166+
}
167+
168+
@Override
169+
public Percentiles percentiles(int percentileTicksPerHalfDistance) {
170+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
171+
}
172+
173+
@Override
174+
public LinearBucketValues linearBucketValues(double valueUnitsPerBucket) {
175+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
176+
}
177+
178+
@Override
179+
public LogarithmicBucketValues logarithmicBucketValues(double valueUnitsInFirstBucket, double logBase) {
180+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
181+
}
182+
183+
@Override
184+
public void outputPercentileDistribution(PrintStream printStream, Double outputValueUnitScalingRatio) {
185+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
186+
}
187+
188+
@Override
189+
public void outputPercentileDistribution(
190+
PrintStream printStream,
191+
int percentileTicksPerHalfDistance,
192+
Double outputValueUnitScalingRatio
193+
) {
194+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
195+
}
196+
197+
@Override
198+
public void outputPercentileDistribution(
199+
PrintStream printStream,
200+
int percentileTicksPerHalfDistance,
201+
Double outputValueUnitScalingRatio,
202+
boolean useCsvFormat
203+
) {
204+
throw new UnsupportedOperationException("Immutable Empty HdrHistogram");
205+
}
206+
207+
}

‎server/src/main/java/org/elasticsearch/search/aggregations/metrics/EmptyTDigestState.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.search.aggregations.metrics;
10+
11+
import com.tdunning.math.stats.Centroid;
12+
import com.tdunning.math.stats.TDigest;
13+
14+
import java.util.List;
15+
16+
public final class EmptyTDigestState extends TDigestState {
17+
public EmptyTDigestState() {
18+
super(1.0D);
19+
}
20+
21+
@Override
22+
public TDigest recordAllData() {
23+
throw new UnsupportedOperationException("Immutable Empty TDigest");
24+
}
25+
26+
@Override
27+
public void add(double x, int w) {
28+
throw new UnsupportedOperationException("Immutable Empty TDigest");
29+
}
30+
31+
@Override
32+
public void add(List<? extends TDigest> others) {
33+
throw new UnsupportedOperationException("Immutable Empty TDigest");
34+
}
35+
36+
@Override
37+
public void add(double x, int w, List<Double> data) {
38+
throw new UnsupportedOperationException("Immutable Empty TDigest");
39+
}
40+
41+
@Override
42+
public void compress() {
43+
throw new UnsupportedOperationException("Immutable Empty TDigest");
44+
}
45+
46+
@Override
47+
public void add(double x) {
48+
throw new UnsupportedOperationException("Immutable Empty TDigest");
49+
}
50+
51+
@Override
52+
public void add(TDigest other) {
53+
throw new UnsupportedOperationException("Immutable Empty TDigest");
54+
}
55+
56+
@Override
57+
protected Centroid createCentroid(double mean, int id) {
58+
throw new UnsupportedOperationException("Immutable Empty TDigest");
59+
}
60+
61+
@Override
62+
public boolean isRecording() {
63+
return false;
64+
}
65+
}

‎server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalTDigestPercentiles.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public Iterator<Percentile> iterator() {
4747

4848
@Override
4949
public double percentile(double percent) {
50-
return state.quantile(percent / 100);
50+
return this.state != null ? state.quantile(percent / 100) : Double.NaN;
5151
}
5252

5353
@Override

0 commit comments

Comments
 (0)