Skip to content

Commit 9938742

Browse files
JonasKunzJonasKunz
authored
Add FieldData implementation for exponential_histogram fields (#135423)
1 parent 50d6518 commit 9938742

File tree

6 files changed

+375
-50
lines changed

6 files changed

+375
-50
lines changed

‎x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/ExponentialHistogramFieldMapper.java‎

Lines changed: 146 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,24 @@
1212
import org.apache.lucene.document.NumericDocValuesField;
1313
import org.apache.lucene.index.BinaryDocValues;
1414
import org.apache.lucene.index.LeafReader;
15+
import org.apache.lucene.index.LeafReaderContext;
1516
import org.apache.lucene.index.NumericDocValues;
1617
import org.apache.lucene.search.Query;
18+
import org.apache.lucene.search.SortField;
1719
import org.apache.lucene.util.BytesRef;
1820
import org.apache.lucene.util.NumericUtils;
1921
import org.elasticsearch.common.Explicit;
2022
import org.elasticsearch.common.io.stream.BytesStreamOutput;
23+
import org.elasticsearch.common.util.BigArrays;
2124
import org.elasticsearch.common.util.FeatureFlag;
25+
import org.elasticsearch.core.Nullable;
2226
import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
2327
import org.elasticsearch.exponentialhistogram.ExponentialHistogramUtils;
2428
import org.elasticsearch.exponentialhistogram.ExponentialHistogramXContent;
2529
import org.elasticsearch.exponentialhistogram.ZeroBucket;
2630
import org.elasticsearch.index.fielddata.FieldDataContext;
2731
import org.elasticsearch.index.fielddata.IndexFieldData;
32+
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
2833
import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader;
2934
import org.elasticsearch.index.mapper.DocumentParserContext;
3035
import org.elasticsearch.index.mapper.DocumentParsingException;
@@ -36,11 +41,19 @@
3641
import org.elasticsearch.index.mapper.SourceValueFetcher;
3742
import org.elasticsearch.index.mapper.ValueFetcher;
3843
import org.elasticsearch.index.query.SearchExecutionContext;
44+
import org.elasticsearch.script.field.DocValuesScriptFieldFactory;
45+
import org.elasticsearch.search.DocValueFormat;
46+
import org.elasticsearch.search.MultiValueMode;
47+
import org.elasticsearch.search.sort.BucketedSort;
48+
import org.elasticsearch.search.sort.SortOrder;
3949
import org.elasticsearch.xcontent.CopyingXContentParser;
4050
import org.elasticsearch.xcontent.ParseField;
4151
import org.elasticsearch.xcontent.XContentBuilder;
4252
import org.elasticsearch.xcontent.XContentParser;
4353
import org.elasticsearch.xcontent.XContentSubParser;
54+
import org.elasticsearch.xpack.exponentialhistogram.fielddata.ExponentialHistogramValuesReader;
55+
import org.elasticsearch.xpack.exponentialhistogram.fielddata.IndexExponentialHistogramFieldData;
56+
import org.elasticsearch.xpack.exponentialhistogram.fielddata.LeafExponentialHistogramFieldData;
4457

4558
import java.io.IOException;
4659
import java.util.ArrayList;
@@ -242,12 +255,69 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
242255

243256
@Override
244257
public boolean isAggregatable() {
245-
return false;
258+
return true;
246259
}
247260

248261
@Override
249262
public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) {
250-
throw new IllegalArgumentException("The [" + CONTENT_TYPE + "] field does not support this operation currently");
263+
return (cache, breakerService) -> new IndexExponentialHistogramFieldData(name()) {
264+
@Override
265+
public LeafExponentialHistogramFieldData load(LeafReaderContext context) {
266+
return new LeafExponentialHistogramFieldData() {
267+
@Override
268+
public ExponentialHistogramValuesReader getHistogramValues() throws IOException {
269+
return new DocValuesReader(context.reader(), fieldName);
270+
}
271+
272+
@Override
273+
public DocValuesScriptFieldFactory getScriptFieldFactory(String name) {
274+
throw new UnsupportedOperationException("The [" + CONTENT_TYPE + "] field does not " + "support scripts");
275+
}
276+
277+
@Override
278+
public SortedBinaryDocValues getBytesValues() {
279+
throw new UnsupportedOperationException(
280+
"String representation of doc values " + "for [" + CONTENT_TYPE + "] fields is not supported"
281+
);
282+
}
283+
284+
@Override
285+
public long ramBytesUsed() {
286+
return 0; // No dynamic allocations
287+
}
288+
};
289+
}
290+
291+
@Override
292+
public LeafExponentialHistogramFieldData loadDirect(LeafReaderContext context) throws Exception {
293+
return load(context);
294+
}
295+
296+
@Override
297+
public SortField sortField(
298+
Object missingValue,
299+
MultiValueMode sortMode,
300+
XFieldComparatorSource.Nested nested,
301+
boolean reverse
302+
) {
303+
throw new UnsupportedOperationException("can't sort on the [" + CONTENT_TYPE + "] field");
304+
}
305+
306+
@Override
307+
public BucketedSort newBucketedSort(
308+
BigArrays bigArrays,
309+
Object missingValue,
310+
MultiValueMode sortMode,
311+
XFieldComparatorSource.Nested nested,
312+
SortOrder sortOrder,
313+
DocValueFormat format,
314+
int bucketSize,
315+
BucketedSort.ExtraData extra
316+
) {
317+
throw new IllegalArgumentException("can't sort on the [" + CONTENT_TYPE + "] field");
318+
}
319+
320+
};
251321
}
252322

253323
@Override
@@ -721,76 +791,102 @@ protected FieldMapper.SyntheticSourceSupport syntheticSourceSupport() {
721791
);
722792
}
723793

794+
private static class DocValuesReader implements ExponentialHistogramValuesReader {
795+
796+
private final BinaryDocValues histoDocValues;
797+
private final NumericDocValues zeroThresholds;
798+
private final NumericDocValues valueCounts;
799+
private final NumericDocValues valueSums;
800+
private final NumericDocValues valueMinima;
801+
private final NumericDocValues valueMaxima;
802+
803+
private int currentDocId = -1;
804+
private final CompressedExponentialHistogram tempHistogram = new CompressedExponentialHistogram();
805+
806+
DocValuesReader(LeafReader leafReader, String fullPath) throws IOException {
807+
histoDocValues = leafReader.getBinaryDocValues(fullPath);
808+
zeroThresholds = leafReader.getNumericDocValues(zeroThresholdSubFieldName(fullPath));
809+
valueCounts = leafReader.getNumericDocValues(valuesCountSubFieldName(fullPath));
810+
valueSums = leafReader.getNumericDocValues(valuesSumSubFieldName(fullPath));
811+
valueMinima = leafReader.getNumericDocValues(valuesMinSubFieldName(fullPath));
812+
valueMaxima = leafReader.getNumericDocValues(valuesMaxSubFieldName(fullPath));
813+
}
814+
815+
boolean hasAnyValues() {
816+
return histoDocValues != null;
817+
}
818+
819+
@Override
820+
public boolean advanceExact(int docId) throws IOException {
821+
boolean isPresent = histoDocValues != null && histoDocValues.advanceExact(docId);
822+
currentDocId = isPresent ? docId : -1;
823+
return isPresent;
824+
}
825+
826+
@Override
827+
public ExponentialHistogram histogramValue() throws IOException {
828+
if (currentDocId == -1) {
829+
throw new IllegalStateException("No histogram present for current document");
830+
}
831+
boolean zeroThresholdPresent = zeroThresholds.advanceExact(currentDocId);
832+
boolean valueCountsPresent = valueCounts.advanceExact(currentDocId);
833+
boolean valueSumsPresent = valueSums.advanceExact(currentDocId);
834+
assert zeroThresholdPresent && valueCountsPresent && valueSumsPresent;
835+
836+
BytesRef encodedHistogram = histoDocValues.binaryValue();
837+
double zeroThreshold = NumericUtils.sortableLongToDouble(zeroThresholds.longValue());
838+
long valueCount = valueCounts.longValue();
839+
double valueSum = NumericUtils.sortableLongToDouble(valueSums.longValue());
840+
double valueMin;
841+
if (valueMinima != null && valueMinima.advanceExact(currentDocId)) {
842+
valueMin = NumericUtils.sortableLongToDouble(valueMinima.longValue());
843+
} else {
844+
valueMin = Double.NaN;
845+
}
846+
double valueMax;
847+
if (valueMaxima != null && valueMaxima.advanceExact(currentDocId)) {
848+
valueMax = NumericUtils.sortableLongToDouble(valueMaxima.longValue());
849+
} else {
850+
valueMax = Double.NaN;
851+
}
852+
tempHistogram.reset(zeroThreshold, valueCount, valueSum, valueMin, valueMax, encodedHistogram);
853+
return tempHistogram;
854+
}
855+
}
856+
724857
private class ExponentialHistogramSyntheticFieldLoader implements CompositeSyntheticFieldLoader.DocValuesLayer {
725858

726-
private final CompressedExponentialHistogram histogram = new CompressedExponentialHistogram();
727-
private BytesRef binaryValue;
728-
private double zeroThreshold;
729-
private long valueCount;
730-
private double valueSum;
731-
private double valueMin;
732-
private double valueMax;
859+
@Nullable
860+
private ExponentialHistogram currentHistogram;
733861

734862
@Override
735863
public SourceLoader.SyntheticFieldLoader.DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf)
736864
throws IOException {
737-
BinaryDocValues histoDocValues = leafReader.getBinaryDocValues(fieldType().name());
738-
if (histoDocValues == null) {
739-
// No values in this leaf
740-
binaryValue = null;
865+
DocValuesReader histogramReader = new DocValuesReader(leafReader, fullPath());
866+
if (histogramReader.hasAnyValues() == false) {
741867
return null;
742868
}
743-
NumericDocValues zeroThresholds = leafReader.getNumericDocValues(zeroThresholdSubFieldName(fullPath()));
744-
NumericDocValues valueCounts = leafReader.getNumericDocValues(valuesCountSubFieldName(fullPath()));
745-
NumericDocValues valueSums = leafReader.getNumericDocValues(valuesSumSubFieldName(fullPath()));
746-
NumericDocValues valueMinima = leafReader.getNumericDocValues(valuesMinSubFieldName(fullPath()));
747-
NumericDocValues valueMaxima = leafReader.getNumericDocValues(valuesMaxSubFieldName(fullPath()));
748-
assert zeroThresholds != null;
749-
assert valueCounts != null;
750-
assert valueSums != null;
751869
return docId -> {
752-
if (histoDocValues.advanceExact(docId)) {
753-
754-
boolean zeroThresholdPresent = zeroThresholds.advanceExact(docId);
755-
boolean valueCountsPresent = valueCounts.advanceExact(docId);
756-
boolean valueSumsPresent = valueSums.advanceExact(docId);
757-
assert zeroThresholdPresent && valueCountsPresent && valueSumsPresent;
758-
759-
binaryValue = histoDocValues.binaryValue();
760-
zeroThreshold = NumericUtils.sortableLongToDouble(zeroThresholds.longValue());
761-
valueCount = valueCounts.longValue();
762-
valueSum = NumericUtils.sortableLongToDouble(valueSums.longValue());
763-
764-
if (valueMinima != null && valueMinima.advanceExact(docId)) {
765-
valueMin = NumericUtils.sortableLongToDouble(valueMinima.longValue());
766-
} else {
767-
valueMin = Double.NaN;
768-
}
769-
if (valueMaxima != null && valueMaxima.advanceExact(docId)) {
770-
valueMax = NumericUtils.sortableLongToDouble(valueMaxima.longValue());
771-
} else {
772-
valueMax = Double.NaN;
773-
}
870+
if (histogramReader.advanceExact(docId)) {
871+
currentHistogram = histogramReader.histogramValue();
774872
return true;
775873
}
776-
binaryValue = null;
874+
currentHistogram = null;
777875
return false;
778876
};
779877
}
780878

781879
@Override
782880
public boolean hasValue() {
783-
return binaryValue != null;
881+
return currentHistogram != null;
784882
}
785883

786884
@Override
787885
public void write(XContentBuilder b) throws IOException {
788-
if (binaryValue == null) {
886+
if (currentHistogram == null) {
789887
return;
790888
}
791-
792-
histogram.reset(zeroThreshold, valueCount, valueSum, valueMin, valueMax, binaryValue);
793-
ExponentialHistogramXContent.serialize(b, histogram);
889+
ExponentialHistogramXContent.serialize(b, currentHistogram);
794890
}
795891

796892
@Override
@@ -800,7 +896,7 @@ public String fieldName() {
800896

801897
@Override
802898
public long valueCount() {
803-
return binaryValue != null ? 1 : 0;
899+
return currentHistogram != null ? 1 : 0;
804900
}
805901
};
806902

‎x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/aggregations/support/ExponentialHistogramValuesSource.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
package org.elasticsearch.xpack.exponentialhistogram.aggregations.support;
8+
9+
import org.apache.lucene.index.LeafReaderContext;
10+
import org.elasticsearch.common.Rounding;
11+
import org.elasticsearch.index.fielddata.DocValueBits;
12+
import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
13+
import org.elasticsearch.search.aggregations.AggregationErrors;
14+
import org.elasticsearch.search.aggregations.support.AggregationContext;
15+
import org.elasticsearch.xpack.exponentialhistogram.ExponentialHistogramFieldMapper;
16+
import org.elasticsearch.xpack.exponentialhistogram.fielddata.ExponentialHistogramValuesReader;
17+
import org.elasticsearch.xpack.exponentialhistogram.fielddata.IndexExponentialHistogramFieldData;
18+
19+
import java.io.IOException;
20+
import java.util.function.Function;
21+
22+
public class ExponentialHistogramValuesSource {
23+
24+
public abstract static class ExponentialHistogram extends org.elasticsearch.search.aggregations.support.ValuesSource {
25+
26+
public abstract ExponentialHistogramValuesReader getHistogramValues(LeafReaderContext context) throws IOException;
27+
28+
public static class Fielddata extends ExponentialHistogram {
29+
30+
protected final IndexExponentialHistogramFieldData indexFieldData;
31+
32+
public Fielddata(IndexExponentialHistogramFieldData indexFieldData) {
33+
this.indexFieldData = indexFieldData;
34+
}
35+
36+
@Override
37+
public SortedBinaryDocValues bytesValues(LeafReaderContext context) {
38+
return indexFieldData.load(context).getBytesValues();
39+
}
40+
41+
@Override
42+
public DocValueBits docsWithValue(LeafReaderContext context) throws IOException {
43+
ExponentialHistogramValuesReader values = getHistogramValues(context);
44+
return new DocValueBits() {
45+
@Override
46+
public boolean advanceExact(int doc) throws IOException {
47+
return values.advanceExact(doc);
48+
}
49+
};
50+
}
51+
52+
@Override
53+
protected Function<Rounding, Rounding.Prepared> roundingPreparer(AggregationContext context) {
54+
throw AggregationErrors.unsupportedRounding(ExponentialHistogramFieldMapper.CONTENT_TYPE);
55+
}
56+
57+
public ExponentialHistogramValuesReader getHistogramValues(LeafReaderContext context) throws IOException {
58+
return indexFieldData.load(context).getHistogramValues();
59+
}
60+
}
61+
}
62+
}

0 commit comments

Comments
 (0)