Skip to content

Commit 50c9662

Browse files
martijnvgmartijnvg
committed
Update date_histogram aggregation to make use of doc values skippers.
Additionally, updated the filter-by-filter optimization to make use of the new TimestampQuery.
1 parent 6f621e4 commit 50c9662

File tree

5 files changed

+121
-10
lines changed

5 files changed

+121
-10
lines changed

‎server/src/main/java/org/elasticsearch/lucene/queries/TimestampQuery.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,14 @@ public TimestampQuery(long minTimestamp, long maxTimestamp) {
4545
this.maxTimestamp = maxTimestamp;
4646
}
4747

48+
public long getMinTimestamp() {
49+
return minTimestamp;
50+
}
51+
52+
public long getMaxTimestamp() {
53+
return maxTimestamp;
54+
}
55+
4856
@Override
4957
public Query rewrite(IndexSearcher indexSearcher) throws IOException {
5058
if (minTimestamp == Long.MIN_VALUE && maxTimestamp == Long.MAX_VALUE) {

‎server/src/main/java/org/elasticsearch/lucene/util/DocValuesSkipperUtils.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.lucene.util;
11+
12+
import org.apache.lucene.index.IndexReader;
13+
import org.apache.lucene.index.LeafReaderContext;
14+
15+
import java.io.IOException;
16+
17+
public final class DocValuesSkipperUtils {
18+
19+
private DocValuesSkipperUtils() {}
20+
21+
public static Long getMinValue(IndexReader reader, String field) throws IOException {
22+
Long minValue = null;
23+
for (LeafReaderContext ctx : reader.leaves()) {
24+
var skipper = ctx.reader().getDocValuesSkipper(field);
25+
if (skipper == null) {
26+
continue;
27+
}
28+
long leafMinValue = skipper.minValue();
29+
if (minValue == null) {
30+
minValue = leafMinValue;
31+
} else {
32+
minValue = Long.min(minValue, leafMinValue);
33+
}
34+
}
35+
return minValue;
36+
}
37+
38+
public static Long getMaxValue(IndexReader reader, String field) throws IOException {
39+
Long maxValue = null;
40+
for (LeafReaderContext ctx : reader.leaves()) {
41+
var skipper = ctx.reader().getDocValuesSkipper(field);
42+
if (skipper == null) {
43+
continue;
44+
}
45+
long leafMaxValue = skipper.maxValue();
46+
if (maxValue == null) {
47+
maxValue = leafMaxValue;
48+
} else {
49+
maxValue = Long.max(maxValue, leafMaxValue);
50+
}
51+
}
52+
return maxValue;
53+
}
54+
}

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,12 @@
1515
import org.apache.lucene.search.BulkScorer;
1616
import org.apache.lucene.search.ConstantScoreQuery;
1717
import org.apache.lucene.search.DocIdSetIterator;
18+
import org.apache.lucene.search.FieldExistsQuery;
1819
import org.apache.lucene.search.IndexOrDocValuesQuery;
1920
import org.apache.lucene.search.IndexSearcher;
2021
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
2122
import org.apache.lucene.search.LeafCollector;
23+
import org.apache.lucene.search.MatchAllDocsQuery;
2224
import org.apache.lucene.search.MatchNoDocsQuery;
2325
import org.apache.lucene.search.PointRangeQuery;
2426
import org.apache.lucene.search.Query;
@@ -27,7 +29,9 @@
2729
import org.apache.lucene.search.ScorerSupplier;
2830
import org.apache.lucene.search.Weight;
2931
import org.apache.lucene.util.Bits;
32+
import org.elasticsearch.cluster.metadata.DataStream;
3033
import org.elasticsearch.common.io.stream.StreamOutput;
34+
import org.elasticsearch.lucene.queries.TimestampQuery;
3135
import org.elasticsearch.search.aggregations.Aggregator;
3236
import org.elasticsearch.xcontent.XContentBuilder;
3337

@@ -138,6 +142,28 @@ QueryToFilterAdapter union(Query extraQuery) throws IOException {
138142
return new QueryToFilterAdapter(searcher(), key(), merged);
139143
}
140144
}
145+
if (unwrappedQuery instanceof TimestampQuery first && unwrappedExtraQuery instanceof TimestampQuery second) {
146+
// Merge the ranges:
147+
long min = Math.min(first.getMinTimestamp(), second.getMinTimestamp());
148+
long max = Math.max(first.getMaxTimestamp(), second.getMaxTimestamp());
149+
return new QueryToFilterAdapter(searcher(), key(), new TimestampQuery(min, max));
150+
} else if ((unwrappedQuery instanceof FieldExistsQuery f1 && DataStream.TIMESTAMP_FIELD_NAME.equals(f1.getField()))
151+
|| (unwrappedExtraQuery instanceof FieldExistsQuery f2 && DataStream.TIMESTAMP_FIELD_NAME.equals(f2.getField()))) {
152+
// just pick the query that is a field exists query:
153+
var query = unwrappedQuery instanceof FieldExistsQuery ? unwrappedQuery : unwrappedExtraQuery;
154+
return new QueryToFilterAdapter(searcher(), key(), query);
155+
} else if (unwrappedQuery instanceof MatchNoDocsQuery || unwrappedExtraQuery instanceof MatchNoDocsQuery) {
156+
// Use the query that didn't rewrite to match no docs query:
157+
if (unwrappedQuery instanceof MatchNoDocsQuery && unwrappedExtraQuery instanceof MatchNoDocsQuery) {
158+
return new QueryToFilterAdapter(searcher(), key(), unwrappedQuery);
159+
} else if (unwrappedQuery instanceof MatchAllDocsQuery) {
160+
return new QueryToFilterAdapter(searcher(), key(), unwrappedExtraQuery);
161+
} else if (unwrappedExtraQuery instanceof MatchAllDocsQuery) {
162+
return new QueryToFilterAdapter(searcher(), key(), unwrappedQuery);
163+
}
164+
} else {
165+
// assert false : "unwrapped query: " + unwrappedQuery + ",unwrappedExtraQuery: " + unwrappedExtraQuery;
166+
}
141167
BooleanQuery.Builder builder = new BooleanQuery.Builder();
142168
builder.add(query, BooleanClause.Occur.FILTER);
143169
builder.add(extraQuery, BooleanClause.Occur.FILTER);

‎server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import org.apache.lucene.search.Query;
2020
import org.apache.lucene.search.QueryVisitor;
2121
import org.apache.lucene.util.BytesRef;
22+
import org.elasticsearch.cluster.metadata.DataStream;
2223
import org.elasticsearch.common.Rounding;
2324
import org.elasticsearch.common.geo.GeoPoint;
2425
import org.elasticsearch.common.time.DateFormatter;
@@ -30,6 +31,8 @@
3031
import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
3132
import org.elasticsearch.index.mapper.MappedFieldType;
3233
import org.elasticsearch.index.mapper.RangeFieldMapper;
34+
import org.elasticsearch.lucene.queries.TimestampQuery;
35+
import org.elasticsearch.lucene.util.DocValuesSkipperUtils;
3336
import org.elasticsearch.script.AggregationScript;
3437
import org.elasticsearch.search.DocValueFormat;
3538
import org.elasticsearch.search.aggregations.AggregationErrors;
@@ -315,18 +318,31 @@ public Function<Rounding, Rounding.Prepared> roundingPreparer(AggregationContext
315318
range[1] = dft.resolution().parsePointAsMillis(max);
316319
}
317320
}
321+
322+
if (dft.hasDocValuesSkipper()) {
323+
Long min = DocValuesSkipperUtils.getMinValue(context.searcher().getIndexReader(), fieldContext.field());
324+
if (min != null) {
325+
long max = DocValuesSkipperUtils.getMaxValue(context.searcher().getIndexReader(), fieldContext.field());
326+
range[0] = min;
327+
range[1] = max;
328+
}
329+
}
330+
318331
log.trace("Bounds after index bound date rounding: {}, {}", range[0], range[1]);
319332

320333
boolean isMultiValue = false;
321-
for (LeafReaderContext leaf : context.searcher().getLeafContexts()) {
322-
if (fieldContext.fieldType().isIndexed()) {
323-
PointValues pointValues = leaf.reader().getPointValues(fieldContext.field());
324-
if (pointValues != null && pointValues.size() != pointValues.getDocCount()) {
325-
isMultiValue = true;
326-
}
327-
} else if (fieldContext.fieldType().hasDocValues()) {
328-
if (DocValues.unwrapSingleton(leaf.reader().getSortedNumericDocValues(fieldContext.field())) == null) {
329-
isMultiValue = true;
334+
// The @timestamp field is always a single valued.
335+
if (DataStream.TIMESTAMP_FIELD_NAME.equals(indexFieldData.getFieldName()) == false) {
336+
for (LeafReaderContext leaf : context.searcher().getLeafContexts()) {
337+
if (fieldContext.fieldType().isIndexed()) {
338+
PointValues pointValues = leaf.reader().getPointValues(fieldContext.field());
339+
if (pointValues != null && pointValues.size() != pointValues.getDocCount()) {
340+
isMultiValue = true;
341+
}
342+
} else if (fieldContext.fieldType().hasDocValues()) {
343+
if (DocValues.unwrapSingleton(leaf.reader().getSortedNumericDocValues(fieldContext.field())) == null) {
344+
isMultiValue = true;
345+
}
330346
}
331347
}
332348
}
@@ -355,6 +371,9 @@ public void visitLeaf(Query query) {
355371
if (query instanceof PointRangeQuery prq) {
356372
range[0] = Math.max(range[0], dft.resolution().parsePointAsMillis(prq.getLowerPoint()));
357373
range[1] = Math.min(range[1], dft.resolution().parsePointAsMillis(prq.getUpperPoint()));
374+
} else if (query instanceof TimestampQuery timestampQuery) {
375+
range[0] = Math.max(range[0], timestampQuery.getMinTimestamp());
376+
range[1] = Math.min(range[1], timestampQuery.getMaxTimestamp());
358377
}
359378
};
360379
});

‎server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,13 @@
88
*/
99
package org.elasticsearch.search.aggregations.support;
1010

11+
import org.elasticsearch.cluster.metadata.DataStream;
1112
import org.elasticsearch.common.Rounding;
1213
import org.elasticsearch.core.Nullable;
1314
import org.elasticsearch.index.fielddata.IndexFieldData;
1415
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
1516
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
17+
import org.elasticsearch.index.mapper.DateFieldMapper;
1618
import org.elasticsearch.index.mapper.MappedFieldType;
1719
import org.elasticsearch.index.mapper.NumberFieldMapper;
1820
import org.elasticsearch.index.mapper.RangeFieldMapper;
@@ -429,7 +431,9 @@ public Function<byte[], Number> getPointReaderOrNull() {
429431
* the ordering.
430432
*/
431433
public boolean alignesWithSearchIndex() {
432-
return script() == null && missing() == null && fieldType() != null && fieldType().isIndexed();
434+
boolean isTimestampField = fieldContext() != null && DataStream.TIMESTAMP_FIELD_NAME.equals(fieldContext().field());
435+
boolean hasDocValuesSkipper = isTimestampField && ((DateFieldMapper.DateFieldType) fieldContext.fieldType()).hasDocValuesSkipper();
436+
return script() == null && missing() == null && fieldType() != null && (fieldType().isIndexed() || hasDocValuesSkipper);
433437
}
434438

435439
/**

0 commit comments

Comments
 (0)