Skip to content

Commit a0127c4

Browse files
martijnvgmartijnvg
committed
Re-enable use_doc_values_skipper and add specialized lucene query for @timestamp field filtering.
1 parent 98a3719 commit a0127c4

File tree

4 files changed

+378
-1
lines changed

4 files changed

+378
-1
lines changed

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,7 @@ public boolean isES87TSDBCodecEnabled() {
677677
public static final boolean DOC_VALUES_SKIPPER = new FeatureFlag("doc_values_skipper").isEnabled();
678678
public static final Setting<Boolean> USE_DOC_VALUES_SKIPPER = Setting.boolSetting(
679679
"index.mapping.use_doc_values_skipper",
680-
false,
680+
DOC_VALUES_SKIPPER,
681681
Property.IndexScope,
682682
Property.Final
683683
);

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
2626
import org.apache.lucene.search.Query;
2727
import org.elasticsearch.ElasticsearchParseException;
28+
import org.elasticsearch.cluster.metadata.DataStream;
2829
import org.elasticsearch.common.geo.ShapeRelation;
2930
import org.elasticsearch.common.logging.DeprecationCategory;
3031
import org.elasticsearch.common.logging.DeprecationLogger;
@@ -50,6 +51,7 @@
5051
import org.elasticsearch.index.query.DateRangeIncludingNowQuery;
5152
import org.elasticsearch.index.query.QueryRewriteContext;
5253
import org.elasticsearch.index.query.SearchExecutionContext;
54+
import org.elasticsearch.lucene.queries.TimestampQuery;
5355
import org.elasticsearch.script.DateFieldScript;
5456
import org.elasticsearch.script.Script;
5557
import org.elasticsearch.script.ScriptCompiler;
@@ -739,6 +741,12 @@ public Query rangeQuery(
739741
parser = forcedDateParser;
740742
}
741743
return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> {
744+
var indexMode = context.getIndexSettings().getMode();
745+
if ((indexMode == IndexMode.TIME_SERIES || indexMode == IndexMode.LOGSDB)
746+
&& name().equals(DataStream.TIMESTAMP_FIELD_NAME)) {
747+
return new TimestampQuery(l, u);
748+
}
749+
742750
Query query;
743751
if (isIndexed()) {
744752
query = LongPoint.newRangeQuery(name(), l, u);

‎server/src/main/java/org/elasticsearch/lucene/queries/TimestampIterator.java‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
package org.elasticsearch.lucene.queries;
10+
11+
import org.apache.lucene.index.DocValuesSkipper;
12+
import org.apache.lucene.search.DocIdSetIterator;
13+
import org.apache.lucene.search.TwoPhaseIterator;
14+
15+
import java.io.IOException;
16+
17+
/**
18+
* Based on {@link org.apache.lucene.search.DocValuesRangeIterator} but modified for time series and logsdb use cases.
19+
*/
20+
final class TimestampIterator extends TwoPhaseIterator {
21+
22+
private final RangeNoGapsApproximation approximation;
23+
private final TwoPhaseIterator innerTwoPhase;
24+
25+
TimestampIterator(
26+
TwoPhaseIterator twoPhase,
27+
DocValuesSkipper timestampSkipper,
28+
DocValuesSkipper primaryFieldSkipper,
29+
long minTimestamp,
30+
long maxTimestamp
31+
) {
32+
super(new RangeNoGapsApproximation(twoPhase.approximation(), timestampSkipper, primaryFieldSkipper, minTimestamp, maxTimestamp));
33+
this.approximation = (RangeNoGapsApproximation) approximation();
34+
this.innerTwoPhase = twoPhase;
35+
}
36+
37+
static final class RangeNoGapsApproximation extends DocIdSetIterator {
38+
39+
private final DocIdSetIterator innerApproximation;
40+
41+
final DocValuesSkipper timestampSkipper;
42+
final DocValuesSkipper primaryFieldSkipper;
43+
final long minTimestamp;
44+
final long maxTimestamp;
45+
46+
private int doc = -1;
47+
48+
// Track a decision for all doc IDs between the current doc ID and upTo inclusive.
49+
Match match = Match.MAYBE;
50+
int upTo = -1;
51+
52+
RangeNoGapsApproximation(
53+
DocIdSetIterator innerApproximation,
54+
DocValuesSkipper timestampSkipper,
55+
DocValuesSkipper primaryFieldSkipper,
56+
long minTimestamp,
57+
long maxTimestamp
58+
) {
59+
this.innerApproximation = innerApproximation;
60+
this.timestampSkipper = timestampSkipper;
61+
this.primaryFieldSkipper = primaryFieldSkipper;
62+
this.minTimestamp = minTimestamp;
63+
this.maxTimestamp = maxTimestamp;
64+
}
65+
66+
@Override
67+
public int docID() {
68+
return doc;
69+
}
70+
71+
@Override
72+
public int nextDoc() throws IOException {
73+
return advance(docID() + 1);
74+
}
75+
76+
@Override
77+
public int advance(int target) throws IOException {
78+
while (true) {
79+
if (target > upTo) {
80+
timestampSkipper.advance(target);
81+
// If target doesn't have a value and is between two blocks, it is possible that advance()
82+
// moved to a block that doesn't contain `target`.
83+
target = Math.max(target, timestampSkipper.minDocID(0));
84+
if (target == NO_MORE_DOCS) {
85+
return doc = NO_MORE_DOCS;
86+
}
87+
upTo = timestampSkipper.maxDocID(0);
88+
match = match(0);
89+
90+
// If we have a YES or NO decision, see if we still have the same decision on a higher
91+
// level (= on a wider range of doc IDs)
92+
int nextLevel = 1;
93+
while (match != Match.MAYBE && nextLevel < timestampSkipper.numLevels() && match == match(nextLevel)) {
94+
upTo = timestampSkipper.maxDocID(nextLevel);
95+
nextLevel++;
96+
}
97+
}
98+
switch (match) {
99+
case YES:
100+
return doc = target;
101+
case MAYBE:
102+
case NO:
103+
if (match == Match.NO) {
104+
primaryFieldSkipper.advance(target);
105+
int betterUptTo = -1;
106+
for (int level = 0; level < primaryFieldSkipper.numLevels(); level++) {
107+
if (primaryFieldSkipper.minValue(level) == primaryFieldSkipper.maxValue(level)) {
108+
betterUptTo = primaryFieldSkipper.maxDocID(level);
109+
} else {
110+
break;
111+
}
112+
}
113+
if (betterUptTo > upTo) {
114+
upTo = betterUptTo;
115+
}
116+
}
117+
118+
if (upTo == DocIdSetIterator.NO_MORE_DOCS) {
119+
return doc = NO_MORE_DOCS;
120+
}
121+
target = upTo + 1;
122+
break;
123+
default:
124+
throw new AssertionError("Unknown enum constant: " + match);
125+
}
126+
}
127+
}
128+
129+
@Override
130+
public long cost() {
131+
return innerApproximation.cost();
132+
}
133+
134+
Match match(int level) {
135+
long minValue = timestampSkipper.minValue(level);
136+
long maxValue = timestampSkipper.maxValue(level);
137+
if (minValue > maxTimestamp || maxValue < minTimestamp) {
138+
return Match.NO;
139+
} else if (minValue >= minTimestamp && maxValue <= maxTimestamp) {
140+
return Match.YES;
141+
} else {
142+
return Match.MAYBE;
143+
}
144+
}
145+
146+
}
147+
148+
@Override
149+
public boolean matches() throws IOException {
150+
return switch (approximation.match) {
151+
case YES -> true;
152+
case MAYBE -> innerTwoPhase.matches();
153+
case NO -> throw new IllegalStateException("Unpositioned approximation");
154+
};
155+
}
156+
157+
@Override
158+
public int docIDRunEnd() throws IOException {
159+
if (approximation.match == Match.YES) {
160+
return approximation.upTo + 1;
161+
}
162+
return super.docIDRunEnd();
163+
}
164+
165+
@Override
166+
public float matchCost() {
167+
return innerTwoPhase.matchCost();
168+
}
169+
170+
enum Match {
171+
/** None of the documents in the range match */
172+
NO,
173+
/** Document values need to be checked to verify matches */
174+
MAYBE,
175+
/** All docs in the range match */
176+
YES;
177+
}
178+
}

0 commit comments

Comments
 (0)