Skip to content

Commit 5f04cb2

Browse files
romseygeekromseygeek
committed
Allow re-enabling of sparse indexes on @timestamp field in LogsDB
1 parent 1758533 commit 5f04cb2

File tree

8 files changed

+222
-23
lines changed

8 files changed

+222
-23
lines changed

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,8 @@ private static Version parseUnchecked(String version) {
190190
public static final IndexVersion KEYWORD_MULTI_FIELDS_NOT_STORED_WHEN_IGNORED = def(9_040_0_00, Version.LUCENE_10_3_0);
191191
public static final IndexVersion UPGRADE_TO_LUCENE_10_3_1 = def(9_041_0_00, Version.LUCENE_10_3_1);
192192

193+
public static final IndexVersion REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_042_0_00, Version.LUCENE_10_3_1);
194+
193195
/*
194196
* STOP! READ THIS FIRST! No, really,
195197
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1164,7 +1164,7 @@ private DateFieldMapper(
11641164
* Determines whether the doc values skipper (sparse index) should be used for the {@code @timestamp} field.
11651165
* <p>
11661166
* The doc values skipper is enabled only if {@code index.mapping.use_doc_values_skipper} is set to {@code true},
1167-
* the index was created on or after {@link IndexVersions#TIMESTAMP_DOC_VALUES_SPARSE_INDEX}, and the
1167+
* the index was created on or after {@link IndexVersions#REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX}, and the
11681168
* field has doc values enabled. Additionally, the index mode must be {@link IndexMode#LOGSDB} or {@link IndexMode#TIME_SERIES}, and
11691169
* the index sorting configuration must include the {@code @timestamp} field.
11701170
*
@@ -1186,7 +1186,7 @@ private static boolean shouldUseDocValuesSkipper(
11861186
final IndexSortConfig indexSortConfig,
11871187
final String fullFieldName
11881188
) {
1189-
return indexCreatedVersion.onOrAfter(IndexVersions.TIMESTAMP_DOC_VALUES_SPARSE_INDEX)
1189+
return indexCreatedVersion.onOrAfter(IndexVersions.REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX)
11901190
&& useDocValuesSkipper
11911191
&& hasDocValues
11921192
&& (IndexMode.LOGSDB.equals(indexMode) || IndexMode.TIME_SERIES.equals(indexMode))

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/MergedDocValuesRangeQuery.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.search.aggregations.bucket.filter;
11+
12+
import org.apache.lucene.document.NumericDocValuesField;
13+
import org.apache.lucene.search.MatchNoDocsQuery;
14+
import org.apache.lucene.search.Query;
15+
import org.elasticsearch.core.SuppressForbidden;
16+
17+
import java.lang.reflect.Field;
18+
import java.util.Objects;
19+
20+
class MergedDocValuesRangeQuery {
21+
22+
@SuppressForbidden(reason = "Uses reflection to access package-private lucene class")
23+
public static Query merge(Query query, Query extraQuery) {
24+
Class<? extends Query> queryClass = query.getClass();
25+
Class<? extends Query> extraQueryClass = extraQuery.getClass();
26+
27+
if (queryClass.equals(extraQueryClass) == false
28+
|| queryClass.getCanonicalName().equals("org.apache.lucene.document.SortedNumericDocValuesRangeQuery") == false) {
29+
return null;
30+
}
31+
32+
try {
33+
Field fieldName = queryClass.getDeclaredField("field");
34+
fieldName.setAccessible(true);
35+
36+
String field = fieldName.get(query).toString();
37+
if (Objects.equals(field, fieldName.get(extraQuery)) == false) {
38+
return null;
39+
}
40+
41+
Field lowerValue = queryClass.getDeclaredField("lowerValue");
42+
Field upperValue = queryClass.getDeclaredField("upperValue");
43+
lowerValue.setAccessible(true);
44+
upperValue.setAccessible(true);
45+
46+
long q1LowerValue = lowerValue.getLong(query);
47+
long q1UpperValue = upperValue.getLong(query);
48+
long q2LowerValue = lowerValue.getLong(extraQuery);
49+
long q2UpperValue = upperValue.getLong(extraQuery);
50+
51+
if (q1UpperValue < q2LowerValue || q2UpperValue < q1LowerValue) {
52+
return new MatchNoDocsQuery("Non-overlapping range queries");
53+
}
54+
55+
return NumericDocValuesField.newSlowRangeQuery(
56+
field,
57+
Math.max(q1LowerValue, q2LowerValue),
58+
Math.min(q1UpperValue, q2UpperValue)
59+
);
60+
61+
} catch (NoSuchFieldException | IllegalAccessException e) {
62+
return null;
63+
}
64+
}
65+
}

‎server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/QueryToFilterAdapter.java‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -132,13 +132,12 @@ QueryToFilterAdapter union(Query extraQuery) throws IOException {
132132
extraQuery = searcher().rewrite(new ConstantScoreQuery(extraQuery));
133133
Query unwrappedExtraQuery = unwrap(extraQuery);
134134
Query unwrappedQuery = unwrap(query);
135-
if (unwrappedQuery instanceof PointRangeQuery q1 && unwrappedExtraQuery instanceof PointRangeQuery q2) {
136-
Query merged = MergedPointRangeQuery.merge(q1, q2);
137-
if (merged != null) {
138-
// Should we rewrap here?
139-
return new QueryToFilterAdapter(searcher(), key(), merged);
140-
}
135+
136+
Query merged = maybeMergeRangeQueries(unwrappedQuery, unwrappedExtraQuery);
137+
if (merged != null) {
138+
return new QueryToFilterAdapter(searcher(), key(), merged);
141139
}
140+
142141
BooleanQuery.Builder builder = new BooleanQuery.Builder();
143142
builder.add(query, BooleanClause.Occur.FILTER);
144143
builder.add(extraQuery, BooleanClause.Occur.FILTER);
@@ -155,6 +154,13 @@ public boolean isInefficientUnion() {
155154
};
156155
}
157156

157+
private static Query maybeMergeRangeQueries(Query query, Query extraQuery) throws IOException {
158+
if (query instanceof PointRangeQuery q1 && extraQuery instanceof PointRangeQuery q2) {
159+
return MergedPointRangeQuery.merge(q1, q2);
160+
}
161+
return MergedDocValuesRangeQuery.merge(query, extraQuery);
162+
}
163+
158164
private static Query unwrap(Query query) {
159165
while (true) {
160166
switch (query) {

‎server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.apache.logging.log4j.LogManager;
1313
import org.apache.logging.log4j.Logger;
1414
import org.apache.lucene.index.DocValues;
15+
import org.apache.lucene.index.DocValuesSkipper;
1516
import org.apache.lucene.index.LeafReaderContext;
1617
import org.apache.lucene.index.PointValues;
1718
import org.apache.lucene.search.BooleanClause;
@@ -314,6 +315,12 @@ public Function<Rounding, Rounding.Prepared> roundingPreparer(AggregationContext
314315
range[0] = dft.resolution().parsePointAsMillis(min);
315316
range[1] = dft.resolution().parsePointAsMillis(max);
316317
}
318+
} else if (dft.hasDocValuesSkipper()) {
319+
log.trace("Attempting to apply skipper-based data rounding");
320+
range[0] = dft.resolution()
321+
.roundDownToMillis(DocValuesSkipper.globalMinValue(context.searcher(), fieldContext.field()));
322+
range[1] = dft.resolution()
323+
.roundDownToMillis(DocValuesSkipper.globalMaxValue(context.searcher(), fieldContext.field()));
317324
}
318325
log.trace("Bounds after index bound date rounding: {}, {}", range[0], range[1]);
319326

‎server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.index.fielddata.IndexFieldData;
1414
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
1515
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
16+
import org.elasticsearch.index.mapper.DateFieldMapper;
1617
import org.elasticsearch.index.mapper.MappedFieldType;
1718
import org.elasticsearch.index.mapper.NumberFieldMapper;
1819
import org.elasticsearch.index.mapper.RangeFieldMapper;
@@ -429,7 +430,11 @@ public Function<byte[], Number> getPointReaderOrNull() {
429430
* the ordering.
430431
*/
431432
public boolean alignsWithSearchIndex() {
432-
return script() == null && missing() == null && fieldType() != null && fieldType().indexType().supportsSortShortcuts();
433+
boolean hasDocValuesSkipper = fieldType() instanceof DateFieldMapper.DateFieldType dft && dft.hasDocValuesSkipper();
434+
return script() == null
435+
&& missing() == null
436+
&& fieldType() != null
437+
&& (fieldType().indexType().supportsSortShortcuts() || hasDocValuesSkipper);
433438
}
434439

435440
/**

‎server/src/test/java/org/elasticsearch/search/aggregations/bucket/filter/MergedDocValuesRangeQueryTests.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.search.aggregations.bucket.filter;
11+
12+
import org.apache.lucene.document.LongPoint;
13+
import org.apache.lucene.document.NumericDocValuesField;
14+
import org.apache.lucene.index.Term;
15+
import org.apache.lucene.search.MatchNoDocsQuery;
16+
import org.apache.lucene.search.TermQuery;
17+
import org.elasticsearch.test.ESTestCase;
18+
19+
import static org.hamcrest.Matchers.equalTo;
20+
import static org.hamcrest.Matchers.instanceOf;
21+
import static org.hamcrest.Matchers.nullValue;
22+
23+
public class MergedDocValuesRangeQueryTests extends ESTestCase {
24+
25+
public void testNotRangeQueries() {
26+
assertThat(
27+
MergedDocValuesRangeQuery.merge(LongPoint.newRangeQuery("field", 1, 4), new TermQuery(new Term("field", "foo"))),
28+
nullValue()
29+
);
30+
31+
assertThat(
32+
MergedDocValuesRangeQuery.merge(NumericDocValuesField.newSlowRangeQuery("field", 1, 4), LongPoint.newRangeQuery("field", 2, 4)),
33+
nullValue()
34+
);
35+
36+
assertThat(
37+
MergedDocValuesRangeQuery.merge(LongPoint.newRangeQuery("field", 2, 4), NumericDocValuesField.newSlowRangeQuery("field", 1, 4)),
38+
nullValue()
39+
);
40+
}
41+
42+
public void testDifferentFields() {
43+
assertThat(
44+
MergedDocValuesRangeQuery.merge(
45+
NumericDocValuesField.newSlowRangeQuery("field1", 1, 4),
46+
NumericDocValuesField.newSlowRangeQuery("field2", 2, 4)
47+
),
48+
nullValue()
49+
);
50+
}
51+
52+
public void testNoOverlap() {
53+
assertThat(
54+
MergedDocValuesRangeQuery.merge(
55+
NumericDocValuesField.newSlowRangeQuery("field", 1, 4),
56+
NumericDocValuesField.newSlowRangeQuery("field", 6, 8)
57+
),
58+
instanceOf(MatchNoDocsQuery.class)
59+
);
60+
}
61+
62+
public void testOverlap() {
63+
assertThat(
64+
MergedDocValuesRangeQuery.merge(
65+
NumericDocValuesField.newSlowRangeQuery("field", 1, 6),
66+
NumericDocValuesField.newSlowRangeQuery("field", 4, 8)
67+
),
68+
equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6))
69+
);
70+
71+
assertThat(
72+
MergedDocValuesRangeQuery.merge(
73+
NumericDocValuesField.newSlowRangeQuery("field", 4, 8),
74+
NumericDocValuesField.newSlowRangeQuery("field", 1, 6)
75+
),
76+
equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6))
77+
);
78+
79+
assertThat(
80+
MergedDocValuesRangeQuery.merge(
81+
NumericDocValuesField.newSlowRangeQuery("field", 1, 8),
82+
NumericDocValuesField.newSlowRangeQuery("field", 4, 6)
83+
),
84+
equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6))
85+
);
86+
87+
assertThat(
88+
MergedDocValuesRangeQuery.merge(
89+
NumericDocValuesField.newSlowRangeQuery("field", 4, 6),
90+
NumericDocValuesField.newSlowRangeQuery("field", 1, 8)
91+
),
92+
equalTo(NumericDocValuesField.newSlowRangeQuery("field", 4, 6))
93+
);
94+
}
95+
96+
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java‎

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
package org.elasticsearch.xpack.esql.stats;
99

10+
import org.apache.lucene.index.DocValuesSkipper;
1011
import org.apache.lucene.index.DocValuesType;
1112
import org.apache.lucene.index.FieldInfo;
1213
import org.apache.lucene.index.FieldInfos;
@@ -17,6 +18,7 @@
1718
import org.apache.lucene.index.PointValues;
1819
import org.apache.lucene.index.Term;
1920
import org.apache.lucene.index.Terms;
21+
import org.apache.lucene.search.IndexSearcher;
2022
import org.apache.lucene.util.BytesRef;
2123
import org.apache.lucene.util.NumericUtils;
2224
import org.elasticsearch.index.mapper.ConstantFieldType;
@@ -201,21 +203,29 @@ public Object min(FieldName field) {
201203
var stat = cache.computeIfAbsent(field.string(), this::makeFieldStats);
202204
// Consolidate min for indexed date fields only, skip the others and mixed-typed fields.
203205
MappedFieldType fieldType = stat.config.fieldType;
204-
if (fieldType == null || stat.config.indexed == false || fieldType instanceof DateFieldType == false) {
206+
boolean hasDocValueSkipper = fieldType instanceof DateFieldType dft && dft.hasDocValuesSkipper();
207+
if (fieldType == null
208+
|| (hasDocValueSkipper == false && stat.config.indexed == false)
209+
|| fieldType instanceof DateFieldType == false) {
205210
return null;
206211
}
207212
if (stat.min == null) {
208213
var min = new long[] { Long.MAX_VALUE };
209214
Holder<Boolean> foundMinValue = new Holder<>(false);
210215
doWithContexts(r -> {
211-
byte[] minPackedValue = PointValues.getMinPackedValue(r, field.string());
212-
if (minPackedValue != null && minPackedValue.length == 8) {
213-
long minValue = NumericUtils.sortableBytesToLong(minPackedValue, 0);
214-
if (minValue <= min[0]) {
215-
min[0] = minValue;
216-
foundMinValue.set(true);
216+
long minValue = Long.MAX_VALUE;
217+
if (hasDocValueSkipper) {
218+
minValue = DocValuesSkipper.globalMinValue(new IndexSearcher(r), field.string());
219+
} else {
220+
byte[] minPackedValue = PointValues.getMinPackedValue(r, field.string());
221+
if (minPackedValue != null && minPackedValue.length == 8) {
222+
minValue = NumericUtils.sortableBytesToLong(minPackedValue, 0);
217223
}
218224
}
225+
if (minValue <= min[0]) {
226+
min[0] = minValue;
227+
foundMinValue.set(true);
228+
}
219229
return true;
220230
}, true);
221231
stat.min = foundMinValue.get() ? min[0] : null;
@@ -228,21 +238,29 @@ public Object max(FieldName field) {
228238
var stat = cache.computeIfAbsent(field.string(), this::makeFieldStats);
229239
// Consolidate max for indexed date fields only, skip the others and mixed-typed fields.
230240
MappedFieldType fieldType = stat.config.fieldType;
231-
if (fieldType == null || stat.config.indexed == false || fieldType instanceof DateFieldType == false) {
241+
boolean hasDocValueSkipper = fieldType instanceof DateFieldType dft && dft.hasDocValuesSkipper();
242+
if (fieldType == null
243+
|| (hasDocValueSkipper == false && stat.config.indexed == false)
244+
|| fieldType instanceof DateFieldType == false) {
232245
return null;
233246
}
234247
if (stat.max == null) {
235248
var max = new long[] { Long.MIN_VALUE };
236249
Holder<Boolean> foundMaxValue = new Holder<>(false);
237250
doWithContexts(r -> {
238-
byte[] maxPackedValue = PointValues.getMaxPackedValue(r, field.string());
239-
if (maxPackedValue != null && maxPackedValue.length == 8) {
240-
long maxValue = NumericUtils.sortableBytesToLong(maxPackedValue, 0);
241-
if (maxValue >= max[0]) {
242-
max[0] = maxValue;
243-
foundMaxValue.set(true);
251+
long maxValue = Long.MIN_VALUE;
252+
if (hasDocValueSkipper) {
253+
maxValue = DocValuesSkipper.globalMaxValue(new IndexSearcher(r), field.string());
254+
} else {
255+
byte[] maxPackedValue = PointValues.getMaxPackedValue(r, field.string());
256+
if (maxPackedValue != null && maxPackedValue.length == 8) {
257+
maxValue = NumericUtils.sortableBytesToLong(maxPackedValue, 0);
244258
}
245259
}
260+
if (maxValue >= max[0]) {
261+
max[0] = maxValue;
262+
foundMaxValue.set(true);
263+
}
246264
return true;
247265
}, true);
248266
stat.max = foundMaxValue.get() ? max[0] : null;

0 commit comments

Comments
 (0)