Skip to content

Commit f3069c6

Browse files
romseygeekromseygeek
authored
Replace indexes with skippers for all TSDB numeric/keyword/ip fields (elastic#138092)
Rather than only doing this for fields explicitly configured as dimensions, all numeric/keyword/ip fields in TSDB mode are set to default to index=false and enable skippers on doc values.
1 parent f2d5e71 commit f3069c6

File tree

14 files changed

+77
-62
lines changed

14 files changed

+77
-62
lines changed

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@ private static Version parseUnchecked(String version) {
193193
public static final IndexVersion SKIPPERS_ENABLED_BY_DEFAULT = def(9_043_0_00, Version.LUCENE_10_3_1);
194194
public static final IndexVersion TIME_SERIES_USE_SYNTHETIC_ID = def(9_044_0_00, Version.LUCENE_10_3_1);
195195
public static final IndexVersion TIME_SERIES_DIMENSIONS_USE_SKIPPERS = def(9_045_0_00, Version.LUCENE_10_3_1);
196+
public static final IndexVersion TIME_SERIES_ALL_FIELDS_USE_SKIPPERS = def(9_046_0_00, Version.LUCENE_10_3_1);
196197

197198
/*
198199
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import java.util.Set;
6060

6161
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
62+
import static org.elasticsearch.index.mapper.FieldMapper.Parameter.useTimeSeriesDocValuesSkippers;
6263

6364
/**
6465
* A field mapper for boolean fields.
@@ -148,12 +149,7 @@ private IndexType indexType() {
148149
if (docValues.get() == false) {
149150
return IndexType.NONE;
150151
}
151-
if (dimension.get()
152-
&& indexSettings.useDocValuesSkipper()
153-
&& indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) {
154-
return IndexType.skippers();
155-
}
156-
return IndexType.docValuesOnly();
152+
return useTimeSeriesDocValuesSkippers(indexSettings, dimension.get()) ? IndexType.skippers() : IndexType.docValuesOnly();
157153
}
158154

159155
@Override

‎server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1323,12 +1323,23 @@ public static Parameter<Boolean> indexParam(
13231323
"index",
13241324
false,
13251325
initializer,
1326-
() -> isDimension.get() == false
1327-
|| indexSettings.useDocValuesSkipper() == false
1328-
|| indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)
1326+
() -> useTimeSeriesDocValuesSkippers(indexSettings, isDimension.get()) == false
13291327
);
13301328
}
13311329

1330+
public static boolean useTimeSeriesDocValuesSkippers(IndexSettings indexSettings, boolean isDimension) {
1331+
if (indexSettings.useDocValuesSkipper() == false) {
1332+
return false;
1333+
}
1334+
if (indexSettings.getMode() == IndexMode.TIME_SERIES) {
1335+
if (isDimension) {
1336+
return indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS);
1337+
}
1338+
return indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ALL_FIELDS_USE_SKIPPERS);
1339+
}
1340+
return false;
1341+
}
1342+
13321343
public static Parameter<Boolean> storeParam(Function<FieldMapper, Boolean> initializer, boolean defaultValue) {
13331344
return Parameter.boolParam("store", false, initializer, defaultValue);
13341345
}

‎server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import java.util.function.BiFunction;
5858

5959
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
60+
import static org.elasticsearch.index.mapper.FieldMapper.Parameter.useTimeSeriesDocValuesSkippers;
6061
import static org.elasticsearch.index.mapper.IpPrefixAutomatonUtil.buildIpPrefixAutomaton;
6162

6263
/**
@@ -175,9 +176,7 @@ private IndexType indexType() {
175176
if (indexSettings.getIndexVersionCreated().isLegacyIndexVersion()) {
176177
return hasDocValues.get() ? IndexType.archivedPoints() : IndexType.NONE;
177178
}
178-
if (dimension.get()
179-
&& indexSettings.useDocValuesSkipper()
180-
&& indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) {
179+
if (useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) {
181180
return IndexType.skippers();
182181
}
183182
return IndexType.points(indexed.get(), hasDocValues.get());
@@ -640,11 +639,11 @@ private void indexValue(DocumentParserContext context, ESInetAddressPoint addres
640639
context.getRoutingFields().addIp(fieldType().name(), address.getInetAddress());
641640
}
642641
LuceneDocument doc = context.doc();
643-
if (indexed) {
642+
if (fieldType().indexType.hasPoints()) {
644643
doc.add(address);
645644
}
646-
if (hasDocValues) {
647-
if (indexed == false) {
645+
if (fieldType().indexType.hasDocValues()) {
646+
if (fieldType().indexType.hasDocValuesSkipper()) {
648647
doc.add(SortedSetDocValuesField.indexedField(fieldType().name(), address.binaryValue()));
649648
} else {
650649
doc.add(new SortedSetDocValuesField(fieldType().name(), address.binaryValue()));

‎server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@
9898
import static org.elasticsearch.core.Strings.format;
9999
import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING;
100100
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
101+
import static org.elasticsearch.index.mapper.FieldMapper.Parameter.useTimeSeriesDocValuesSkippers;
101102

102103
/**
103104
* A field mapper for keywords. This mapper accepts strings and indexes them as-is.
@@ -435,7 +436,7 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
435436

436437
private FieldType resolveFieldType(final boolean forceDocValuesSkipper, final String fullFieldName) {
437438
FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE);
438-
if (forceDocValuesSkipper || shouldUseHostnameSkipper(fullFieldName) || shouldUseDimensionSkipper()) {
439+
if (forceDocValuesSkipper || shouldUseHostnameSkipper(fullFieldName) || shouldUseTimeSeriesSkipper()) {
439440
fieldtype = new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
440441
}
441442
fieldtype.setOmitNorms(this.hasNorms.getValue() == false);
@@ -455,12 +456,8 @@ private FieldType resolveFieldType(final boolean forceDocValuesSkipper, final St
455456
return fieldtype;
456457
}
457458

458-
private boolean shouldUseDimensionSkipper() {
459-
return hasDocValues.get()
460-
&& dimension.get()
461-
&& indexed.get() == false
462-
&& indexSettings.useDocValuesSkipper()
463-
&& indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS);
459+
private boolean shouldUseTimeSeriesSkipper() {
460+
return hasDocValues.get() && indexed.get() == false && useTimeSeriesDocValuesSkippers(indexSettings, dimension.get());
464461
}
465462

466463
private boolean shouldUseHostnameSkipper(final String fullFieldName) {

‎server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
import java.util.function.Function;
8686

8787
import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName;
88+
import static org.elasticsearch.index.mapper.FieldMapper.Parameter.useTimeSeriesDocValuesSkippers;
8889

8990
/** A {@link FieldMapper} for numeric types: byte, short, int, long, float and double. */
9091
public class NumberFieldMapper extends FieldMapper {
@@ -168,9 +169,8 @@ public Builder(String name, NumberType type, ScriptCompiler compiler, IndexSetti
168169
).acceptsNull();
169170
this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension, hasDocValues::get);
170171
this.indexed = Parameter.indexParam(m -> toType(m).indexed, () -> {
171-
if (dimension.get()) {
172-
return indexSettings.useDocValuesSkipper() == false
173-
|| indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS);
172+
if (useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) {
173+
return false;
174174
}
175175
if (indexSettings.getMode() == IndexMode.TIME_SERIES) {
176176
var metricType = getMetric().getValue();
@@ -216,11 +216,7 @@ private IndexType indexType() {
216216
if (indexSettings.getIndexVersionCreated().isLegacyIndexVersion()) {
217217
return IndexType.archivedPoints();
218218
}
219-
if (indexSettings.useDocValuesSkipper()
220-
&& indexed.get() == false
221-
&& hasDocValues.get()
222-
&& dimension.get()
223-
&& indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) {
219+
if (indexed.get() == false && hasDocValues.get() && useTimeSeriesDocValuesSkippers(indexSettings, dimension.get())) {
224220
return IndexType.skippers();
225221
}
226222
return IndexType.points(indexed.get(), hasDocValues.get());

‎server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ public void testDimension() throws IOException {
227227
assertDimension(true, BooleanFieldMapper.BooleanFieldType::isDimension);
228228
assertDimension(false, BooleanFieldMapper.BooleanFieldType::isDimension);
229229

230-
assertDimensionIndexing();
230+
assertTimeSeriesIndexing();
231231
}
232232

233233
public void testDimensionIndexedAndDocvalues() {

‎server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ public void testDimension() throws IOException {
225225
assertDimension(true, IpFieldMapper.IpFieldType::isDimension);
226226
assertDimension(false, IpFieldMapper.IpFieldType::isDimension);
227227

228-
assertDimensionIndexing();
228+
assertTimeSeriesIndexing();
229229
}
230230

231231
public void testDimensionIndexedAndDocvalues() {

‎server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -428,7 +428,7 @@ public void testDimension() throws IOException {
428428
assertDimension(true, KeywordFieldMapper.KeywordFieldType::isDimension);
429429
assertDimension(false, KeywordFieldMapper.KeywordFieldType::isDimension);
430430

431-
assertDimensionIndexing();
431+
assertTimeSeriesIndexing();
432432
}
433433

434434
public void testDimensionAndIgnoreAbove() throws IOException {

‎test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java‎

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.elasticsearch.common.unit.ByteSizeValue;
4444
import org.elasticsearch.common.xcontent.XContentHelper;
4545
import org.elasticsearch.core.CheckedConsumer;
46+
import org.elasticsearch.index.IndexMode;
4647
import org.elasticsearch.index.IndexSettings;
4748
import org.elasticsearch.index.IndexVersion;
4849
import org.elasticsearch.index.IndexVersions;
@@ -390,32 +391,50 @@ protected <T> void assertDimension(boolean isDimension, Function<T, Boolean> che
390391
@SuppressWarnings("unchecked") // Syntactic sugar in tests
391392
T fieldType = (T) mapperService.fieldType("field");
392393
assertThat(checker.apply(fieldType), equalTo(isDimension));
394+
}
393395

394-
Settings settings = Settings.builder().put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), true).build();
395-
mapperService = createMapperService(settings, fieldMapping(b -> {
396-
minimalMapping(b);
397-
b.field("time_series_dimension", isDimension);
398-
}));
399-
400-
assertThat(
401-
mapperService.fieldType("field").indexType().hasDocValuesSkipper(),
402-
equalTo(mapperService.getIndexSettings().useDocValuesSkipper() && isDimension)
403-
);
396+
public void assertTimeSeriesIndexing() throws IOException {
397+
assertTimeSeriesIndexing(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS, true, true);
398+
assertTimeSeriesIndexing(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS, false, false);
399+
assertTimeSeriesIndexing(IndexVersions.TIME_SERIES_ALL_FIELDS_USE_SKIPPERS, true, true);
400+
assertTimeSeriesIndexing(IndexVersions.TIME_SERIES_ALL_FIELDS_USE_SKIPPERS, false, true);
401+
assertTimeSeriesIndexing(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID, true, false);
402+
assertTimeSeriesIndexing(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID, false, false);
404403
}
405404

406-
public void assertDimensionIndexing() throws IOException {
407-
Settings settings = Settings.builder().put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), true).build();
408-
MapperService mapperService = createMapperService(settings, fieldMapping(b -> {
405+
public void assertTimeSeriesIndexing(IndexVersion indexVersion, boolean isDimension, boolean hasSkippers) throws IOException {
406+
407+
// In time series mode, index=false and skippers=true for all fields,
408+
// regardless of their dimension status
409+
410+
Settings settings = Settings.builder()
411+
.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), true)
412+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.getName())
413+
.put("index.routing_path", "dim")
414+
.build();
415+
MapperService mapperService = createMapperService(indexVersion, settings, fieldMapping(b -> {
409416
minimalMapping(b);
410-
b.field("time_series_dimension", true);
417+
b.field("time_series_dimension", isDimension);
411418
}));
412419
assumeTrue("Skippers disabled by feature flag", mapperService.getIndexSettings().useDocValuesSkipper());
413420

414-
ParsedDocument doc = mapperService.documentMapper().parse(source(this::writeField));
421+
ParsedDocument doc = mapperService.documentMapper().parse(source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {
422+
writeField(b);
423+
b.field("@timestamp", "2025-11-14T12:00:00.000Z");
424+
b.field("dim", "foo");
425+
}, null));
415426
IndexableField field = doc.rootDoc().getField("field");
416-
assertSame(DocValuesSkipIndexType.RANGE, field.fieldType().docValuesSkipIndexType());
417-
assertSame(IndexOptions.NONE, field.fieldType().indexOptions());
418-
assertEquals(0, field.fieldType().pointDimensionCount());
427+
428+
if (hasSkippers) {
429+
assertSame(DocValuesSkipIndexType.RANGE, field.fieldType().docValuesSkipIndexType());
430+
assertSame(IndexOptions.NONE, field.fieldType().indexOptions());
431+
assertEquals(0, field.fieldType().pointDimensionCount());
432+
assertEquals(IndexType.skippers(), mapperService.fieldType("field").indexType());
433+
} else {
434+
assertSame(DocValuesSkipIndexType.NONE, field.fieldType().docValuesSkipIndexType());
435+
assertTrue(mapperService.fieldType("field").indexType().hasDenseIndex());
436+
}
437+
419438
}
420439

421440
protected <T> void assertMetricType(String metricType, Function<T, Enum<TimeSeriesParams.MetricType>> checker) throws IOException {

0 commit comments

Comments
 (0)