Skip to content

Commit fb28191

Browse files
jordan-powersgeorgewallace
jordan-powers
authored and
georgewallace
committed
Enable a sparse doc values index for @timestamp in time-series indices (elastic#123191)
This patch builds on the work done in elastic#122161 by also enabling the sparse doc values index for @timestamp in time-series indices.
1 parent 7aca7e2 commit fb28191

File tree

6 files changed

+261
-31
lines changed

6 files changed

+261
-31
lines changed

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
*/
99
package org.elasticsearch.datastreams.mapper;
1010

11+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1112
import org.elasticsearch.common.settings.Settings;
1213
import org.elasticsearch.core.CheckedConsumer;
1314
import org.elasticsearch.datastreams.DataStreamsPlugin;
@@ -537,4 +538,199 @@ public void testFieldTypeWithDocValuesSkipper_CustomTimestampField() throws IOEx
537538
assertFalse(defaultTimestamp.fieldType().hasDocValuesSkipper());
538539
}
539540
}
541+
542+
public void testFieldTypeWithDocValuesSkipper_TSDBModeDisabledDocValuesSkipper() throws IOException {
543+
final Settings settings = Settings.builder()
544+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
545+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
546+
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
547+
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
548+
.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false)
549+
.build();
550+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
551+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
552+
b.field("type", "date");
553+
b.endObject();
554+
}));
555+
556+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
557+
.mappers()
558+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
559+
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
560+
assertTrue(timestampMapper.fieldType().hasDocValues());
561+
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
562+
assertTrue(timestampMapper.fieldType().isIndexed());
563+
}
564+
565+
public void testFieldTypeWithDocValuesSkipper_TSDBMode() throws IOException {
566+
final Settings settings = Settings.builder()
567+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
568+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
569+
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
570+
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
571+
.build();
572+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
573+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
574+
b.field("type", "date");
575+
b.endObject();
576+
}));
577+
578+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
579+
.mappers()
580+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
581+
assertTrue(timestampMapper.fieldType().hasDocValues());
582+
if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) {
583+
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
584+
assertFalse(timestampMapper.fieldType().isIndexed());
585+
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
586+
} else {
587+
// TODO: remove this 'else' branch when removing the `doc_values_skipper` feature flag
588+
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
589+
assertTrue(timestampMapper.fieldType().isIndexed());
590+
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
591+
}
592+
}
593+
594+
public void testFieldTypeWithDocValuesSkipper_TSDBModeNoTimestampMapping() throws IOException {
595+
final Settings settings = Settings.builder()
596+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
597+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
598+
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
599+
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
600+
.build();
601+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {}));
602+
603+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
604+
.mappers()
605+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
606+
assertTrue(timestampMapper.fieldType().hasDocValues());
607+
if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) {
608+
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
609+
assertFalse(timestampMapper.fieldType().isIndexed());
610+
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
611+
} else {
612+
// TODO: remove this 'else' branch when removing the `doc_values_skipper` feature flag
613+
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
614+
assertTrue(timestampMapper.fieldType().isIndexed());
615+
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
616+
}
617+
}
618+
619+
public void testFieldTypeWithDocValuesSkipper_TSDBModeTimestampDateNanos() throws IOException {
620+
final Settings settings = Settings.builder()
621+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
622+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
623+
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
624+
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
625+
.build();
626+
final MapperService mapperService = withMapping(
627+
new TestMapperServiceBuilder().settings(settings).applyDefaultMapping(false).build(),
628+
timestampMapping(true, b -> {
629+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
630+
b.field("type", "date_nanos");
631+
b.endObject();
632+
})
633+
);
634+
635+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
636+
.mappers()
637+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
638+
assertTrue(timestampMapper.fieldType().hasDocValues());
639+
if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) {
640+
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
641+
assertFalse(timestampMapper.fieldType().isIndexed());
642+
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
643+
} else {
644+
// TODO: remove this 'else' branch when removing the `doc_values_skipper` feature flag
645+
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
646+
assertTrue(timestampMapper.fieldType().isIndexed());
647+
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
648+
}
649+
}
650+
651+
public void testFieldTypeWithDocValuesSkipper_TSDBModeExplicitTimestampIndexEnabledDocValuesSkipper() throws IOException {
652+
final Settings settings = Settings.builder()
653+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
654+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
655+
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
656+
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
657+
.build();
658+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
659+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
660+
b.field("type", "date");
661+
b.field("index", true);
662+
b.endObject();
663+
}));
664+
665+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
666+
.mappers()
667+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
668+
assertTrue(timestampMapper.fieldType().hasDocValues());
669+
if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) {
670+
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
671+
assertFalse(timestampMapper.fieldType().isIndexed());
672+
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
673+
} else {
674+
// TODO: remove this 'else' branch when removing the `doc_values_skipper` feature flag
675+
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
676+
assertTrue(timestampMapper.fieldType().isIndexed());
677+
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
678+
}
679+
}
680+
681+
public void testFieldTypeWithDocValuesSkipper_TSDBModeExplicitTimestampIndexDisabledDocValuesSkipper() throws IOException {
682+
final Settings settings = Settings.builder()
683+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
684+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
685+
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
686+
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
687+
.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false)
688+
.build();
689+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
690+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
691+
b.field("type", "date");
692+
b.field("index", true);
693+
b.endObject();
694+
}));
695+
696+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
697+
.mappers()
698+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
699+
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
700+
assertTrue(timestampMapper.fieldType().hasDocValues());
701+
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
702+
assertTrue(timestampMapper.fieldType().isIndexed());
703+
}
704+
705+
public void testFieldTypeWithDocValuesSkipper_TSDBModeWithoutDefaultMapping() throws IOException {
706+
final Settings settings = Settings.builder()
707+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
708+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
709+
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
710+
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
711+
.build();
712+
final MapperService mapperService = withMapping(
713+
new TestMapperServiceBuilder().settings(settings).applyDefaultMapping(false).build(),
714+
timestampMapping(true, b -> {
715+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
716+
b.field("type", "date");
717+
b.endObject();
718+
})
719+
);
720+
721+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
722+
.mappers()
723+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
724+
assertTrue(timestampMapper.fieldType().hasDocValues());
725+
if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) {
726+
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
727+
assertFalse(timestampMapper.fieldType().isIndexed());
728+
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
729+
} else {
730+
// TODO: remove this 'else' branch when removing the `doc_values_skipper` feature flag
731+
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
732+
assertTrue(timestampMapper.fieldType().isIndexed());
733+
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
734+
}
735+
}
540736
}

‎server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -242,16 +242,31 @@ public void testSearchIdleBoolQueryMatchOneIndex() throws InterruptedException {
242242
// are executed only if we have enough shards.
243243
int idleIndexShardsCount = 3;
244244
int activeIndexShardsCount = 3;
245+
246+
var idleIndexSettingsBuilder = Settings.builder()
247+
.put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
248+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, idleIndexShardsCount)
249+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
250+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
251+
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-10T00:00:00.000Z")
252+
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-11T00:00:00.000Z");
253+
254+
var activeIndexSettingsBuilder = Settings.builder()
255+
.put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
256+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, activeIndexShardsCount)
257+
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
258+
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
259+
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-12T00:00:00.000Z")
260+
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-13T23:59:59.999Z");
261+
262+
if (IndexSettings.DOC_VALUES_SKIPPER.isEnabled()) {
263+
idleIndexSettingsBuilder.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false);
264+
activeIndexSettingsBuilder.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false);
265+
}
266+
245267
createIndex(
246268
idleIndex,
247-
Settings.builder()
248-
.put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
249-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, idleIndexShardsCount)
250-
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
251-
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
252-
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-10T00:00:00.000Z")
253-
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-11T00:00:00.000Z")
254-
.build(),
269+
idleIndexSettingsBuilder.build(),
255270
"doc",
256271
"keyword",
257272
"type=keyword",
@@ -262,14 +277,7 @@ public void testSearchIdleBoolQueryMatchOneIndex() throws InterruptedException {
262277
);
263278
createIndex(
264279
activeIndex,
265-
Settings.builder()
266-
.put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
267-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, activeIndexShardsCount)
268-
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
269-
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
270-
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-12T00:00:00.000Z")
271-
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-13T23:59:59.999Z")
272-
.build(),
280+
activeIndexSettingsBuilder.build(),
273281
"doc",
274282
"keyword",
275283
"type=keyword",

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
package org.elasticsearch.cluster.metadata;
1010

1111
import org.apache.lucene.document.LongPoint;
12+
import org.apache.lucene.index.DocValuesSkipIndexType;
13+
import org.apache.lucene.index.DocValuesSkipper;
14+
import org.apache.lucene.index.FieldInfo;
1215
import org.apache.lucene.index.LeafReader;
1316
import org.apache.lucene.index.PointValues;
1417
import org.elasticsearch.ElasticsearchException;
@@ -84,6 +87,12 @@ public static boolean isFailureStoreFeatureFlagEnabled() {
8487
// Timeseries indices' leaf readers should be sorted by desc order of their timestamp field, as it allows search time optimizations
8588
public static final Comparator<LeafReader> TIMESERIES_LEAF_READERS_SORTER = Comparator.comparingLong((LeafReader r) -> {
8689
try {
90+
FieldInfo info = r.getFieldInfos().fieldInfo(TIMESTAMP_FIELD_NAME);
91+
if (info != null && info.docValuesSkipIndexType() == DocValuesSkipIndexType.RANGE) {
92+
DocValuesSkipper skipper = r.getDocValuesSkipper(TIMESTAMP_FIELD_NAME);
93+
return skipper.maxValue();
94+
}
95+
8796
PointValues points = r.getPointValues(TIMESTAMP_FIELD_NAME);
8897
if (points != null) {
8998
byte[] sortValue = points.getMaxPackedValue();

‎server/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@
1010
package org.elasticsearch.common.lucene.uid;
1111

1212
import org.apache.lucene.document.LongPoint;
13+
import org.apache.lucene.index.DocValuesSkipIndexType;
14+
import org.apache.lucene.index.DocValuesSkipper;
15+
import org.apache.lucene.index.FieldInfo;
1316
import org.apache.lucene.index.LeafReader;
1417
import org.apache.lucene.index.LeafReaderContext;
1518
import org.apache.lucene.index.NumericDocValues;
@@ -94,15 +97,27 @@ final class PerThreadIDVersionAndSeqNoLookup {
9497
this.loadedTimestampRange = loadTimestampRange;
9598
// Also check for the existence of the timestamp field, because sometimes a segment can only contain tombstone documents,
9699
// which don't have any mapped fields (also not the timestamp field) and just some meta fields like _id, _seq_no etc.
97-
if (loadTimestampRange && reader.getFieldInfos().fieldInfo(DataStream.TIMESTAMP_FIELD_NAME) != null) {
98-
PointValues tsPointValues = reader.getPointValues(DataStream.TIMESTAMP_FIELD_NAME);
99-
assert tsPointValues != null : "no timestamp field for reader:" + reader + " and parent:" + reader.getContext().parent.reader();
100-
minTimestamp = LongPoint.decodeDimension(tsPointValues.getMinPackedValue(), 0);
101-
maxTimestamp = LongPoint.decodeDimension(tsPointValues.getMaxPackedValue(), 0);
102-
} else {
103-
minTimestamp = 0;
104-
maxTimestamp = Long.MAX_VALUE;
100+
long minTimestamp = 0;
101+
long maxTimestamp = Long.MAX_VALUE;
102+
if (loadTimestampRange) {
103+
FieldInfo info = reader.getFieldInfos().fieldInfo(DataStream.TIMESTAMP_FIELD_NAME);
104+
if (info != null) {
105+
if (info.docValuesSkipIndexType() == DocValuesSkipIndexType.RANGE) {
106+
DocValuesSkipper skipper = reader.getDocValuesSkipper(DataStream.TIMESTAMP_FIELD_NAME);
107+
assert skipper != null : "no skipper for reader:" + reader + " and parent:" + reader.getContext().parent.reader();
108+
minTimestamp = skipper.minValue();
109+
maxTimestamp = skipper.maxValue();
110+
} else {
111+
PointValues tsPointValues = reader.getPointValues(DataStream.TIMESTAMP_FIELD_NAME);
112+
assert tsPointValues != null
113+
: "no timestamp field for reader:" + reader + " and parent:" + reader.getContext().parent.reader();
114+
minTimestamp = LongPoint.decodeDimension(tsPointValues.getMinPackedValue(), 0);
115+
maxTimestamp = LongPoint.decodeDimension(tsPointValues.getMaxPackedValue(), 0);
116+
}
117+
}
105118
}
119+
this.minTimestamp = minTimestamp;
120+
this.maxTimestamp = maxTimestamp;
106121
}
107122

108123
PerThreadIDVersionAndSeqNoLookup(LeafReader reader, boolean loadTimestampRange) throws IOException {

0 commit comments

Comments
 (0)