Skip to content

Enable a sparse doc values index for @timestamp in time-series indices #123191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Mar 3, 2025
Merged

Enable a sparse doc values index for @timestamp in time-series indices #123191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
e251d86
Test that @timestamp's DateFieldMapper is using docValuesSkipper
jordan-powers Feb 21, 2025
d3ad4e6
Update DateFieldMapper#shouldUseDocValuesSkipper to support TSDB
jordan-powers Feb 21, 2025
9e51083
Add comment explaining tsdb sorting
jordan-powers Feb 24, 2025
d1ca991
Merge remote-tracking branch 'upstream/main' into tsdb-timestamp-spar…
jordan-powers Feb 24, 2025
7cd7974
Update hasTimestampField() check to consider doc values skipper
jordan-powers Feb 24, 2025
bd40507
Get min/max timestamp from DocValuesSkipper if enabled
jordan-powers Feb 24, 2025
ce9f792
Get max value for LeafReader sorting from docvalues skipper
jordan-powers Feb 24, 2025
b325699
Merge remote-tracking branch 'upstream/main' into tsdb-timestamp-spar…
jordan-powers Feb 24, 2025
da28201
Add missing FieldInfo null check
jordan-powers Feb 24, 2025
0b3d514
Merge remote-tracking branch 'upstream/main' into tsdb-timestamp-spar…
jordan-powers Feb 24, 2025
645949a
Add non-null assertion to doc values skipper lookup
jordan-powers Feb 25, 2025
677f2f7
Avoid FieldInfo lookup if loadTimestampRange is false
jordan-powers Feb 25, 2025
26cb18c
Simplify DateFieldMapper#shouldUseDocValuesSkipper check
jordan-powers Feb 25, 2025
c8607b2
Merge remote-tracking branch 'upstream/main' into tsdb-timestamp-spar…
jordan-powers Feb 25, 2025
bc89576
Use doc values skipper for DateFieldType#isFieldWithinQuery
jordan-powers Feb 25, 2025
e7cee43
Merge remote-tracking branch 'upstream/main' into tsdb-timestamp-spar…
jordan-powers Feb 25, 2025
675e175
DateFieldTypeTests#isFieldWithinRangeTestCase include doc values skipper
jordan-powers Feb 25, 2025
3bee698
Merge remote-tracking branch 'upstream/main' into tsdb-timestamp-spar…
jordan-powers Feb 26, 2025
752881a
Break out isFieldWithinQuery optimization into separate PR
jordan-powers Feb 28, 2025
6ffac04
Update SearchIdleIT test to disable doc values skipper
jordan-powers Mar 3, 2025
e9c619a
Merge remote-tracking branch 'upstream/main' into tsdb-timestamp-spar…
jordan-powers Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
196 changes: 196 additions & 0 deletions ...c/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
*/
package org.elasticsearch.datastreams.mapper;

import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.CheckedConsumer;
import org.elasticsearch.datastreams.DataStreamsPlugin;
Expand Down Expand Up @@ -537,4 +538,199 @@ public void testFieldTypeWithDocValuesSkipper_CustomTimestampField() throws IOEx
assertFalse(defaultTimestamp.fieldType().hasDocValuesSkipper());
}
}

public void testFieldTypeWithDocValuesSkipper_TSDBModeDisabledDocValuesSkipper() throws IOException {
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false)
.build();
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
b.field("type", "date");
b.endObject();
}));

final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
.mappers()
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
assertTrue(timestampMapper.fieldType().hasDocValues());
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
assertTrue(timestampMapper.fieldType().isIndexed());
}

public void testFieldTypeWithDocValuesSkipper_TSDBMode() throws IOException {
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
.build();
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
b.field("type", "date");
b.endObject();
}));

final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
.mappers()
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
assertTrue(timestampMapper.fieldType().hasDocValues());
if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) {
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
assertFalse(timestampMapper.fieldType().isIndexed());
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
} else {
// TODO: remove this 'else' branch when removing the `doc_values_skipper` feature flag
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
assertTrue(timestampMapper.fieldType().isIndexed());
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
}
}

public void testFieldTypeWithDocValuesSkipper_TSDBModeNoTimestampMapping() throws IOException {
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
.build();
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {}));

final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
.mappers()
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
assertTrue(timestampMapper.fieldType().hasDocValues());
if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) {
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
assertFalse(timestampMapper.fieldType().isIndexed());
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
} else {
// TODO: remove this 'else' branch when removing the `doc_values_skipper` feature flag
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
assertTrue(timestampMapper.fieldType().isIndexed());
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
}
}

public void testFieldTypeWithDocValuesSkipper_TSDBModeTimestampDateNanos() throws IOException {
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
.build();
final MapperService mapperService = withMapping(
new TestMapperServiceBuilder().settings(settings).applyDefaultMapping(false).build(),
timestampMapping(true, b -> {
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
b.field("type", "date_nanos");
b.endObject();
})
);

final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
.mappers()
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
assertTrue(timestampMapper.fieldType().hasDocValues());
if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) {
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
assertFalse(timestampMapper.fieldType().isIndexed());
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
} else {
// TODO: remove this 'else' branch when removing the `doc_values_skipper` feature flag
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
assertTrue(timestampMapper.fieldType().isIndexed());
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
}
}

public void testFieldTypeWithDocValuesSkipper_TSDBModeExplicitTimestampIndexEnabledDocValuesSkipper() throws IOException {
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
.build();
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
b.field("type", "date");
b.field("index", true);
b.endObject();
}));

final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
.mappers()
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
assertTrue(timestampMapper.fieldType().hasDocValues());
if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) {
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
assertFalse(timestampMapper.fieldType().isIndexed());
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
} else {
// TODO: remove this 'else' branch when removing the `doc_values_skipper` feature flag
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
assertTrue(timestampMapper.fieldType().isIndexed());
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
}
}

public void testFieldTypeWithDocValuesSkipper_TSDBModeExplicitTimestampIndexDisabledDocValuesSkipper() throws IOException {
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false)
.build();
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
b.field("type", "date");
b.field("index", true);
b.endObject();
}));

final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
.mappers()
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
assertTrue(timestampMapper.fieldType().hasDocValues());
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
assertTrue(timestampMapper.fieldType().isIndexed());
}

public void testFieldTypeWithDocValuesSkipper_TSDBModeWithoutDefaultMapping() throws IOException {
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
.build();
final MapperService mapperService = withMapping(
new TestMapperServiceBuilder().settings(settings).applyDefaultMapping(false).build(),
timestampMapping(true, b -> {
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
b.field("type", "date");
b.endObject();
})
);

final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
.mappers()
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
assertTrue(timestampMapper.fieldType().hasDocValues());
if (IndexSettings.USE_DOC_VALUES_SKIPPER.get(settings)) {
assumeTrue("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled());
assertFalse(timestampMapper.fieldType().isIndexed());
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
} else {
// TODO: remove this 'else' branch when removing the `doc_values_skipper` feature flag
assumeFalse("doc_values_skipper feature flag enabled", IndexSettings.DOC_VALUES_SKIPPER.isEnabled() == false);
assertTrue(timestampMapper.fieldType().isIndexed());
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
}
}
}
40 changes: 24 additions & 16 deletions server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,16 +242,31 @@ public void testSearchIdleBoolQueryMatchOneIndex() throws InterruptedException {
// are executed only if we have enough shards.
int idleIndexShardsCount = 3;
int activeIndexShardsCount = 3;

var idleIndexSettingsBuilder = Settings.builder()
.put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, idleIndexShardsCount)
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-10T00:00:00.000Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-11T00:00:00.000Z");

var activeIndexSettingsBuilder = Settings.builder()
.put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, activeIndexShardsCount)
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-12T00:00:00.000Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-13T23:59:59.999Z");

if (IndexSettings.DOC_VALUES_SKIPPER.isEnabled()) {
idleIndexSettingsBuilder.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false);
activeIndexSettingsBuilder.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false);
}

createIndex(
idleIndex,
Settings.builder()
.put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, idleIndexShardsCount)
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-10T00:00:00.000Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-11T00:00:00.000Z")
.build(),
idleIndexSettingsBuilder.build(),
"doc",
"keyword",
"type=keyword",
Expand All @@ -262,14 +277,7 @@ public void testSearchIdleBoolQueryMatchOneIndex() throws InterruptedException {
);
createIndex(
activeIndex,
Settings.builder()
.put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), "500ms")
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, activeIndexShardsCount)
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
.put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-05-12T00:00:00.000Z")
.put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-05-13T23:59:59.999Z")
.build(),
activeIndexSettingsBuilder.build(),
"doc",
"keyword",
"type=keyword",
Expand Down
9 changes: 9 additions & 0 deletions server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
package org.elasticsearch.cluster.metadata;

import org.apache.lucene.document.LongPoint;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.PointValues;
import org.elasticsearch.ElasticsearchException;
Expand Down Expand Up @@ -84,6 +87,12 @@ public static boolean isFailureStoreFeatureFlagEnabled() {
// Timeseries indices' leaf readers should be sorted by desc order of their timestamp field, as it allows search time optimizations
public static final Comparator<LeafReader> TIMESERIES_LEAF_READERS_SORTER = Comparator.comparingLong((LeafReader r) -> {
try {
FieldInfo info = r.getFieldInfos().fieldInfo(TIMESTAMP_FIELD_NAME);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

if (info != null && info.docValuesSkipIndexType() == DocValuesSkipIndexType.RANGE) {
DocValuesSkipper skipper = r.getDocValuesSkipper(TIMESTAMP_FIELD_NAME);
return skipper.maxValue();
}

PointValues points = r.getPointValues(TIMESTAMP_FIELD_NAME);
if (points != null) {
byte[] sortValue = points.getMaxPackedValue();
Expand Down
31 changes: 23 additions & 8 deletions ...r/src/main/java/org/elasticsearch/common/lucene/uid/PerThreadIDVersionAndSeqNoLookup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
package org.elasticsearch.common.lucene.uid;

import org.apache.lucene.document.LongPoint;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesSkipper;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
Expand Down Expand Up @@ -94,15 +97,27 @@ final class PerThreadIDVersionAndSeqNoLookup {
this.loadedTimestampRange = loadTimestampRange;
// Also check for the existence of the timestamp field, because sometimes a segment can only contain tombstone documents,
// which don't have any mapped fields (also not the timestamp field) and just some meta fields like _id, _seq_no etc.
if (loadTimestampRange && reader.getFieldInfos().fieldInfo(DataStream.TIMESTAMP_FIELD_NAME) != null) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe leave this if statement in tact and add the following check below here:

if (IndexSettings.DOC_VALUES_SKIPPER.isEnabled()) {
                DocValuesSkipper skipper = reader.getDocValuesSkipper(DataStream.TIMESTAMP_FIELD_NAME);
                assert skipper != null : "no skipper for reader:" + reader + " and parent:" + reader.getContext().parent.reader();
                minTimestamp = skipper.minValue();
                maxTimestamp = skipper.maxValue();
            } else {
                PointValues tsPointValues = reader.getPointValues(DataStream.TIMESTAMP_FIELD_NAME);
                assert tsPointValues != null
                    : "no timestamp field for reader:" + reader + " and parent:" + reader.getContext().parent.reader();
                minTimestamp = LongPoint.decodeDimension(tsPointValues.getMinPackedValue(), 0);
                maxTimestamp = LongPoint.decodeDimension(tsPointValues.getMaxPackedValue(), 0);
            }

The reason here is that when loadTimestampRange is requested and timestamp field exists only two scenarios can be true. Either there is PointValues for timestamp field or if feature flag is enabled skipper should exist for timestamp field.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From what I can tell, there isn't an easy way to get the current IndexSettings instance into this method. I could update the constructor signature to take some extra info (either the whole IndexSettings instance or just the boolean indicating if doc values skipper is enabled), but it seems easier to just check the FieldInfo.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No need to get an IndexSettings instance, IndexSettings.DOC_VALUES_SKIPPER is a static field.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh I see, you're saying I should check the feature flag, not the index setting. But what if the feature flag is enabled but the skipper is disabled by the index setting? Then we still want to check the point values

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point. Let's keep this as is then.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I actually still had to update this check. I'm keeping the same logic, still checking the FieldInfo to see if the doc values skipper is enabled, but it seems retrieving the FieldInfo even if loadTimestampRange is false was causing the CI to fail (something to do with the translogInMemorySegmentCount that I don't fully understand).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there are cases where we fake the reader. I think when we read from translog, for realtime get. I suspect those IndexReaders sometimes don't work well with field infos.

PointValues tsPointValues = reader.getPointValues(DataStream.TIMESTAMP_FIELD_NAME);
assert tsPointValues != null : "no timestamp field for reader:" + reader + " and parent:" + reader.getContext().parent.reader();
minTimestamp = LongPoint.decodeDimension(tsPointValues.getMinPackedValue(), 0);
maxTimestamp = LongPoint.decodeDimension(tsPointValues.getMaxPackedValue(), 0);
} else {
minTimestamp = 0;
maxTimestamp = Long.MAX_VALUE;
long minTimestamp = 0;
long maxTimestamp = Long.MAX_VALUE;
if (loadTimestampRange) {
FieldInfo info = reader.getFieldInfos().fieldInfo(DataStream.TIMESTAMP_FIELD_NAME);
if (info != null) {
if (info.docValuesSkipIndexType() == DocValuesSkipIndexType.RANGE) {
DocValuesSkipper skipper = reader.getDocValuesSkipper(DataStream.TIMESTAMP_FIELD_NAME);
assert skipper != null : "no skipper for reader:" + reader + " and parent:" + reader.getContext().parent.reader();
minTimestamp = skipper.minValue();
maxTimestamp = skipper.maxValue();
} else {
PointValues tsPointValues = reader.getPointValues(DataStream.TIMESTAMP_FIELD_NAME);
assert tsPointValues != null
: "no timestamp field for reader:" + reader + " and parent:" + reader.getContext().parent.reader();
minTimestamp = LongPoint.decodeDimension(tsPointValues.getMinPackedValue(), 0);
maxTimestamp = LongPoint.decodeDimension(tsPointValues.getMaxPackedValue(), 0);
}
}
}
this.minTimestamp = minTimestamp;
this.maxTimestamp = maxTimestamp;
}

PerThreadIDVersionAndSeqNoLookup(LeafReader reader, boolean loadTimestampRange) throws IOException {
Expand Down
Loading