Skip to content

Commit 62bec03

Browse files
salvatore-campagnasalvatore-campagna
committed
Enable a doc values sparse idnex on the timestamp field
1 parent 95c9b4b commit 62bec03

File tree

7 files changed

+337
-20
lines changed

7 files changed

+337
-20
lines changed

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
import org.elasticsearch.common.settings.Settings;
1212
import org.elasticsearch.core.CheckedConsumer;
1313
import org.elasticsearch.datastreams.DataStreamsPlugin;
14+
import org.elasticsearch.index.IndexMode;
15+
import org.elasticsearch.index.IndexSettings;
16+
import org.elasticsearch.index.IndexSortConfig;
1417
import org.elasticsearch.index.IndexVersion;
1518
import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
1619
import org.elasticsearch.index.mapper.DateFieldMapper;
@@ -231,4 +234,186 @@ public void testValidateDefaultIgnoreMalformed() throws Exception {
231234
assertThat(summaryTimestamp.ignoreMalformed(), is(false));
232235
}
233236
}
237+
238+
public void testFieldTypeWithSkipDocValues_LogsDBMode() throws IOException {
239+
final Settings settings = Settings.builder()
240+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
241+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
242+
.build();
243+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
244+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
245+
b.field("type", "date");
246+
b.endObject();
247+
}));
248+
249+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
250+
.mappers()
251+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
252+
assertTrue(timestampMapper.fieldType().hasDocValues());
253+
assertTrue(timestampMapper.fieldType().hasDocValuesSparseIndex());
254+
assertFalse(timestampMapper.fieldType().isIndexed());
255+
}
256+
257+
public void testFieldTypeWithSkipDocValues_LogsDBModeWithoutDefaultMapping() throws IOException {
258+
final Settings settings = Settings.builder()
259+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
260+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
261+
.build();
262+
final MapperService mapperService = withMapping(
263+
new TestMapperServiceBuilder().settings(settings).applyDefaultMapping(false).build(),
264+
timestampMapping(true, b -> {
265+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
266+
b.field("type", "date");
267+
b.endObject();
268+
})
269+
);
270+
271+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
272+
.mappers()
273+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
274+
assertTrue(timestampMapper.fieldType().hasDocValues());
275+
assertFalse(timestampMapper.fieldType().isIndexed());
276+
assertTrue(timestampMapper.fieldType().hasDocValuesSparseIndex());
277+
}
278+
279+
public void testFieldTypeWithSkipDocValues_ExplicitIndex() throws IOException {
280+
final Settings settings = Settings.builder()
281+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
282+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
283+
.build();
284+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
285+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
286+
b.field("type", "date");
287+
b.field("index", true);
288+
b.endObject();
289+
}));
290+
291+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
292+
.mappers()
293+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
294+
assertTrue(timestampMapper.fieldType().hasDocValues());
295+
assertTrue(timestampMapper.fieldType().isIndexed());
296+
assertFalse(timestampMapper.fieldType().hasDocValuesSparseIndex());
297+
}
298+
299+
public void testFieldTypeWithSkipDocValues_IndexFalseWithoutDefaultMapping() throws IOException {
300+
final Settings settings = Settings.builder()
301+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
302+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
303+
.build();
304+
final IllegalArgumentException ex = assertThrows(
305+
IllegalArgumentException.class,
306+
() -> withMapping(
307+
new TestMapperServiceBuilder().settings(settings).applyDefaultMapping(false).build(),
308+
timestampMapping(true, b -> {
309+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
310+
b.field("type", "date");
311+
b.field("index", false);
312+
b.endObject();
313+
})
314+
)
315+
);
316+
317+
assertEquals("data stream timestamp field [@timestamp] is not indexed", ex.getMessage());
318+
}
319+
320+
public void testFieldTypeWithSkipDocValues_IndexFalseWithDefaultMapping() throws IOException {
321+
final Settings settings = Settings.builder()
322+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
323+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
324+
.build();
325+
final IllegalArgumentException ex = assertThrows(
326+
IllegalArgumentException.class,
327+
() -> createMapperService(settings, timestampMapping(true, b -> {
328+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
329+
b.field("type", "date");
330+
b.field("index", false);
331+
b.endObject();
332+
}))
333+
);
334+
335+
// A data stream always has a mapped @timestamp field with `index: true`
336+
assertTrue(ex.getMessage().contains("Cannot update parameter [index] from [true] to [false]"));
337+
}
338+
339+
public void testFieldTypeWithSkipDocValues_DocValuesFalse() {
340+
final Settings settings = Settings.builder()
341+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
342+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
343+
.build();
344+
final IllegalArgumentException ex = expectThrows(
345+
IllegalArgumentException.class,
346+
() -> withMapping(
347+
new TestMapperServiceBuilder().settings(settings).applyDefaultMapping(false).build(),
348+
timestampMapping(true, b -> {
349+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
350+
b.field("type", "date");
351+
b.field("doc_values", false);
352+
b.endObject();
353+
})
354+
)
355+
);
356+
assertEquals(
357+
ex.getMessage(),
358+
"data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] doesn't have doc values"
359+
);
360+
}
361+
362+
public void testFieldTypeWithSkipDocValues_WithoutTimestampSorting() throws IOException {
363+
final Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build();
364+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
365+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
366+
b.field("type", "date");
367+
b.endObject();
368+
}));
369+
370+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
371+
.mappers()
372+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
373+
assertTrue(timestampMapper.fieldType().hasDocValues());
374+
// NOTE: in LogsDB we always sort on @timestamp (and maybe also on host.name) by default
375+
assertFalse(timestampMapper.fieldType().isIndexed());
376+
assertTrue(timestampMapper.fieldType().hasDocValuesSparseIndex());
377+
}
378+
379+
public void testFieldTypeWithSkipDocValues_StandardMode() throws IOException {
380+
final Settings settings = Settings.builder()
381+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
382+
.build();
383+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
384+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
385+
b.field("type", "date");
386+
b.endObject();
387+
}));
388+
389+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
390+
.mappers()
391+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
392+
assertTrue(timestampMapper.fieldType().hasDocValues());
393+
assertTrue(timestampMapper.fieldType().isIndexed());
394+
assertFalse(timestampMapper.fieldType().hasDocValuesSparseIndex());
395+
}
396+
397+
public void testFieldTypeWithSkipDocValues_InvalidFieldName() throws IOException {
398+
final Settings settings = Settings.builder()
399+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
400+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
401+
.build();
402+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
403+
b.startObject("timestamp");
404+
b.field("type", "date");
405+
b.endObject();
406+
}));
407+
408+
final DateFieldMapper customTimestamp = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("timestamp");
409+
assertTrue(customTimestamp.fieldType().hasDocValues());
410+
assertTrue(customTimestamp.fieldType().isIndexed());
411+
assertFalse(customTimestamp.fieldType().hasDocValuesSparseIndex());
412+
413+
// Default LogsDB mapping including @timestamp field is used
414+
final DateFieldMapper defaultTimestamp = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("@timestamp");
415+
assertTrue(defaultTimestamp.fieldType().hasDocValues());
416+
assertFalse(defaultTimestamp.fieldType().isIndexed());
417+
assertTrue(defaultTimestamp.fieldType().hasDocValuesSparseIndex());
418+
}
234419
}

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ private static Version parseUnchecked(String version) {
134134
public static final IndexVersion UPGRADE_TO_LUCENE_9_12_1 = def(8_523_0_00, parseUnchecked("9.12.1"));
135135
public static final IndexVersion INFERENCE_METADATA_FIELDS_BACKPORT = def(8_524_0_00, parseUnchecked("9.12.1"));
136136
public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_0_00, parseUnchecked("9.12.1"));
137+
public static final IndexVersion TIMESTAMP_DOC_VALUES_SPARSE_INDEX_BACKPORT = def(8_526_0_00, parseUnchecked("9.12.1"));
137138
public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_0_00, Version.LUCENE_10_0_0);
138139
public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_0_00, Version.LUCENE_10_0_0);
139140
public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_0_00, Version.LUCENE_10_0_0);
@@ -144,6 +145,7 @@ private static Version parseUnchecked(String version) {
144145
public static final IndexVersion SOURCE_MAPPER_MODE_ATTRIBUTE_NOOP = def(9_007_0_00, Version.LUCENE_10_0_0);
145146
public static final IndexVersion HOSTNAME_DOC_VALUES_SPARSE_INDEX = def(9_008_0_00, Version.LUCENE_10_0_0);
146147
public static final IndexVersion UPGRADE_TO_LUCENE_10_1_0 = def(9_009_0_00, Version.LUCENE_10_1_0);
148+
public static final IndexVersion TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_010_0_00, Version.LUCENE_10_1_0);
147149
/*
148150
* STOP! READ THIS FIRST! No, really,
149151
* ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _

‎server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java‎

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
import org.apache.lucene.search.Query;
1616
import org.elasticsearch.common.bytes.BytesReference;
1717
import org.elasticsearch.common.xcontent.XContentHelper;
18+
import org.elasticsearch.index.IndexMode;
19+
import org.elasticsearch.index.IndexSettings;
20+
import org.elasticsearch.index.IndexVersions;
1821
import org.elasticsearch.index.TimestampBounds;
1922
import org.elasticsearch.index.mapper.DateFieldMapper.Resolution;
2023
import org.elasticsearch.index.query.SearchExecutionContext;
@@ -25,6 +28,7 @@
2528
import java.io.UncheckedIOException;
2629
import java.time.Instant;
2730
import java.util.Map;
31+
import java.util.function.Function;
2832

2933
import static org.elasticsearch.core.TimeValue.NSEC_PER_MSEC;
3034
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
@@ -39,8 +43,10 @@ public class DataStreamTimestampFieldMapper extends MetadataFieldMapper {
3943
public static final String DEFAULT_PATH = "@timestamp";
4044
public static final String TIMESTAMP_VALUE_KEY = "@timestamp._value";
4145

42-
public static final DataStreamTimestampFieldMapper ENABLED_INSTANCE = new DataStreamTimestampFieldMapper(true);
43-
private static final DataStreamTimestampFieldMapper DISABLED_INSTANCE = new DataStreamTimestampFieldMapper(false);
46+
public static final Function<IndexSettings, DataStreamTimestampFieldMapper> ENABLED_INSTANCE =
47+
indexSettings -> new DataStreamTimestampFieldMapper(true, indexSettings);
48+
private static final Function<IndexSettings, DataStreamTimestampFieldMapper> DISABLED_INSTANCE =
49+
indexSettings -> new DataStreamTimestampFieldMapper(false, indexSettings);
4450

4551
// For now the field shouldn't be useable in searches.
4652
// In the future it should act as an alias to the actual data stream timestamp field.
@@ -82,9 +88,11 @@ public static class Builder extends MetadataFieldMapper.Builder {
8288
private final Parameter<Boolean> enabled = Parameter.boolParam("enabled", true, m -> toType(m).enabled, false)
8389
// this field mapper may be enabled but once enabled, may not be disabled
8490
.setMergeValidator((previous, current, conflicts) -> (previous == current) || (previous == false && current));
91+
private final IndexSettings indexSettings;
8592

86-
public Builder() {
93+
public Builder(final IndexSettings indexSettings) {
8794
super(NAME);
95+
this.indexSettings = indexSettings;
8896
}
8997

9098
@Override
@@ -94,22 +102,27 @@ protected Parameter<?>[] getParameters() {
94102

95103
@Override
96104
public MetadataFieldMapper build() {
97-
return enabled.getValue() ? ENABLED_INSTANCE : DISABLED_INSTANCE;
105+
return enabled.getValue() ? ENABLED_INSTANCE.apply(indexSettings) : DISABLED_INSTANCE.apply(indexSettings);
98106
}
99107
}
100108

101-
public static final TypeParser PARSER = new ConfigurableTypeParser(c -> DISABLED_INSTANCE, c -> new Builder());
109+
public static final TypeParser PARSER = new ConfigurableTypeParser(
110+
c -> DISABLED_INSTANCE.apply(c.getIndexSettings()),
111+
c -> new Builder(c.getIndexSettings())
112+
);
102113

103114
private final boolean enabled;
115+
private final IndexSettings indexSettings;
104116

105-
private DataStreamTimestampFieldMapper(boolean enabled) {
117+
private DataStreamTimestampFieldMapper(boolean enabled, final IndexSettings indexSettings) {
106118
super(TimestampFieldType.INSTANCE);
107119
this.enabled = enabled;
120+
this.indexSettings = indexSettings;
108121
}
109122

110123
@Override
111124
public FieldMapper.Builder getMergeBuilder() {
112-
return new Builder().init(this);
125+
return new Builder(indexSettings).init(this);
113126
}
114127

115128
public void doValidate(MappingLookup lookup) {
@@ -139,7 +152,9 @@ public void doValidate(MappingLookup lookup) {
139152
}
140153

141154
DateFieldMapper dateFieldMapper = (DateFieldMapper) mapper;
142-
if (dateFieldMapper.fieldType().isIndexed() == false) {
155+
boolean hasNoIndex = dateFieldMapper.fieldType().isIndexed() == false
156+
&& dateFieldMapper.fieldType().hasDocValuesSparseIndex() == false;
157+
if (hasNoIndex) {
143158
throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] is not indexed");
144159
}
145160
if (dateFieldMapper.fieldType().hasDocValues() == false) {
@@ -170,6 +185,9 @@ public void doValidate(MappingLookup lookup) {
170185
configuredSettings.remove("meta");
171186
configuredSettings.remove("format");
172187
configuredSettings.remove("locale");
188+
if (isIndexParamExplicitOverrideAllowed()) {
189+
configuredSettings.remove("index");
190+
}
173191

174192
// ignoring malformed values is disallowed (see previous check),
175193
// however if `index.mapping.ignore_malformed` has been set to true then
@@ -192,6 +210,14 @@ public void doValidate(MappingLookup lookup) {
192210
}
193211
}
194212

213+
private boolean isIndexParamExplicitOverrideAllowed() {
214+
return FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()
215+
&& (indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIMESTAMP_DOC_VALUES_SPARSE_INDEX)
216+
|| (indexSettings.getIndexVersionCreated()
217+
.between(IndexVersions.TIMESTAMP_DOC_VALUES_SPARSE_INDEX_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0)))
218+
&& IndexMode.LOGSDB.equals(indexSettings.getMode());
219+
}
220+
195221
public static void storeTimestampValueForReuse(LuceneDocument document, long timestamp) {
196222
var existingField = document.getByKey(DataStreamTimestampFieldMapper.TIMESTAMP_VALUE_KEY);
197223
if (existingField != null) {

0 commit comments

Comments
 (0)