Skip to content

Commit 6999ec7

Browse files
salvatore-campagnasalvatore-campagna
committed
feature: enable a sparse doc values index for @timestamp in LogsDB
1 parent 9209341 commit 6999ec7

File tree

10 files changed

+567
-201
lines changed

10 files changed

+567
-201
lines changed

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/mapper/DataStreamTimestampFieldMapperTests.java‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
import org.elasticsearch.common.settings.Settings;
1212
import org.elasticsearch.core.CheckedConsumer;
1313
import org.elasticsearch.datastreams.DataStreamsPlugin;
14+
import org.elasticsearch.index.IndexMode;
15+
import org.elasticsearch.index.IndexSettings;
16+
import org.elasticsearch.index.IndexSortConfig;
1417
import org.elasticsearch.index.IndexVersion;
1518
import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
1619
import org.elasticsearch.index.mapper.DateFieldMapper;
@@ -231,4 +234,212 @@ public void testValidateDefaultIgnoreMalformed() throws Exception {
231234
assertThat(summaryTimestamp.ignoreMalformed(), is(false));
232235
}
233236
}
237+
238+
public void testFieldTypeWithDocValuesSkipper_LogsDBModeDisabledDocValuesSkipper() throws IOException {
239+
final Settings settings = Settings.builder()
240+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
241+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
242+
.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false)
243+
.build();
244+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
245+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
246+
b.field("type", "date");
247+
b.endObject();
248+
}));
249+
250+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
251+
.mappers()
252+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
253+
assertTrue(timestampMapper.fieldType().hasDocValues());
254+
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
255+
assertTrue(timestampMapper.fieldType().isIndexed());
256+
}
257+
258+
public void testFieldTypeWithDocValuesSkipper_LogsDBMode() throws IOException {
259+
final Settings settings = Settings.builder()
260+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
261+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
262+
.build();
263+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
264+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
265+
b.field("type", "date");
266+
b.endObject();
267+
}));
268+
269+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
270+
.mappers()
271+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
272+
assertTrue(timestampMapper.fieldType().hasDocValues());
273+
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
274+
assertFalse(timestampMapper.fieldType().isIndexed());
275+
}
276+
277+
public void testFieldTypeWithDocValuesSkipper_LogsDBModeExplicitTimestampIndexEnabledDocValuesSkipper() throws IOException {
278+
final Settings settings = Settings.builder()
279+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
280+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
281+
.build();
282+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
283+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
284+
b.field("type", "date");
285+
b.field("index", true);
286+
b.endObject();
287+
}));
288+
289+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
290+
.mappers()
291+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
292+
assertTrue(timestampMapper.fieldType().hasDocValues());
293+
// NOTE: setting `index.mapping.use_doc_values_skipper` true overrides `index` for @timestamp in LogsDB
294+
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
295+
assertFalse(timestampMapper.fieldType().isIndexed());
296+
}
297+
298+
public void testFieldTypeWithDocValuesSkipper_LogsDBModeExplicitTimestampIndexDisabledDocValuesSkipper() throws IOException {
299+
final Settings settings = Settings.builder()
300+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
301+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
302+
.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false)
303+
.build();
304+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
305+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
306+
b.field("type", "date");
307+
b.field("index", true); // NOTE: setting `index.mapping.use_doc_values_skipper` false does not override `index`
308+
b.endObject();
309+
}));
310+
311+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
312+
.mappers()
313+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
314+
assertTrue(timestampMapper.fieldType().hasDocValues());
315+
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
316+
assertTrue(timestampMapper.fieldType().isIndexed());
317+
}
318+
319+
public void testFieldTypeWithDocValuesSkipper_LogsDBModeWithoutDefaultMapping() throws IOException {
320+
final Settings settings = Settings.builder()
321+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
322+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
323+
.build();
324+
final MapperService mapperService = withMapping(
325+
new TestMapperServiceBuilder().settings(settings).applyDefaultMapping(false).build(),
326+
timestampMapping(true, b -> {
327+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
328+
b.field("type", "date");
329+
b.endObject();
330+
})
331+
);
332+
333+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
334+
.mappers()
335+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
336+
assertTrue(timestampMapper.fieldType().hasDocValues());
337+
assertFalse(timestampMapper.fieldType().isIndexed());
338+
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
339+
}
340+
341+
public void testFieldTypeWithDocValuesSkipper_DocValuesFalseEnabledDocValuesSkipper() {
342+
final Settings settings = Settings.builder()
343+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
344+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
345+
.build();
346+
final IllegalArgumentException ex = expectThrows(
347+
IllegalArgumentException.class,
348+
() -> withMapping(
349+
new TestMapperServiceBuilder().settings(settings).applyDefaultMapping(false).build(),
350+
timestampMapping(true, b -> {
351+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
352+
b.field("type", "date");
353+
b.field("doc_values", false);
354+
b.endObject();
355+
})
356+
)
357+
);
358+
assertEquals(
359+
ex.getMessage(),
360+
"data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] doesn't have doc values"
361+
);
362+
}
363+
364+
public void testFieldTypeWithDocValuesSkipper_DocValuesFalseDisabledDocValuesSkipper() {
365+
final Settings settings = Settings.builder()
366+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
367+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
368+
.put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), false)
369+
.build();
370+
final IllegalArgumentException ex = expectThrows(
371+
IllegalArgumentException.class,
372+
() -> withMapping(
373+
new TestMapperServiceBuilder().settings(settings).applyDefaultMapping(false).build(),
374+
timestampMapping(true, b -> {
375+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
376+
b.field("type", "date");
377+
b.field("doc_values", false);
378+
b.endObject();
379+
})
380+
)
381+
);
382+
assertEquals(
383+
ex.getMessage(),
384+
"data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] doesn't have doc values"
385+
);
386+
}
387+
388+
public void testFieldTypeWithDocValuesSkipper_WithoutTimestampSorting() throws IOException {
389+
final Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build();
390+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
391+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
392+
b.field("type", "date");
393+
b.endObject();
394+
}));
395+
396+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
397+
.mappers()
398+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
399+
assertTrue(timestampMapper.fieldType().hasDocValues());
400+
// NOTE: in LogsDB we always sort on @timestamp (and maybe also on host.name) by default
401+
assertFalse(timestampMapper.fieldType().isIndexed());
402+
assertTrue(timestampMapper.fieldType().hasDocValuesSkipper());
403+
}
404+
405+
public void testFieldTypeWithDocValuesSkipper_StandardMode() throws IOException {
406+
final Settings settings = Settings.builder()
407+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
408+
.build();
409+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
410+
b.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH);
411+
b.field("type", "date");
412+
b.endObject();
413+
}));
414+
415+
final DateFieldMapper timestampMapper = (DateFieldMapper) mapperService.documentMapper()
416+
.mappers()
417+
.getMapper(DataStreamTimestampFieldMapper.DEFAULT_PATH);
418+
assertTrue(timestampMapper.fieldType().hasDocValues());
419+
assertTrue(timestampMapper.fieldType().isIndexed());
420+
assertFalse(timestampMapper.fieldType().hasDocValuesSkipper());
421+
}
422+
423+
public void testFieldTypeWithDocValuesSkipper_CustomTimestampField() throws IOException {
424+
final Settings settings = Settings.builder()
425+
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
426+
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), DataStreamTimestampFieldMapper.DEFAULT_PATH)
427+
.build();
428+
final MapperService mapperService = createMapperService(settings, timestampMapping(true, b -> {
429+
b.startObject("timestamp");
430+
b.field("type", "date");
431+
b.endObject();
432+
}));
433+
434+
final DateFieldMapper customTimestamp = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("timestamp");
435+
assertTrue(customTimestamp.fieldType().hasDocValues());
436+
assertTrue(customTimestamp.fieldType().isIndexed());
437+
assertFalse(customTimestamp.fieldType().hasDocValuesSkipper());
438+
439+
// Default LogsDB mapping including @timestamp field is used
440+
final DateFieldMapper defaultTimestamp = (DateFieldMapper) mapperService.documentMapper().mappers().getMapper("@timestamp");
441+
assertTrue(defaultTimestamp.fieldType().hasDocValues());
442+
assertFalse(defaultTimestamp.fieldType().isIndexed());
443+
assertTrue(defaultTimestamp.fieldType().hasDocValuesSkipper());
444+
}
234445
}

0 commit comments

Comments
 (0)