Stop iterating over all fields to extract @timestamp value (#110603)

lkts · web-flow · commit 349dac284a0e · 2024-07-11T09:24:53.000-07:00
diff --git a/docs/changelog/110603.yaml b/docs/changelog/110603.yaml
@@ -0,0 +1,6 @@
+pr: 110603
+summary: Stop iterating over all fields to extract @timestamp value
+area: TSDB
+type: enhancement
+issues:
+ - 92297
diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java
@@ -335,7 +335,7 @@ public void testStoreParameterDefaults() throws IOException {
         var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {
             b.field("field", "1234");
             if (timeSeriesIndexMode) {
-                b.field("@timestamp", randomMillisUpToYear9999());
+                b.field("@timestamp", "2000-10-10T23:40:53.384Z");
                 b.field("dimension", "dimension1");
             }
         }, null);
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java
@@ -8,7 +8,8 @@
 
 package org.elasticsearch.index.mapper;
 
-import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -22,7 +23,6 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.time.Instant;
-import java.util.List;
 import java.util.Map;
 
 import static org.elasticsearch.core.TimeValue.NSEC_PER_MSEC;
@@ -36,6 +36,7 @@ public class DataStreamTimestampFieldMapper extends MetadataFieldMapper {
 
     public static final String NAME = "_data_stream_timestamp";
     public static final String DEFAULT_PATH = "@timestamp";
+    public static final String TIMESTAMP_VALUE_KEY = "@timestamp._value";
 
     public static final DataStreamTimestampFieldMapper ENABLED_INSTANCE = new DataStreamTimestampFieldMapper(true);
     private static final DataStreamTimestampFieldMapper DISABLED_INSTANCE = new DataStreamTimestampFieldMapper(false);
@@ -189,43 +190,44 @@ public void doValidate(MappingLookup lookup) {
         }
     }
 
+    public static void storeTimestampValueForReuse(LuceneDocument document, long timestamp) {
+        var existingField = document.getByKey(DataStreamTimestampFieldMapper.TIMESTAMP_VALUE_KEY);
+        if (existingField != null) {
+            throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] encountered multiple values");
+        }
+
+        document.onlyAddKey(
+            DataStreamTimestampFieldMapper.TIMESTAMP_VALUE_KEY,
+            new LongField(DataStreamTimestampFieldMapper.TIMESTAMP_VALUE_KEY, timestamp, Field.Store.NO)
+        );
+    }
+
+    public static long extractTimestampValue(LuceneDocument document) {
+        IndexableField timestampValueField = document.getByKey(TIMESTAMP_VALUE_KEY);
+        if (timestampValueField == null) {
+            throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] is missing");
+        }
+
+        return timestampValueField.numericValue().longValue();
+    }
+
     @Override
     public void postParse(DocumentParserContext context) throws IOException {
         if (enabled == false) {
             // not configured, so skip the validation
             return;
         }
-        boolean foundFsTimestampField = false;
-        IndexableField first = null;
-        final List<IndexableField> fields = context.rootDoc().getFields();
-        for (int i = 0; i < fields.size(); i++) {
-            IndexableField indexableField = fields.get(i);
-            if (DEFAULT_PATH.equals(indexableField.name()) == false) {
-                continue;
-            }
-            if (first == null) {
-                first = indexableField;
-            }
-            if (indexableField.fieldType().docValuesType() == DocValuesType.SORTED_NUMERIC) {
-                if (foundFsTimestampField) {
-                    throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] encountered multiple values");
-                }
-                foundFsTimestampField = true;
-            }
-        }
 
-        if (first == null) {
-            throw new IllegalArgumentException("data stream timestamp field [" + DEFAULT_PATH + "] is missing");
-        }
+        long timestamp = extractTimestampValue(context.doc());
+
         var indexMode = context.indexSettings().getMode();
         if (indexMode.shouldValidateTimestamp()) {
             TimestampBounds bounds = context.indexSettings().getTimestampBounds();
-            validateTimestamp(bounds, first, context);
+            validateTimestamp(bounds, timestamp, context);
         }
     }
 
-    private static void validateTimestamp(TimestampBounds bounds, IndexableField field, DocumentParserContext context) {
-        long originValue = field.numericValue().longValue();
+    private static void validateTimestamp(TimestampBounds bounds, long originValue, DocumentParserContext context) {
         long value = originValue;
 
         Resolution resolution;
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
@@ -364,6 +364,7 @@ public DateFieldMapper build(MapperBuilderContext context) {
                 && ignoreMalformed.isConfigured() == false) {
                 ignoreMalformed.setValue(false);
             }
+
             return new DateFieldMapper(
                 leafName(),
                 ft,
@@ -868,8 +869,10 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
     private final ScriptCompiler scriptCompiler;
     private final FieldValues<Long> scriptValues;
 
+    private final boolean isDataStreamTimestampField;
+
     private DateFieldMapper(
-        String simpleName,
+        String leafName,
         MappedFieldType mappedFieldType,
         MultiFields multiFields,
         CopyTo copyTo,
@@ -878,7 +881,7 @@ private DateFieldMapper(
         boolean isSourceSynthetic,
         Builder builder
     ) {
-        super(simpleName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.get());
+        super(leafName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.get());
         this.store = builder.store.getValue();
         this.indexed = builder.index.getValue();
         this.hasDocValues = builder.docValues.getValue();
@@ -894,6 +897,7 @@ private DateFieldMapper(
         this.script = builder.script.get();
         this.scriptCompiler = builder.scriptCompiler;
         this.scriptValues = builder.scriptValues();
+        this.isDataStreamTimestampField = mappedFieldType.name().equals(DataStreamTimestampFieldMapper.DEFAULT_PATH);
     }
 
     @Override
@@ -942,6 +946,16 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
     }
 
     private void indexValue(DocumentParserContext context, long timestamp) {
+        // DataStreamTimestampFieldMapper and TsidExtractingFieldMapper need to use timestamp value,
+        // so when this is true we store it in a well-known place
+        // instead of forcing them to iterate over all fields.
+        //
+        // DataStreamTimestampFieldMapper is present and enabled both
+        // in data streams and standalone indices in time_series mode
+        if (isDataStreamTimestampField && context.mappingLookup().isDataStreamTimestampFieldEnabled()) {
+            DataStreamTimestampFieldMapper.storeTimestampValueForReuse(context.doc(), timestamp);
+        }
+
         if (indexed && hasDocValues) {
             context.doc().add(new LongField(fieldType().name(), timestamp));
         } else if (hasDocValues) {
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java
@@ -46,13 +46,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
     private static final long SEED = 0;
 
     public static void createField(DocumentParserContext context, IndexRouting.ExtractFromSource.Builder routingBuilder, BytesRef tsid) {
-        final IndexableField timestampField = context.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH);
-        if (timestampField == null) {
-            throw new IllegalArgumentException(
-                "data stream timestamp field [" + DataStreamTimestampFieldMapper.DEFAULT_PATH + "] is missing"
-            );
-        }
-        long timestamp = timestampField.numericValue().longValue();
+        final long timestamp = DataStreamTimestampFieldMapper.extractTimestampValue(context.doc());
         String id;
         if (routingBuilder != null) {
             byte[] suffix = new byte[16];
diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java
@@ -121,9 +121,10 @@ public void testSetDefaultTimeRangeValue() {
 
     public void testRequiredRouting() {
         Settings s = getSettings();
+        var mapperService = new TestMapperServiceBuilder().settings(s).applyDefaultMapping(false).build();
         Exception e = expectThrows(
             IllegalArgumentException.class,
-            () -> createMapperService(s, topMapping(b -> b.startObject("_routing").field("required", true).endObject()))
+            () -> withMapping(mapperService, topMapping(b -> b.startObject("_routing").field("required", true).endObject()))
         );
         assertThat(e.getMessage(), equalTo("routing is forbidden on CRUD operations that target indices in [index.mode=time_series]"));
     }
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java
@@ -167,7 +167,7 @@ public void testDefaultsForTimeSeriesIndex() throws IOException {
 
         var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {
             b.field("field", Base64.getEncoder().encodeToString(randomByteArrayOfLength(10)));
-            b.field("@timestamp", randomMillisUpToYear9999());
+            b.field("@timestamp", "2000-10-10T23:40:53.384Z");
             b.field("dimension", "dimension1");
         }, null);
         ParsedDocument doc = mapper.parse(source);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java
@@ -294,7 +294,7 @@ public void testStoreParameterDefaults() throws IOException {
         var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {
             b.field("field", "1234");
             if (timeSeriesIndexMode) {
-                b.field("@timestamp", randomMillisUpToYear9999());
+                b.field("@timestamp", "2000-10-10T23:40:53.384Z");
                 b.field("dimension", "dimension1");
             }
         }, null);
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java
@@ -67,7 +67,7 @@ private DocumentMapper createDocumentMapper(String routingPath, XContentBuilder
                 .put(MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING.getKey(), 200) // Allow tests that use many dimensions
                 .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), routingPath)
                 .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
-                .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
+                .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-10-29T00:00:00Z")
                 .build(),
             mappings
         ).documentMapper();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java
@@ -44,7 +44,7 @@ private DocumentMapper createMapper(XContentBuilder mappings) throws IOException
             getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
                 .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing path is required")
                 .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z")
-                .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z")
+                .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-10-29T00:00:00Z")
                 .build(),
             mappings
         ).documentMapper();
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
@@ -225,13 +225,15 @@ protected class TestMapperServiceBuilder {
         private BooleanSupplier idFieldDataEnabled;
         private ScriptCompiler scriptCompiler;
         private MapperMetrics mapperMetrics;
+        private boolean applyDefaultMapping;
 
         public TestMapperServiceBuilder() {
             indexVersion = getVersion();
             settings = getIndexSettings();
             idFieldDataEnabled = () -> true;
             scriptCompiler = MapperServiceTestCase.this::compileScript;
             mapperMetrics = MapperMetrics.NOOP;
+            applyDefaultMapping = true;
         }
 
         public TestMapperServiceBuilder indexVersion(IndexVersion indexVersion) {
@@ -254,6 +256,11 @@ public TestMapperServiceBuilder mapperMetrics(MapperMetrics mapperMetrics) {
             return this;
         }
 
+        public TestMapperServiceBuilder applyDefaultMapping(boolean applyDefaultMapping) {
+            this.applyDefaultMapping = applyDefaultMapping;
+            return this;
+        }
+
         public MapperService build() {
             IndexSettings indexSettings = createIndexSettings(indexVersion, settings);
             SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
@@ -269,7 +276,7 @@ public void onCache(ShardId shardId, Accountable accountable) {}
                 public void onRemoval(ShardId shardId, Accountable accountable) {}
             });
 
-            return new MapperService(
+            var mapperService = new MapperService(
                 () -> TransportVersion.current(),
                 indexSettings,
                 createIndexAnalyzers(indexSettings),
@@ -284,6 +291,12 @@ public void onRemoval(ShardId shardId, Accountable accountable) {}
                 bitsetFilterCache::getBitSetProducer,
                 mapperMetrics
             );
+
+            if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping() != null) {
+                mapperService.merge(null, indexSettings.getMode().getDefaultMapping(), MapperService.MergeReason.MAPPING_UPDATE);
+            }
+
+            return mapperService;
         }
     }
 

Original file line number	Diff line number	Diff line change
`@@ -335,7 +335,7 @@ public void testStoreParameterDefaults() throws IOException {`
`335`	`335`	`var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {`
`336`	`336`	`b.field("field", "1234");`
`337`	`337`	`if (timeSeriesIndexMode) {`
`338`		`- b.field("@timestamp", randomMillisUpToYear9999());`
	`338`	`+ b.field("@timestamp", "2000-10-10T23:40:53.384Z");`
`339`	`339`	`b.field("dimension", "dimension1");`
`340`	`340`	`}`
`341`	`341`	`}, null);`
Original file line number	Diff line number	Diff line change
`@@ -294,7 +294,7 @@ public void testStoreParameterDefaults() throws IOException {`
`294`	`294`	`var source = source(TimeSeriesRoutingHashFieldMapper.DUMMY_ENCODED_VALUE, b -> {`
`295`	`295`	`b.field("field", "1234");`
`296`	`296`	`if (timeSeriesIndexMode) {`
`297`		`- b.field("@timestamp", randomMillisUpToYear9999());`
	`297`	`+ b.field("@timestamp", "2000-10-10T23:40:53.384Z");`
`298`	`298`	`b.field("dimension", "dimension1");`
`299`	`299`	`}`
`300`	`300`	`}, null);`