Enable the use of nested field type with index.mode=time_series

docs/changelog/122224.yaml

@@ -0,0 +1,6 @@
+pr: 122224
+summary: Enable the use of nested field type with index.mode=time_series
+area: Mapping
+type: enhancement
+issues:
+ - 120874

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml

@@ -0,0 +1,233 @@
+setup:
+  - requires:
+      cluster_features: ["mapper.tsdb_nested_field_support"]
+      reason: "tsdb index with nested field support enabled"
+
+---
+"Create TSDB index with field of nested type":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mode: time_series
+              number_of_replicas: 1
+              number_of_shards: 1
+              routing_path: [department]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              department:
+                type: keyword
+                time_series_dimension: true
+              staff:
+                type: integer
+              courses:
+                type: nested
+                properties:
+                  name:
+                    type: keyword
+                  credits:
+                    type: integer
+
+  - do:
+      index:
+        index: test
+        body: { "@timestamp": "2021-04-28T01:00:00Z", "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4 } ] }
+
+  - do:
+      index:
+        index: test
+        body: { "@timestamp": "2021-04-28T02:00:00Z", "department": "math", "staff": 20, "courses": [ { "name": "Precalculus", "credits": 1 }, { "name": "Linear Algebra", "credits": 3 } ] }
+
+  - do:
+      indices.refresh:
+        index: [ test ]
+
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          query:
+            nested:
+              path: "courses"
+              query:
+                bool:
+                  must:
+                    - term:
+                        courses.name: Precalculus
+                    - term:
+                        courses.credits: 3
+
+  - match: { hits.total.value: 0 }
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            nested:
+              path: "courses"
+              query:
+                bool:
+                  must:
+                    - term:
+                        courses.name: "Object Oriented Programming"
+                    - term:
+                        courses.credits: 3
+
+  - match: { hits.total.value: 1 }
+  - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" }
+  - match: { hits.hits.0._source.department: "compsci" }
+  - match: { hits.hits.0._source.courses:  [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4, } ] }
+
+---
+
+"TSDB index with multi-level nested fields":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mode: time_series
+              number_of_replicas: 1
+              number_of_shards: 1
+              routing_path: [department]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              department:
+                type: keyword
+                time_series_dimension: true
+              staff:
+                type: integer
+              courses:
+                type: nested
+                properties:
+                  name:
+                    type: keyword
+                  credits:
+                    type: integer
+                  students:
+                    type: nested
+                    properties:
+                      name:
+                        type: text
+                      major:
+                        type: keyword
+
+  - do:
+      index:
+        index: test
+        body:
+          "@timestamp": "2021-04-28T01:00:00Z"
+          department: "compsci"
+          staff: 12
+          courses:
+            - name: "Object Oriented Programming"
+              credits: 3
+              students:
+                - name: "Kimora Tanner"
+                  major: "Computer Science"
+                - name: "Bruno Garrett"
+                  major: "Software Engineering"
+            - name: "Theory of Computation"
+              credits: 4
+              students:
+                - name: "Elliott Booker"
+                  major: "Computer Engineering"
+                - name: "Kimora Tanner"
+                  major: "Software Engineering"
+
+  - do:
+     index:
+       index: test
+       body:
+         "@timestamp": "2021-04-28T02:00:00Z"
+         department: "math"
+         staff: 20
+         courses:
+           - name: "Precalculus"
+             credits: 4
+             students:
+               - name: "Elliott Ayers"
+                 major: "Software Engineering"
+               - name: "Sylvie Howe"
+                 major: "Computer Engineering"
+           - name: "Linear Algebra"
+             credits: 3
+             students:
+               - name: "Kimora Tanner"
+                 major: "Computer Science"
+               - name: "Bruno Garett"
+                 major: "Software Engineering"
+               - name: "Amelia Booker"
+                 major: "Psychology"
+
+  - do:
+      index:
+        index: test
+        body:
+          "@timestamp": "2021-04-28T03:00:00Z"
+          department: "compsci"
+          staff: 12
+          courses:
+            - name: "Object Oriented Programming"
+              credits: 3
+              students:
+                - name: "Kimora Tanner"
+                  major: "Computer Science"
+                - name: "Bruno Garrett"
+                  major: "Software Engineering"
+                - name: "Elliott Booker"
+                  major: "Computer Engineering"
+            - name: "Theory of Computation"
+              credits: 4
+              students:
+                - name: "Kimora Tanner"
+                  major: "Software Engineering"
+                - name: "Elliott Ayers"
+                  major: "Software Engineering"
+                - name: "Apollo Pittman"
+                  major: "Computer Engineering"
+
+  - do:
+      indices.refresh:
+        index: [ test ]
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            nested:
+              path: "courses"
+              query:
+                bool:
+                  must:
+                    - nested:
+                        path: "courses.students"
+                        query:
+                          bool:
+                            must:
+                              - match:
+                                  courses.students.name: "Elliott"
+                              - term:
+                                  courses.students.major: "Computer Engineering"
+                    - term:
+                        courses.name: "Theory of Computation"
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._source.department: "compsci" }
+  - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -347,11 +347,10 @@ nested dimensions:
 ---
 nested fields:
   - requires:
-      cluster_features: ["gte_v8.2.0"]
+      cluster_features: ["gte_v8.2.0", "mapper.tsdb_nested_field_support"]
       reason: message changed in 8.2.0
 
   - do:
-      catch: /cannot have nested fields when index is in \[index.mode=time_series\]/
       indices.create:
         index: test
         body:

server/src/main/java/org/elasticsearch/index/IndexMode.java

@@ -29,7 +29,6 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
-import org.elasticsearch.index.mapper.NestedLookup;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFields;
@@ -156,9 +155,6 @@ private static String error(Setting<?> unsupported) {
 
         @Override
         public void validateMapping(MappingLookup lookup) {
-            if (lookup.nestedLookup() != NestedLookup.EMPTY) {
-                throw new IllegalArgumentException("cannot have nested fields when index is in " + tsdbMode());
-            }
             if (((RoutingFieldMapper) lookup.getMapper(RoutingFieldMapper.NAME)).required()) {
                 throw new IllegalArgumentException(routingRequiredBad());
             }

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -665,8 +665,14 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested
         if (idField != null) {
             // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then
             // delete it when the root document is deleted too.
-            // NOTE: we don't support nested fields in tsdb so it's safe to assume the standard id mapper.
             doc.add(new StringField(IdFieldMapper.NAME, idField.binaryValue(), Field.Store.NO));
+        } else if (indexSettings().getMode() == IndexMode.TIME_SERIES) {
+            // For time series indices, the _id is generated from the _tsid, which in turn is generated from the values of the configured
+            // routing fields. At this point in document parsing, we can't guarantee that we've parsed all the routing fields yet, so the
+            // parent document's _id is not yet available.
+            // So we just add the child document without the parent _id, then in TimeSeriesIdFieldMapper#postParse we set the _id on all
+            // child documents once we've calculated it.
+            assert getRoutingFields().equals(RoutingFields.Noop.INSTANCE) == false;
         } else {
             throw new IllegalStateException("The root document of a nested document should have an _id field");
         }

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -27,6 +27,7 @@ public class MapperFeatures implements FeatureSpecification {
         "mapper.counted_keyword.synthetic_source_native_support"
     );
 
+    public static final NodeFeature TSDB_NESTED_FIELD_SUPPORT = new NodeFeature("mapper.tsdb_nested_field_support");
     public static final NodeFeature META_FETCH_FIELDS_ERROR_CODE_CHANGED = new NodeFeature("meta_fetch_fields_error_code_changed");
     public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support");
     public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix");
@@ -49,6 +50,7 @@ public Set<NodeFeature> getTestFeatures() {
             COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT,
             SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX,
             DYNAMIC_HANDLING_IN_COPY_TO,
+            TSDB_NESTED_FIELD_SUPPORT,
             SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE,
             ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX
         );

server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java

@@ -9,7 +9,9 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.StringField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
@@ -142,6 +144,15 @@ public void postParse(DocumentParserContext context) throws IOException {
                 : null,
             timeSeriesId
         );
+
+        // We need to add the uid or id to nested Lucene documents so that when a document gets deleted, the nested documents are
+        // also deleted. Usually this happens when the nested document is created (in DocumentParserContext#createNestedContext), but
+        // for time-series indices the _id isn't available at that point.
+        var binaryId = context.doc().getField(IdFieldMapper.NAME).binaryValue();
+        for (LuceneDocument doc : context.nonRootDocuments()) {
+            assert doc.getField(IdFieldMapper.NAME) == null;
+            doc.add(new StringField(IdFieldMapper.NAME, binaryId, Field.Store.NO));
+        }
     }
 
     private IndexVersion getIndexVersionCreated(final DocumentParserContext context) {

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java

@@ -136,11 +136,6 @@ protected void minimalMapping(XContentBuilder b) throws IOException {
         b.field("type", "semantic_text");
     }
 
-    @Override
-    protected String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) {
-        return "cannot have nested fields when index is in [index.mode=time_series]";
-    }
-
     @Override
     protected void metaMapping(XContentBuilder b) throws IOException {
         super.metaMapping(b);