Enable the use of nested field type with index.mode=time_series

docs/changelog/122224.yaml

@@ -0,0 +1,6 @@
+pr: 122224
+summary: Enable the use of nested field type with index.mode=time_series
+area: Mapping
+type: enhancement
+issues:
+ - 120874

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml

@@ -0,0 +1,61 @@
+---
+"Create TSDB index with field of nested type":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mode: time_series
+              number_of_replicas: 1
+              number_of_shards: 1
+              routing_path: [department]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              department:
+                type: keyword
+                time_series_dimension: true
+              staff:
+                type: integer
+              courses:
+                type: nested
+                properties:
+                  name:
+                    type: keyword
+                  credits:
+                    type: integer
+
+  - do:
+      index:
+        index: test
+        body: { "@timestamp": "2021-04-28T01:00:00Z", "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4, } ] }
+
+  - do:
+      index:
+        index: test
+        body: { "@timestamp": "2021-04-28T02:00:00Z", "department": "math", "staff": 20, "courses": [ { "name": "Precalculus", "credits": 1 }, { "name": "Linear Algebra", "credits": 3 } ] }
+
+  - do:
+      indices.refresh:
+        index: [ test ]
+
+  - do:
+      search:
+        index: test
+        body:
+          size: 0
+          query:
+            bool:
+              must:
+                - term:
+                    courses.name: Precalculus
+                - term:
+                    courses.credits: 3
+
+  - match:
+     hits.total.value: 0

server/src/main/java/org/elasticsearch/index/IndexMode.java

@@ -29,7 +29,6 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.MetadataFieldMapper;
-import org.elasticsearch.index.mapper.NestedLookup;
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFields;
@@ -156,9 +155,6 @@ private static String error(Setting<?> unsupported) {
 
         @Override
         public void validateMapping(MappingLookup lookup) {
-            if (lookup.nestedLookup() != NestedLookup.EMPTY) {
-                throw new IllegalArgumentException("cannot have nested fields when index is in " + tsdbMode());
-            }
             if (((RoutingFieldMapper) lookup.getMapper(RoutingFieldMapper.NAME)).required()) {
                 throw new IllegalArgumentException(routingRequiredBad());
             }

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -665,8 +665,13 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested
         if (idField != null) {
             // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then
             // delete it when the root document is deleted too.
-            // NOTE: we don't support nested fields in tsdb so it's safe to assume the standard id mapper.
             doc.add(new StringField(IdFieldMapper.NAME, idField.binaryValue(), Field.Store.NO));
+        } else if (indexSettings().getMode() == IndexMode.TIME_SERIES) {
+            // For time series indices, the _id is generated from the _tsid, which in turn is generated from the values of the configured
+            // routing fields. At this point in document parsing, we can't guarantee that we've parsed all the routing fields yet, so the
+            // parent document's _id is not yet available.
+            // So we just add the child document without the parent _id, then in TimeSeriesIdFieldMapper#postParse we set the _id on all
+            // child documents once we've calculated it.
         } else {
             throw new IllegalStateException("The root document of a nested document should have an _id field");
         }

server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java

@@ -9,7 +9,9 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.StringField;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
@@ -142,6 +144,14 @@ public void postParse(DocumentParserContext context) throws IOException {
                 : null,
             timeSeriesId
         );
+
+        // We need to add the uid or id to nested Lucene documents so that when a document gets deleted, the nested documents are
+        // also deleted. Usually this happens when the nested document is created (in DocumentParser#createNestedContext), but
+        // for time-series indices the _id isn't available at that point.
+        assert context.id() != null;
+        for (LuceneDocument doc : context.nonRootDocuments()) {
+            doc.add(new StringField(IdFieldMapper.NAME, Uid.encodeId(context.id()), Field.Store.NO));
+        }
     }
 
     private IndexVersion getIndexVersionCreated(final DocumentParserContext context) {