From 59b2d57f6d8c394af8505cf6c489240f99ba05fd Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Mon, 10 Feb 2025 17:37:36 -0800 Subject: [PATCH 01/13] Add REST test with nested field in time-series index --- .../test/tsdb/160_nested_fields.yml | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml new file mode 100644 index 0000000000000..171ff2612cbc3 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml @@ -0,0 +1,61 @@ +--- +"Create TSDB index with field of nested type": + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 1 + number_of_shards: 1 + routing_path: [department] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + department: + type: keyword + time_series_dimension: true + staff: + type: integer + courses: + type: nested + properties: + name: + type: keyword + credits: + type: integer + + - do: + index: + index: test + body: { "@timestamp": "2021-04-28T01:00:00Z", "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4, } ] } + + - do: + index: + index: test + body: { "@timestamp": "2021-04-28T02:00:00Z", "department": "math", "staff": 20, "courses": [ { "name": "Precalculus", "credits": 1 }, { "name": "Linear Algebra", "credits": 3 } ] } + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + size: 0 + query: + bool: + must: + - term: + courses.name: Precalculus + - term: + courses.credits: 3 + + - match: + hits.total.value: 0 From 412f12ec19f29384a7b6f8f2d1180463b3d67b85 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Mon, 10 Feb 2025 17:41:50 -0800 Subject: [PATCH 02/13] Enable nested fields in time-series mode indices --- .../main/java/org/elasticsearch/index/IndexMode.java | 4 ---- .../index/mapper/DocumentParserContext.java | 7 ++++++- .../index/mapper/TimeSeriesIdFieldMapper.java | 10 ++++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 7287a0bf307b9..04bf820926660 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -29,7 +29,6 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.ProvidedIdFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.RoutingFields; @@ -156,9 +155,6 @@ private static String error(Setting unsupported) { @Override public void validateMapping(MappingLookup lookup) { - if (lookup.nestedLookup() != NestedLookup.EMPTY) { - throw new IllegalArgumentException("cannot have nested fields when index is in " + tsdbMode()); - } if (((RoutingFieldMapper) lookup.getMapper(RoutingFieldMapper.NAME)).required()) { throw new IllegalArgumentException(routingRequiredBad()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 127ec05b25e63..cf6bf435a4725 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -665,8 +665,13 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested if (idField != null) { // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then // delete it when the root document is deleted too. - // NOTE: we don't support nested fields in tsdb so it's safe to assume the standard id mapper. doc.add(new StringField(IdFieldMapper.NAME, idField.binaryValue(), Field.Store.NO)); + } else if (indexSettings().getMode() == IndexMode.TIME_SERIES) { + // For time series indices, the _id is generated from the _tsid, which in turn is generated from the values of the configured + // routing fields. At this point in document parsing, we can't guarantee that we've parsed all the routing fields yet, so the + // parent document's _id is not yet available. + // So we just add the child document without the parent _id, then in TimeSeriesIdFieldMapper#postParse we set the _id on all + // child documents once we've calculated it. } else { throw new IllegalStateException("The root document of a nested document should have an _id field"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 8af3c3e6ec270..734f5bc6531cd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -9,7 +9,9 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StringField; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; @@ -142,6 +144,14 @@ public void postParse(DocumentParserContext context) throws IOException { : null, timeSeriesId ); + + // We need to add the uid or id to nested Lucene documents so that when a document gets deleted, the nested documents are + // also deleted. Usually this happens when the nested document is created (in DocumentParser#createNestedContext), but + // for time-series indices the _id isn't available at that point. + assert context.id() != null; + for (LuceneDocument doc : context.nonRootDocuments()) { + doc.add(new StringField(IdFieldMapper.NAME, Uid.encodeId(context.id()), Field.Store.NO)); + } } private IndexVersion getIndexVersionCreated(final DocumentParserContext context) { From c6097c746ca0faaa4ae38bffa04f854ce5ec1c34 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Mon, 10 Feb 2025 17:57:56 -0800 Subject: [PATCH 03/13] Update docs/changelog/122224.yaml --- docs/changelog/122224.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/122224.yaml diff --git a/docs/changelog/122224.yaml b/docs/changelog/122224.yaml new file mode 100644 index 0000000000000..41ae8c6578600 --- /dev/null +++ b/docs/changelog/122224.yaml @@ -0,0 +1,6 @@ +pr: 122224 +summary: Enable the use of nested field type with index.mode=time_series +area: Mapping +type: enhancement +issues: + - 120874 From ed6f965f4d14dd6e0f9ff99bd3403a4befd73245 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 11 Feb 2025 10:42:23 -0800 Subject: [PATCH 04/13] Use nested query in nested field test Also, add a test that actually returns a document. --- .../test/tsdb/160_nested_fields.yml | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml index 171ff2612cbc3..ca1e2b02e3213 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml @@ -33,7 +33,7 @@ - do: index: index: test - body: { "@timestamp": "2021-04-28T01:00:00Z", "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4, } ] } + body: { "@timestamp": "2021-04-28T01:00:00Z", "department": "compsci", "staff": 12, "courses": [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4 } ] } - do: index: @@ -50,12 +50,34 @@ body: size: 0 query: - bool: - must: - - term: - courses.name: Precalculus - - term: - courses.credits: 3 + nested: + path: "courses" + query: + bool: + must: + - term: + courses.name: Precalculus + - term: + courses.credits: 3 - - match: - hits.total.value: 0 + - match: { hits.total.value: 0 } + + - do: + search: + index: test + body: + query: + nested: + path: "courses" + query: + bool: + must: + - term: + courses.name: "Object Oriented Programming" + - term: + courses.credits: 3 + + - match: { hits.total.value: 1 } + - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" } + - match: { hits.hits.0._source.department: "compsci" } + - match: { hits.hits.0._source.courses: [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4, } ] } From 40334dba0c198a45b7399865172a877a9b97d8bb Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 11 Feb 2025 10:47:32 -0800 Subject: [PATCH 05/13] Iter --- .../elasticsearch/index/mapper/DocumentParserContext.java | 1 + .../index/mapper/TimeSeriesIdFieldMapper.java | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index cf6bf435a4725..7b5e28dc1fbef 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -672,6 +672,7 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested // parent document's _id is not yet available. // So we just add the child document without the parent _id, then in TimeSeriesIdFieldMapper#postParse we set the _id on all // child documents once we've calculated it. + assert getRoutingFields().equals(RoutingFields.Noop.INSTANCE) == false; } else { throw new IllegalStateException("The root document of a nested document should have an _id field"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 734f5bc6531cd..3d5b477ef3fe2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -146,11 +146,12 @@ public void postParse(DocumentParserContext context) throws IOException { ); // We need to add the uid or id to nested Lucene documents so that when a document gets deleted, the nested documents are - // also deleted. Usually this happens when the nested document is created (in DocumentParser#createNestedContext), but + // also deleted. Usually this happens when the nested document is created (in DocumentParserContext#createNestedContext), but // for time-series indices the _id isn't available at that point. - assert context.id() != null; + var binaryId = context.doc().getField(IdFieldMapper.NAME).binaryValue(); for (LuceneDocument doc : context.nonRootDocuments()) { - doc.add(new StringField(IdFieldMapper.NAME, Uid.encodeId(context.id()), Field.Store.NO)); + assert doc.getField(IdFieldMapper.NAME) == null; + doc.add(new StringField(IdFieldMapper.NAME, binaryId, Field.Store.NO)); } } From 42a20f9ecd6767e55fcfed61ce01090515cc9db8 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 11 Feb 2025 11:47:38 -0800 Subject: [PATCH 06/13] Add test for tsdb with multi-level nested fields --- .../test/tsdb/160_nested_fields.yml | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml index ca1e2b02e3213..9b54ba50fe659 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml @@ -81,3 +81,148 @@ - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" } - match: { hits.hits.0._source.department: "compsci" } - match: { hits.hits.0._source.courses: [ { "name": "Object Oriented Programming", "credits": 3 }, { "name": "Theory of Computation", "credits": 4, } ] } + +--- + +"TSDB index with multi-level nested fields": + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + number_of_replicas: 1 + number_of_shards: 1 + routing_path: [department] + time_series: + start_time: 2021-04-28T00:00:00Z + end_time: 2021-04-29T00:00:00Z + mappings: + properties: + "@timestamp": + type: date + department: + type: keyword + time_series_dimension: true + staff: + type: integer + courses: + type: nested + properties: + name: + type: keyword + credits: + type: integer + students: + type: nested + properties: + name: + type: text + major: + type: keyword + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T01:00:00Z" + department: "compsci" + staff: 12 + courses: + - name: "Object Oriented Programming" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garrett" + major: "Software Engineering" + - name: "Theory of Computation" + credits: 4 + students: + - name: "Elliott Booker" + major: "Computer Engineering" + - name: "Kimora Tanner" + major: "Software Engineering" + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T02:00:00Z" + department: "math" + staff: 20 + courses: + - name: "Precalculus" + credits: 4 + students: + - name: "Elliott Ayers" + major: "Software Engineering" + - name: "Sylvie Howe" + major: "Computer Engineering" + - name: "Linear Algebra" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garett" + major: "Software Engineering" + - name: "Amelia Booker" + major: "Psychology" + + - do: + index: + index: test + body: + "@timestamp": "2021-04-28T03:00:00Z" + department: "compsci" + staff: 12 + courses: + - name: "Object Oriented Programming" + credits: 3 + students: + - name: "Kimora Tanner" + major: "Computer Science" + - name: "Bruno Garrett" + major: "Software Engineering" + - name: "Elliott Booker" + major: "Computer Engineering" + - name: "Theory of Computation" + credits: 4 + students: + - name: "Kimora Tanner" + major: "Software Engineering" + - name: "Elliott Ayers" + major: "Software Engineering" + - name: "Apollo Pittman" + major: "Computer Engineering" + + - do: + indices.refresh: + index: [ test ] + + - do: + search: + index: test + body: + query: + nested: + path: "courses" + query: + bool: + must: + - nested: + path: "courses.students" + query: + bool: + must: + - match: + courses.students.name: "Elliott" + - term: + courses.students.major: "Computer Engineering" + - term: + courses.name: "Theory of Computation" + + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.department: "compsci" } + - match: { "hits.hits.0._source.@timestamp": "2021-04-28T01:00:00.000Z" } From ab7935d8cc0f4eb2132d470c09dd066d66ab09d5 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 11 Feb 2025 12:38:39 -0800 Subject: [PATCH 07/13] Update tests to support nested fields --- .../resources/rest-api-spec/test/tsdb/20_mapping.yml | 2 -- .../xpack/inference/mapper/SemanticTextFieldMapperTests.java | 5 ----- 2 files changed, 7 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index f25601fc2e228..911799a838ae5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -322,7 +322,6 @@ nested dimensions: reason: message changed in 8.2.0 - do: - catch: /time_series_dimension can't be configured in nested field \[nested.dim\]/ indices.create: index: test body: @@ -351,7 +350,6 @@ nested fields: reason: message changed in 8.2.0 - do: - catch: /cannot have nested fields when index is in \[index.mode=time_series\]/ indices.create: index: test body: diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index e837e1b0db989..5d1c058c89da0 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -136,11 +136,6 @@ protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "semantic_text"); } - @Override - protected String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { - return "cannot have nested fields when index is in [index.mode=time_series]"; - } - @Override protected void metaMapping(XContentBuilder b) throws IOException { super.metaMapping(b); From d7acbe04a425d5f4ed03e1c0b7a2191c1c14ebad Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 11 Feb 2025 12:52:12 -0800 Subject: [PATCH 08/13] Add cluster feature --- .../resources/rest-api-spec/test/tsdb/160_nested_fields.yml | 5 +++++ .../java/org/elasticsearch/index/mapper/MapperFeatures.java | 2 ++ 2 files changed, 7 insertions(+) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml index 9b54ba50fe659..f4aca5ab264e8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/160_nested_fields.yml @@ -1,3 +1,8 @@ +setup: + - requires: + cluster_features: ["mapper.tsdb_nested_field_support"] + reason: "tsdb index with nested field support enabled" + --- "Create TSDB index with field of nested type": - do: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 7567fae7d73e6..42d35cb2ef09f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -27,6 +27,7 @@ public class MapperFeatures implements FeatureSpecification { "mapper.counted_keyword.synthetic_source_native_support" ); + public static final NodeFeature TSDB_NESTED_FIELD_SUPPORT = new NodeFeature("mapper.tsdb_nested_field_support"); public static final NodeFeature META_FETCH_FIELDS_ERROR_CODE_CHANGED = new NodeFeature("meta_fetch_fields_error_code_changed"); public static final NodeFeature SPARSE_VECTOR_STORE_SUPPORT = new NodeFeature("mapper.sparse_vector.store_support"); public static final NodeFeature SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX = new NodeFeature("mapper.nested.sorting_fields_check_fix"); @@ -49,6 +50,7 @@ public Set getTestFeatures() { COUNTED_KEYWORD_SYNTHETIC_SOURCE_NATIVE_SUPPORT, SORT_FIELDS_CHECK_FOR_NESTED_OBJECT_FIX, DYNAMIC_HANDLING_IN_COPY_TO, + TSDB_NESTED_FIELD_SUPPORT, SourceFieldMapper.SYNTHETIC_RECOVERY_SOURCE, ObjectMapper.SUBOBJECTS_FALSE_MAPPING_UPDATE_FIX ); From 117217cf3d2bc372e9340e5ee57b9dcc72e659b5 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 11 Feb 2025 13:18:06 -0800 Subject: [PATCH 09/13] Add cluster feature to 20_mapping tests --- .../resources/rest-api-spec/test/tsdb/20_mapping.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 911799a838ae5..434b4787babe7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -318,7 +318,7 @@ runtime field matching routing path: --- nested dimensions: - requires: - cluster_features: ["gte_v8.2.0"] + cluster_features: ["gte_v8.2.0", "mapper.tsdb_nested_field_support"] reason: message changed in 8.2.0 - do: @@ -346,7 +346,7 @@ nested dimensions: --- nested fields: - requires: - cluster_features: ["gte_v8.2.0"] + cluster_features: ["gte_v8.2.0", "mapper.tsdb_nested_field_support"] reason: message changed in 8.2.0 - do: From ff1a4ea8c7331ab20b5fcf208759776fcb20b0eb Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Tue, 11 Feb 2025 14:06:49 -0800 Subject: [PATCH 10/13] Fix yaml=tsdb/20_mapping/nested dimensions --- .../resources/rest-api-spec/test/tsdb/20_mapping.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 434b4787babe7..53d15484de433 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -318,10 +318,11 @@ runtime field matching routing path: --- nested dimensions: - requires: - cluster_features: ["gte_v8.2.0", "mapper.tsdb_nested_field_support"] + cluster_features: ["gte_v8.2.0"] reason: message changed in 8.2.0 - do: + catch: /time_series_dimension can't be configured in nested field \[nested.dim\]/ indices.create: index: test body: From 07b7a576bee0eb3f3efa052f421a69fa625cf401 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Wed, 12 Feb 2025 15:13:48 -0800 Subject: [PATCH 11/13] Avoid use of LuceneDocument::getField --- .../index/mapper/TimeSeriesIdFieldMapper.java | 5 ++--- .../index/mapper/TsidExtractingIdFieldMapper.java | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 3d5b477ef3fe2..3325ad4e96eb9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -137,7 +137,7 @@ public void postParse(DocumentParserContext context) throws IOException { } context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); - TsidExtractingIdFieldMapper.createField( + BytesRef uidEncoded = TsidExtractingIdFieldMapper.createField( context, getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) ? routingPathFields.routingBuilder() @@ -148,10 +148,9 @@ public void postParse(DocumentParserContext context) throws IOException { // We need to add the uid or id to nested Lucene documents so that when a document gets deleted, the nested documents are // also deleted. Usually this happens when the nested document is created (in DocumentParserContext#createNestedContext), but // for time-series indices the _id isn't available at that point. - var binaryId = context.doc().getField(IdFieldMapper.NAME).binaryValue(); for (LuceneDocument doc : context.nonRootDocuments()) { assert doc.getField(IdFieldMapper.NAME) == null; - doc.add(new StringField(IdFieldMapper.NAME, binaryId, Field.Store.NO)); + doc.add(new StringField(IdFieldMapper.NAME, uidEncoded, Field.Store.NO)); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 821b11410f93b..8aca004949209 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -46,7 +46,11 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext private static final long SEED = 0; - public static void createField(DocumentParserContext context, IndexRouting.ExtractFromSource.Builder routingBuilder, BytesRef tsid) { + public static BytesRef createField( + DocumentParserContext context, + IndexRouting.ExtractFromSource.Builder routingBuilder, + BytesRef tsid + ) { final long timestamp = DataStreamTimestampFieldMapper.extractTimestampValue(context.doc()); String id; if (routingBuilder != null) { @@ -94,6 +98,7 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra BytesRef uidEncoded = Uid.encodeId(context.id()); context.doc().add(new StringField(NAME, uidEncoded, Field.Store.YES)); + return uidEncoded; } public static String createId(int routingHash, BytesRef tsid, long timestamp) { From 84bd7dd8fceca9cf86d93166719069b72e65b175 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Wed, 12 Feb 2025 15:17:38 -0800 Subject: [PATCH 12/13] Remove redundant test {tsdb/20_mapping/nested fields} --- .../rest-api-spec/test/tsdb/20_mapping.yml | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 53d15484de433..5963ddb46e0b3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -344,36 +344,6 @@ nested dimensions: type: keyword time_series_dimension: true ---- -nested fields: - - requires: - cluster_features: ["gte_v8.2.0", "mapper.tsdb_nested_field_support"] - reason: message changed in 8.2.0 - - - do: - indices.create: - index: test - body: - settings: - index: - mode: time_series - routing_path: [dim] - time_series: - start_time: 2021-04-28T00:00:00Z - end_time: 2021-04-29T00:00:00Z - mappings: - properties: - "@timestamp": - type: date - dim: - type: keyword - time_series_dimension: true - nested: - type: nested - properties: - foo: - type: keyword - --- "Unable to define a metric type for a runtime field": - requires: From 905534f35b1ef576234889db4e6cfa5c847c6421 Mon Sep 17 00:00:00 2001 From: Jordan Powers Date: Thu, 13 Feb 2025 08:18:18 -0800 Subject: [PATCH 13/13] Mute tsdb/20_mapping/nested fields --- rest-api-spec/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 3bdaf029c3641..205b02a8936bb 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -82,4 +82,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> "cluster.desired_nodes/10_basic/Test update desired nodes with node_version generates a warning", "node_version warning is removed in 9.0" ) + task.skipTest("tsdb/20_mapping/nested fields", "nested field support in tsdb indices is now supported") })