Skip to content

Commit 10d088d

Browse files
kkrik-eskkrik-es
authored
[8.13] Handle pass-through subfields with deep nesting (#106798)
1 parent ad39c3f commit 10d088d

File tree

3 files changed

+373
-16
lines changed

3 files changed

+373
-16
lines changed

‎modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml‎

Lines changed: 222 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -463,13 +463,13 @@ dynamic templates with nesting:
463463
refresh: true
464464
body:
465465
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
466-
- '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10" }'
466+
- '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5", "attributes.a.much.deeper.nested.dim": "AC" }'
467467
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
468-
- '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10" }'
468+
- '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5", "attributes.a.much.deeper.nested.dim": "AC" }'
469469
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
470-
- '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20" }'
470+
- '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5", "attributes.a.much.deeper.nested.dim": "BD" }'
471471
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
472-
- '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20" }'
472+
- '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5", "attributes.a.much.deeper.nested.dim": "BD" }'
473473

474474
- do:
475475
search:
@@ -495,7 +495,7 @@ dynamic templates with nesting:
495495
field: _tsid
496496

497497
- length: { aggregations.filterA.tsids.buckets: 1 }
498-
- match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
498+
- match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
499499
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
500500

501501
- do:
@@ -514,7 +514,7 @@ dynamic templates with nesting:
514514
field: _tsid
515515

516516
- length: { aggregations.filterA.tsids.buckets: 1 }
517-
- match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
517+
- match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
518518
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
519519

520520
- do:
@@ -533,7 +533,7 @@ dynamic templates with nesting:
533533
field: _tsid
534534

535535
- length: { aggregations.filterA.tsids.buckets: 1 }
536-
- match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
536+
- match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
537537
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
538538

539539
- do:
@@ -545,14 +545,227 @@ dynamic templates with nesting:
545545
filterA:
546546
filter:
547547
term:
548-
another.dim2: 10
548+
another.dim2: 10.5
549+
aggs:
550+
tsids:
551+
terms:
552+
field: _tsid
553+
554+
- length: { aggregations.filterA.tsids.buckets: 1 }
555+
- match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
556+
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
557+
558+
- do:
559+
search:
560+
index: k9s
561+
body:
562+
size: 0
563+
aggs:
564+
filterA:
565+
filter:
566+
term:
567+
a.much.deeper.nested.dim: AC
568+
aggs:
569+
tsids:
570+
terms:
571+
field: _tsid
572+
573+
- length: { aggregations.filterA.tsids.buckets: 1 }
574+
- match: { aggregations.filterA.tsids.buckets.0.key: "NNnsRFDTqKogyRBhOBQclM4BkssYqVppKiBimIqLDvTuF9nXxZWMD04YHQKL09tJYL5G4yo" }
575+
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
576+
577+
---
578+
dynamic templates with incremental indexing:
579+
- skip:
580+
version: " - 8.12.99"
581+
reason: "Support for dynamic fields was added in 8.13"
582+
- do:
583+
allowed_warnings:
584+
- "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
585+
indices.put_index_template:
586+
name: my-dynamic-template
587+
body:
588+
index_patterns: [k9s*]
589+
data_stream: {}
590+
template:
591+
settings:
592+
index:
593+
number_of_shards: 1
594+
mode: time_series
595+
time_series:
596+
start_time: 2023-08-31T13:03:08.138Z
597+
598+
mappings:
599+
properties:
600+
attributes:
601+
type: passthrough
602+
dynamic: true
603+
time_series_dimension: true
604+
resource:
605+
type: object
606+
properties:
607+
attributes:
608+
type: passthrough
609+
dynamic: true
610+
time_series_dimension: true
611+
dynamic_templates:
612+
- counter_metric:
613+
mapping:
614+
type: integer
615+
time_series_metric: counter
616+
617+
- do:
618+
bulk:
619+
index: k9s
620+
refresh: true
621+
body:
622+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
623+
- '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "attributes.dim2": "C" }'
624+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
625+
- '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "attributes.dim2": "C" }'
626+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
627+
- '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "attributes.dim2": "D" }'
628+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
629+
- '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "attributes.dim2": "D" }'
630+
631+
- do:
632+
bulk:
633+
index: k9s
634+
refresh: true
635+
body:
636+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
637+
- '{ "@timestamp": "2023-09-01T13:04:08.138Z","data": "110", "resource.attributes.another.dim1": "1", "attributes.another.dim2": "10.5" }'
638+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
639+
- '{ "@timestamp": "2023-09-01T13:04:09.138Z","data": "120", "resource.attributes.another.dim1": "1", "attributes.another.dim2": "10.5" }'
640+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
641+
- '{ "@timestamp": "2023-09-01T13:04:10.138Z","data": "130", "resource.attributes.another.dim1": "2", "attributes.another.dim2": "20.5" }'
642+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
643+
- '{ "@timestamp": "2023-09-01T13:04:10.238Z","data": "140", "resource.attributes.another.dim1": "2", "attributes.another.dim2": "20.5" }'
644+
645+
- do:
646+
bulk:
647+
index: k9s
648+
refresh: true
649+
body:
650+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
651+
- '{ "@timestamp": "2023-09-01T13:05:08.138Z","data": "210", "resource.attributes.another.deeper.dim1": "1", "attributes.another.deeper.dim2": "10.5" }'
652+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
653+
- '{ "@timestamp": "2023-09-01T13:05:09.138Z","data": "220", "resource.attributes.another.deeper.dim1": "1", "attributes.another.deeper.dim2": "10.5" }'
654+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
655+
- '{ "@timestamp": "2023-09-01T13:05:10.138Z","data": "230", "resource.attributes.another.deeper.dim1": "2", "attributes.another.deeper.dim2": "20.5" }'
656+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
657+
- '{ "@timestamp": "2023-09-01T13:05:10.238Z","data": "240", "resource.attributes.another.deeper.dim1": "2", "attributes.another.deeper.dim2": "20.5" }'
658+
659+
- do:
660+
bulk:
661+
index: k9s
662+
refresh: true
663+
body:
664+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
665+
- '{ "@timestamp": "2023-09-01T13:06:08.138Z","data": "310", "attributes.a.much.deeper.nested.dim": "AC" }'
666+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
667+
- '{ "@timestamp": "2023-09-01T13:06:09.138Z","data": "320", "attributes.a.much.deeper.nested.dim": "AC" }'
668+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
669+
- '{ "@timestamp": "2023-09-01T13:06:10.138Z","data": "330", "attributes.a.much.deeper.nested.dim": "BD" }'
670+
- '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
671+
- '{ "@timestamp": "2023-09-01T13:06:10.238Z","data": "340", "attributes.a.much.deeper.nested.dim": "BD" }'
672+
673+
- do:
674+
search:
675+
index: k9s
676+
body:
677+
size: 0
678+
679+
- match: { hits.total.value: 16 }
680+
681+
- do:
682+
search:
683+
index: k9s
684+
body:
685+
size: 0
686+
aggs:
687+
filterA:
688+
filter:
689+
term:
690+
dim1: A
691+
aggs:
692+
tsids:
693+
terms:
694+
field: _tsid
695+
696+
- length: { aggregations.filterA.tsids.buckets: 1 }
697+
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
698+
699+
- do:
700+
search:
701+
index: k9s
702+
body:
703+
size: 0
704+
aggs:
705+
filterA:
706+
filter:
707+
term:
708+
dim2: C
709+
aggs:
710+
tsids:
711+
terms:
712+
field: _tsid
713+
714+
- length: { aggregations.filterA.tsids.buckets: 1 }
715+
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
716+
717+
- do:
718+
search:
719+
index: k9s
720+
body:
721+
size: 0
722+
aggs:
723+
filterA:
724+
filter:
725+
term:
726+
another.deeper.dim1: 1
727+
aggs:
728+
tsids:
729+
terms:
730+
field: _tsid
731+
732+
- length: { aggregations.filterA.tsids.buckets: 1 }
733+
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
734+
735+
- do:
736+
search:
737+
index: k9s
738+
body:
739+
size: 0
740+
aggs:
741+
filterA:
742+
filter:
743+
term:
744+
another.deeper.dim2: 10.5
745+
aggs:
746+
tsids:
747+
terms:
748+
field: _tsid
749+
750+
- length: { aggregations.filterA.tsids.buckets: 1 }
751+
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
752+
753+
- do:
754+
search:
755+
index: k9s
756+
body:
757+
size: 0
758+
aggs:
759+
filterA:
760+
filter:
761+
term:
762+
a.much.deeper.nested.dim: AC
549763
aggs:
550764
tsids:
551765
terms:
552766
field: _tsid
553767

554768
- length: { aggregations.filterA.tsids.buckets: 1 }
555-
- match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" }
556769
- match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
557770

558771
---

‎server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java‎

Lines changed: 54 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -128,19 +128,22 @@ public RootObjectMapper build(MapperBuilderContext context) {
128128
}
129129

130130
Map<String, Mapper> getAliasMappers(Map<String, Mapper> mappers, MapperBuilderContext context) {
131-
Map<String, Mapper> aliasMappers = new HashMap<>();
131+
Map<String, Mapper> newMappers = new HashMap<>();
132132
Map<String, ObjectMapper.Builder> objectIntermediates = new HashMap<>(1);
133-
getAliasMappers(mappers, aliasMappers, objectIntermediates, context, 0);
133+
Map<String, ObjectMapper.Builder> objectIntermediatesFullName = new HashMap<>(1);
134+
getAliasMappers(mappers, mappers, newMappers, objectIntermediates, objectIntermediatesFullName, context, 0);
134135
for (var entry : objectIntermediates.entrySet()) {
135-
aliasMappers.put(entry.getKey(), entry.getValue().build(context));
136+
newMappers.put(entry.getKey(), entry.getValue().build(context));
136137
}
137-
return aliasMappers;
138+
return newMappers;
138139
}
139140

140141
void getAliasMappers(
141142
Map<String, Mapper> mappers,
143+
Map<String, Mapper> topLevelMappers,
142144
Map<String, Mapper> aliasMappers,
143145
Map<String, ObjectMapper.Builder> objectIntermediates,
146+
Map<String, ObjectMapper.Builder> objectIntermediatesFullName,
144147
MapperBuilderContext context,
145148
int level
146149
) {
@@ -179,32 +182,76 @@ void getAliasMappers(
179182
).build(context);
180183
aliasMappers.put(aliasMapper.simpleName(), aliasMapper);
181184
} else {
185+
conflict = topLevelMappers.get(fieldNameParts[0]);
186+
if (conflict != null) {
187+
if (isConflictingObject(conflict, fieldNameParts)) {
188+
throw new IllegalArgumentException(
189+
"Conflicting objects created during alias generation for pass-through field: ["
190+
+ conflict.name()
191+
+ "]"
192+
);
193+
}
194+
}
195+
182196
// Nest the alias within object(s).
183197
String realFieldName = fieldNameParts[fieldNameParts.length - 1];
184198
Mapper.Builder fieldBuilder = new FieldAliasMapper.Builder(realFieldName).path(
185199
fieldMapper.mappedFieldType.name()
186200
);
201+
ObjectMapper.Builder intermediate = null;
187202
for (int i = fieldNameParts.length - 2; i >= 0; --i) {
188203
String intermediateObjectName = fieldNameParts[i];
189-
ObjectMapper.Builder intermediate = objectIntermediates.computeIfAbsent(
190-
intermediateObjectName,
204+
intermediate = objectIntermediatesFullName.computeIfAbsent(
205+
concatStrings(fieldNameParts, i),
191206
s -> new ObjectMapper.Builder(intermediateObjectName, ObjectMapper.Defaults.SUBOBJECTS)
192207
);
193208
intermediate.add(fieldBuilder);
194209
fieldBuilder = intermediate;
195210
}
211+
objectIntermediates.putIfAbsent(fieldNameParts[0], intermediate);
196212
}
197213
}
198214
}
199215
}
200216
} else if (mapper instanceof ObjectMapper objectMapper) {
201217
// Call recursively to check child fields. The level guards against long recursive call sequences.
202-
getAliasMappers(objectMapper.mappers, aliasMappers, objectIntermediates, context, level + 1);
218+
getAliasMappers(
219+
objectMapper.mappers,
220+
topLevelMappers,
221+
aliasMappers,
222+
objectIntermediates,
223+
objectIntermediatesFullName,
224+
context,
225+
level + 1
226+
);
203227
}
204228
}
205229
}
206230
}
207231

232+
private static String concatStrings(String[] parts, int last) {
233+
StringBuilder builder = new StringBuilder();
234+
for (int i = 0; i <= last; i++) {
235+
builder.append('.');
236+
builder.append(parts[i]);
237+
}
238+
return builder.toString();
239+
}
240+
241+
private static boolean isConflictingObject(Mapper mapper, String[] parts) {
242+
for (int i = 0; i < parts.length - 1; i++) {
243+
if (mapper == null) {
244+
return true;
245+
}
246+
if (mapper instanceof ObjectMapper objectMapper) {
247+
mapper = objectMapper.getMapper(parts[i + 1]);
248+
} else {
249+
return true;
250+
}
251+
}
252+
return mapper == null;
253+
}
254+
208255
private final Explicit<DateFormatter[]> dynamicDateTimeFormatters;
209256
private final Explicit<Boolean> dateDetection;
210257
private final Explicit<Boolean> numericDetection;

0 commit comments

Comments
 (0)