Skip to content

Commit 406f65f

Browse files
kkrik-eskkrik-es
authored
[8.16] Use flattened names in ignored source (#115822) (#115899)
* Use flattened names in ignored source (#115822) * Use flattened names in ignored source * spotless * fix rest compat * fix unittests * expand dots (cherry picked from commit 06eb072) # Conflicts: # rest-api-spec/build.gradle * Update 20_synthetic_source.yml * Update 21_synthetic_source_stored.yml
1 parent 94158e1 commit 406f65f

File tree

7 files changed

+203
-14
lines changed

7 files changed

+203
-14
lines changed

‎rest-api-spec/build.gradle‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,5 +247,4 @@ tasks.named("precommit").configure {
247247
tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
248248
task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility")
249249
task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility")
250-
task.skipTest("range/20_synthetic_source/Date range", "date range breaking change causes tests to produce incorrect values for compatibility")
251250
})

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,13 @@ object with unmapped fields:
4141
- match: { hits.hits.0._source.some_string: AaAa }
4242
- match: { hits.hits.0._source.some_int: 1000 }
4343
- match: { hits.hits.0._source.some_double: 123.456789 }
44-
- match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA }
44+
- match: { hits.hits.0._source.a: { very.deeply.nested.field: AAAA } }
4545
- match: { hits.hits.0._source.some_bool: true }
4646
- match: { hits.hits.1._source.name: bbbb }
4747
- match: { hits.hits.1._source.some_string: BbBb }
4848
- match: { hits.hits.1._source.some_int: 2000 }
4949
- match: { hits.hits.1._source.some_double: 321.987654 }
50-
- match: { hits.hits.1._source.a.very.deeply.nested.field: BBBB }
50+
- match: { hits.hits.1._source.a: { very.deeply.nested.field: BBBB } }
5151

5252

5353
---
@@ -143,10 +143,10 @@ nested object with unmapped fields:
143143
- match: { hits.total.value: 2 }
144144
- match: { hits.hits.0._source.path.to.name: aaaa }
145145
- match: { hits.hits.0._source.path.to.surname: AaAa }
146-
- match: { hits.hits.0._source.path.some.other.name: AaAaAa }
146+
- match: { hits.hits.0._source.path.some.other\.name: AaAaAa }
147147
- match: { hits.hits.1._source.path.to.name: bbbb }
148148
- match: { hits.hits.1._source.path.to.surname: BbBb }
149-
- match: { hits.hits.1._source.path.some.other.name: BbBbBb }
149+
- match: { hits.hits.1._source.path.some.other\.name: BbBbBb }
150150

151151

152152
---
@@ -191,7 +191,7 @@ empty object with unmapped fields:
191191

192192
- match: { hits.total.value: 1 }
193193
- match: { hits.hits.0._source.path.to.surname: AaAa }
194-
- match: { hits.hits.0._source.path.some.other.name: AaAaAa }
194+
- match: { hits.hits.0._source.path.some.other\.name: AaAaAa }
195195

196196

197197
---
@@ -475,7 +475,7 @@ object with dynamic override:
475475
- match: { hits.hits.0._source.path_no.to: { a.very.deeply.nested.field: A } }
476476
- match: { hits.hits.0._source.path_runtime.name: bar }
477477
- match: { hits.hits.0._source.path_runtime.some_int: 20 }
478-
- match: { hits.hits.0._source.path_runtime.to.a.very.deeply.nested.field: B }
478+
- match: { hits.hits.0._source.path_runtime.to: { a.very.deeply.nested.field: B } }
479479

480480

481481
---

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1246,3 +1246,52 @@ index param - nested object with stored array:
12461246
- match: { hits.hits.1._source.nested.0.b.1.c: 300 }
12471247
- match: { hits.hits.1._source.nested.1.b.0.c: 40 }
12481248
- match: { hits.hits.1._source.nested.1.b.1.c: 400 }
1249+
1250+
1251+
---
1252+
index param - flattened fields:
1253+
- requires:
1254+
cluster_features: ["mapper.synthetic_source_keep"]
1255+
reason: requires keeping array source
1256+
1257+
- do:
1258+
indices.create:
1259+
index: test
1260+
body:
1261+
settings:
1262+
index:
1263+
mapping:
1264+
synthetic_source_keep: arrays
1265+
mappings:
1266+
_source:
1267+
mode: synthetic
1268+
properties:
1269+
name:
1270+
type: keyword
1271+
outer:
1272+
properties:
1273+
inner:
1274+
type: object
1275+
1276+
- do:
1277+
bulk:
1278+
index: test
1279+
refresh: true
1280+
body:
1281+
- '{ "create": { } }'
1282+
- '{ "name": "A", "outer": { "inner": [ { "a.b": "AA", "a.c": "AAA" } ] } }'
1283+
- '{ "create": { } }'
1284+
- '{ "name": "B", "outer": { "inner": [ { "a.x.y.z": "BB", "a.z.y.x": "BBB" } ] } }'
1285+
1286+
1287+
- match: { errors: false }
1288+
1289+
- do:
1290+
search:
1291+
index: test
1292+
sort: name
1293+
- match: { hits.total.value: 2 }
1294+
- match: { hits.hits.0._source.name: A }
1295+
- match: { hits.hits.0._source.outer.inner: [{ a.b: AA, a.c: AAA }] }
1296+
- match: { hits.hits.1._source.name: B }
1297+
- match: { hits.hits.1._source.outer.inner: [{ a.x.y.z: BB, a.z.y.x: BBB }] }

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -518,11 +518,7 @@ public final boolean addDynamicMapper(Mapper mapper) {
518518
if (canAddIgnoredField()) {
519519
try {
520520
addIgnoredField(
521-
IgnoredSourceFieldMapper.NameValue.fromContext(
522-
this,
523-
mapper.fullPath(),
524-
XContentDataHelper.encodeToken(parser())
525-
)
521+
IgnoredSourceFieldMapper.NameValue.fromContext(this, mapper.fullPath(), encodeFlattenedToken())
526522
);
527523
} catch (IOException e) {
528524
throw new IllegalArgumentException("failed to parse field [" + mapper.fullPath() + " ]", e);

‎server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@
3434
*/
3535
class DotExpandingXContentParser extends FilterXContentParserWrapper {
3636

37+
static boolean isInstance(XContentParser parser) {
38+
return parser instanceof WrappingParser;
39+
}
40+
3741
private static final class WrappingParser extends FilterXContentParser {
3842

3943
private final ContentPath contentPath;

‎server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -221,8 +221,11 @@ static Tuple<DocumentParserContext, XContentParser> cloneSubContextWithParser(Do
221221
private static Tuple<XContentParserConfiguration, XContentBuilder> cloneSubContextParserConfiguration(DocumentParserContext context)
222222
throws IOException {
223223
XContentParser parser = context.parser();
224+
var oldValue = context.path().isWithinLeafObject();
225+
context.path().setWithinLeafObject(true);
224226
XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent());
225227
builder.copyCurrentStructure(parser);
228+
context.path().setWithinLeafObject(oldValue);
226229

227230
XContentParserConfiguration configuration = XContentParserConfiguration.EMPTY.withRegistry(parser.getXContentRegistry())
228231
.withDeprecationHandler(parser.getDeprecationHandler())
@@ -235,9 +238,17 @@ private static DocumentParserContext cloneDocumentParserContext(
235238
XContentParserConfiguration configuration,
236239
XContentBuilder builder
237240
) throws IOException {
238-
DocumentParserContext subcontext = context.switchParser(
239-
XContentHelper.createParserNotCompressed(configuration, BytesReference.bytes(builder), context.parser().contentType())
241+
XContentParser newParser = XContentHelper.createParserNotCompressed(
242+
configuration,
243+
BytesReference.bytes(builder),
244+
context.parser().contentType()
240245
);
246+
if (DotExpandingXContentParser.isInstance(context.parser())) {
247+
// If we performed dot expanding originally we need to continue to do so when we replace the parser.
248+
newParser = DotExpandingXContentParser.expandDots(newParser, context.path());
249+
}
250+
251+
DocumentParserContext subcontext = context.switchParser(newParser);
241252
subcontext.setRecordedSource(); // Avoids double-storing parts of the source for the same parser subtree.
242253
subcontext.parser().nextToken();
243254
return subcontext;

‎server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1987,6 +1987,136 @@ public void testDisabledObjectWithFlatFields() throws IOException {
19871987
{"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray);
19881988
}
19891989

1990+
public void testRegularObjectWithFlatFields() throws IOException {
1991+
DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
1992+
b.startObject("top").field("type", "object").field("synthetic_source_keep", "all").endObject();
1993+
})).documentMapper();
1994+
1995+
CheckedConsumer<XContentBuilder, IOException> document = b -> {
1996+
b.startObject("top");
1997+
b.field("file.name", "A");
1998+
b.field("file.line", 10);
1999+
b.endObject();
2000+
};
2001+
2002+
var syntheticSource = syntheticSource(documentMapper, document);
2003+
assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource);
2004+
2005+
CheckedConsumer<XContentBuilder, IOException> documentWithArray = b -> {
2006+
b.startArray("top");
2007+
b.startObject();
2008+
b.field("file.name", "A");
2009+
b.field("file.line", 10);
2010+
b.endObject();
2011+
b.startObject();
2012+
b.field("file.name", "B");
2013+
b.field("file.line", 20);
2014+
b.endObject();
2015+
b.endArray();
2016+
};
2017+
2018+
var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray);
2019+
assertEquals("""
2020+
{"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray);
2021+
}
2022+
2023+
public void testRegularObjectWithFlatFieldsInsideAnArray() throws IOException {
2024+
DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
2025+
b.startObject("top");
2026+
b.startObject("properties");
2027+
{
2028+
b.startObject("inner").field("type", "object").field("synthetic_source_keep", "all").endObject();
2029+
}
2030+
b.endObject();
2031+
b.endObject();
2032+
})).documentMapper();
2033+
2034+
CheckedConsumer<XContentBuilder, IOException> document = b -> {
2035+
b.startArray("top");
2036+
b.startObject();
2037+
{
2038+
b.startObject("inner");
2039+
b.field("file.name", "A");
2040+
b.field("file.line", 10);
2041+
b.endObject();
2042+
}
2043+
b.endObject();
2044+
b.endArray();
2045+
};
2046+
2047+
var syntheticSource = syntheticSource(documentMapper, document);
2048+
assertEquals("{\"top\":{\"inner\":{\"file.name\":\"A\",\"file.line\":10}}}", syntheticSource);
2049+
2050+
CheckedConsumer<XContentBuilder, IOException> documentWithArray = b -> {
2051+
b.startArray("top");
2052+
b.startObject();
2053+
{
2054+
b.startObject("inner");
2055+
b.field("file.name", "A");
2056+
b.field("file.line", 10);
2057+
b.endObject();
2058+
}
2059+
b.endObject();
2060+
b.startObject();
2061+
{
2062+
b.startObject("inner");
2063+
b.field("file.name", "B");
2064+
b.field("file.line", 20);
2065+
b.endObject();
2066+
}
2067+
b.endObject();
2068+
b.endArray();
2069+
};
2070+
2071+
var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray);
2072+
assertEquals("""
2073+
{"top":{"inner":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}}""", syntheticSourceWithArray);
2074+
}
2075+
2076+
public void testIgnoredDynamicObjectWithFlatFields() throws IOException {
2077+
var syntheticSource = getSyntheticSourceWithFieldLimit(b -> {
2078+
b.startObject("top");
2079+
b.field("file.name", "A");
2080+
b.field("file.line", 10);
2081+
b.endObject();
2082+
});
2083+
assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource);
2084+
2085+
var syntheticSourceWithArray = getSyntheticSourceWithFieldLimit(b -> {
2086+
b.startArray("top");
2087+
b.startObject();
2088+
b.field("file.name", "A");
2089+
b.field("file.line", 10);
2090+
b.endObject();
2091+
b.startObject();
2092+
b.field("file.name", "B");
2093+
b.field("file.line", 20);
2094+
b.endObject();
2095+
b.endArray();
2096+
});
2097+
assertEquals("""
2098+
{"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray);
2099+
}
2100+
2101+
public void testStoredArrayWithFlatFields() throws IOException {
2102+
DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> {
2103+
b.startObject("outer").startObject("properties");
2104+
{
2105+
b.startObject("inner").field("type", "object").endObject();
2106+
}
2107+
b.endObject().endObject();
2108+
})).documentMapper();
2109+
var syntheticSource = syntheticSource(documentMapper, b -> {
2110+
b.startObject("outer").startArray("inner");
2111+
{
2112+
b.startObject().field("a.b", "a.b").field("a.c", "a.c").endObject();
2113+
}
2114+
b.endArray().endObject();
2115+
});
2116+
assertEquals("""
2117+
{"outer":{"inner":[{"a.b":"a.b","a.c":"a.c"}]}}""", syntheticSource);
2118+
}
2119+
19902120
protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader)
19912121
throws IOException {
19922122
// We exclude ignored source field since in some cases it contains an exact copy of a part of document source.

0 commit comments

Comments
 (0)