Skip to content

Commit 06eb072

Browse files
kkrik-eskkrik-es
authored
Use flattened names in ignored source (#115822)
* Use flattened names in ignored source * spotless * fix rest compat * fix unittests * expand dots
1 parent 853f51f commit 06eb072

File tree

7 files changed

+211
-17
lines changed

7 files changed

+211
-17
lines changed

‎rest-api-spec/build.gradle‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,10 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
5959
task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling with vector_operations_count")
6060
task.skipTest("indices.sort/10_basic/Index Sort", "warning does not exist for compatibility")
6161
task.skipTest("search/330_fetch_fields/Test search rewrite", "warning does not exist for compatibility")
62+
task.skipTest("indices.create/20_synthetic_source/object with dynamic override", "temporary until backported")
63+
task.skipTest("indices.create/20_synthetic_source/object with unmapped fields", "temporary until backported")
64+
task.skipTest("indices.create/20_synthetic_source/empty object with unmapped fields", "temporary until backported")
65+
task.skipTest("indices.create/20_synthetic_source/nested object with unmapped fields", "temporary until backported")
6266
task.skipTest("indices.create/21_synthetic_source_stored/object param - nested object with stored array", "temporary until backported")
6367
task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
6468
})

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
object with unmapped fields:
22
- requires:
3-
cluster_features: ["mapper.track_ignored_source"]
3+
cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"]
44
reason: requires tracking ignored source
55

66
- do:
@@ -41,13 +41,13 @@ object with unmapped fields:
4141
- match: { hits.hits.0._source.some_string: AaAa }
4242
- match: { hits.hits.0._source.some_int: 1000 }
4343
- match: { hits.hits.0._source.some_double: 123.456789 }
44-
- match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA }
44+
- match: { hits.hits.0._source.a: { very.deeply.nested.field: AAAA } }
4545
- match: { hits.hits.0._source.some_bool: true }
4646
- match: { hits.hits.1._source.name: bbbb }
4747
- match: { hits.hits.1._source.some_string: BbBb }
4848
- match: { hits.hits.1._source.some_int: 2000 }
4949
- match: { hits.hits.1._source.some_double: 321.987654 }
50-
- match: { hits.hits.1._source.a.very.deeply.nested.field: BBBB }
50+
- match: { hits.hits.1._source.a: { very.deeply.nested.field: BBBB } }
5151

5252

5353
---
@@ -100,7 +100,7 @@ unmapped arrays:
100100
---
101101
nested object with unmapped fields:
102102
- requires:
103-
cluster_features: ["mapper.track_ignored_source"]
103+
cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"]
104104
reason: requires tracking ignored source
105105

106106
- do:
@@ -143,16 +143,16 @@ nested object with unmapped fields:
143143
- match: { hits.total.value: 2 }
144144
- match: { hits.hits.0._source.path.to.name: aaaa }
145145
- match: { hits.hits.0._source.path.to.surname: AaAa }
146-
- match: { hits.hits.0._source.path.some.other.name: AaAaAa }
146+
- match: { hits.hits.0._source.path.some.other\.name: AaAaAa }
147147
- match: { hits.hits.1._source.path.to.name: bbbb }
148148
- match: { hits.hits.1._source.path.to.surname: BbBb }
149-
- match: { hits.hits.1._source.path.some.other.name: BbBbBb }
149+
- match: { hits.hits.1._source.path.some.other\.name: BbBbBb }
150150

151151

152152
---
153153
empty object with unmapped fields:
154154
- requires:
155-
cluster_features: ["mapper.track_ignored_source"]
155+
cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"]
156156
reason: requires tracking ignored source
157157

158158
- do:
@@ -191,7 +191,7 @@ empty object with unmapped fields:
191191

192192
- match: { hits.total.value: 1 }
193193
- match: { hits.hits.0._source.path.to.surname: AaAa }
194-
- match: { hits.hits.0._source.path.some.other.name: AaAaAa }
194+
- match: { hits.hits.0._source.path.some.other\.name: AaAaAa }
195195

196196

197197
---
@@ -434,7 +434,7 @@ mixed disabled and enabled objects:
434434
---
435435
object with dynamic override:
436436
- requires:
437-
cluster_features: ["mapper.ignored_source.dont_expand_dots"]
437+
cluster_features: ["mapper.ignored_source.dont_expand_dots", "mapper.bwc_workaround_9_0"]
438438
reason: requires tracking ignored source
439439

440440
- do:
@@ -475,7 +475,7 @@ object with dynamic override:
475475
- match: { hits.hits.0._source.path_no.to: { a.very.deeply.nested.field: A } }
476476
- match: { hits.hits.0._source.path_runtime.name: bar }
477477
- match: { hits.hits.0._source.path_runtime.some_int: 20 }
478-
- match: { hits.hits.0._source.path_runtime.to.a.very.deeply.nested.field: B }
478+
- match: { hits.hits.0._source.path_runtime.to: { a.very.deeply.nested.field: B } }
479479

480480

481481
---

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1249,3 +1249,52 @@ index param - nested object with stored array:
12491249
- match: { hits.hits.1._source.nested.0.b.1.c: 300 }
12501250
- match: { hits.hits.1._source.nested.1.b.0.c: 40 }
12511251
- match: { hits.hits.1._source.nested.1.b.1.c: 400 }
1252+
1253+
1254+
---
1255+
index param - flattened fields:
1256+
- requires:
1257+
cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"]
1258+
reason: requires keeping array source
1259+
1260+
- do:
1261+
indices.create:
1262+
index: test
1263+
body:
1264+
settings:
1265+
index:
1266+
mapping:
1267+
synthetic_source_keep: arrays
1268+
mappings:
1269+
_source:
1270+
mode: synthetic
1271+
properties:
1272+
name:
1273+
type: keyword
1274+
outer:
1275+
properties:
1276+
inner:
1277+
type: object
1278+
1279+
- do:
1280+
bulk:
1281+
index: test
1282+
refresh: true
1283+
body:
1284+
- '{ "create": { } }'
1285+
- '{ "name": "A", "outer": { "inner": [ { "a.b": "AA", "a.c": "AAA" } ] } }'
1286+
- '{ "create": { } }'
1287+
- '{ "name": "B", "outer": { "inner": [ { "a.x.y.z": "BB", "a.z.y.x": "BBB" } ] } }'
1288+
1289+
1290+
- match: { errors: false }
1291+
1292+
- do:
1293+
search:
1294+
index: test
1295+
sort: name
1296+
- match: { hits.total.value: 2 }
1297+
- match: { hits.hits.0._source.name: A }
1298+
- match: { hits.hits.0._source.outer.inner: [{ a.b: AA, a.c: AAA }] }
1299+
- match: { hits.hits.1._source.name: B }
1300+
- match: { hits.hits.1._source.outer.inner: [{ a.x.y.z: BB, a.z.y.x: BBB }] }

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -528,11 +528,7 @@ public final boolean addDynamicMapper(Mapper mapper) {
528528
if (canAddIgnoredField()) {
529529
try {
530530
addIgnoredField(
531-
IgnoredSourceFieldMapper.NameValue.fromContext(
532-
this,
533-
mapper.fullPath(),
534-
XContentDataHelper.encodeToken(parser())
535-
)
531+
IgnoredSourceFieldMapper.NameValue.fromContext(this, mapper.fullPath(), encodeFlattenedToken())
536532
);
537533
} catch (IOException e) {
538534
throw new IllegalArgumentException("failed to parse field [" + mapper.fullPath() + " ]", e);

‎server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@
3434
*/
3535
class DotExpandingXContentParser extends FilterXContentParserWrapper {
3636

37+
static boolean isInstance(XContentParser parser) {
38+
return parser instanceof WrappingParser;
39+
}
40+
3741
private static final class WrappingParser extends FilterXContentParser {
3842

3943
private final ContentPath contentPath;

‎server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -221,8 +221,11 @@ static Tuple<DocumentParserContext, XContentParser> cloneSubContextWithParser(Do
221221
private static Tuple<XContentParserConfiguration, XContentBuilder> cloneSubContextParserConfiguration(DocumentParserContext context)
222222
throws IOException {
223223
XContentParser parser = context.parser();
224+
var oldValue = context.path().isWithinLeafObject();
225+
context.path().setWithinLeafObject(true);
224226
XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent());
225227
builder.copyCurrentStructure(parser);
228+
context.path().setWithinLeafObject(oldValue);
226229

227230
XContentParserConfiguration configuration = XContentParserConfiguration.EMPTY.withRegistry(parser.getXContentRegistry())
228231
.withDeprecationHandler(parser.getDeprecationHandler())
@@ -235,9 +238,17 @@ private static DocumentParserContext cloneDocumentParserContext(
235238
XContentParserConfiguration configuration,
236239
XContentBuilder builder
237240
) throws IOException {
238-
DocumentParserContext subcontext = context.switchParser(
239-
XContentHelper.createParserNotCompressed(configuration, BytesReference.bytes(builder), context.parser().contentType())
241+
XContentParser newParser = XContentHelper.createParserNotCompressed(
242+
configuration,
243+
BytesReference.bytes(builder),
244+
context.parser().contentType()
240245
);
246+
if (DotExpandingXContentParser.isInstance(context.parser())) {
247+
// If we performed dot expanding originally we need to continue to do so when we replace the parser.
248+
newParser = DotExpandingXContentParser.expandDots(newParser, context.path());
249+
}
250+
251+
DocumentParserContext subcontext = context.switchParser(newParser);
241252
subcontext.setRecordedSource(); // Avoids double-storing parts of the source for the same parser subtree.
242253
subcontext.parser().nextToken();
243254
return subcontext;

‎server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2075,6 +2075,136 @@ public void testDisabledObjectWithFlatFields() throws IOException {
20752075
{"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray);
20762076
}
20772077

2078+
public void testRegularObjectWithFlatFields() throws IOException {
2079+
DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
2080+
b.startObject("top").field("type", "object").field("synthetic_source_keep", "all").endObject();
2081+
})).documentMapper();
2082+
2083+
CheckedConsumer<XContentBuilder, IOException> document = b -> {
2084+
b.startObject("top");
2085+
b.field("file.name", "A");
2086+
b.field("file.line", 10);
2087+
b.endObject();
2088+
};
2089+
2090+
var syntheticSource = syntheticSource(documentMapper, document);
2091+
assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource);
2092+
2093+
CheckedConsumer<XContentBuilder, IOException> documentWithArray = b -> {
2094+
b.startArray("top");
2095+
b.startObject();
2096+
b.field("file.name", "A");
2097+
b.field("file.line", 10);
2098+
b.endObject();
2099+
b.startObject();
2100+
b.field("file.name", "B");
2101+
b.field("file.line", 20);
2102+
b.endObject();
2103+
b.endArray();
2104+
};
2105+
2106+
var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray);
2107+
assertEquals("""
2108+
{"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray);
2109+
}
2110+
2111+
public void testRegularObjectWithFlatFieldsInsideAnArray() throws IOException {
2112+
DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
2113+
b.startObject("top");
2114+
b.startObject("properties");
2115+
{
2116+
b.startObject("inner").field("type", "object").field("synthetic_source_keep", "all").endObject();
2117+
}
2118+
b.endObject();
2119+
b.endObject();
2120+
})).documentMapper();
2121+
2122+
CheckedConsumer<XContentBuilder, IOException> document = b -> {
2123+
b.startArray("top");
2124+
b.startObject();
2125+
{
2126+
b.startObject("inner");
2127+
b.field("file.name", "A");
2128+
b.field("file.line", 10);
2129+
b.endObject();
2130+
}
2131+
b.endObject();
2132+
b.endArray();
2133+
};
2134+
2135+
var syntheticSource = syntheticSource(documentMapper, document);
2136+
assertEquals("{\"top\":{\"inner\":{\"file.name\":\"A\",\"file.line\":10}}}", syntheticSource);
2137+
2138+
CheckedConsumer<XContentBuilder, IOException> documentWithArray = b -> {
2139+
b.startArray("top");
2140+
b.startObject();
2141+
{
2142+
b.startObject("inner");
2143+
b.field("file.name", "A");
2144+
b.field("file.line", 10);
2145+
b.endObject();
2146+
}
2147+
b.endObject();
2148+
b.startObject();
2149+
{
2150+
b.startObject("inner");
2151+
b.field("file.name", "B");
2152+
b.field("file.line", 20);
2153+
b.endObject();
2154+
}
2155+
b.endObject();
2156+
b.endArray();
2157+
};
2158+
2159+
var syntheticSourceWithArray = syntheticSource(documentMapper, documentWithArray);
2160+
assertEquals("""
2161+
{"top":{"inner":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}}""", syntheticSourceWithArray);
2162+
}
2163+
2164+
public void testIgnoredDynamicObjectWithFlatFields() throws IOException {
2165+
var syntheticSource = getSyntheticSourceWithFieldLimit(b -> {
2166+
b.startObject("top");
2167+
b.field("file.name", "A");
2168+
b.field("file.line", 10);
2169+
b.endObject();
2170+
});
2171+
assertEquals("{\"top\":{\"file.name\":\"A\",\"file.line\":10}}", syntheticSource);
2172+
2173+
var syntheticSourceWithArray = getSyntheticSourceWithFieldLimit(b -> {
2174+
b.startArray("top");
2175+
b.startObject();
2176+
b.field("file.name", "A");
2177+
b.field("file.line", 10);
2178+
b.endObject();
2179+
b.startObject();
2180+
b.field("file.name", "B");
2181+
b.field("file.line", 20);
2182+
b.endObject();
2183+
b.endArray();
2184+
});
2185+
assertEquals("""
2186+
{"top":[{"file.name":"A","file.line":10},{"file.name":"B","file.line":20}]}""", syntheticSourceWithArray);
2187+
}
2188+
2189+
public void testStoredArrayWithFlatFields() throws IOException {
2190+
DocumentMapper documentMapper = createMapperServiceWithStoredArraySource(syntheticSourceMapping(b -> {
2191+
b.startObject("outer").startObject("properties");
2192+
{
2193+
b.startObject("inner").field("type", "object").endObject();
2194+
}
2195+
b.endObject().endObject();
2196+
})).documentMapper();
2197+
var syntheticSource = syntheticSource(documentMapper, b -> {
2198+
b.startObject("outer").startArray("inner");
2199+
{
2200+
b.startObject().field("a.b", "a.b").field("a.c", "a.c").endObject();
2201+
}
2202+
b.endArray().endObject();
2203+
});
2204+
assertEquals("""
2205+
{"outer":{"inner":[{"a.b":"a.b","a.c":"a.c"}]}}""", syntheticSource);
2206+
}
2207+
20782208
protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader)
20792209
throws IOException {
20802210
// We exclude ignored source field since in some cases it contains an exact copy of a part of document source.

0 commit comments

Comments
 (0)