Skip to content

Commit e223df1

Browse files
masseykemasseyke
authored
Adding a merge_type parameter to the ingest simulate API (#132210)
1 parent 75507f7 commit e223df1

File tree

11 files changed

+282
-40
lines changed

11 files changed

+282
-40
lines changed

‎docs/changelog/132210.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132210
2+
summary: Adding a `merge_type` parameter to the ingest simulate API
3+
area: Ingest Node
4+
type: enhancement
5+
issues:
6+
- 131608

‎qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1931,3 +1931,157 @@ setup:
19311931
- match: { docs.0.doc._index: "simple-data-stream1" }
19321932
- match: { docs.0.doc._source.bar: "baz" }
19331933
- match: { docs.0.doc.error.type: "document_parsing_exception" }
1934+
1935+
---
1936+
"Test ingest simulate with mapping addition on subobjects":
1937+
1938+
- skip:
1939+
features:
1940+
- headers
1941+
- allowed_warnings
1942+
1943+
- do:
1944+
indices.put_index_template:
1945+
name: subobject-template
1946+
body:
1947+
index_patterns: subobject-index*
1948+
template:
1949+
mappings:
1950+
properties:
1951+
a.b:
1952+
type: match_only_text
1953+
1954+
- do:
1955+
headers:
1956+
Content-Type: application/json
1957+
simulate.ingest:
1958+
body: >
1959+
{
1960+
"docs": [
1961+
{
1962+
"_index": "subobject-index-1",
1963+
"_id": "AZgsHA0B41JjTOmNiBKC",
1964+
"_source": {
1965+
"a.b": "some text"
1966+
}
1967+
}
1968+
],
1969+
"mapping_addition": {
1970+
"properties": {
1971+
"a.b": {
1972+
"type": "keyword"
1973+
}
1974+
}
1975+
}
1976+
}
1977+
- length: { docs: 1 }
1978+
- match: { docs.0.doc._index: "subobject-index-1" }
1979+
- match: { docs.0.doc._source.a\.b: "some text" }
1980+
- match: { docs.0.doc.error.type: "mapper_parsing_exception" }
1981+
1982+
# Here we provide a mapping_substitution to the subobject, and make sure that it is applied rather than throwing an
1983+
# exception.
1984+
- do:
1985+
headers:
1986+
Content-Type: application/json
1987+
simulate.ingest:
1988+
merge_type: "template"
1989+
body: >
1990+
{
1991+
"docs": [
1992+
{
1993+
"_index": "subobject-index-1",
1994+
"_id": "AZgsHA0B41JjTOmNiBKC",
1995+
"_source": {
1996+
"a.b": "some text"
1997+
}
1998+
}
1999+
],
2000+
"mapping_addition": {
2001+
"properties": {
2002+
"a.b": {
2003+
"type": "keyword"
2004+
}
2005+
}
2006+
}
2007+
}
2008+
- length: { docs: 1 }
2009+
- match: { docs.0.doc._index: "subobject-index-1" }
2010+
- match: { docs.0.doc._source.a\.b: "some text" }
2011+
- not_exists: docs.0.doc.error
2012+
2013+
# Now we run the same test but with index_template_substitutions rather than mapping_addition. In this case though,
2014+
# the mappings are _substituted_, not merged. That is, the original template and its mappings are completely replaced
2015+
# with the new one. So the merge_type has no impact.
2016+
- do:
2017+
headers:
2018+
Content-Type: application/json
2019+
simulate.ingest:
2020+
merge_type: "index"
2021+
body: >
2022+
{
2023+
"docs": [
2024+
{
2025+
"_index": "subobject-index-1",
2026+
"_id": "AZgsHA0B41JjTOmNiBKC",
2027+
"_source": {
2028+
"a.b": "some text"
2029+
}
2030+
}
2031+
],
2032+
"index_template_substitutions": {
2033+
"subobject-template": {
2034+
"index_patterns": ["subobject-index*"],
2035+
"template": {
2036+
"mappings": {
2037+
"properties": {
2038+
"a.b": {
2039+
"type": "keyword"
2040+
}
2041+
}
2042+
}
2043+
}
2044+
}
2045+
}
2046+
}
2047+
- length: { docs: 1 }
2048+
- match: { docs.0.doc._index: "subobject-index-1" }
2049+
- match: { docs.0.doc._source.a\.b: "some text" }
2050+
- not_exists: docs.0.doc.error
2051+
2052+
# This makes sure that we get the same result for merge_type "template" for index_template_substitutions
2053+
- do:
2054+
headers:
2055+
Content-Type: application/json
2056+
simulate.ingest:
2057+
merge_type: "template"
2058+
body: >
2059+
{
2060+
"docs": [
2061+
{
2062+
"_index": "subobject-index-1",
2063+
"_id": "AZgsHA0B41JjTOmNiBKC",
2064+
"_source": {
2065+
"a.b": "some text"
2066+
}
2067+
}
2068+
],
2069+
"index_template_substitutions": {
2070+
"subobject-template": {
2071+
"index_patterns": ["subobject-index*"],
2072+
"template": {
2073+
"mappings": {
2074+
"properties": {
2075+
"a.b": {
2076+
"type": "keyword"
2077+
}
2078+
}
2079+
}
2080+
}
2081+
}
2082+
}
2083+
}
2084+
- length: { docs: 1 }
2085+
- match: { docs.0.doc._index: "subobject-index-1" }
2086+
- match: { docs.0.doc._source.a\.b: "some text" }
2087+
- not_exists: docs.0.doc.error

‎rest-api-spec/src/main/resources/rest-api-spec/api/simulate.ingest.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@
3838
"pipeline":{
3939
"type":"string",
4040
"description":"The pipeline id to preprocess incoming documents with if no pipeline is given for a particular document"
41+
},
42+
"merge_type":{
43+
"type":"string",
44+
"description":"The mapping merge type if mapping overrides are being provided in mapping_addition. The allowed values are one of index or template. The index option merges mappings the way they would be merged into an existing index. The template option merges mappings the way they would be merged into a template.",
45+
"default": "index"
4146
}
4247
},
4348
"body":{

‎server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ public void testMappingValidationIndexExists() {
6161
}
6262
""";
6363
indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet();
64-
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
64+
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null);
6565
bulkRequest.add(new IndexRequest(indexName).source("""
6666
{
6767
"foo1": "baz"
@@ -163,7 +163,7 @@ private void assertMappingsUpdatedFromSubstitutions(String indexName, String ind
163163
""", XContentType.JSON).id(randomUUID());
164164
{
165165
// First we use the original component template, and expect a failure in the second document:
166-
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
166+
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null);
167167
bulkRequest.add(indexRequest1);
168168
bulkRequest.add(indexRequest2);
169169
BulkResponse response = client().execute(new ActionType<BulkResponse>(SimulateBulkAction.NAME), bulkRequest).actionGet();
@@ -197,7 +197,8 @@ private void assertMappingsUpdatedFromSubstitutions(String indexName, String ind
197197
)
198198
),
199199
Map.of(),
200-
Map.of()
200+
Map.of(),
201+
null
201202
);
202203
bulkRequest.add(indexRequest1);
203204
bulkRequest.add(indexRequest2);
@@ -235,7 +236,8 @@ private void assertMappingsUpdatedFromSubstitutions(String indexName, String ind
235236
indexTemplateName,
236237
Map.of("index_patterns", List.of(indexName), "composed_of", List.of("test-component-template-2"))
237238
),
238-
Map.of()
239+
Map.of(),
240+
null
239241
);
240242
bulkRequest.add(indexRequest1);
241243
bulkRequest.add(indexRequest2);
@@ -258,7 +260,8 @@ private void assertMappingsUpdatedFromSubstitutions(String indexName, String ind
258260
Map.of(
259261
"_doc",
260262
Map.of("dynamic", "strict", "properties", Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text")))
261-
)
263+
),
264+
null
262265
);
263266
bulkRequest.add(indexRequest1);
264267
bulkRequest.add(indexRequest2);
@@ -277,7 +280,7 @@ public void testMappingValidationIndexDoesNotExistsNoTemplate() {
277280
* mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure.
278281
*/
279282
String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
280-
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
283+
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null);
281284
bulkRequest.add(new IndexRequest(indexName).source("""
282285
{
283286
"foo1": "baz"
@@ -324,7 +327,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti
324327
request.indexTemplate(composableIndexTemplate);
325328

326329
client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet();
327-
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
330+
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null);
328331
bulkRequest.add(new IndexRequest(indexName).source("""
329332
{
330333
"foo1": "baz"
@@ -356,7 +359,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() {
356359
indicesAdmin().putTemplate(
357360
new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer")
358361
).actionGet();
359-
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
362+
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null);
360363
bulkRequest.add(new IndexRequest(indexName).source("""
361364
{
362365
"foo1": "baz"
@@ -410,7 +413,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti
410413
client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet();
411414
{
412415
// First, try with no @timestamp to make sure we're picking up data-stream-specific templates
413-
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
416+
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null);
414417
bulkRequest.add(new IndexRequest(indexName).source("""
415418
{
416419
"foo1": "baz"
@@ -437,7 +440,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti
437440
}
438441
{
439442
// Now with @timestamp
440-
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of());
443+
BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null);
441444
bulkRequest.add(new IndexRequest(indexName).source("""
442445
{
443446
"@timestamp": "2024-08-27",

‎server/src/main/java/org/elasticsearch/TransportVersions.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,7 @@ static TransportVersion def(int id) {
359359
public static final TransportVersion TRANSPORT_NODE_USAGE_STATS_FOR_THREAD_POOLS_ACTION = def(9_135_0_00);
360360
public static final TransportVersion INDEX_TEMPLATE_TRACKING_INFO = def(9_136_0_00);
361361
public static final TransportVersion EXTENDED_SNAPSHOT_STATS_IN_NODE_INFO = def(9_137_0_00);
362+
public static final TransportVersion SIMULATE_INGEST_MAPPING_MERGE_TYPE = def(9_138_0_00);
362363

363364
/*
364365
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ public class SimulateBulkRequest extends BulkRequest {
103103
private final Map<String, Map<String, Object>> componentTemplateSubstitutions;
104104
private final Map<String, Map<String, Object>> indexTemplateSubstitutions;
105105
private final Map<String, Object> mappingAddition;
106+
private final String mappingMergeType;
106107

107108
/**
108109
* @param pipelineSubstitutions The pipeline definitions that are to be used in place of any pre-existing pipeline definitions with
@@ -118,7 +119,8 @@ public SimulateBulkRequest(
118119
Map<String, Map<String, Object>> pipelineSubstitutions,
119120
Map<String, Map<String, Object>> componentTemplateSubstitutions,
120121
Map<String, Map<String, Object>> indexTemplateSubstitutions,
121-
Map<String, Object> mappingAddition
122+
Map<String, Object> mappingAddition,
123+
String mappingMergeType
122124
) {
123125
super();
124126
Objects.requireNonNull(pipelineSubstitutions);
@@ -129,6 +131,7 @@ public SimulateBulkRequest(
129131
this.componentTemplateSubstitutions = componentTemplateSubstitutions;
130132
this.indexTemplateSubstitutions = indexTemplateSubstitutions;
131133
this.mappingAddition = mappingAddition;
134+
this.mappingMergeType = mappingMergeType;
132135
}
133136

134137
@SuppressWarnings("unchecked")
@@ -147,6 +150,11 @@ public SimulateBulkRequest(StreamInput in) throws IOException {
147150
} else {
148151
mappingAddition = Map.of();
149152
}
153+
if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INGEST_MAPPING_MERGE_TYPE)) {
154+
mappingMergeType = in.readOptionalString();
155+
} else {
156+
mappingMergeType = null;
157+
}
150158
}
151159

152160
@Override
@@ -160,6 +168,9 @@ public void writeTo(StreamOutput out) throws IOException {
160168
if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_17_0)) {
161169
out.writeGenericValue(mappingAddition);
162170
}
171+
if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INGEST_MAPPING_MERGE_TYPE)) {
172+
out.writeOptionalString(mappingMergeType);
173+
}
163174
}
164175

165176
public Map<String, Map<String, Object>> getPipelineSubstitutions() {
@@ -189,6 +200,10 @@ public Map<String, Object> getMappingAddition() {
189200
return mappingAddition;
190201
}
191202

203+
public String getMappingMergeType() {
204+
return mappingMergeType;
205+
}
206+
192207
private static ComponentTemplate convertRawTemplateToComponentTemplate(Map<String, Object> rawTemplate) {
193208
ComponentTemplate componentTemplate;
194209
try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) {
@@ -215,7 +230,8 @@ public BulkRequest shallowClone() {
215230
pipelineSubstitutions,
216231
componentTemplateSubstitutions,
217232
indexTemplateSubstitutions,
218-
mappingAddition
233+
mappingAddition,
234+
mappingMergeType
219235
);
220236
bulkRequest.setRefreshPolicy(getRefreshPolicy());
221237
bulkRequest.waitForActiveShards(waitForActiveShards());

0 commit comments

Comments
 (0)