diff --git a/docs/changelog/132210.yaml b/docs/changelog/132210.yaml new file mode 100644 index 0000000000000..f8bb674b7652f --- /dev/null +++ b/docs/changelog/132210.yaml @@ -0,0 +1,6 @@ +pr: 132210 +summary: Adding a `merge_type` parameter to the ingest simulate API +area: Ingest Node +type: enhancement +issues: + - 131608 diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index 2cccc8988ff0e..94d9622334c91 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -1931,3 +1931,157 @@ setup: - match: { docs.0.doc._index: "simple-data-stream1" } - match: { docs.0.doc._source.bar: "baz" } - match: { docs.0.doc.error.type: "document_parsing_exception" } + +--- +"Test ingest simulate with mapping addition on subobjects": + + - skip: + features: + - headers + - allowed_warnings + + - do: + indices.put_index_template: + name: subobject-template + body: + index_patterns: subobject-index* + template: + mappings: + properties: + a.b: + type: match_only_text + + - do: + headers: + Content-Type: application/json + simulate.ingest: + body: > + { + "docs": [ + { + "_index": "subobject-index-1", + "_id": "AZgsHA0B41JjTOmNiBKC", + "_source": { + "a.b": "some text" + } + } + ], + "mapping_addition": { + "properties": { + "a.b": { + "type": "keyword" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "subobject-index-1" } + - match: { docs.0.doc._source.a\.b: "some text" } + - match: { docs.0.doc.error.type: "mapper_parsing_exception" } + + # Here we provide a mapping_substitution to the subobject, and make sure that it is applied rather than throwing an + # exception. + - do: + headers: + Content-Type: application/json + simulate.ingest: + merge_type: "template" + body: > + { + "docs": [ + { + "_index": "subobject-index-1", + "_id": "AZgsHA0B41JjTOmNiBKC", + "_source": { + "a.b": "some text" + } + } + ], + "mapping_addition": { + "properties": { + "a.b": { + "type": "keyword" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "subobject-index-1" } + - match: { docs.0.doc._source.a\.b: "some text" } + - not_exists: docs.0.doc.error + + # Now we run the same test but with index_template_substitutions rather than mapping_addition. In this case though, + # the mappings are _substituted_, not merged. That is, the original template and its mappings are completely replaced + # with the new one. So the merge_type has no impact. + - do: + headers: + Content-Type: application/json + simulate.ingest: + merge_type: "index" + body: > + { + "docs": [ + { + "_index": "subobject-index-1", + "_id": "AZgsHA0B41JjTOmNiBKC", + "_source": { + "a.b": "some text" + } + } + ], + "index_template_substitutions": { + "subobject-template": { + "index_patterns": ["subobject-index*"], + "template": { + "mappings": { + "properties": { + "a.b": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "subobject-index-1" } + - match: { docs.0.doc._source.a\.b: "some text" } + - not_exists: docs.0.doc.error + + # This makes sure that we get the same result for merge_type "template" for index_template_substitutions + - do: + headers: + Content-Type: application/json + simulate.ingest: + merge_type: "template" + body: > + { + "docs": [ + { + "_index": "subobject-index-1", + "_id": "AZgsHA0B41JjTOmNiBKC", + "_source": { + "a.b": "some text" + } + } + ], + "index_template_substitutions": { + "subobject-template": { + "index_patterns": ["subobject-index*"], + "template": { + "mappings": { + "properties": { + "a.b": { + "type": "keyword" + } + } + } + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "subobject-index-1" } + - match: { docs.0.doc._source.a\.b: "some text" } + - not_exists: docs.0.doc.error diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/simulate.ingest.json b/rest-api-spec/src/main/resources/rest-api-spec/api/simulate.ingest.json index 91e7153d466da..0c90514692f95 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/simulate.ingest.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/simulate.ingest.json @@ -38,6 +38,11 @@ "pipeline":{ "type":"string", "description":"The pipeline id to preprocess incoming documents with if no pipeline is given for a particular document" + }, + "merge_type":{ + "type":"string", + "description":"The mapping merge type if mapping overrides are being provided in mapping_addition. The allowed values are one of index or template. The index option merges mappings the way they would be merged into an existing index. The template option merges mappings the way they would be merged into a template.", + "default": "index" } }, "body":{ diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index cabe428a7487c..a0242f88629fa 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -61,7 +61,7 @@ public void testMappingValidationIndexExists() { } """; indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -163,7 +163,7 @@ private void assertMappingsUpdatedFromSubstitutions(String indexName, String ind """, XContentType.JSON).id(randomUUID()); { // First we use the original component template, and expect a failure in the second document: - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); @@ -197,7 +197,8 @@ private void assertMappingsUpdatedFromSubstitutions(String indexName, String ind ) ), Map.of(), - Map.of() + Map.of(), + null ); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); @@ -235,7 +236,8 @@ private void assertMappingsUpdatedFromSubstitutions(String indexName, String ind indexTemplateName, Map.of("index_patterns", List.of(indexName), "composed_of", List.of("test-component-template-2")) ), - Map.of() + Map.of(), + null ); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); @@ -258,7 +260,8 @@ private void assertMappingsUpdatedFromSubstitutions(String indexName, String ind Map.of( "_doc", Map.of("dynamic", "strict", "properties", Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text"))) - ) + ), + null ); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); @@ -277,7 +280,7 @@ public void testMappingValidationIndexDoesNotExistsNoTemplate() { * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. */ String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -324,7 +327,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -356,7 +359,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() { indicesAdmin().putTemplate( new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") ).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -410,7 +413,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); { // First, try with no @timestamp to make sure we're picking up data-stream-specific templates - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -437,7 +440,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti } { // Now with @timestamp - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null); bulkRequest.add(new IndexRequest(indexName).source(""" { "@timestamp": "2024-08-27", diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index c5562f187b4ab..f1f892f11ecf1 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -359,6 +359,7 @@ static TransportVersion def(int id) { public static final TransportVersion TRANSPORT_NODE_USAGE_STATS_FOR_THREAD_POOLS_ACTION = def(9_135_0_00); public static final TransportVersion INDEX_TEMPLATE_TRACKING_INFO = def(9_136_0_00); public static final TransportVersion EXTENDED_SNAPSHOT_STATS_IN_NODE_INFO = def(9_137_0_00); + public static final TransportVersion SIMULATE_INGEST_MAPPING_MERGE_TYPE = def(9_138_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index 60485047e70a1..87f71dfce99fd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -103,6 +103,7 @@ public class SimulateBulkRequest extends BulkRequest { private final Map> componentTemplateSubstitutions; private final Map> indexTemplateSubstitutions; private final Map mappingAddition; + private final String mappingMergeType; /** * @param pipelineSubstitutions The pipeline definitions that are to be used in place of any pre-existing pipeline definitions with @@ -118,7 +119,8 @@ public SimulateBulkRequest( Map> pipelineSubstitutions, Map> componentTemplateSubstitutions, Map> indexTemplateSubstitutions, - Map mappingAddition + Map mappingAddition, + String mappingMergeType ) { super(); Objects.requireNonNull(pipelineSubstitutions); @@ -129,6 +131,7 @@ public SimulateBulkRequest( this.componentTemplateSubstitutions = componentTemplateSubstitutions; this.indexTemplateSubstitutions = indexTemplateSubstitutions; this.mappingAddition = mappingAddition; + this.mappingMergeType = mappingMergeType; } @SuppressWarnings("unchecked") @@ -147,6 +150,11 @@ public SimulateBulkRequest(StreamInput in) throws IOException { } else { mappingAddition = Map.of(); } + if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INGEST_MAPPING_MERGE_TYPE)) { + mappingMergeType = in.readOptionalString(); + } else { + mappingMergeType = null; + } } @Override @@ -160,6 +168,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_17_0)) { out.writeGenericValue(mappingAddition); } + if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INGEST_MAPPING_MERGE_TYPE)) { + out.writeOptionalString(mappingMergeType); + } } public Map> getPipelineSubstitutions() { @@ -189,6 +200,10 @@ public Map getMappingAddition() { return mappingAddition; } + public String getMappingMergeType() { + return mappingMergeType; + } + private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) { ComponentTemplate componentTemplate; try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) { @@ -215,7 +230,8 @@ public BulkRequest shallowClone() { pipelineSubstitutions, componentTemplateSubstitutions, indexTemplateSubstitutions, - mappingAddition + mappingAddition, + mappingMergeType ); bulkRequest.setRefreshPolicy(getRefreshPolicy()); bulkRequest.waitForActiveShards(waitForActiveShards()); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 85e9f2c5084de..1cd3a25b37140 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -136,6 +136,7 @@ protected void doInternalExecute( Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); Map indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions(); Map mappingAddition = ((SimulateBulkRequest) bulkRequest).getMappingAddition(); + MapperService.MergeReason mappingMergeReason = getMergeReason(((SimulateBulkRequest) bulkRequest).getMappingMergeType()); for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; @@ -144,7 +145,8 @@ protected void doInternalExecute( componentTemplateSubstitutions, indexTemplateSubstitutions, mappingAddition, - request + request, + mappingMergeReason ); Exception mappingValidationException = validationResult.v2(); responses.set( @@ -170,6 +172,16 @@ protected void doInternalExecute( ); } + private MapperService.MergeReason getMergeReason(String mergeType) { + return Optional.ofNullable(mergeType).map(type -> switch (type) { + case "index" -> MapperService.MergeReason.MAPPING_UPDATE; + case "template" -> MapperService.MergeReason.INDEX_TEMPLATE; + default -> throw new IllegalArgumentException( + "Unsupported merge type '" + mergeType + "'. Valid values are 'index' and 'template'." + ); + }).orElse(MapperService.MergeReason.MAPPING_UPDATE); + } + /** * This creates a temporary index with the mappings of the index in the request, and then attempts to index the source from the request * into it. If there is a mapping exception, that exception is returned. On success the returned exception is null. @@ -182,7 +194,8 @@ private Tuple, Exception> validateMappings( Map componentTemplateSubstitutions, Map indexTemplateSubstitutions, Map mappingAddition, - IndexRequest request + IndexRequest request, + MapperService.MergeReason mappingMergeReason ) { final SourceToParse sourceToParse = new SourceToParse( request.id(), @@ -207,7 +220,7 @@ private Tuple, Exception> validateMappings( IndexMetadata imd = project.getIndexSafe(indexAbstraction.getWriteIndex(request, project)); CompressedXContent mappings = Optional.ofNullable(imd.mapping()).map(MappingMetadata::source).orElse(null); CompressedXContent mergedMappings = mappingAddition == null ? null : mergeMappings(mappings, mappingAddition); - ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, mergedMappings, request, sourceToParse); + ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, mergedMappings, request, sourceToParse, mappingMergeReason); } else { /* * The index did not exist, or we have component template substitutions, so we put together the mappings from existing @@ -265,7 +278,7 @@ private Tuple, Exception> validateMappings( ); CompressedXContent mappings = template.mappings(); CompressedXContent mergedMappings = mergeMappings(mappings, mappingAddition); - ignoredFields = validateUpdatedMappings(mappings, mergedMappings, request, sourceToParse); + ignoredFields = validateUpdatedMappings(mappings, mergedMappings, request, sourceToParse, mappingMergeReason); } else { List matchingTemplates = findV1Templates(simulatedProjectMetadata, request.index(), false); if (matchingTemplates.isEmpty() == false) { @@ -279,7 +292,7 @@ private Tuple, Exception> validateMappings( xContentRegistry ); final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition); - ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse); + ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse, mappingMergeReason); } else { /* * The index matched no templates and had no mapping of its own. If there were component template substitutions @@ -287,7 +300,7 @@ private Tuple, Exception> validateMappings( * and validate. */ final CompressedXContent combinedMappings = mergeMappings(null, mappingAddition); - ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse); + ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse, mappingMergeReason); } } } @@ -305,7 +318,8 @@ private Collection validateUpdatedMappings( @Nullable CompressedXContent originalMappings, @Nullable CompressedXContent updatedMappings, IndexRequest request, - SourceToParse sourceToParse + SourceToParse sourceToParse, + MapperService.MergeReason mappingMergeReason ) throws IOException { Settings dummySettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) @@ -318,14 +332,15 @@ private Collection validateUpdatedMappings( originalIndexMetadataBuilder.putMapping(new MappingMetadata(originalMappings)); } final IndexMetadata originalIndexMetadata = originalIndexMetadataBuilder.build(); - return validateUpdatedMappingsFromIndexMetadata(originalIndexMetadata, updatedMappings, request, sourceToParse); + return validateUpdatedMappingsFromIndexMetadata(originalIndexMetadata, updatedMappings, request, sourceToParse, mappingMergeReason); } private Collection validateUpdatedMappingsFromIndexMetadata( IndexMetadata originalIndexMetadata, @Nullable CompressedXContent updatedMappings, IndexRequest request, - SourceToParse sourceToParse + SourceToParse sourceToParse, + MapperService.MergeReason mappingMergeReason ) throws IOException { if (updatedMappings == null) { return List.of(); // no validation to do @@ -335,7 +350,7 @@ private Collection validateUpdatedMappingsFromIndexMetadata( .putMapping(new MappingMetadata(updatedMappings)) .build(); Engine.Index result = indicesService.withTempIndexService(originalIndexMetadata, indexService -> { - indexService.mapperService().merge(updatedIndexMetadata, MapperService.MergeReason.MAPPING_UPDATE); + indexService.mapperService().merge(updatedIndexMetadata, mappingMergeReason); return IndexShard.prepareIndex( indexService.mapperService(), sourceToParse, diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java index 978b6d1c3a92d..32296da94520a 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java @@ -85,11 +85,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC "index_template_substitutions" ); Object mappingAddition = sourceMap.remove("mapping_addition"); + String mappingMergeType = request.param("merge_type"); SimulateBulkRequest bulkRequest = new SimulateBulkRequest( pipelineSubstitutions == null ? Map.of() : pipelineSubstitutions, componentTemplateSubstitutions == null ? Map.of() : componentTemplateSubstitutions, indexTemplateSubstitutions == null ? Map.of() : indexTemplateSubstitutions, - mappingAddition == null ? Map.of() : Map.of("_doc", mappingAddition) + mappingAddition == null ? Map.of() : Map.of("_doc", mappingAddition), + mappingMergeType ); BytesReference transformedData = convertToBulkRequestXContentBytes(sourceMap); bulkRequest.add( diff --git a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java index 1e651791eb18a..30f0ca0ee338a 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java @@ -33,7 +33,8 @@ public void testSerialization() throws Exception { getMapOrEmpty(getTestPipelineSubstitutions()), getMapOrEmpty(getTestComponentTemplateSubstitutions()), getMapOrEmpty(getTestIndexTemplateSubstitutions()), - getMapOrEmpty(getTestMappingAddition()) + getMapOrEmpty(getTestMappingAddition()), + getTestMappingMergeType() ); } @@ -52,7 +53,8 @@ public void testNullsNotAllowed() { null, getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), - getTestMappingAddition() + getTestMappingAddition(), + getTestMappingMergeType() ) ); assertThrows( @@ -61,12 +63,19 @@ public void testNullsNotAllowed() { getTestPipelineSubstitutions(), null, getTestComponentTemplateSubstitutions(), - getTestMappingAddition() + getTestMappingAddition(), + getTestMappingMergeType() ) ); assertThrows( NullPointerException.class, - () -> new SimulateBulkRequest(getTestPipelineSubstitutions(), getTestPipelineSubstitutions(), null, getTestMappingAddition()) + () -> new SimulateBulkRequest( + getTestPipelineSubstitutions(), + getTestPipelineSubstitutions(), + null, + getTestMappingAddition(), + getTestMappingMergeType() + ) ); assertThrows( NullPointerException.class, @@ -74,7 +83,8 @@ public void testNullsNotAllowed() { getTestPipelineSubstitutions(), getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), - null + null, + getTestMappingMergeType() ) ); } @@ -83,13 +93,15 @@ private void testSerialization( Map> pipelineSubstitutions, Map> componentTemplateSubstitutions, Map> indexTemplateSubstitutions, - Map mappingAddition + Map mappingAddition, + String mappingMergeType ) throws IOException { SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( pipelineSubstitutions, componentTemplateSubstitutions, indexTemplateSubstitutions, - mappingAddition + mappingAddition, + mappingMergeType ); /* * Note: SimulateBulkRequest does not implement equals or hashCode, so we can't test serialization in the usual way for a @@ -97,11 +109,18 @@ private void testSerialization( */ SimulateBulkRequest copy = copyWriteable(simulateBulkRequest, null, SimulateBulkRequest::new); assertThat(copy.getPipelineSubstitutions(), equalTo(simulateBulkRequest.getPipelineSubstitutions())); + assertThat(copy.getMappingMergeType(), equalTo(simulateBulkRequest.getMappingMergeType())); } @SuppressWarnings({ "unchecked", "rawtypes" }) public void testGetComponentTemplateSubstitutions() throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of(), + Map.of(), + Map.of(), + getTestMappingMergeType() + ); assertThat(simulateBulkRequest.getComponentTemplateSubstitutions(), equalTo(Map.of())); String substituteComponentTemplatesString = """ { @@ -135,7 +154,13 @@ public void testGetComponentTemplateSubstitutions() throws IOException { XContentType.JSON ).v2(); Map> substituteComponentTemplates = (Map>) tempMap; - simulateBulkRequest = new SimulateBulkRequest(Map.of(), substituteComponentTemplates, Map.of(), Map.of()); + simulateBulkRequest = new SimulateBulkRequest( + Map.of(), + substituteComponentTemplates, + Map.of(), + Map.of(), + getTestMappingMergeType() + ); Map componentTemplateSubstitutions = simulateBulkRequest.getComponentTemplateSubstitutions(); assertThat(componentTemplateSubstitutions.size(), equalTo(2)); assertThat( @@ -160,7 +185,13 @@ public void testGetComponentTemplateSubstitutions() throws IOException { } public void testGetIndexTemplateSubstitutions() throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of(), + Map.of(), + Map.of(), + getTestMappingMergeType() + ); assertThat(simulateBulkRequest.getIndexTemplateSubstitutions(), equalTo(Map.of())); String substituteIndexTemplatesString = """ { @@ -196,7 +227,7 @@ public void testGetIndexTemplateSubstitutions() throws IOException { randomBoolean(), XContentType.JSON ).v2(); - simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), substituteIndexTemplates, Map.of()); + simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), substituteIndexTemplates, Map.of(), getTestMappingMergeType()); Map indexTemplateSubstitutions = simulateBulkRequest.getIndexTemplateSubstitutions(); assertThat(indexTemplateSubstitutions.size(), equalTo(2)); assertThat( @@ -222,7 +253,8 @@ public void testShallowClone() throws IOException { getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), getTestIndexTemplateSubstitutions(), - getTestMappingAddition() + getTestMappingAddition(), + getTestMappingMergeType() ); simulateBulkRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); simulateBulkRequest.waitForActiveShards(randomIntBetween(1, 10)); @@ -308,4 +340,12 @@ private static Map getTestMappingAddition() { ) ); } + + private static String getTestMappingMergeType() { + if (randomBoolean()) { + return null; + } else { + return randomFrom("index", "template"); + } + } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index eab44f1c56b16..8f9e134b98a8e 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -138,7 +138,7 @@ public void tearDown() throws Exception { public void testIndexData() throws IOException { Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null); int bulkItemCount = randomIntBetween(0, 200); for (int i = 0; i < bulkItemCount; i++) { Map source = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(5)); @@ -225,7 +225,7 @@ public void testIndexDataWithValidation() throws IOException { * Here we only add a mapping_addition because if there is no mapping at all TransportSimulateBulkAction skips mapping validation * altogether, and we need it to run for this test to pass. */ - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of("_doc", Map.of("dynamic", "strict"))); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of("_doc", Map.of("dynamic", "strict")), null); int bulkItemCount = randomIntBetween(0, 200); Map indicesMap = new HashMap<>(); Map v1Templates = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java index 9cd3a74d9e73a..b75111b83884a 100644 --- a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java @@ -71,7 +71,7 @@ public void testGetPipeline() { ingestService.innerUpdatePipelines(projectId, ingestMetadata); { // First we make sure that if there are no substitutions that we get our original pipeline back: - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of(), Map.of(), null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline = simulateIngestService.getPipeline(projectId, "pipeline1"); assertThat(pipeline.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1")))); @@ -89,7 +89,7 @@ public void testGetPipeline() { ); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, Map.of(), Map.of(), Map.of()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, Map.of(), Map.of(), Map.of(), null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline(projectId, "pipeline1"); assertThat( @@ -109,7 +109,7 @@ public void testGetPipeline() { */ Map> pipelineSubstitutions = new HashMap<>(); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, Map.of(), Map.of(), Map.of()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, Map.of(), Map.of(), Map.of(), null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline(projectId, "pipeline1"); assertThat(pipeline1.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1"))));