From 63b5c579766d75d11a2e467f1497b777fcf2357f Mon Sep 17 00:00:00 2001 From: Keith Massey Date: Wed, 30 Jul 2025 15:03:48 -0500 Subject: [PATCH] Simulate ingest API uses existing index mapping when mapping_addition is given (#132101) (cherry picked from commit 593f48fe1b5767dcb40a8ad27f2ee1d23a2ad121) # Conflicts: # server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java --- docs/changelog/132101.yaml | 6 + .../test/ingest/80_ingest_simulate.yml | 191 ++++++++++++++++++ .../bulk/TransportSimulateBulkAction.java | 56 ++--- 3 files changed, 217 insertions(+), 36 deletions(-) create mode 100644 docs/changelog/132101.yaml diff --git a/docs/changelog/132101.yaml b/docs/changelog/132101.yaml new file mode 100644 index 0000000000000..84ff4fbefaafc --- /dev/null +++ b/docs/changelog/132101.yaml @@ -0,0 +1,6 @@ +pr: 132101 +summary: Simulate ingest API uses existing index mapping when `mapping_addition` is + given +area: Ingest Node +type: bug +issues: [] diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index d602607bf7515..28c713d3eba6d 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -1776,3 +1776,194 @@ setup: - match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" } - match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] } - not_exists: docs.0.doc.error + +--- +"Test mapping addition correctly respects mapping of indices without templates": + # In this test, we make sure that when we have an index that has mapping but was not built with a template, that the + # additional_mapping respects the existing mapping for validation. + + - skip: + features: + - headers + - allowed_warnings + + # A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists + # because this test is making sure we get correct behavior when an index matches *no* template: + - do: + indices.delete_template: + name: '*' + ignore: 404 + + # We create the index no-template-index with an implicit mapping that has a foo field with type long: + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "no-template-index"}}' + - '{"foo": 3}' + + # Now we make sure that the existing mapping is taken into account when we simulate with a mapping_addition. Since + # the pre-existing mapping has foo mapped as a long, this ought to fail with a document_parsing_exception because + # we are attempting to write a boolean foo. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: no-template-index + body: > + { + "docs": [ + { + "_id": "test-id", + "_index": "no-template-index", + "_source": { + "@timestamp": "2025-07-25T09:06:06.929Z", + "is_valid": true, + "foo": true + } + } + ], + "mapping_addition": { + "properties": { + "is_valid": { + "type": "boolean" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "no-template-index" } + - match: { docs.0.doc._source.foo: true } + - match: { docs.0.doc._source.is_valid: true } + - match: { docs.0.doc.error.type: "document_parsing_exception" } + + # Now we add a template for this index. + - do: + indices.put_template: + name: my-template-1 + body: + index_patterns: no-template-index + mappings: + properties: + foo: + type: boolean + + # And we still expect the index's mapping to be used rather than the template: + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: no-template-index + body: > + { + "docs": [ + { + "_id": "test-id", + "_index": "no-template-index", + "_source": { + "@timestamp": "2025-07-25T09:06:06.929Z", + "is_valid": true, + "foo": true + } + } + ], + "mapping_addition": { + "properties": { + "is_valid": { + "type": "boolean" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "no-template-index" } + - match: { docs.0.doc._source.foo: true } + - match: { docs.0.doc._source.is_valid: true } + - match: { docs.0.doc.error.type: "document_parsing_exception" } + +--- +"Test ingest simulate with mapping addition for data streams when write index has different mapping": + # In this test, we make sure that when a data stream's write index has a mapping that is different from the mapping + # in its template, and a mapping_override is given, then the mapping_override is applied to the mapping of the write + # index rather than the mapping of the template. + + - skip: + features: + - headers + - allowed_warnings + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: boolean + bar: + type: boolean + + - do: + allowed_warnings: + - "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation" + indices.put_index_template: + name: my-template-1 + body: + index_patterns: [simple-data-stream1] + composed_of: + - mappings_template + data_stream: {} + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + # Now that the data stream exists, we change the template to remove the mapping for bar. The write index still has the + # old mapping. + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + properties: + foo: + type: boolean + + # We expect the mapping_addition to be added to the mapping of the write index, which has a boolean bar field. So this + # simulate ingest ought to fail. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "bar": "baz" + } + } + ], + "mapping_addition": { + "properties": { + "baz": { + "type": "keyword" + } + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.bar: "baz" } + - match: { docs.0.doc.error.type: "document_parsing_exception" } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index a3eab93b94d6e..aa21956da9159 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -69,6 +69,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.Executor; @@ -205,32 +206,15 @@ private Tuple, Exception> validateMappings( Collection ignoredFields = List.of(); IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); try { - if (indexAbstraction != null - && componentTemplateSubstitutions.isEmpty() - && indexTemplateSubstitutions.isEmpty() - && mappingAddition.isEmpty()) { + if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) { /* - * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService - * to do the mapping validation, using all the existing logic for validation. + * In this case the index exists and we don't have any template overrides. So we can just merge the mappingAddition (which + * might not exist) into the existing index mapping. */ IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata())); - indicesService.withTempIndexService(imd, indexService -> { - indexService.mapperService().updateMapping(null, imd); - return IndexShard.prepareIndex( - indexService.mapperService(), - sourceToParse, - SequenceNumbers.UNASSIGNED_SEQ_NO, - -1, - -1, - VersionType.INTERNAL, - Engine.Operation.Origin.PRIMARY, - Long.MIN_VALUE, - false, - request.ifSeqNo(), - request.ifPrimaryTerm(), - 0 - ); - }); + CompressedXContent mappings = Optional.ofNullable(imd.mapping()).map(MappingMetadata::source).orElse(null); + CompressedXContent mergedMappings = mappingAddition == null ? null : mergeMappings(mappings, mappingAddition); + ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, mergedMappings, request, sourceToParse); } else { /* * The index did not exist, or we have component template substitutions, so we put together the mappings from existing @@ -304,15 +288,6 @@ private Tuple, Exception> validateMappings( ); final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition); ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse); - } else if (indexAbstraction != null && mappingAddition.isEmpty() == false) { - /* - * The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we - * merge in the mapping addition if it exists, and validate. - */ - MappingMetadata mappingFromIndex = clusterService.state().metadata().index(indexAbstraction.getName()).mapping(); - CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source(); - CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition); - ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse); } else { /* * The index matched no templates and had no mapping of its own. If there were component template substitutions @@ -340,9 +315,6 @@ private Collection validateUpdatedMappings( IndexRequest request, SourceToParse sourceToParse ) throws IOException { - if (updatedMappings == null) { - return List.of(); // no validation to do - } Settings dummySettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) @@ -354,8 +326,20 @@ private Collection validateUpdatedMappings( originalIndexMetadataBuilder.putMapping(new MappingMetadata(originalMappings)); } final IndexMetadata originalIndexMetadata = originalIndexMetadataBuilder.build(); + return validateUpdatedMappingsFromIndexMetadata(originalIndexMetadata, updatedMappings, request, sourceToParse); + } + + private Collection validateUpdatedMappingsFromIndexMetadata( + IndexMetadata originalIndexMetadata, + @Nullable CompressedXContent updatedMappings, + IndexRequest request, + SourceToParse sourceToParse + ) throws IOException { + if (updatedMappings == null) { + return List.of(); // no validation to do + } final IndexMetadata updatedIndexMetadata = IndexMetadata.builder(request.index()) - .settings(dummySettings) + .settings(originalIndexMetadata.getSettings()) .putMapping(new MappingMetadata(updatedMappings)) .build(); Engine.Index result = indicesService.withTempIndexService(originalIndexMetadata, indexService -> {