[9.0] Simulate ingest API uses existing index mapping when mapping_addition is given (#132101)

docs/changelog/132101.yaml

@@ -0,0 +1,6 @@
+pr: 132101
+summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
+  given
+area: Ingest Node
+type: bug
+issues: []

qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml

@@ -1740,3 +1740,194 @@ setup:
   - match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" }
   - match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] }
   - not_exists: docs.0.doc.error
+
+---
+"Test mapping addition correctly respects mapping of indices without templates":
+  # In this test, we make sure that when we have an index that has mapping but was not built with a template, that the
+  # additional_mapping respects the existing mapping for validation.
+
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+
+  # A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists
+  # because this test is making sure we get correct behavior when an index matches *no* template:
+  - do:
+      indices.delete_template:
+        name:   '*'
+        ignore: 404
+
+  # We create the index no-template-index with an implicit mapping that has a foo field with type long:
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "no-template-index"}}'
+          - '{"foo": 3}'
+
+  # Now we make sure that the existing mapping is taken into account when we simulate with a mapping_addition. Since
+  # the pre-existing mapping has foo mapped as a long, this ought to fail with a document_parsing_exception because
+  # we are attempting to write a boolean foo.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: no-template-index
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "test-id",
+                "_index": "no-template-index",
+                "_source": {
+                  "@timestamp": "2025-07-25T09:06:06.929Z",
+                  "is_valid": true,
+                  "foo": true
+                }
+              }
+            ],
+            "mapping_addition": {
+              "properties": {
+                "is_valid": {
+                  "type": "boolean"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "no-template-index" }
+  - match: { docs.0.doc._source.foo: true }
+  - match: { docs.0.doc._source.is_valid: true }
+  - match: { docs.0.doc.error.type: "document_parsing_exception" }
+
+  # Now we add a template for this index.
+  - do:
+      indices.put_template:
+        name: my-template-1
+        body:
+          index_patterns: no-template-index
+          mappings:
+            properties:
+              foo:
+                type: boolean
+
+  # And we still expect the index's mapping to be used rather than the template:
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: no-template-index
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "test-id",
+                "_index": "no-template-index",
+                "_source": {
+                  "@timestamp": "2025-07-25T09:06:06.929Z",
+                  "is_valid": true,
+                  "foo": true
+                }
+              }
+            ],
+            "mapping_addition": {
+              "properties": {
+                "is_valid": {
+                  "type": "boolean"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "no-template-index" }
+  - match: { docs.0.doc._source.foo: true }
+  - match: { docs.0.doc._source.is_valid: true }
+  - match: { docs.0.doc.error.type: "document_parsing_exception" }
+
+---
+"Test ingest simulate with mapping addition for data streams when write index has different mapping":
+  # In this test, we make sure that when a data stream's write index has a mapping that is different from the mapping
+  # in its template, and a mapping_override is given, then the mapping_override is applied to the mapping of the write
+  # index rather than the mapping of the template.
+
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+
+  - do:
+      cluster.put_component_template:
+        name: mappings_template
+        body:
+          template:
+            mappings:
+              dynamic: strict
+              properties:
+                foo:
+                  type: boolean
+                bar:
+                  type: boolean
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template-1
+        body:
+          index_patterns: [simple-data-stream1]
+          composed_of:
+            - mappings_template
+          data_stream: {}
+
+  - do:
+      indices.create_data_stream:
+        name: simple-data-stream1
+  - is_true: acknowledged
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  # Now that the data stream exists, we change the template to remove the mapping for bar. The write index still has the
+  # old mapping.
+  - do:
+      cluster.put_component_template:
+        name: mappings_template
+        body:
+          template:
+            mappings:
+              properties:
+                foo:
+                  type: boolean
+
+  # We expect the mapping_addition to be added to the mapping of the write index, which has a boolean bar field. So this
+  # simulate ingest ought to fail.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: simple-data-stream1
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "@timestamp": 1234,
+                  "bar": "baz"
+                }
+              }
+            ],
+            "mapping_addition": {
+              "properties": {
+                "baz": {
+                  "type": "keyword"
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "simple-data-stream1" }
+  - match: { docs.0.doc._source.bar: "baz" }
+  - match: { docs.0.doc.error.type: "document_parsing_exception" }

server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java

@@ -69,6 +69,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Executor;
 
@@ -196,32 +197,15 @@ private Tuple<Collection<String>, Exception> validateMappings(
         Collection<String> ignoredFields = List.of();
         IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index());
         try {
-            if (indexAbstraction != null
-                && componentTemplateSubstitutions.isEmpty()
-                && indexTemplateSubstitutions.isEmpty()
-                && mappingAddition.isEmpty()) {
+            if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) {
                 /*
-                 * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService
-                 * to do the mapping validation, using all the existing logic for validation.
+                 * In this case the index exists and we don't have any template overrides. So we can just merge the mappingAddition (which
+                 * might not exist) into the existing index mapping.
                  */
                 IndexMetadata imd = state.metadata().getIndexSafe(indexAbstraction.getWriteIndex(request, state.metadata()));
-                indicesService.withTempIndexService(imd, indexService -> {
-                    indexService.mapperService().updateMapping(null, imd);
-                    return IndexShard.prepareIndex(
-                        indexService.mapperService(),
-                        sourceToParse,
-                        SequenceNumbers.UNASSIGNED_SEQ_NO,
-                        -1,
-                        -1,
-                        VersionType.INTERNAL,
-                        Engine.Operation.Origin.PRIMARY,
-                        Long.MIN_VALUE,
-                        false,
-                        request.ifSeqNo(),
-                        request.ifPrimaryTerm(),
-                        0
-                    );
-                });
+                CompressedXContent mappings = Optional.ofNullable(imd.mapping()).map(MappingMetadata::source).orElse(null);
+                CompressedXContent mergedMappings = mappingAddition == null ? null : mergeMappings(mappings, mappingAddition);
+                ignoredFields = validateUpdatedMappingsFromIndexMetadata(imd, mergedMappings, request, sourceToParse);
             } else {
                 /*
                  * The index did not exist, or we have component template substitutions, so we put together the mappings from existing
@@ -295,15 +279,6 @@ private Tuple<Collection<String>, Exception> validateMappings(
                         );
                         final CompressedXContent combinedMappings = mergeMappings(new CompressedXContent(mappingsMap), mappingAddition);
                         ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
-                    } else if (indexAbstraction != null && mappingAddition.isEmpty() == false) {
-                        /*
-                         * The index matched no templates of any kind, including the substitutions. But it might have a mapping. So we
-                         * merge in the mapping addition if it exists, and validate.
-                         */
-                        MappingMetadata mappingFromIndex = clusterService.state().metadata().index(indexAbstraction.getName()).mapping();
-                        CompressedXContent currentIndexCompressedXContent = mappingFromIndex == null ? null : mappingFromIndex.source();
-                        CompressedXContent combinedMappings = mergeMappings(currentIndexCompressedXContent, mappingAddition);
-                        ignoredFields = validateUpdatedMappings(null, combinedMappings, request, sourceToParse);
                     } else {
                         /*
                          * The index matched no templates and had no mapping of its own. If there were component template substitutions
@@ -331,9 +306,6 @@ private Collection<String> validateUpdatedMappings(
         IndexRequest request,
         SourceToParse sourceToParse
     ) throws IOException {
-        if (updatedMappings == null) {
-            return List.of(); // no validation to do
-        }
         Settings dummySettings = Settings.builder()
             .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
@@ -345,8 +317,20 @@ private Collection<String> validateUpdatedMappings(
             originalIndexMetadataBuilder.putMapping(new MappingMetadata(originalMappings));
         }
         final IndexMetadata originalIndexMetadata = originalIndexMetadataBuilder.build();
+        return validateUpdatedMappingsFromIndexMetadata(originalIndexMetadata, updatedMappings, request, sourceToParse);
+    }
+
+    private Collection<String> validateUpdatedMappingsFromIndexMetadata(
+        IndexMetadata originalIndexMetadata,
+        @Nullable CompressedXContent updatedMappings,
+        IndexRequest request,
+        SourceToParse sourceToParse
+    ) throws IOException {
+        if (updatedMappings == null) {
+            return List.of(); // no validation to do
+        }
         final IndexMetadata updatedIndexMetadata = IndexMetadata.builder(request.index())
-            .settings(dummySettings)
+            .settings(originalIndexMetadata.getSettings())
             .putMapping(new MappingMetadata(updatedMappings))
             .build();
         Engine.Index result = indicesService.withTempIndexService(originalIndexMetadata, indexService -> {