Updating TransportSimulateIndexTemplateAction.resolveTemplate() to account for data stream overrides

Author: masseyke

qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml

@@ -1740,3 +1740,100 @@ setup:
   - match: { docs.0.doc._source.abc: "sfdsfsfdsfsfdsfsfdsfsfdsfsfdsf" }
   - match: { docs.0.doc.ignored_fields: [ {"field": "abc"} ] }
   - not_exists: docs.0.doc.error
+
+---
+"Simulate ingest with data stream with mapping and setting overrides":
+  - skip:
+      features: headers
+
+  - do:
+      indices.put_index_template:
+        name: test
+        body:
+          index_patterns: test*
+          template:
+            lifecycle:
+              data_retention: "7d"
+          data_stream: {}
+
+  - do:
+      indices.create_data_stream:
+        name: test
+  - is_true: acknowledged
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  - do:
+      indices.put_data_stream_mappings:
+        name: test
+        body:
+          properties:
+            foo:
+              type: boolean
+
+  - match: { data_streams.0.applied_to_data_stream: true }
+
+  # For an ordinary simulate request with no overrides, we fetch the mapping from the write index, and do not take the
+  # data stream mapping override into account. So the foo field is unmapped, and we can write text to it.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: test
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "@timestamp": 1234,
+                  "foo": "bar"
+                }
+              }
+            ]
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "test" }
+  - match: { docs.0.doc._source.foo: "bar" }
+  - not_exists: docs.0.doc.error
+
+  # If template overrides are given, we go to the data stream's template and mapping override (even if the given
+  # template overrides are irrelevant as is the case in this test). Now we see that the foo field is mapped as a
+  # boolean, so we get a validation error when trying to write text to that field.
+  - do:
+      headers:
+        Content-Type: application/json
+      simulate.ingest:
+        index: test
+        body: >
+          {
+            "docs": [
+              {
+                "_id": "asdf",
+                "_source": {
+                  "@timestamp": 1234,
+                  "foo": "bar"
+                }
+              }
+            ],
+            "component_template_substitutions": {
+              "mappings_template": {
+                "template": {
+                  "mappings": {
+                    "dynamic": "true",
+                    "properties": {
+                      "foo": {
+                        "type": "keyword"
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
+  - length: { docs: 1 }
+  - match: { docs.0.doc._index: "test" }
+  - match: { docs.0.doc._source.foo: "bar" }
+  - match: { docs.0.doc.error.type: "document_parsing_exception" }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.simulate_index_template/10_basic.yml

@@ -250,3 +250,51 @@
   - match: {template.lifecycle.data_retention: "7d"}
   - is_true: template.lifecycle.rollover
   - match: {overlapping: []}
+
+---
+"Simulate index template with data stream with mapping and setting overrides":
+  - do:
+      indices.put_index_template:
+        name: test
+        body:
+          index_patterns: test*
+          template:
+            lifecycle:
+              data_retention: "7d"
+          data_stream: {}
+
+  - do:
+      indices.create_data_stream:
+        name: test
+  - is_true: acknowledged
+
+  - do:
+      cluster.health:
+        wait_for_status: yellow
+
+  - do:
+      indices.put_data_stream_mappings:
+        name: test
+        body:
+          properties:
+            foo:
+              type: keyword
+
+  - match: { data_streams.0.applied_to_data_stream: true }
+
+  - do:
+      indices.put_data_stream_settings:
+        name: test
+        body:
+          index:
+            refresh_interval: "47s"
+
+  - match: { data_streams.0.applied_to_data_stream: true }
+
+  - do:
+      indices.simulate_index_template:
+        name: test
+        include_defaults: true
+
+  - match: {template.mappings.properties.foo.type: "keyword"}
+  - match: {template.settings.index.refresh_interval: "47s"}

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java

@@ -49,6 +49,7 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -158,7 +159,6 @@ protected void localClusterStateOperation(
             listener.onResponse(new SimulateIndexTemplateResponse(null, null));
             return;
         }
-
         final ProjectMetadata tempProjectMetadata = resolveTemporaryState(matchingTemplate, request.getIndexName(), projectWithTemplate);
         ComposableIndexTemplate templateV2 = tempProjectMetadata.templatesV2().get(matchingTemplate);
         assert templateV2 != null : "the matched template must exist";
@@ -167,6 +167,7 @@ protected void localClusterStateOperation(
             matchingTemplate,
             request.getIndexName(),
             projectWithTemplate,
+            state.metadata(),
             isDslOnlyMode,
             xContentRegistry,
             indicesService,
@@ -238,14 +239,14 @@ public static Template resolveTemplate(
         final String matchingTemplate,
         final String indexName,
         final ProjectMetadata simulatedProject,
+        final ProjectMetadata actualProject,
         final boolean isDslOnlyMode,
         final NamedXContentRegistry xContentRegistry,
         final IndicesService indicesService,
         final SystemIndices systemIndices,
         Set<IndexSettingProvider> indexSettingProviders
     ) throws Exception {
         Settings templateSettings = resolveSettings(simulatedProject, matchingTemplate);
-
         List<Map<String, AliasMetadata>> resolvedAliases = MetadataIndexTemplateService.resolveAliases(simulatedProject, matchingTemplate);
 
         ComposableIndexTemplate template = simulatedProject.templatesV2().get(matchingTemplate);
@@ -271,6 +272,18 @@ public static Template resolveTemplate(
             xContentRegistry,
             simulatedIndexName
         );
+        if (template.getDataStreamTemplate() != null) {
+            DataStream dataStream = actualProject.dataStreams().get(indexName);
+            if (dataStream != null) {
+                CompressedXContent dataStreamMappingOverrides = dataStream.getMappings();
+                if (ComposableIndexTemplate.EMPTY_MAPPINGS.equals(dataStreamMappingOverrides) == false) {
+                    mappings = new ArrayList<>(mappings);
+                    mappings.add(dataStreamMappingOverrides);
+                }
+                templateSettings = templateSettings.merge(dataStream.getSettings());
+            }
+        }
+        final List<CompressedXContent> finalMappings = mappings;
 
         // First apply settings sourced from index settings providers
         final var now = Instant.now();
@@ -337,7 +350,7 @@ public static Template resolveTemplate(
             indexMetadata,
             tempIndexService -> {
                 MapperService mapperService = tempIndexService.mapperService();
-                mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE);
+                mapperService.merge(MapperService.SINGLE_MAPPING_NAME, finalMappings, MapperService.MergeReason.INDEX_TEMPLATE);
 
                 DocumentMapper documentMapper = mapperService.documentMapper();
                 return documentMapper != null ? documentMapper.mappingSource() : null;

server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java

@@ -179,6 +179,7 @@ protected void localClusterStateOperation(
             matchingTemplate,
             temporaryIndexName,
             projectWithTemplate,
+            projectWithTemplate, // this arg does not matter -- we never need data stream overrides because we never match a data stream
             isDslOnlyMode,
             xContentRegistry,
             indicesService,

server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java

@@ -273,6 +273,7 @@ private Tuple<Collection<String>, Exception> validateMappings(
                         matchingTemplate,
                         request.index(),
                         simulatedProjectMetadata,
+                        project,
                         isDataStreamsLifecycleOnlyMode(clusterService.getSettings()),
                         xContentRegistry,
                         indicesService,

server/src/test/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateActionTests.java

@@ -97,6 +97,7 @@ public boolean overrulesTemplateAndRequestSettings() {
             matchingTemplate,
             indexName,
             simulatedProject,
+            simulatedProject,
             isDslOnlyMode,
             xContentRegistry(),
             indicesService,