Removing _doc from data stream effective mappings

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -2429,24 +2429,20 @@ public void testGetEffectiveMappings() throws Exception {
         {
             ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent("""
                     {
-                      "_doc":{
-                        "dynamic":"strict",
-                        "properties":{
-                          "field1":{
-                            "type":"text"
-                          }
+                      "dynamic":"strict",
+                      "properties":{
+                        "field1":{
+                          "type":"text"
                         }
                       }
                     }
                 """), null), 3L, null);
             ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent("""
                     {
-                      "_doc":{
-                        "dynamic":"strict",
-                        "properties":{
-                          "field2":{
-                            "type":"text"
-                          }
+                      "dynamic":"strict",
+                      "properties":{
+                        "field2":{
+                          "type":"text"
                         }
                       }
                     }
@@ -2464,12 +2460,10 @@ public void testGetEffectiveMappings() throws Exception {
                     .indexPatterns(List.of("effective-*"))
                     .template(Template.builder().mappings(CompressedXContent.fromJSON("""
                         {
-                          "_doc":{
-                            "dynamic":"strict",
-                            "properties":{
-                              "field3":{
-                                "type":"text"
-                              }
+                          "dynamic":"strict",
+                          "properties":{
+                            "field3":{
+                              "type":"text"
                             }
                           }
                         }
@@ -2530,25 +2524,22 @@ public void testGetEffectiveMappings() throws Exception {
         Map<String, Object> effectiveMappingMap = XContentHelper.convertToMap(effectiveMappings.uncompressed(), true, XContentType.JSON)
             .v2();
         Map<String, Object> expectedEffectiveMappingMap = Map.of(
-            "_doc",
+            "dynamic",
+            "strict",
+            "_data_stream_timestamp",
+            Map.of("enabled", true),
+            "properties",
             Map.of(
-                "dynamic",
-                "strict",
-                "_data_stream_timestamp",
-                Map.of("enabled", true),
-                "properties",
-                Map.of(
-                    "@timestamp",
-                    Map.of("type", "date"),
-                    "field1",
-                    Map.of("type", "keyword"),
-                    "field2",
-                    Map.of("type", "text"),
-                    "field3",
-                    Map.of("type", "text"),
-                    "field4",
-                    Map.of("type", "keyword")
-                )
+                "@timestamp",
+                Map.of("type", "date"),
+                "field1",
+                Map.of("type", "keyword"),
+                "field2",
+                Map.of("type", "text"),
+                "field3",
+                Map.of("type", "text"),
+                "field4",
+                Map.of("type", "keyword")
             )
         );
         assertThat(effectiveMappingMap, equalTo(expectedEffectiveMappingMap));

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TransportUpdateDataStreamMappingsActionIT.java

@@ -55,38 +55,32 @@ public void testGetAndUpdateMappings() throws IOException {
         createDataStream(dataStreamName);
 
         Map<String, Object> originalMappings = Map.of(
-            "_doc",
-            Map.of(
-                "dynamic",
-                "strict",
-                "_data_stream_timestamp",
-                Map.of("enabled", true),
-                "properties",
-                Map.of("@timestamp", Map.of("type", "date"), "foo1", Map.of("type", "text"), "foo2", Map.of("type", "text"))
-            )
+            "dynamic",
+            "strict",
+            "_data_stream_timestamp",
+            Map.of("enabled", true),
+            "properties",
+            Map.of("@timestamp", Map.of("type", "date"), "foo1", Map.of("type", "text"), "foo2", Map.of("type", "text"))
         );
         Map<String, Object> mappingOverrides = Map.of(
             "properties",
             Map.of("foo2", Map.of("type", "keyword"), "foo3", Map.of("type", "text"))
         );
         Map<String, Object> expectedEffectiveMappings = Map.of(
-            "_doc",
+            "dynamic",
+            "strict",
+            "_data_stream_timestamp",
+            Map.of("enabled", true),
+            "properties",
             Map.of(
-                "dynamic",
-                "strict",
-                "_data_stream_timestamp",
-                Map.of("enabled", true),
-                "properties",
-                Map.of(
-                    "@timestamp",
-                    Map.of("type", "date"),
-                    "foo1",
-                    Map.of("type", "text"),
-                    "foo2",
-                    Map.of("type", "keyword"),
-                    "foo3",
-                    Map.of("type", "text")
-                )
+                "@timestamp",
+                Map.of("type", "date"),
+                "foo1",
+                Map.of("type", "text"),
+                "foo2",
+                Map.of("type", "keyword"),
+                "foo3",
+                Map.of("type", "text")
             )
         );
         assertExpectedMappings(dataStreamName, Map.of(), originalMappings);
@@ -137,15 +131,13 @@ public void testGetAndUpdateMappings() throws IOException {
     private void createDataStream(String dataStreamName) throws IOException {
         String mappingString = """
             {
-              "_doc":{
-                "dynamic":"strict",
-                "properties":{
-                  "foo1":{
-                    "type":"text"
-                  },
-                  "foo2":{
-                    "type":"text"
-                  }
+              "dynamic":"strict",
+              "properties":{
+                "foo1":{
+                  "type":"text"
+                },
+                "foo2":{
+                  "type":"text"
                 }
               }
             }

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_data_stream_mappings.yml

@@ -34,7 +34,7 @@ setup:
         name: my-data-stream-1
   - match: { data_streams.0.name: my-data-stream-1 }
   - match: { data_streams.0.mappings: {} }
-  - length: { data_streams.0.effective_mappings._doc.properties: 2 }
+  - length: { data_streams.0.effective_mappings.properties: 2 }
 
   - do:
       indices.get_data_stream:
@@ -56,7 +56,7 @@ setup:
   - match: { data_streams.0.name: my-data-stream-1 }
   - match: { data_streams.0.applied_to_data_stream: true }
   - match: { data_streams.0.mappings.properties.name.type: "keyword" }
-  - match: { data_streams.0.effective_mappings._doc.properties.name.type: "keyword" }
+  - match: { data_streams.0.effective_mappings.properties.name.type: "keyword" }
 
   - do:
       indices.rollover:
@@ -71,9 +71,9 @@ setup:
       indices.get_data_stream_mappings:
         name: my-data-stream-1
   - match: { data_streams.0.name: my-data-stream-1 }
-  - length: { data_streams.0.effective_mappings._doc.properties: 3 }
+  - length: { data_streams.0.effective_mappings.properties: 3 }
   - match: { data_streams.0.mappings.properties.name.type: "keyword" }
-  - match: { data_streams.0.effective_mappings._doc.properties.name.type: "keyword" }
+  - match: { data_streams.0.effective_mappings.properties.name.type: "keyword" }
 
   - do:
       indices.get_data_stream:
@@ -149,9 +149,9 @@ setup:
   - match: { data_streams.0.mappings.properties.field1.type: "text" }
   - match: { data_streams.0.mappings.properties.field2: null }
   - match: { data_streams.0.mappings.properties.field3.type: "text" }
-  - match: { data_streams.0.effective_mappings._doc.properties.field1.type: "text" }
-  - match: { data_streams.0.effective_mappings._doc.properties.field2.type: "keyword" }
-  - match: { data_streams.0.effective_mappings._doc.properties.field3.type: "text" }
+  - match: { data_streams.0.effective_mappings.properties.field1.type: "text" }
+  - match: { data_streams.0.effective_mappings.properties.field2.type: "keyword" }
+  - match: { data_streams.0.effective_mappings.properties.field3.type: "text" }
 
   - do:
       cluster.put_component_template:
@@ -180,10 +180,10 @@ setup:
       indices.get_data_stream_mappings:
         name: my-component-only-data-stream-1
   - match: { data_streams.0.name: my-component-only-data-stream-1 }
-  - length: { data_streams.0.effective_mappings._doc.properties: 5 }
+  - length: { data_streams.0.effective_mappings.properties: 5 }
   - match: { data_streams.0.mappings.properties.field1.type: "text" }
-  - match: { data_streams.0.effective_mappings._doc.properties.field3.type: "text" }
-  - match: { data_streams.0.effective_mappings._doc.properties.field4.type: "keyword" }
+  - match: { data_streams.0.effective_mappings.properties.field3.type: "text" }
+  - match: { data_streams.0.effective_mappings.properties.field4.type: "keyword" }
 
   - do:
       indices.get_data_stream:
@@ -212,7 +212,7 @@ setup:
   - match: { data_streams.0.name: my-component-only-data-stream-1 }
   - match: { data_streams.0.applied_to_data_stream: true }
   - match: { data_streams.0.mappings null }
-  - match: { data_streams.0.effective_mappings._doc.properties.field1.type: "keyword" }
-  - match: { data_streams.0.effective_mappings._doc.properties.field2.type: "keyword" }
-  - match: { data_streams.0.effective_mappings._doc.properties.field3: null }
-  - match: { data_streams.0.effective_mappings._doc.properties.field4.type: "keyword" }
+  - match: { data_streams.0.effective_mappings.properties.field1.type: "keyword" }
+  - match: { data_streams.0.effective_mappings.properties.field2.type: "keyword" }
+  - match: { data_streams.0.effective_mappings.properties.field3: null }
+  - match: { data_streams.0.effective_mappings.properties.field4.type: "keyword" }

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -46,7 +46,6 @@
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
-import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
@@ -500,13 +499,20 @@ public static CompressedXContent getEffectiveMappings(
             writeIndex.getName()
         );
         return indicesService.withTempIndexService(projectMetadata.index(writeIndex), indexService -> {
-            MapperService mapperService = indexService.mapperService();
-            DocumentMapper documentMapper = mapperService.merge(
-                MapperService.SINGLE_MAPPING_NAME,
-                mappings,
-                MapperService.MergeReason.INDEX_TEMPLATE
-            );
-            return documentMapper.mappingSource();
+            CompressedXContent mergedMapping = indexService.mapperService()
+                .merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE)
+                .mappingSource();
+            /*
+             * If the merged mapping contains the old "_doc" type placeholder, we remove it to make things more straightforward for the
+             * client:
+             */
+            Map<String, Object> mergedMappingMap = XContentHelper.convertToMap(mergedMapping.uncompressed(), true, XContentType.JSON).v2();
+            if (mergedMappingMap.containsKey(MapperService.SINGLE_MAPPING_NAME)) {
+                mergedMapping = ComposableIndexTemplate.convertMappingMapToXContent(
+                    (Map<String, ?>) mergedMappingMap.get(MapperService.SINGLE_MAPPING_NAME)
+                );
+            }
+            return mergedMapping;
         });
     }