Deprecate _source.mode in mappings (#117172)

Re-introduce #116689

Author: dnhatn

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java

@@ -137,7 +137,7 @@ public void skipTest(String fullTestName, String reason) {
         // However, the folder can be arbitrarily nest so, a == a1/a2/a3, and the test name can include forward slashes, so c == c1/c2/c3
         // So we also need to support a1/a2/a3/b/c1/c2/c3
 
-        String[] testParts = fullTestName.split("/");
+        String[] testParts = fullTestName.split("/", 3);
         if (testParts.length < 3) {
             throw new IllegalArgumentException(
                 "To skip tests, all 3 parts [folder/file/test name] must be defined. found [" + fullTestName + "]"

docs/changelog/116689.yaml

@@ -0,0 +1,10 @@
+pr: 116689
+summary: Deprecate `_source.mode` in mappings
+area: Mapping
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate `_source.mode` in mappings
+  area: Mapping
+  details: Configuring `_source.mode` in mappings is deprecated and will be removed in future versions. Use `index.mapping.source.mode` index setting instead.
+  impact: Use `index.mapping.source.mode` index setting instead

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java

@@ -17,7 +17,6 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
-import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.hamcrest.Matcher;
@@ -30,9 +29,6 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.test.MapMatcher.assertMap;
-import static org.elasticsearch.test.MapMatcher.matchesMap;
-
 public class LogsIndexModeFullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {
 
     @ClassRule
@@ -169,22 +165,16 @@ public void testLogsIndexing() throws IOException {
             assertOK(bulkIndexResponse);
             assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
 
-            assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk());
-            assertIndexMappingsAndSettings(
-                1,
-                Matchers.equalTo("logsdb"),
-                matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic"))
-            );
+            assertIndexSettings(0, Matchers.nullValue());
+            assertIndexSettings(1, Matchers.equalTo("logsdb"));
         }
     }
 
-    private void assertIndexMappingsAndSettings(int backingIndex, final Matcher<Object> indexModeMatcher, final MapMatcher mappingsMatcher)
-        throws IOException {
+    private void assertIndexSettings(int backingIndex, final Matcher<Object> indexModeMatcher) throws IOException {
         assertThat(
             getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"),
             indexModeMatcher
         );
-        assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher);
     }
 
     private static Request createDataStream(final String dataStreamName) {

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java

@@ -17,7 +17,6 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
-import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.hamcrest.Matcher;
@@ -30,9 +29,6 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.test.MapMatcher.assertMap;
-import static org.elasticsearch.test.MapMatcher.matchesMap;
-
 public class LogsIndexModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
 
     @ClassRule()
@@ -160,14 +156,10 @@ public void testLogsIndexing() throws IOException {
             assertOK(bulkIndexResponse);
             assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
 
-            assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk());
-            assertIndexMappingsAndSettings(1, Matchers.nullValue(), matchesMap().extraOk());
-            assertIndexMappingsAndSettings(2, Matchers.nullValue(), matchesMap().extraOk());
-            assertIndexMappingsAndSettings(
-                3,
-                Matchers.equalTo("logsdb"),
-                matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic"))
-            );
+            assertIndexSettings(0, Matchers.nullValue());
+            assertIndexSettings(1, Matchers.nullValue());
+            assertIndexSettings(2, Matchers.nullValue());
+            assertIndexSettings(3, Matchers.equalTo("logsdb"));
         }
     }
 
@@ -183,13 +175,11 @@ static void enableLogsdbByDefault() throws IOException {
         assertOK(client().performRequest(request));
     }
 
-    private void assertIndexMappingsAndSettings(int backingIndex, final Matcher<Object> indexModeMatcher, final MapMatcher mappingsMatcher)
-        throws IOException {
+    private void assertIndexSettings(int backingIndex, final Matcher<Object> indexModeMatcher) throws IOException {
         assertThat(
             getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"),
             indexModeMatcher
         );
-        assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher);
     }
 
     private static Request createDataStream(final String dataStreamName) {

rest-api-spec/build.gradle

@@ -60,4 +60,10 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
   task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0")
+  task.skipTest("tsdb/20_mapping/stored source is supported", "no longer serialize source_mode")
+  task.skipTest("tsdb/20_mapping/Synthetic source", "no longer serialize source_mode")
+  task.skipTest("logsdb/10_settings/create logs index", "no longer serialize source_mode")
+  task.skipTest("logsdb/20_source_mapping/stored _source mode is supported", "no longer serialize source_mode")
+  task.skipTest("logsdb/20_source_mapping/include/exclude is supported with stored _source", "no longer serialize source_mode")
+  task.skipTest("logsdb/20_source_mapping/synthetic _source is default", "no longer serialize source_mode")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml

@@ -76,11 +76,6 @@ create logs index:
   - is_true: test
   - match: { test.settings.index.mode: "logsdb" }
 
-  - do:
-      indices.get_mapping:
-        index: test
-  - match: { test.mappings._source.mode: synthetic }
-
 ---
 using default timestamp field mapping:
   - requires:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml

@@ -13,10 +13,10 @@ synthetic _source is default:
             index:
               mode: logsdb
   - do:
-      indices.get:
+      indices.get_settings:
         index: test-default-source
-
-  - match: { test-default-source.mappings._source.mode: "synthetic" }
+  - match: { test-default-source.settings.index.mode: logsdb }
+  - match: { test-default-source.settings.index.mapping.source.mode: null }
 
 ---
 stored _source mode is supported:
@@ -28,11 +28,12 @@ stored _source mode is supported:
             index:
               mode: logsdb
               mapping.source.mode: stored
+
   - do:
-      indices.get:
+      indices.get_settings:
         index: test-stored-source
-
-  - match: { test-stored-source.mappings._source.mode: "stored" }
+  - match: { test-stored-source.settings.index.mode: logsdb }
+  - match: { test-stored-source.settings.index.mapping.source.mode: stored }
 
 ---
 disabled _source is not supported:
@@ -110,7 +111,6 @@ include/exclude is supported with stored _source:
       indices.get:
         index: test-includes
 
-  - match: { test-includes.mappings._source.mode: "stored" }
   - match: { test-includes.mappings._source.includes: ["a"] }
 
   - do:
@@ -129,5 +129,4 @@ include/exclude is supported with stored _source:
       indices.get:
         index: test-excludes
 
-  - match: { test-excludes.mappings._source.mode: "stored" }
   - match: { test-excludes.mappings._source.excludes: ["b"] }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -450,11 +450,6 @@ nested fields:
                 type: long
                 time_series_metric: gauge
 
-  - do:
-      indices.get_mapping: {}
-
-  - match: {tsdb-synthetic.mappings._source.mode: synthetic}
-
 ---
 stored source is supported:
   - requires:
@@ -486,12 +481,6 @@ stored source is supported:
                         type: keyword
                         time_series_dimension: true
 
-  - do:
-      indices.get:
-        index: tsdb_index
-
-  - match: { tsdb_index.mappings._source.mode: "stored" }
-
 ---
 disabled source is not supported:
   - requires:

server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedFunction;
@@ -54,7 +55,7 @@ Settings getAdditionalIndexSettings(
     /**
      * Infrastructure class that holds services that can be used by {@link IndexSettingProvider} instances.
      */
-    record Parameters(CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory) {
+    record Parameters(ClusterService clusterService, CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory) {
 
     }
 

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -134,6 +134,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0);
+    public static final IndexVersion DEPRECATE_SOURCE_MODE_MAPPER = def(9_003_00_0, Version.LUCENE_10_0_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _