Revert "Deprecate _source.mode in mappings (#116689)" (#117150)

This reverts commit 0d7b90e, because of bwc testing failures.

Author: martijnvg

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java

@@ -137,7 +137,7 @@ public void skipTest(String fullTestName, String reason) {
         // However, the folder can be arbitrarily nest so, a == a1/a2/a3, and the test name can include forward slashes, so c == c1/c2/c3
         // So we also need to support a1/a2/a3/b/c1/c2/c3
 
-        String[] testParts = fullTestName.split("/", 3);
+        String[] testParts = fullTestName.split("/");
         if (testParts.length < 3) {
             throw new IllegalArgumentException(
                 "To skip tests, all 3 parts [folder/file/test name] must be defined. found [" + fullTestName + "]"

docs/changelog/116689.yaml

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.hamcrest.Matcher;
@@ -29,6 +30,9 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.elasticsearch.test.MapMatcher.matchesMap;
+
 public class LogsIndexModeFullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {
 
     @ClassRule
@@ -165,16 +169,22 @@ public void testLogsIndexing() throws IOException {
             assertOK(bulkIndexResponse);
             assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
 
-            assertIndexSettings(0, Matchers.nullValue());
-            assertIndexSettings(1, Matchers.equalTo("logsdb"));
+            assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(
+                1,
+                Matchers.equalTo("logsdb"),
+                matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic"))
+            );
         }
     }
 
-    private void assertIndexSettings(int backingIndex, final Matcher<Object> indexModeMatcher) throws IOException {
+    private void assertIndexMappingsAndSettings(int backingIndex, final Matcher<Object> indexModeMatcher, final MapMatcher mappingsMatcher)
+        throws IOException {
         assertThat(
             getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"),
             indexModeMatcher
         );
+        assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher);
     }
 
     private static Request createDataStream(final String dataStreamName) {

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.hamcrest.Matcher;
@@ -29,6 +30,9 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.elasticsearch.test.MapMatcher.matchesMap;
+
 public class LogsIndexModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
 
     @ClassRule()
@@ -156,10 +160,14 @@ public void testLogsIndexing() throws IOException {
             assertOK(bulkIndexResponse);
             assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
 
-            assertIndexSettings(0, Matchers.nullValue());
-            assertIndexSettings(1, Matchers.nullValue());
-            assertIndexSettings(2, Matchers.nullValue());
-            assertIndexSettings(3, Matchers.equalTo("logsdb"));
+            assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(1, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(2, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(
+                3,
+                Matchers.equalTo("logsdb"),
+                matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic"))
+            );
         }
     }
 
@@ -175,11 +183,13 @@ static void enableLogsdbByDefault() throws IOException {
         assertOK(client().performRequest(request));
     }
 
-    private void assertIndexSettings(int backingIndex, final Matcher<Object> indexModeMatcher) throws IOException {
+    private void assertIndexMappingsAndSettings(int backingIndex, final Matcher<Object> indexModeMatcher, final MapMatcher mappingsMatcher)
+        throws IOException {
         assertThat(
             getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"),
             indexModeMatcher
         );
+        assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher);
     }
 
     private static Request createDataStream(final String dataStreamName) {

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java

@@ -60,10 +60,4 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
   task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0")
-  task.skipTest("tsdb/20_mapping/stored source is supported", "no longer serialize source_mode")
-  task.skipTest("tsdb/20_mapping/Synthetic source", "no longer serialize source_mode")
-  task.skipTest("logsdb/10_settings/create logs index", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/stored _source mode is supported", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/include/exclude is supported with stored _source", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/synthetic _source is default", "no longer serialize source_mode")
 })

rest-api-spec/build.gradle

@@ -76,6 +76,11 @@ create logs index:
   - is_true: test
   - match: { test.settings.index.mode: "logsdb" }
 
+  - do:
+      indices.get_mapping:
+        index: test
+  - match: { test.mappings._source.mode: synthetic }
+
 ---
 using default timestamp field mapping:
   - requires:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml

@@ -13,10 +13,10 @@ synthetic _source is default:
             index:
               mode: logsdb
   - do:
-      indices.get_settings:
+      indices.get:
         index: test-default-source
-  - match: { test-default-source.settings.index.mode: logsdb }
-  - match: { test-default-source.settings.index.mapping.source.mode: null }
+
+  - match: { test-default-source.mappings._source.mode: "synthetic" }
 
 ---
 stored _source mode is supported:
@@ -28,12 +28,11 @@ stored _source mode is supported:
             index:
               mode: logsdb
               mapping.source.mode: stored
-
   - do:
-      indices.get_settings:
+      indices.get:
         index: test-stored-source
-  - match: { test-stored-source.settings.index.mode: logsdb }
-  - match: { test-stored-source.settings.index.mapping.source.mode: stored }
+
+  - match: { test-stored-source.mappings._source.mode: "stored" }
 
 ---
 disabled _source is not supported:
@@ -111,6 +110,7 @@ include/exclude is supported with stored _source:
       indices.get:
         index: test-includes
 
+  - match: { test-includes.mappings._source.mode: "stored" }
   - match: { test-includes.mappings._source.includes: ["a"] }
 
   - do:
@@ -129,4 +129,5 @@ include/exclude is supported with stored _source:
       indices.get:
         index: test-excludes
 
+  - match: { test-excludes.mappings._source.mode: "stored" }
   - match: { test-excludes.mappings._source.excludes: ["b"] }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml

@@ -450,6 +450,11 @@ nested fields:
                 type: long
                 time_series_metric: gauge
 
+  - do:
+      indices.get_mapping: {}
+
+  - match: {tsdb-synthetic.mappings._source.mode: synthetic}
+
 ---
 stored source is supported:
   - requires:
@@ -481,6 +486,12 @@ stored source is supported:
                         type: keyword
                         time_series_dimension: true
 
+  - do:
+      indices.get:
+        index: tsdb_index
+
+  - match: { tsdb_index.mappings._source.mode: "stored" }
+
 ---
 disabled source is not supported:
   - requires:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedFunction;
@@ -55,7 +54,7 @@ Settings getAdditionalIndexSettings(
     /**
      * Infrastructure class that holds services that can be used by {@link IndexSettingProvider} instances.
      */
-    record Parameters(ClusterService clusterService, CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory) {
+    record Parameters(CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory) {
 
     }
 

server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java

@@ -134,7 +134,6 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0);
-    public static final IndexVersion DEPRECATE_SOURCE_MODE_MAPPER = def(9_003_00_0, Version.LUCENE_10_0_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _