Enable synthetic_id by default under feature flag (#144026)

* Enable synthetic_id by default if settings are valid

Newly created time_series indices use synthetic id by default.

The new IndexVersion, TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT,  prevent old indices
(with older index version) from changing default behavior.

`index.mapping.synthetic_id` defaults to true if
- Feature flag TSDB_SYNTHETIC_ID_FEATURE_FLAG is enabled
- index mode is time_series
- index version is TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT or later
- codec is default or unset

and setting is still allowed to be explicitly set if index version
is TIME_SERIES_USE_SYNTHETIC_ID_94 or later. Only the default behavior
is controlled by TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT.

This default logic is duplicated by IndexMetadata to make sure metadata always
agree with Settings.

MODE setting moved above SYNTHETIC_ID to allow access.

* NodeFeature index.time_series_synthetic_id_default

Making sure to add this node feature in the same PR as adding the new
index version, `TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT` to make sure the
node feature can be used as a shortcut / replacement for checking
version in tests.

* Skip tsdb/25_id... and delete/70_tsdb in yamlRestCompatTest

The compat tests run with a copy of the yaml file from the BWC branch,
which doesn't contain the skip for index.time_series_synthetic_id.

This means the hardcoded ids in the test doesn't match with actual
default behavior anymore, so we skip those tests in the compatibility
run.

This is safe because the behavior is not changed for existing indices
that don't use synthetic id. Only new indices will have the changed
behavior, so there is no compatibility issue.

To make this work we nedd to make it possible to skip compatibility
tests with overloaded names.

If the name of the test is also a name of an operation such as `get` or
`delete` in below examples, then the build would fail with

```
Execution failed for task ':rest-api-spec:yamlRestCompatTestTransform'.
> class com.fasterxml.jackson.databind.node.ObjectNode cannot be cast to
 class com.fasterxml.jackson.databind.node.ArrayNode
```

Instead of throwing an (assert) exception if the node to transform
(skip) is of the wrong type, we now just ignore it instead.

Example of test that could not be skipped before, but that can be
skipped now:
```
get:
  - do:
      get:
        index:   id_generation_test
        id:      cZZNs7B9sSWsyrL5AAABeRnSA5M
```
---
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com> and Cursor

Author: burqen

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java

@@ -87,8 +87,14 @@ private void addSkip(ArrayNode skipParent) {
     @Override
     public void transformTest(ObjectNode parent) {
         if (testName.isBlank() == false) {
-            assert parent.get(testName) instanceof ArrayNode;
-            addSkip((ArrayNode) parent.get(testName));
+            JsonNode value = parent.get(testName);
+            // Only apply skip to test documents where the key is the test name and value is the steps array.
+            // Do not apply to nested keys with the same name (e.g. "do: get: { ... }" request body).
+            // This makes it possible to skip tests where the test name is an overloaded term such as
+            // task.skipTest("tsdb/25_id_generation/delete",...)
+            if (value instanceof ArrayNode) {
+                addSkip((ArrayNode) value);
+            }
         }
     }
 

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBSyntheticIdsIT.java

@@ -24,6 +24,7 @@
 import org.elasticsearch.action.admin.indices.diskusage.TransportAnalyzeIndexDiskUsageAction;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.admin.indices.rollover.RolloverResponse;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.support.WriteRequest;
@@ -1297,6 +1298,70 @@ public void testMerge() throws Exception {
         assertShardsHaveNoIdStoredFieldValuesOnDisk(indices);
     }
 
+    public void testDefaultSetting() throws Exception {
+        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
+
+        String indexName = randomIndexName();
+
+        // Don't set IndexSettings.SYNTHETIC_ID to test default behavior.
+        // Use default codec so the SYNTHETIC_ID default is true
+        // (codec will be randomised by ESIntegTestCase.randomIndexTemplate if not explicitly set)
+        Settings.Builder settingsBuilder = Settings.builder()
+            .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES)
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "hostname")
+            .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), CodecService.DEFAULT_CODEC);
+        final var mapping = """
+            {
+                "properties": {
+                    "@timestamp": {
+                        "type": "date"
+                    },
+                    "hostname": {
+                        "type": "keyword",
+                        "time_series_dimension": true
+                    },
+                    "metric": {
+                        "properties": {
+                            "field": {
+                                "type": "keyword"
+                            },
+                            "value": {
+                                "type": "integer",
+                                "time_series_metric": "counter"
+                            }
+                        }
+                    }
+                }
+            }
+            """;
+        assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(settingsBuilder).setMapping(mapping).get());
+
+        var timestamp = Instant.now();
+        createDocuments(
+            indexName,
+            document(timestamp, "vm-dev01", "cpu-load", 0),
+            document(timestamp.plus(1, ChronoUnit.SECONDS), "vm-dev02", "cpu-load", 1)
+        );
+        ensureGreen(indexName);
+        flushAndRefresh(indexName);
+
+        GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings(TEST_REQUEST_TIMEOUT, indexName).get();
+        String versionSetting = getSettingsResponse.getSetting(indexName, IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey());
+        IndexVersion version = IndexVersion.fromId(Integer.parseInt(versionSetting));
+        assertTrue(version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT));
+        String syntheticIdSetting = getSettingsResponse.getSetting(indexName, IndexSettings.SYNTHETIC_ID.getKey());
+        assertThat(syntheticIdSetting, Matchers.nullValue());
+
+        var diskUsage = diskUsage(indexName);
+        var diskUsageIdField = AnalyzeIndexDiskUsageTestUtils.getPerFieldDiskUsage(diskUsage, IdFieldMapper.NAME);
+        assertThat("_id field should not have postings on disk", diskUsageIdField.getInvertedIndexBytes(), equalTo(0L));
+        assertThat("_id field should have bloom filter usage", diskUsageIdField.getBloomFilterBytes(), greaterThan(0L));
+
+        var indices = new HashSet<String>();
+        indices.add(indexName);
+        assertShardsHaveNoIdStoredFieldValuesOnDisk(indices);
+    }
+
     /**
      * This test verifies that index with synthetic id cannot be created
      * if index version is too low. Imagine a mixed cluster where node A has

rest-api-spec/build.gradle

@@ -158,4 +158,16 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
     "get/100_synthetic_source/fields with ignore_malformed",
     "Malformed values are now stored in binary doc values which sort differently than stored fields"
   )
+  task.skipTest("delete/70_tsdb/basic tsdb delete", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/routing_path matches object", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/ids query", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/create operation on top of old document fails over bulk", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/delete over _bulk", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/delete", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/index a new document on top of an old one", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/generates a consistent id", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/create operation on top of old document fails", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/get", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/routing_path matches deep object", "ids have changed after introduction of synthetic id")
+  task.skipTest("tsdb/25_id_generation/index a new document on top of an old one over bulk", "ids have changed after introduction of synthetic id")
 })

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -47,6 +47,8 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.index.codec.CodecService;
+import org.elasticsearch.index.engine.EngineConfig;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.search.QueryParserHelper;
@@ -2574,15 +2576,7 @@ IndexMetadata build(boolean repair) {
             String indexModeString = settings.get(IndexSettings.MODE.getKey());
             final IndexMode indexMode = indexModeString != null ? IndexMode.fromString(indexModeString.toLowerCase(Locale.ROOT)) : null;
             final boolean isTsdb = indexMode == IndexMode.TIME_SERIES;
-            boolean useTimeSeriesSyntheticId = false;
-            if (isTsdb
-                && IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG
-                && indexCreatedVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)) {
-                var setting = settings.get(IndexSettings.SYNTHETIC_ID.getKey());
-                if (setting != null && setting.equalsIgnoreCase(Boolean.TRUE.toString())) {
-                    useTimeSeriesSyntheticId = true;
-                }
-            }
+            boolean useTimeSeriesSyntheticId = shouldUseTimeSeriesSyntheticId(isTsdb, indexCreatedVersion, settings);
             final boolean sequenceNumbersDisabled = IndexSettings.DISABLE_SEQUENCE_NUMBERS_FEATURE_FLAG
                 && indexCreatedVersion.onOrAfter(IndexVersions.DISABLE_SEQUENCE_NUMBERS)
                 && settings.getAsBoolean(IndexSettings.DISABLE_SEQUENCE_NUMBERS.getKey(), false);
@@ -2642,6 +2636,18 @@ IndexMetadata build(boolean repair) {
             );
         }
 
+        private static boolean shouldUseTimeSeriesSyntheticId(boolean isTsdb, IndexVersion version, Settings settings) {
+            String codecSetting = settings.get(EngineConfig.INDEX_CODEC_SETTING.getKey());
+            if (IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG
+                && isTsdb
+                && version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94)
+                && (codecSetting == null || codecSetting.equalsIgnoreCase(CodecService.DEFAULT_CODEC))) {
+                boolean defaultValue = version.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT);
+                return settings.getAsBoolean(IndexSettings.SYNTHETIC_ID.getKey(), defaultValue);
+            }
+            return false;
+        }
+
         @SuppressWarnings("unchecked")
         public static void toXContent(IndexMetadata indexMetadata, XContentBuilder builder, ToXContent.Params params) throws IOException {
             Metadata.XContentContext context = Metadata.XContentContext.valueOf(

server/src/main/java/org/elasticsearch/index/IndexFeatures.java

@@ -25,6 +25,7 @@ public Set<NodeFeature> getFeatures() {
     public static final NodeFeature LOGSDB_NO_HOST_NAME_FIELD = new NodeFeature("index.logsdb_no_host_name_field");
 
     public static final NodeFeature TIME_SERIES_SYNTHETIC_ID = new NodeFeature("index.time_series_synthetic_id");
+    public static final NodeFeature TIME_SERIES_SYNTHETIC_ID_DEFAULT = new NodeFeature("index.time_series_synthetic_id_default");
 
     public static final NodeFeature TIME_SERIES_NO_SEQNO = new NodeFeature("index.time_series_no_seqno");
 
@@ -51,6 +52,7 @@ public Set<NodeFeature> getTestFeatures() {
         return Set.of(
             LOGSDB_NO_HOST_NAME_FIELD,
             TIME_SERIES_SYNTHETIC_ID,
+            TIME_SERIES_SYNTHETIC_ID_DEFAULT,
             TIME_SERIES_NO_SEQNO,
             SYNONYMS_SET_LENIENT_ON_NON_EXISTING,
             THROW_EXCEPTION_FOR_UNKNOWN_TOKEN_IN_REST_INDEX_PUT_ALIAS_ACTION,

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -681,8 +681,43 @@ public boolean isES87TSDBCodecEnabled() {
         Property.Final
     );
 
+    /**
+    * The {@link IndexMode "mode"} of the index.
+    */
+    public static final Setting<IndexMode> MODE = Setting.enumSetting(
+        IndexMode.class,
+        "index.mode",
+        IndexMode.STANDARD,
+        new Setting.Validator<>() {
+            @Override
+            public void validate(IndexMode value) {}
+
+            @Override
+            public void validate(IndexMode value, Map<Setting<?>, Object> settings) {
+                value.validateWithOtherSettings(settings);
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                return IndexMode.VALIDATE_WITH_SETTINGS.iterator();
+            }
+        },
+        Property.IndexScope,
+        Property.Final,
+        Property.ServerlessPublic
+    );
+
     public static final boolean TSDB_SYNTHETIC_ID_FEATURE_FLAG = new FeatureFlag("tsdb_synthetic_id").isEnabled();
-    public static final Setting<Boolean> SYNTHETIC_ID = Setting.boolSetting("index.mapping.synthetic_id", false, new Setting.Validator<>() {
+    public static final Setting<Boolean> SYNTHETIC_ID = Setting.boolSetting("index.mapping.synthetic_id", settings -> {
+        IndexVersion indexVersion = SETTING_INDEX_VERSION_CREATED.get(settings);
+        IndexMode indexMode = MODE.get(settings);
+        String codec = INDEX_CODEC_SETTING.get(settings);
+        boolean onByDefault = indexVersion.onOrAfter(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT);
+        return TSDB_SYNTHETIC_ID_FEATURE_FLAG
+            && IndexMode.TIME_SERIES.equals(indexMode)
+            && CodecService.DEFAULT_CODEC.equalsIgnoreCase(codec)
+            && onByDefault ? Boolean.TRUE.toString() : Boolean.FALSE.toString();
+    }, new Setting.Validator<>() {
         @Override
         public void validate(Boolean enabled) {
             if (enabled) {
@@ -717,7 +752,7 @@ public void validate(Boolean enabled, Map<Setting<?>, Object> settings) {
                 }
 
                 var codecName = (String) settings.get(INDEX_CODEC_SETTING);
-                if (codecName.equals(CodecService.DEFAULT_CODEC) == false) {
+                if (codecName.equalsIgnoreCase(CodecService.DEFAULT_CODEC) == false) {
                     throw new IllegalArgumentException(
                         String.format(
                             Locale.ROOT,
@@ -757,32 +792,6 @@ public Iterator<Setting<?>> settings() {
         }
     }, Property.IndexScope, Property.Final);
 
-    /**
-     * The {@link IndexMode "mode"} of the index.
-     */
-    public static final Setting<IndexMode> MODE = Setting.enumSetting(
-        IndexMode.class,
-        "index.mode",
-        IndexMode.STANDARD,
-        new Setting.Validator<>() {
-            @Override
-            public void validate(IndexMode value) {}
-
-            @Override
-            public void validate(IndexMode value, Map<Setting<?>, Object> settings) {
-                value.validateWithOtherSettings(settings);
-            }
-
-            @Override
-            public Iterator<Setting<?>> settings() {
-                return IndexMode.VALIDATE_WITH_SETTINGS.iterator();
-            }
-        },
-        Property.IndexScope,
-        Property.Final,
-        Property.ServerlessPublic
-    );
-
     public static final Setting<Boolean> USE_DOC_VALUES_SKIPPER = Setting.boolSetting("index.mapping.use_doc_values_skipper", s -> {
         IndexVersion iv = SETTING_INDEX_VERSION_CREATED.get(s);
         if (MODE.get(s) == IndexMode.TIME_SERIES) {

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -237,6 +237,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion UPGRADE_DISKBBQ_ES940 = def(9_076_00_0, Version.LUCENE_10_4_0);
     public static final IndexVersion FLATTENED_FIELD_NO_ROOT_DOC_VALUES = def(9_077_0_00, Version.LUCENE_10_4_0);
     public static final IndexVersion IGNORED_SOURCE_AS_DOC_VALUES = def(9_078_0_00, Version.LUCENE_10_4_0);
+    public static final IndexVersion TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT = def(9_079_0_00, Version.LUCENE_10_4_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java

@@ -978,6 +978,45 @@ public void testSyntheticIdCorrectSettings() {
         assertTrue(indexMetadata.useTimeSeriesSyntheticId());
     }
 
+    public void testSyntheticIdDefaultValueTrue() {
+        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
+        IndexVersion version = IndexVersionUtils.randomVersionBetween(
+            IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT,
+            IndexVersion.current()
+        );
+        IndexMode mode = IndexMode.TIME_SERIES;
+        String codec = CodecService.DEFAULT_CODEC;
+
+        Settings settings = Settings.builder()
+            .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec)
+            .put(IndexSettings.MODE.getKey(), mode)
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing")
+            .build();
+        IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version);
+
+        IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
+        assertTrue(indexSettings.useTimeSeriesSyntheticId());
+        assertTrue(indexMetadata.useTimeSeriesSyntheticId());
+    }
+
+    public void testSyntheticIdDefaultValueFalse() {
+        assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
+        IndexVersion version = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_DEFAULT);
+        IndexMode mode = IndexMode.TIME_SERIES;
+        String codec = CodecService.DEFAULT_CODEC;
+
+        Settings settings = Settings.builder()
+            .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codec)
+            .put(IndexSettings.MODE.getKey(), mode)
+            .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "some-routing")
+            .build();
+        IndexMetadata indexMetadata = newIndexMeta("some-index", settings, version);
+
+        IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
+        assertFalse(indexSettings.useTimeSeriesSyntheticId());
+        assertFalse(indexMetadata.useTimeSeriesSyntheticId());
+    }
+
     public void testSyntheticIdBadVersion() {
         assumeTrue("Test should only run with feature flag", IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
         IndexVersion badVersion = IndexVersionUtils.getPreviousVersion(IndexVersions.TIME_SERIES_USE_SYNTHETIC_ID_94);