Enable synthetic _id randomly in TsdbDataStreamRestIT (#138751)

Relates ES-13605

Author: tlrx

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/DisabledSecurityDataStreamTestCase.java

@@ -14,6 +14,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.junit.ClassRule;
@@ -24,12 +25,15 @@
  */
 public abstract class DisabledSecurityDataStreamTestCase extends ESRestTestCase {
 
+    protected static LocalClusterConfigProvider clusterConfig = c -> {};
+
     @ClassRule
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
         .setting("xpack.security.enabled", "false")
         .setting("xpack.watcher.enabled", "false")
         .feature(FeatureFlag.INDEX_DIMENSIONS_TSID_OPTIMIZATION_FEATURE_FLAG)
+        .apply(() -> clusterConfig) // allows sub-classes to apply a custom cluster configuration
         .build();
 
     @Override

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/TsdbDataStreamRestIT.java

@@ -8,12 +8,15 @@
  */
 package org.elasticsearch.datastreams;
 
+import org.elasticsearch.Build;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.DateFormatters;
 import org.elasticsearch.common.time.FormatNames;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.junit.Before;
 
@@ -25,6 +28,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.backingIndexEqualTo;
+import static org.elasticsearch.index.IndexSettings.USE_SYNTHETIC_ID;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
@@ -36,6 +40,10 @@
 
 public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase {
 
+    static {
+        clusterConfig = config -> config.feature(FeatureFlag.TSDB_SYNTHETIC_ID_FEATURE_FLAG);
+    }
+
     private static final String COMPONENT_TEMPLATE = """
         {
             "template": {
@@ -53,7 +61,7 @@ public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase {
                         "number_of_replicas": 1,
                         "number_of_shards": 2,
                         "mode": "time_series"
-                        SOURCEMODE
+                        RANDOM_INDEX_SETTINGS
                     }
                 },
                 "mappings":{
@@ -202,9 +210,51 @@ public class TsdbDataStreamRestIT extends DisabledSecurityDataStreamTestCase {
             """;
 
     private static String getTemplate() {
-        return TEMPLATE.replace("SOURCEMODE", randomFrom("", """
-            , "mapping": { "source": { "mode": "stored" } }""", """
-            , "mapping": { "source": { "mode": "synthetic" } }"""));
+        String sourceMode = switch (randomInt(2)) {
+            case 0 -> null;
+            case 1 -> """
+                "source": { "mode": "stored" }
+                """;
+            case 2 -> """
+                "source": { "mode": "synthetic" }
+                """;
+            default -> throw new AssertionError("Unknown mode");
+        };
+        String idMode = null;
+        if (Build.current().isSnapshot()) {
+            idMode = switch (randomInt(2)) {
+                case 0 -> null;
+                case 1 -> """
+                    "use_synthetic_id": "false"
+                    """;
+                case 2 -> """
+                    "use_synthetic_id": "true"
+                    """;
+                default -> throw new AssertionError("Unknown mode");
+            };
+        } else {
+            assertFalse(
+                "Setting is enabled by default and must now be tested on non-snapshot build too ",
+                USE_SYNTHETIC_ID.getDefault(Settings.EMPTY)
+            );
+        }
+        if (sourceMode == null && idMode == null) {
+            return TEMPLATE.replace("RANDOM_INDEX_SETTINGS", "");
+        }
+        var mapping = new StringBuilder("""
+            , "mapping": {
+            """);
+        if (sourceMode != null) {
+            mapping.append(sourceMode);
+        }
+        if (idMode != null) {
+            if (sourceMode != null) {
+                mapping.append(',');
+            }
+            mapping.append(idMode);
+        }
+        mapping.append('}');
+        return TEMPLATE.replace("RANDOM_INDEX_SETTINGS", mapping.toString());
     }
 
     private static boolean trialStarted = false;

server/src/main/java/module-info.java

@@ -445,6 +445,7 @@
 
     provides org.apache.lucene.codecs.PostingsFormat
         with
+            org.elasticsearch.index.codec.tsdb.TSDBSyntheticIdPostingsFormat,
             org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.postings.ES812PostingsFormat;

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java

@@ -27,7 +27,8 @@ public enum FeatureFlag {
     ),
     RANDOM_SAMPLING("es.random_sampling_feature_flag_enabled=true", Version.fromString("9.2.0"), null),
     INFERENCE_API_CCM("es.inference_api_ccm_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
-    GENERIC_VECTOR_FORMAT("es.generic_vector_format_feature_flag_enabled=true", Version.fromString("9.3.0"), null);
+    GENERIC_VECTOR_FORMAT("es.generic_vector_format_feature_flag_enabled=true", Version.fromString("9.3.0"), null),
+    TSDB_SYNTHETIC_ID_FEATURE_FLAG("es.tsdb_synthetic_id_feature_flag_enabled=true", Version.fromString("9.3.0"), null);
 
     public final String systemProperty;
     public final Version from;