Revert node setting for jvm env variable.

Author: martijnvg

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -107,7 +107,7 @@ private IndexWriter createIndex(final Directory directory, final boolean optimiz
                 new SortedNumericSortField(TIMESTAMP_FIELD, SortField.Type.LONG, true)
             )
         );
-        ES87TSDBDocValuesFormat docValuesFormat = new ES87TSDBDocValuesFormat(optimizedMergeEnabled);
+        ES87TSDBDocValuesFormat docValuesFormat = new ES87TSDBDocValuesFormat(4096, optimizedMergeEnabled);
         config.setCodec(new Lucene101Codec() {
 
             @Override

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -89,7 +89,6 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexingPressure;
 import org.elasticsearch.index.MergePolicyConfig;
-import org.elasticsearch.index.codec.CodecService;
 import org.elasticsearch.index.engine.ThreadPoolMergeScheduler;
 import org.elasticsearch.index.shard.IndexingStatsSettings;
 import org.elasticsearch.indices.IndexingMemoryController;
@@ -636,7 +635,6 @@ public void apply(Settings value, Settings current, Settings previous) {
         ShardsAvailabilityHealthIndicatorService.REPLICA_UNASSIGNED_BUFFER_TIME,
         DataStream.isFailureStoreFeatureFlagEnabled() ? DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING : null,
         IndexingStatsSettings.RECENT_WRITE_LOAD_HALF_LIFE_SETTING,
-        TransportGetAllocationStatsAction.CACHE_TTL_SETTING,
-        CodecService.TSDB_DOC_VALUES_OPTIMIZED_MERGE_SETTING
+        TransportGetAllocationStatsAction.CACHE_TTL_SETTING
     ).filter(Objects::nonNull).collect(toSet());
 }

server/src/main/java/org/elasticsearch/index/codec/CodecService.java

@@ -13,7 +13,6 @@
 import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.FilterCodec;
 import org.apache.lucene.codecs.lucene101.Lucene101Codec;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.core.Nullable;
@@ -33,12 +32,6 @@
 public class CodecService implements CodecProvider {
 
     public static final FeatureFlag ZSTD_STORED_FIELDS_FEATURE_FLAG = new FeatureFlag("zstd_stored_fields");
-    public static final FeatureFlag TSDB_DOC_VALUES_OPTIMIZED_MERGE = new FeatureFlag("tsdb_doc_values_codec_optimized_merge");
-    public static Setting<Boolean> TSDB_DOC_VALUES_OPTIMIZED_MERGE_SETTING = Setting.boolSetting(
-        "indices.time_series.doc_values_codec_optimized_merge",
-        TSDB_DOC_VALUES_OPTIMIZED_MERGE.isEnabled(),
-        Setting.Property.NodeScope
-    );
 
     private final Map<String, Codec> codecs;
 

server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java

@@ -14,7 +14,6 @@
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
-import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
@@ -35,17 +34,15 @@ public class PerFieldFormatSupplier {
 
     private static final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
     private static final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
+    private static final ES87TSDBDocValuesFormat tsdbDocValuesFormat = new ES87TSDBDocValuesFormat();
     private static final ES812PostingsFormat es812PostingsFormat = new ES812PostingsFormat();
     private static final PostingsFormat completionPostingsFormat = PostingsFormat.forName("Completion101");
 
-    private final ES87TSDBDocValuesFormat tsdbDocValuesFormat;
     private final ES87BloomFilterPostingsFormat bloomFilterPostingsFormat;
     private final MapperService mapperService;
 
     public PerFieldFormatSupplier(MapperService mapperService, BigArrays bigArrays) {
         this.mapperService = mapperService;
-        var nodeSettings = mapperService != null ? mapperService.getIndexSettings().getNodeSettings() : Settings.EMPTY;
-        this.tsdbDocValuesFormat = new ES87TSDBDocValuesFormat(CodecService.TSDB_DOC_VALUES_OPTIMIZED_MERGE_SETTING.get(nodeSettings));
         this.bloomFilterPostingsFormat = new ES87BloomFilterPostingsFormat(bigArrays, this::internalGetPostingsFormatForField);
     }
 

server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java

@@ -13,11 +13,10 @@
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.index.SegmentWriteState;
+import org.elasticsearch.common.util.FeatureFlag;
 
 import java.io.IOException;
 
-import static org.elasticsearch.index.codec.CodecService.TSDB_DOC_VALUES_OPTIMIZED_MERGE;
-
 public class ES87TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValuesFormat {
 
     static final int NUMERIC_BLOCK_SHIFT = 7;
@@ -77,16 +76,25 @@ public class ES87TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValuesF
         }
     }
 
+    // Default for escape hatch:
+    static final boolean OPTIMIZED_MERGE_ENABLE_DEFAULT;
+    static final FeatureFlag TSDB_DOC_VALUES_OPTIMIZED_MERGE = new FeatureFlag("tsdb_doc_values_optimized_merge");
+    static final String OPTIMIZED_MERGE_ENABLED_NAME = ES87TSDBDocValuesConsumer.class.getName() + ".enableOptimizedMerge";
+
+    static {
+        boolean optimizedMergeDefault = TSDB_DOC_VALUES_OPTIMIZED_MERGE.isEnabled();
+        OPTIMIZED_MERGE_ENABLE_DEFAULT = Boolean.parseBoolean(
+            System.getProperty(OPTIMIZED_MERGE_ENABLED_NAME, Boolean.toString(optimizedMergeDefault))
+        );
+    }
+
     private final int skipIndexIntervalSize;
+    // TODO: remove escape hatch? Is useful now when testing/benchmarking, but current optimized merge logic currently do too scary things.
     private final boolean enableOptimizedMerge;
 
     /** Default constructor. */
     public ES87TSDBDocValuesFormat() {
-        this(TSDB_DOC_VALUES_OPTIMIZED_MERGE.isEnabled());
-    }
-
-    public ES87TSDBDocValuesFormat(boolean enableOptimizedMerge) {
-        this(DEFAULT_SKIP_INDEX_INTERVAL_SIZE, enableOptimizedMerge);
+        this(DEFAULT_SKIP_INDEX_INTERVAL_SIZE, OPTIMIZED_MERGE_ENABLE_DEFAULT);
     }
 
     /** Doc values fields format with specified skipIndexIntervalSize. */

server/src/test/java/org/elasticsearch/index/codec/tsdb/DocValuesCodecDuelTests.java

@@ -48,7 +48,7 @@ public void testDuel() throws IOException {
             baselineConfig.setMergePolicy(mergePolicy);
             baselineConfig.setCodec(TestUtil.alwaysDocValuesFormat(new Lucene90DocValuesFormat()));
             var contenderConf = newIndexWriterConfig();
-            contenderConf.setCodec(TestUtil.alwaysDocValuesFormat(new ES87TSDBDocValuesFormat(true)));
+            contenderConf.setCodec(TestUtil.alwaysDocValuesFormat(new ES87TSDBDocValuesFormat()));
             contenderConf.setMergePolicy(mergePolicy);
 
             try (

server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatTests.java

@@ -54,7 +54,7 @@ public class ES87TSDBDocValuesFormatTests extends BaseDocValuesFormatTestCase {
 
     private static final int NUM_DOCS = 10;
 
-    private final Codec codec = TestUtil.alwaysDocValuesFormat(new ES87TSDBDocValuesFormat(true));
+    private final Codec codec = TestUtil.alwaysDocValuesFormat(new ES87TSDBDocValuesFormat());
 
     @Override
     protected Codec getCodec() {

server/src/test/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormatVariableSkipIntervalTests.java

@@ -32,13 +32,13 @@ public class ES87TSDBDocValuesFormatVariableSkipIntervalTests extends BaseDocVal
     @Override
     protected Codec getCodec() {
         // small interval size to test with many intervals
-        return TestUtil.alwaysDocValuesFormat(new ES87TSDBDocValuesFormat(random().nextInt(4, 16), true));
+        return TestUtil.alwaysDocValuesFormat(new ES87TSDBDocValuesFormat(random().nextInt(4, 16), false));
     }
 
     public void testSkipIndexIntervalSize() {
         IllegalArgumentException ex = expectThrows(
             IllegalArgumentException.class,
-            () -> new ES87TSDBDocValuesFormat(random().nextInt(Integer.MIN_VALUE, 2), true)
+            () -> new ES87TSDBDocValuesFormat(random().nextInt(Integer.MIN_VALUE, 2), false)
         );
         assertTrue(ex.getMessage().contains("skipIndexIntervalSize must be > 1"));
     }