Enable large numeric blocks for TSDB (#138744)

* Enable large numeric blocks for TSDB

* test fix

* fix sync issue

* comments

* fix test

Author: kkrik-es

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -209,6 +209,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING,
                 IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING,
                 IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING,
+                IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BLOCK_SIZE,
                 InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT,
                 IndexSettings.USE_ES_812_POSTINGS_FORMAT,
 

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -842,6 +842,19 @@ public Iterator<Setting<?>> settings() {
         Property.Final
     );
 
+    public static final Setting<Boolean> USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BLOCK_SIZE = Setting.boolSetting(
+        "index.use_time_series_doc_values_format_large_block_size",
+        settings -> {
+            if (settings == null) {
+                return Boolean.FALSE.toString();
+            }
+            var indexMode = IndexSettings.MODE.get(settings);
+            return Boolean.toString(indexMode == IndexMode.TIME_SERIES);
+        },
+        Property.IndexScope,
+        Property.Final
+    );
+
     /**
      * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use.
      * TODO: Remove in 9.0
@@ -1021,6 +1034,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
     private final boolean useDocValuesSkipper;
     private final boolean useTimeSeriesSyntheticId;
     private final boolean useTimeSeriesDocValuesFormat;
+    private final boolean useTimeSeriesDocValuesFormatLargeBlockSize;
     private final boolean useEs812PostingsFormat;
 
     /**
@@ -1210,6 +1224,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER);
         seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING);
         useTimeSeriesDocValuesFormat = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING);
+        useTimeSeriesDocValuesFormatLargeBlockSize = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_LARGE_BLOCK_SIZE);
         useEs812PostingsFormat = scopedSettings.get(USE_ES_812_POSTINGS_FORMAT);
         final var useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && scopedSettings.get(USE_SYNTHETIC_ID);
         if (indexMetadata.useTimeSeriesSyntheticId() != useSyntheticId) {
@@ -1988,6 +2003,13 @@ public boolean useTimeSeriesDocValuesFormat() {
         return useTimeSeriesDocValuesFormat;
     }
 
+    /**
+     * @return Whether the time series doc value format with large numeric block size should be used.
+     */
+    public boolean isUseTimeSeriesDocValuesFormatLargeBlockSize() {
+        return useTimeSeriesDocValuesFormatLargeBlockSize;
+    }
+
     /**
      * @return Whether the ES 8.12 postings format should be used.
      */

server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java

@@ -15,6 +15,7 @@
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersions;
@@ -43,6 +44,8 @@ public class PerFieldFormatSupplier {
     private static final Set<String> INCLUDE_META_FIELDS;
     private static final Set<String> EXCLUDE_MAPPER_TYPES;
 
+    private static final boolean TSDB_USE_LARGE_NUMERIC_BLOCKS = new FeatureFlag("tsdb_large_numeric_blocks").isEnabled();
+
     static {
         // TODO: should we just allow all fields to use tsdb doc values codec?
         // Avoid using tsdb codec for fields like _seq_no, _primary_term.
@@ -59,7 +62,10 @@ public class PerFieldFormatSupplier {
 
     private static final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
     private static final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
-    private static final ES819TSDBDocValuesFormat tsdbDocValuesFormat = new ES819TSDBDocValuesFormat();
+    private static final ES819TSDBDocValuesFormat tsdbDocValuesFormat = ES819TSDBDocValuesFormat.getInstance(false);
+    private static final ES819TSDBDocValuesFormat tsdbDocValuesFormatLargeNumericBlock = ES819TSDBDocValuesFormat.getInstance(
+        TSDB_USE_LARGE_NUMERIC_BLOCKS
+    );
     private static final ES812PostingsFormat es812PostingsFormat = new ES812PostingsFormat();
     private static final PostingsFormat completionPostingsFormat = PostingsFormat.forName("Completion101");
 
@@ -137,7 +143,9 @@ public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
 
     public DocValuesFormat getDocValuesFormatForField(String field) {
         if (useTSDBDocValuesFormat(field)) {
-            return tsdbDocValuesFormat;
+            return (mapperService != null && mapperService.getIndexSettings().isUseTimeSeriesDocValuesFormatLargeBlockSize())
+                ? tsdbDocValuesFormatLargeNumericBlock
+                : tsdbDocValuesFormat;
         }
         return docValuesFormat;
     }

server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java

@@ -140,6 +140,10 @@ private static boolean getOptimizedMergeEnabledDefault() {
     final BinaryDVCompressionMode binaryDVCompressionMode;
     final boolean enablePerBlockCompression;
 
+    public static ES819TSDBDocValuesFormat getInstance(boolean useLargeNumericBlock) {
+        return useLargeNumericBlock ? new ES819TSDBDocValuesFormat(NUMERIC_LARGE_BLOCK_SHIFT) : new ES819TSDBDocValuesFormat();
+    }
+
     public ES819TSDBDocValuesFormat() {
         this(NUMERIC_BLOCK_SHIFT);
     }