Introduce large block doc values format (#138182)

* Add accessors for block size constants in tsdb codec

* add large block format

* update codec name

* sync

* use new version

* [CI] Auto commit changes from spotless

* fix

* fix and tests

* assert

* refactor

* fix

---------

Co-authored-by: elasticsearchmachine <[email protected]>

Author: kkrik-es

server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesConsumer.java

@@ -73,6 +73,8 @@ final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer {
     private final int minDocsPerOrdinalForOrdinalRangeEncoding;
     final boolean enableOptimizedMerge;
     private final int primarySortFieldNumber;
+    private final int numericBlockShift;
+    private final int numericBlockSize;
     final SegmentWriteState state;
     final BinaryDVCompressionMode binaryDVCompressionMode;
     private final boolean enablePerBlockCompression; // only false for testing
@@ -84,6 +86,7 @@ final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer {
         int skipIndexIntervalSize,
         int minDocsPerOrdinalForOrdinalRangeEncoding,
         boolean enableOptimizedMerge,
+        int numericBlockShift,
         String dataCodec,
         String dataExtension,
         String metaCodec,
@@ -97,6 +100,9 @@ final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer {
         this.minDocsPerOrdinalForOrdinalRangeEncoding = minDocsPerOrdinalForOrdinalRangeEncoding;
         this.primarySortFieldNumber = ES819TSDBDocValuesProducer.primarySortFieldNumber(state.segmentInfo, state.fieldInfos);
         this.context = state.context;
+        this.numericBlockShift = numericBlockShift;
+        this.numericBlockSize = 1 << numericBlockShift;
+
         boolean success = false;
         try {
             final String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
@@ -108,6 +114,7 @@ final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer {
                 state.segmentInfo.getId(),
                 state.segmentSuffix
             );
+
             String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
             meta = state.directory.createOutput(metaName, state.context);
             CodecUtil.writeIndexHeader(
@@ -117,6 +124,8 @@ final class ES819TSDBDocValuesConsumer extends XDocValuesConsumer {
                 state.segmentInfo.getId(),
                 state.segmentSuffix
             );
+            meta.writeByte((byte) numericBlockShift);
+
             maxDoc = state.segmentInfo.maxDoc();
             this.skipIndexIntervalSize = skipIndexIntervalSize;
             this.enableOptimizedMerge = enableOptimizedMerge;
@@ -218,13 +227,13 @@ private long[] writeField(FieldInfo field, TsdbDocValuesProducer valuesProducer,
                     indexWriter = DirectMonotonicWriter.getInstance(
                         meta,
                         new ByteBuffersIndexOutput(indexOut, "temp-dv-index", "temp-dv-index"),
-                        1L + ((numValues - 1) >>> ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SHIFT),
+                        1L + ((numValues - 1) >>> numericBlockShift),
                         ES819TSDBDocValuesFormat.DIRECT_MONOTONIC_BLOCK_SHIFT
                     );
                     meta.writeInt(DIRECT_MONOTONIC_BLOCK_SHIFT);
-                    final long[] buffer = new long[ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE];
+                    final long[] buffer = new long[numericBlockSize];
                     int bufferSize = 0;
-                    final TSDBDocValuesEncoder encoder = new TSDBDocValuesEncoder(ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE);
+                    final TSDBDocValuesEncoder encoder = new TSDBDocValuesEncoder(numericBlockSize);
                     values = valuesProducer.getSortedNumeric(field);
                     final int bitsPerOrd = maxOrd >= 0 ? PackedInts.bitsRequired(maxOrd - 1) : -1;
                     if (valuesProducer.mergeStats.supported() && numDocsWithValue < maxDoc) {
@@ -240,7 +249,7 @@ private long[] writeField(FieldInfo field, TsdbDocValuesProducer valuesProducer,
                         }
                         for (int i = 0; i < count; ++i) {
                             buffer[bufferSize++] = values.nextValue();
-                            if (bufferSize == ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE) {
+                            if (bufferSize == numericBlockSize) {
                                 indexWriter.add(data.getFilePointer() - valuesDataOffset);
                                 if (maxOrd >= 0) {
                                     encoder.encodeOrdinals(buffer, data, bitsPerOrd);
@@ -254,7 +263,7 @@ private long[] writeField(FieldInfo field, TsdbDocValuesProducer valuesProducer,
                     if (bufferSize > 0) {
                         indexWriter.add(data.getFilePointer() - valuesDataOffset);
                         // Fill unused slots in the block with zeroes rather than junk
-                        Arrays.fill(buffer, bufferSize, ES819TSDBDocValuesFormat.NUMERIC_BLOCK_SIZE, 0L);
+                        Arrays.fill(buffer, bufferSize, numericBlockSize, 0L);
                         if (maxOrd >= 0) {
                             encoder.encodeOrdinals(buffer, data, bitsPerOrd);
                         } else {

server/src/main/java/org/elasticsearch/index/codec/tsdb/es819/ES819TSDBDocValuesFormat.java

@@ -37,8 +37,7 @@
 public class ES819TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValuesFormat {
 
     static final int NUMERIC_BLOCK_SHIFT = 7;
-    public static final int NUMERIC_BLOCK_SIZE = 1 << NUMERIC_BLOCK_SHIFT;
-    static final int NUMERIC_BLOCK_MASK = NUMERIC_BLOCK_SIZE - 1;
+    static final int NUMERIC_LARGE_BLOCK_SHIFT = 9;
     static final int DIRECT_MONOTONIC_BLOCK_SHIFT = 16;
     static final String CODEC_NAME = "ES819TSDB";
     static final String DATA_CODEC = "ES819TSDBDocValuesData";
@@ -53,7 +52,8 @@ public class ES819TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValues
 
     static final int VERSION_START = 0;
     static final int VERSION_BINARY_DV_COMPRESSION = 1;
-    static final int VERSION_CURRENT = VERSION_BINARY_DV_COMPRESSION;
+    static final int VERSION_NUMERIC_LARGE_BLOCKS = 2;
+    static final int VERSION_CURRENT = VERSION_NUMERIC_LARGE_BLOCKS;
 
     static final int TERMS_DICT_BLOCK_LZ4_SHIFT = 6;
     static final int TERMS_DICT_BLOCK_LZ4_SIZE = 1 << TERMS_DICT_BLOCK_LZ4_SHIFT;
@@ -130,14 +130,18 @@ private static boolean getOptimizedMergeEnabledDefault() {
      */
     public static final int ORDINAL_RANGE_ENCODING_BLOCK_SHIFT = 12;
 
+    final int numericBlockShift;
     final int skipIndexIntervalSize;
     final int minDocsPerOrdinalForRangeEncoding;
     final boolean enableOptimizedMerge;
     final BinaryDVCompressionMode binaryDVCompressionMode;
     final boolean enablePerBlockCompression;
 
-    /** Default constructor. */
     public ES819TSDBDocValuesFormat() {
+        this(NUMERIC_BLOCK_SHIFT);
+    }
+
+    public ES819TSDBDocValuesFormat(int numericBlockShift) {
         this(
             DEFAULT_SKIP_INDEX_INTERVAL_SIZE,
             ORDINAL_RANGE_ENCODING_MIN_DOC_PER_ORDINAL,
@@ -153,7 +157,8 @@ public ES819TSDBDocValuesFormat(BinaryDVCompressionMode binaryDVCompressionMode)
             ORDINAL_RANGE_ENCODING_MIN_DOC_PER_ORDINAL,
             OPTIMIZED_MERGE_ENABLE_DEFAULT,
             binaryDVCompressionMode,
-            true
+            true,
+            NUMERIC_BLOCK_SHIFT
         );
     }
 
@@ -163,7 +168,8 @@ public ES819TSDBDocValuesFormat(BinaryDVCompressionMode binaryDVCompressionMode,
             ORDINAL_RANGE_ENCODING_MIN_DOC_PER_ORDINAL,
             OPTIMIZED_MERGE_ENABLE_DEFAULT,
             binaryDVCompressionMode,
-            enablePerBlockCompression
+            enablePerBlockCompression,
+            NUMERIC_BLOCK_SHIFT
         );
     }
 
@@ -174,8 +180,27 @@ public ES819TSDBDocValuesFormat(
         boolean enableOptimizedMerge,
         BinaryDVCompressionMode binaryDVCompressionMode,
         final boolean enablePerBlockCompression
+    ) {
+        this(
+            skipIndexIntervalSize,
+            minDocsPerOrdinalForRangeEncoding,
+            enableOptimizedMerge,
+            binaryDVCompressionMode,
+            enablePerBlockCompression,
+            NUMERIC_BLOCK_SHIFT
+        );
+    }
+
+    public ES819TSDBDocValuesFormat(
+        int skipIndexIntervalSize,
+        int minDocsPerOrdinalForRangeEncoding,
+        boolean enableOptimizedMerge,
+        BinaryDVCompressionMode binaryDVCompressionMode,
+        final boolean enablePerBlockCompression,
+        final int numericBlockShift
     ) {
         super(CODEC_NAME);
+        assert numericBlockShift == NUMERIC_BLOCK_SHIFT || numericBlockShift == NUMERIC_LARGE_BLOCK_SHIFT : numericBlockShift;
         if (skipIndexIntervalSize < 2) {
             throw new IllegalArgumentException("skipIndexIntervalSize must be > 1, got [" + skipIndexIntervalSize + "]");
         }
@@ -184,6 +209,7 @@ public ES819TSDBDocValuesFormat(
         this.enableOptimizedMerge = enableOptimizedMerge;
         this.binaryDVCompressionMode = binaryDVCompressionMode;
         this.enablePerBlockCompression = enablePerBlockCompression;
+        this.numericBlockShift = numericBlockShift;
     }
 
     @Override
@@ -195,6 +221,7 @@ public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOExcept
             skipIndexIntervalSize,
             minDocsPerOrdinalForRangeEncoding,
             enableOptimizedMerge,
+            numericBlockShift,
             DATA_CODEC,
             DATA_EXTENSION,
             META_CODEC,