Prepare tsdb doc values format for merging optimizations. (#125933)

The change contains the following changes:

- The numDocsWithField field moved from SortedNumericEntry to NumericEntry. Making this statistic always available.
- Store jump table after values in ES87TSDBDocValuesConsumer#writeField(...). Currently it is stored before storing values. This will allow us later to iterate over the SortedNumericDocValues once. When merging, this is expensive as a merge sort on the fly is being executed.

This change will allow all the optimizations that are listed in #125403

Author: martijnvg

server/src/main/java/module-info.java

@@ -443,7 +443,10 @@
             org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.postings.ES812PostingsFormat;
-    provides org.apache.lucene.codecs.DocValuesFormat with org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
+    provides org.apache.lucene.codecs.DocValuesFormat
+        with
+            org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat,
+            org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
     provides org.apache.lucene.codecs.KnnVectorsFormat
         with
             org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat,

server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java

@@ -19,7 +19,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat;
 import org.elasticsearch.index.codec.postings.ES812PostingsFormat;
-import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
+import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
 import org.elasticsearch.index.mapper.CompletionFieldMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
@@ -34,7 +34,7 @@ public class PerFieldFormatSupplier {
 
     private static final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
     private static final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
-    private static final ES87TSDBDocValuesFormat tsdbDocValuesFormat = new ES87TSDBDocValuesFormat();
+    private static final ES819TSDBDocValuesFormat tsdbDocValuesFormat = new ES819TSDBDocValuesFormat();
     private static final ES812PostingsFormat es812PostingsFormat = new ES812PostingsFormat();
     private static final PostingsFormat completionPostingsFormat = PostingsFormat.forName("Completion101");
 

server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java

@@ -75,7 +75,7 @@ public class ES87TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValuesF
         }
     }
 
-    private final int skipIndexIntervalSize;
+    final int skipIndexIntervalSize;
 
     /** Default constructor. */
     public ES87TSDBDocValuesFormat() {
@@ -93,7 +93,7 @@ public ES87TSDBDocValuesFormat(int skipIndexIntervalSize) {
 
     @Override
     public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-        return new ES87TSDBDocValuesConsumer(state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+        throw new UnsupportedOperationException("writing es87 doc values is no longer supported");
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java

@@ -49,7 +49,7 @@
 import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.SKIP_INDEX_MAX_LEVEL;
 import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT;
 
-public class ES87TSDBDocValuesProducer extends DocValuesProducer {
+final class ES87TSDBDocValuesProducer extends DocValuesProducer {
     private final IntObjectHashMap<NumericEntry> numerics;
     private final IntObjectHashMap<BinaryEntry> binaries;
     private final IntObjectHashMap<SortedEntry> sorted;

server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesEncoder.java

@@ -175,7 +175,7 @@ private void forEncode(int token, int tokenBits, long[] in, DataOutput out) thro
     /**
      * Encode the given longs using a combination of delta-coding, GCD factorization and bit packing.
      */
-    void encode(long[] in, DataOutput out) throws IOException {
+    public void encode(long[] in, DataOutput out) throws IOException {
         assert in.length == numericBlockSize;
 
         deltaEncode(0, 0, in, out);
@@ -193,7 +193,7 @@ void encode(long[] in, DataOutput out) throws IOException {
      *   <li>3: cycle</li>
      * </ul>
      */
-    void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOException {
+    public void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOException {
         assert in.length == numericBlockSize;
         int numRuns = 1;
         long firstValue = in[0];
@@ -260,7 +260,7 @@ void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOExceptio
         }
     }
 
-    void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException {
+    public void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException {
         assert out.length == numericBlockSize : out.length;
 
         long v1 = in.readVLong();
@@ -294,7 +294,7 @@ void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException
     }
 
     /** Decode longs that have been encoded with {@link #encode}. */
-    void decode(DataInput in, long[] out) throws IOException {
+    public void decode(DataInput in, long[] out) throws IOException {
         assert out.length == numericBlockSize : out.length;
 
         final int token = in.readVInt();