[8.x] Prepare tsdb doc values format for merging optimizations. (#126261)

Backporting #125933 to 8.x branch.

The change contains the following changes:

- The numDocsWithField field moved from SortedNumericEntry to NumericEntry. Making this statistic always available.
- Store jump table after values in ES87TSDBDocValuesConsumer#writeField(...). Currently it is stored before storing values. This will allow us later to iterate over the SortedNumericDocValues once. When merging, this is expensive as a merge sort on the fly is being executed.

This change will allow all the optimizations that are listed in #125403

Author: martijnvg

server/src/main/java/module-info.java

@@ -447,7 +447,10 @@
             org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.postings.ES812PostingsFormat;
-    provides org.apache.lucene.codecs.DocValuesFormat with org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
+    provides org.apache.lucene.codecs.DocValuesFormat
+        with
+            org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat,
+            org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
     provides org.apache.lucene.codecs.KnnVectorsFormat
         with
             org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat,

server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java

@@ -19,7 +19,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat;
 import org.elasticsearch.index.codec.postings.ES812PostingsFormat;
-import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
+import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperService;
@@ -31,18 +31,17 @@
  */
 public class PerFieldFormatSupplier {
 
+    private static final ES819TSDBDocValuesFormat tsdbDocValuesFormat = new ES819TSDBDocValuesFormat();
     private final MapperService mapperService;
     private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
     private final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
     private final ES87BloomFilterPostingsFormat bloomFilterPostingsFormat;
-    private final ES87TSDBDocValuesFormat tsdbDocValuesFormat;
 
     private final ES812PostingsFormat es812PostingsFormat;
 
     public PerFieldFormatSupplier(MapperService mapperService, BigArrays bigArrays) {
         this.mapperService = mapperService;
         this.bloomFilterPostingsFormat = new ES87BloomFilterPostingsFormat(bigArrays, this::internalGetPostingsFormatForField);
-        this.tsdbDocValuesFormat = new ES87TSDBDocValuesFormat();
         this.es812PostingsFormat = new ES812PostingsFormat();
     }
 

server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java

@@ -49,7 +49,7 @@ public ES87TSDBDocValuesFormat() {
 
     @Override
     public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-        return new ES87TSDBDocValuesConsumer(state, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+        throw new UnsupportedOperationException("writing es87 doc values is no longer supported");
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java

@@ -44,7 +44,7 @@
 
 import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT;
 
-public class ES87TSDBDocValuesProducer extends DocValuesProducer {
+final class ES87TSDBDocValuesProducer extends DocValuesProducer {
     private final IntObjectHashMap<NumericEntry> numerics;
     private final IntObjectHashMap<BinaryEntry> binaries;
     private final IntObjectHashMap<SortedEntry> sorted;

server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesEncoder.java

@@ -175,7 +175,7 @@ private void forEncode(int token, int tokenBits, long[] in, DataOutput out) thro
     /**
      * Encode the given longs using a combination of delta-coding, GCD factorization and bit packing.
      */
-    void encode(long[] in, DataOutput out) throws IOException {
+    public void encode(long[] in, DataOutput out) throws IOException {
         assert in.length == numericBlockSize;
 
         deltaEncode(0, 0, in, out);
@@ -193,7 +193,7 @@ void encode(long[] in, DataOutput out) throws IOException {
      *   <li>3: cycle</li>
      * </ul>
      */
-    void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOException {
+    public void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOException {
         assert in.length == numericBlockSize;
         int numRuns = 1;
         long firstValue = in[0];
@@ -260,7 +260,7 @@ void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOExceptio
         }
     }
 
-    void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException {
+    public void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException {
         assert out.length == numericBlockSize : out.length;
 
         long v1 = in.readVLong();
@@ -294,7 +294,7 @@ void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException
     }
 
     /** Decode longs that have been encoded with {@link #encode}. */
-    void decode(DataInput in, long[] out) throws IOException {
+    public void decode(DataInput in, long[] out) throws IOException {
         assert out.length == numericBlockSize : out.length;
 
         final int token = in.readVInt();