Merge branch 'main' into data-stream-options-convert-to-yml

Author: gmarouli

muted-tests.yml

@@ -341,9 +341,6 @@ tests:
 - class: org.elasticsearch.xpack.migrate.action.ReindexDataStreamTransportActionIT
   method: testAlreadyUpToDateDataStream
   issue: https://github.com/elastic/elasticsearch/issues/125727
-- class: org.elasticsearch.index.engine.ThreadPoolMergeSchedulerStressTestIT
-  method: testMergingFallsBehindAndThenCatchesUp
-  issue: https://github.com/elastic/elasticsearch/issues/125744
 - class: org.elasticsearch.xpack.esql.spatial.SpatialExtentAggregationNoLicenseIT
   method: testStExtentAggregationWithPoints
   issue: https://github.com/elastic/elasticsearch/issues/125735
@@ -389,18 +386,18 @@ tests:
 - class: org.elasticsearch.action.RejectionActionIT
   method: testSimulatedSearchRejectionLoad
   issue: https://github.com/elastic/elasticsearch/issues/125901
-- class: org.elasticsearch.xpack.security.authz.RBACEngineTests
-  method: testGetRoleDescriptorsIntersectionForRemoteClusterHasDeterministicOrderForIndicesPrivileges
-  issue: https://github.com/elastic/elasticsearch/issues/126061
 - class: org.elasticsearch.search.CCSDuelIT
   method: testTerminateAfter
   issue: https://github.com/elastic/elasticsearch/issues/126085
 - class: org.elasticsearch.index.mapper.blockloader.KeywordFieldBlockLoaderTests
   method: testBlockLoaderForFieldInObject {preference=Params[syntheticSource=false, preference=DOC_VALUES]}
   issue: https://github.com/elastic/elasticsearch/issues/126088
+- class: org.elasticsearch.search.sort.GeoDistanceIT
+  method: testDistanceSortingWithUnmappedField
+  issue: https://github.com/elastic/elasticsearch/issues/126118
 - class: org.elasticsearch.index.mapper.blockloader.KeywordFieldBlockLoaderTests
-  method: testBlockLoader {preference=Params[syntheticSource=false, preference=DOC_VALUES]}"
-  issue: https://github.com/elastic/elasticsearch/issues/126088
+  method: testBlockLoader {preference=Params[syntheticSource=false, preference=DOC_VALUES]}
+  issue: https://github.com/elastic/elasticsearch/issues/126121
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/index/engine/ThreadPoolMergeSchedulerStressTestIT.java

@@ -79,12 +79,6 @@ public static class TestEnginePlugin extends Plugin implements EnginePlugin {
         final Semaphore runMergeSemaphore = new Semaphore(initialRunMergesCount);
         final int waitMergesEnqueuedCount = randomIntBetween(50, 100);
 
-        void allowAllMerging() {
-            // even when indexing is done, queued and backlogged merges can themselves trigger further merging
-            // don't let this test be bothered by that, and simply let all merging run unhindered
-            runMergeSemaphore.release(Integer.MAX_VALUE - initialRunMergesCount);
-        }
-
         class TestInternalEngine extends org.elasticsearch.index.engine.InternalEngine {
 
             TestInternalEngine(EngineConfig engineConfig) {
@@ -265,10 +259,13 @@ public void testMergingFallsBehindAndThenCatchesUp() throws Exception {
         for (Thread indexingThread : indexingThreads) {
             indexingThread.join();
         }
-        // unblock merge threads
-        testEnginePlugin.allowAllMerging();
+        // even when indexing is done, queued and backlogged merges can themselves trigger further merging
+        // don't let this test be bothered by that, and simply unblock all merges
+        // 100k is a fudge value, but there's no easy way to find a smartest one here
+        testEnginePlugin.runMergeSemaphore.release(100_000);
         // await all merging to catch up
         assertBusy(() -> {
+            assert testEnginePlugin.runMergeSemaphore.availablePermits() > 0 : "some merges are blocked, test is broken";
             assertThat(testEnginePlugin.runningMergesSet.size(), is(0));
             assertThat(testEnginePlugin.enqueuedMergesSet.size(), is(0));
             testEnginePlugin.mergeExecutorServiceReference.get().allDone();

server/src/main/java/module-info.java

@@ -443,7 +443,10 @@
             org.elasticsearch.index.codec.bloomfilter.ES85BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat,
             org.elasticsearch.index.codec.postings.ES812PostingsFormat;
-    provides org.apache.lucene.codecs.DocValuesFormat with org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
+    provides org.apache.lucene.codecs.DocValuesFormat
+        with
+            org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat,
+            org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
     provides org.apache.lucene.codecs.KnnVectorsFormat
         with
             org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat,

server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java

@@ -19,7 +19,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat;
 import org.elasticsearch.index.codec.postings.ES812PostingsFormat;
-import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat;
+import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
 import org.elasticsearch.index.mapper.CompletionFieldMapper;
 import org.elasticsearch.index.mapper.IdFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
@@ -34,7 +34,7 @@ public class PerFieldFormatSupplier {
 
     private static final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat();
     private static final KnnVectorsFormat knnVectorsFormat = new Lucene99HnswVectorsFormat();
-    private static final ES87TSDBDocValuesFormat tsdbDocValuesFormat = new ES87TSDBDocValuesFormat();
+    private static final ES819TSDBDocValuesFormat tsdbDocValuesFormat = new ES819TSDBDocValuesFormat();
     private static final ES812PostingsFormat es812PostingsFormat = new ES812PostingsFormat();
     private static final PostingsFormat completionPostingsFormat = PostingsFormat.forName("Completion101");
 

server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesFormat.java

@@ -75,7 +75,7 @@ public class ES87TSDBDocValuesFormat extends org.apache.lucene.codecs.DocValuesF
         }
     }
 
-    private final int skipIndexIntervalSize;
+    final int skipIndexIntervalSize;
 
     /** Default constructor. */
     public ES87TSDBDocValuesFormat() {
@@ -93,7 +93,7 @@ public ES87TSDBDocValuesFormat(int skipIndexIntervalSize) {
 
     @Override
     public DocValuesConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
-        return new ES87TSDBDocValuesConsumer(state, skipIndexIntervalSize, DATA_CODEC, DATA_EXTENSION, META_CODEC, META_EXTENSION);
+        throw new UnsupportedOperationException("writing es87 doc values is no longer supported");
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/codec/tsdb/ES87TSDBDocValuesProducer.java

@@ -49,7 +49,7 @@
 import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.SKIP_INDEX_MAX_LEVEL;
 import static org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT;
 
-public class ES87TSDBDocValuesProducer extends DocValuesProducer {
+final class ES87TSDBDocValuesProducer extends DocValuesProducer {
     private final IntObjectHashMap<NumericEntry> numerics;
     private final IntObjectHashMap<BinaryEntry> binaries;
     private final IntObjectHashMap<SortedEntry> sorted;

server/src/main/java/org/elasticsearch/index/codec/tsdb/TSDBDocValuesEncoder.java

@@ -175,7 +175,7 @@ private void forEncode(int token, int tokenBits, long[] in, DataOutput out) thro
     /**
      * Encode the given longs using a combination of delta-coding, GCD factorization and bit packing.
      */
-    void encode(long[] in, DataOutput out) throws IOException {
+    public void encode(long[] in, DataOutput out) throws IOException {
         assert in.length == numericBlockSize;
 
         deltaEncode(0, 0, in, out);
@@ -193,7 +193,7 @@ void encode(long[] in, DataOutput out) throws IOException {
      *   <li>3: cycle</li>
      * </ul>
      */
-    void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOException {
+    public void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOException {
         assert in.length == numericBlockSize;
         int numRuns = 1;
         long firstValue = in[0];
@@ -260,7 +260,7 @@ void encodeOrdinals(long[] in, DataOutput out, int bitsPerOrd) throws IOExceptio
         }
     }
 
-    void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException {
+    public void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException {
         assert out.length == numericBlockSize : out.length;
 
         long v1 = in.readVLong();
@@ -294,7 +294,7 @@ void decodeOrdinals(DataInput in, long[] out, int bitsPerOrd) throws IOException
     }
 
     /** Decode longs that have been encoded with {@link #encode}. */
-    void decode(DataInput in, long[] out) throws IOException {
+    public void decode(DataInput in, long[] out) throws IOException {
         assert out.length == numericBlockSize : out.length;
 
         final int token = in.readVInt();