Merge branch 'main' into fork

Author: ChrisHegarty

muted-tests.yml

@@ -407,6 +407,15 @@ tests:
 - class: org.elasticsearch.search.SearchCancellationIT
   method: testCancelFailedSearchWhenPartialResultDisallowed
   issue: https://github.com/elastic/elasticsearch/issues/121719
+- class: org.elasticsearch.xpack.esql.analysis.VerifierTests
+  method: testChangePoint
+  issue: https://github.com/elastic/elasticsearch/issues/122179
+- class: org.elasticsearch.xpack.esql.analysis.VerifierTests
+  method: testChangePoint_keySortable
+  issue: https://github.com/elastic/elasticsearch/issues/122180
+- class: org.elasticsearch.xpack.esql.analysis.VerifierTests
+  method: testChangePoint_valueNumeric
+  issue: https://github.com/elastic/elasticsearch/issues/122181
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshot.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.index.engine;
 
+import com.carrotsearch.hppc.IntArrayList;
+
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.FieldDoc;
 import org.apache.lucene.search.ScoreDoc;
@@ -191,8 +193,28 @@ private Translog.Operation[] loadDocuments(List<SearchRecord> documentRecords) t
                     maxDoc = leafReaderContext.reader().maxDoc();
                 } while (docRecord.docID() >= docBase + maxDoc);
 
-                leafFieldLoader = storedFieldLoader.getLoader(leafReaderContext, null);
-                leafSourceLoader = sourceLoader.leaf(leafReaderContext.reader(), null);
+                // TODO: instead of building an array, consider just checking whether doc ids are dense.
+                // Note, field loaders then would lose the ability to optionally eagerly loading values.
+                IntArrayList nextDocIds = new IntArrayList();
+                for (int j = i; j < documentRecords.size(); j++) {
+                    var record = documentRecords.get(j);
+                    if (record.isTombstone()) {
+                        continue;
+                    }
+                    int docID = record.docID();
+                    if (docID >= docBase + maxDoc) {
+                        break;
+                    }
+                    int segmentDocID = docID - docBase;
+                    nextDocIds.add(segmentDocID);
+                }
+
+                // This computed doc ids arrays us used by stored field loader as a heuristic to determine whether to use a sequential
+                // stored field reader (which bulk loads stored fields and avoids decompressing the same blocks multiple times). For
+                // source loader, it is also used as a heuristic for bulk reading doc values (E.g. SingletonDocValuesLoader).
+                int[] nextDocIdArray = nextDocIds.toArray();
+                leafFieldLoader = storedFieldLoader.getLoader(leafReaderContext, nextDocIdArray);
+                leafSourceLoader = sourceLoader.leaf(leafReaderContext.reader(), nextDocIdArray);
                 setNextSourceMetadataReader(leafReaderContext);
             }
             int segmentDocID = docRecord.docID() - docBase;

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java

@@ -2018,6 +2018,7 @@ public void testCategorizeWithFilteredAggregations() {
     }
 
     public void testChangePoint() {
+        assumeTrue("change_point must be enabled", EsqlCapabilities.Cap.CHANGE_POINT.isEnabled());
         var airports = AnalyzerTestUtils.analyzer(loadMapping("mapping-airports.json", "airports"));
         assertEquals("1:30: Unknown column [blahblah]", error("FROM airports | CHANGE_POINT blahblah ON scalerank", airports));
         assertEquals("1:43: Unknown column [blahblah]", error("FROM airports | CHANGE_POINT scalerank ON blahblah", airports));
@@ -2026,6 +2027,7 @@ public void testChangePoint() {
     }
 
     public void testChangePoint_keySortable() {
+        assumeTrue("change_point must be enabled", EsqlCapabilities.Cap.CHANGE_POINT.isEnabled());
         List<DataType> sortableTypes = List.of(BOOLEAN, DOUBLE, DATE_NANOS, DATETIME, INTEGER, IP, KEYWORD, LONG, UNSIGNED_LONG, VERSION);
         List<DataType> unsortableTypes = List.of(CARTESIAN_POINT, CARTESIAN_SHAPE, GEO_POINT, GEO_SHAPE);
         for (DataType type : sortableTypes) {
@@ -2040,6 +2042,7 @@ public void testChangePoint_keySortable() {
     }
 
     public void testChangePoint_valueNumeric() {
+        assumeTrue("change_point must be enabled", EsqlCapabilities.Cap.CHANGE_POINT.isEnabled());
         List<DataType> numericTypes = List.of(DOUBLE, INTEGER, LONG, UNSIGNED_LONG);
         List<DataType> nonNumericTypes = List.of(
             BOOLEAN,