Rework page_size

nik9000 · nik9000 · commit c2f656dd2b55 · 2025-07-15T12:22:13.000-04:00
Don't need it any more
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockDocValuesReader.java
@@ -645,9 +645,9 @@ private BlockLoader.Block readSingleDoc(BlockFactory factory, int docId) throws
             if (ordinals.advanceExact(docId)) {
                 BytesRef v = ordinals.lookupOrd(ordinals.ordValue());
                 // the returned BytesRef can be reused
-                return factory.constantBytes(BytesRef.deepCopyOf(v));
+                return factory.constantBytes(BytesRef.deepCopyOf(v), 1);
             } else {
-                return factory.constantNulls();
+                return factory.constantNulls(1);
             }
         }
 
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java
@@ -150,7 +150,7 @@ public String toString() {
     class ConstantNullsReader implements AllReader {
         @Override
         public Block read(BlockFactory factory, Docs docs, int offset) throws IOException {
-            return factory.constantNulls();
+            return factory.constantNulls(docs.count() - offset);
         }
 
         @Override
@@ -184,7 +184,7 @@ public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) {
                 return new ColumnAtATimeReader() {
                     @Override
                     public Block read(BlockFactory factory, Docs docs, int offset) {
-                        return factory.constantBytes(value);
+                        return factory.constantBytes(value, docs.count() - offset);
                     }
 
                     @Override
@@ -408,13 +408,13 @@ interface BlockFactory {
         /**
          * Build a block that contains only {@code null}.
          */
-        Block constantNulls();
+        Block constantNulls(int count);
 
         /**
          * Build a block that contains {@code value} repeated
          * {@code size} times.
          */
-        Block constantBytes(BytesRef value);
+        Block constantBytes(BytesRef value, int count);
 
         /**
          * Build a reader for reading keyword ordinals.
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapperTests.java
@@ -126,7 +126,7 @@ private static void testBoundsBlockLoaderAux(
                         expected.add(visitor.apply(geometries.get(j + currentIndex)).get());
                     }
                     try (
-                        var block = (TestBlock) loader.reader(leaf).read(TestBlock.factory(leafReader.numDocs()), TestBlock.docs(array), 0)
+                        var block = (TestBlock) loader.reader(leaf).read(TestBlock.factory(), TestBlock.docs(array), 0)
                     ) {
                         for (int i = 0; i < block.size(); i++) {
                             intArrayResults.add(block.get(i));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java
@@ -59,7 +59,7 @@ private void loadBlock(LeafReaderContext ctx, Consumer<TestBlock> test) throws I
             StoredFieldLoader.fromSpec(loader.rowStrideStoredFieldSpec()).getLoader(ctx, null),
             loader.rowStrideStoredFieldSpec().requiresSource() ? SourceLoader.FROM_STORED_SOURCE.leaf(ctx.reader(), null) : null
         );
-        BlockLoader.Builder builder = loader.builder(TestBlock.factory(ctx.reader().numDocs()), 1);
+        BlockLoader.Builder builder = loader.builder(TestBlock.factory(), 1);
         storedFields.advanceTo(0);
         reader.read(0, storedFields, builder);
         TestBlock block = (TestBlock) builder.build();
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/DateFieldBlockLoaderTests.java
@@ -22,7 +22,7 @@
 import java.util.Map;
 import java.util.Objects;
 
-wpublic class DateFieldBlockLoaderTests extends BlockLoaderTestCase {
+public class DateFieldBlockLoaderTests extends BlockLoaderTestCase {
     public DateFieldBlockLoaderTests(Params params) {
         super(FieldType.DATE.toString(), params);
     }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java
@@ -425,8 +425,7 @@ protected final List<Object> blockLoaderReadValuesFromColumnAtATimeReader(Direct
         BlockLoader loader = fieldType.blockLoader(blContext());
         List<Object> all = new ArrayList<>();
         for (LeafReaderContext ctx : reader.leaves()) {
-            TestBlock block = (TestBlock) loader.columnAtATimeReader(ctx)
-                .read(TestBlock.factory(ctx.reader().numDocs()), TestBlock.docs(ctx), offset);
+            TestBlock block = (TestBlock) loader.columnAtATimeReader(ctx).read(TestBlock.factory(), TestBlock.docs(ctx), offset);
             for (int i = 0; i < block.size(); i++) {
                 all.add(block.get(i));
             }
@@ -440,7 +439,7 @@ protected final List<Object> blockLoaderReadValuesFromRowStrideReader(DirectoryR
         List<Object> all = new ArrayList<>();
         for (LeafReaderContext ctx : reader.leaves()) {
             BlockLoader.RowStrideReader blockReader = loader.rowStrideReader(ctx);
-            BlockLoader.Builder builder = loader.builder(TestBlock.factory(ctx.reader().numDocs()), ctx.reader().numDocs());
+            BlockLoader.Builder builder = loader.builder(TestBlock.factory(), ctx.reader().numDocs());
             for (int i = 0; i < ctx.reader().numDocs(); i++) {
                 blockReader.read(i, null, builder);
             }
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestRunner.java
@@ -113,7 +113,7 @@ private Object load(BlockLoader blockLoader, LeafReaderContext context, MapperSe
                 }
             }
             BlockLoader.Docs docs = TestBlock.docs(docArray);
-            var block = (TestBlock) columnAtATimeReader.read(TestBlock.factory(docArray.length - offset), docs, offset);
+            var block = (TestBlock) columnAtATimeReader.read(TestBlock.factory(), docs, offset);
             assertThat(block.size(), equalTo(docArray.length - offset));
             return block.get(0);
         }
@@ -133,7 +133,7 @@ private Object load(BlockLoader blockLoader, LeafReaderContext context, MapperSe
         );
         storedFieldsLoader.advanceTo(1);
 
-        BlockLoader.Builder builder = blockLoader.builder(TestBlock.factory(1), 1);
+        BlockLoader.Builder builder = blockLoader.builder(TestBlock.factory(), 1);
         blockLoader.rowStrideReader(context).read(1, storedFieldsLoader, builder);
         var block = (TestBlock) builder.build();
         assertThat(block.size(), equalTo(1));
diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestBlock.java
@@ -27,7 +27,7 @@
 import static org.junit.Assert.assertNull;
 
 public class TestBlock implements BlockLoader.Block {
-    public static BlockLoader.BlockFactory factory(int pageSize) {
+    public static BlockLoader.BlockFactory factory() {
         return new BlockLoader.BlockFactory() {
             @Override
             public BlockLoader.BooleanBuilder booleansFromDocValues(int expectedCount) {
@@ -177,18 +177,18 @@ public BlockLoader.Builder nulls(int expectedCount) {
             }
 
             @Override
-            public BlockLoader.Block constantNulls() {
-                BlockLoader.LongBuilder builder = longs(pageSize);
-                for (int i = 0; i < pageSize; i++) {
+            public BlockLoader.Block constantNulls(int count) {
+                BlockLoader.LongBuilder builder = longs(count);
+                for (int i = 0; i < count; i++) {
                     builder.appendNull();
                 }
                 return builder.build();
             }
 
             @Override
-            public BlockLoader.Block constantBytes(BytesRef value) {
-                BlockLoader.BytesRefBuilder builder = bytesRefs(pageSize);
-                for (int i = 0; i < pageSize; i++) {
+            public BlockLoader.Block constantBytes(BytesRef value, int count) {
+                BlockLoader.BytesRefBuilder builder = bytesRefs(count);
+                for (int i = 0; i < count; i++) {
                     builder.appendBytesRef(value);
                 }
                 return builder.build();
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ComputeBlockLoaderFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ComputeBlockLoaderFactory.java
@@ -14,18 +14,16 @@
 import org.elasticsearch.core.Releasable;
 
 class ComputeBlockLoaderFactory extends DelegatingBlockLoaderFactory implements Releasable {
-    private final int pageSize;
     private Block nullBlock;
 
-    ComputeBlockLoaderFactory(BlockFactory factory, int pageSize) {
+    ComputeBlockLoaderFactory(BlockFactory factory) {
         super(factory);
-        this.pageSize = pageSize;
     }
 
     @Override
-    public Block constantNulls() {
+    public Block constantNulls(int count) {
         if (nullBlock == null) {
-            nullBlock = factory.newConstantNullBlock(pageSize);
+            nullBlock = factory.newConstantNullBlock(count);
         }
         nullBlock.incRef();
         return nullBlock;
@@ -39,7 +37,7 @@ public void close() {
     }
 
     @Override
-    public BytesRefBlock constantBytes(BytesRef value) {
-        return factory.newConstantBytesRefBlockWith(value, pageSize);
+    public BytesRefBlock constantBytes(BytesRef value, int count) {
+        return factory.newConstantBytesRefBlockWith(value, count);
     }
 }
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/TimeSeriesExtractFieldOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/TimeSeriesExtractFieldOperator.java
@@ -198,12 +198,12 @@ static class BlockLoaderFactory extends DelegatingBlockLoaderFactory {
         }
 
         @Override
-        public BlockLoader.Block constantNulls() {
+        public BlockLoader.Block constantNulls(int count) {
             throw new UnsupportedOperationException("must not be used by column readers");
         }
 
         @Override
-        public BlockLoader.Block constantBytes(BytesRef value) {
+        public BlockLoader.Block constantBytes(BytesRef value, int count) {
             throw new UnsupportedOperationException("must not be used by column readers");
         }
 
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromManyReader.java
@@ -76,7 +76,7 @@ void run(int offset) throws IOException {
                 converters[f] = new BlockLoader[operator.shardContexts.size()];
             }
             try (
-                ComputeBlockLoaderFactory loaderBlockFactory = new ComputeBlockLoaderFactory(operator.blockFactory, docs.getPositionCount())
+                ComputeBlockLoaderFactory loaderBlockFactory = new ComputeBlockLoaderFactory(operator.blockFactory)
             ) {
                 int p = forwards[offset];
                 int shard = docs.shards().getInt(p);
diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesFromSingleReader.java
@@ -89,7 +89,7 @@ private void loadFromSingleLeaf(long jumboBytes, Block[] target, ValuesReaderDoc
 
         List<ColumnAtATimeWork> columnAtATimeReaders = new ArrayList<>(operator.fields.length);
         List<RowStrideReaderWork> rowStrideReaders = new ArrayList<>(operator.fields.length);
-        try (ComputeBlockLoaderFactory loaderBlockFactory = new ComputeBlockLoaderFactory(operator.blockFactory, docs.count() - offset)) {
+        try (ComputeBlockLoaderFactory loaderBlockFactory = new ComputeBlockLoaderFactory(operator.blockFactory)) {
             for (int f = 0; f < operator.fields.length; f++) {
                 ValuesSourceReaderOperator.FieldWork field = operator.fields[f];
                 BlockLoader.ColumnAtATimeReader columnAtATime = field.columnAtATime(ctx);
@@ -113,7 +113,7 @@ private void loadFromSingleLeaf(long jumboBytes, Block[] target, ValuesReaderDoc
             }
             for (ColumnAtATimeWork r : columnAtATimeReaders) {
                 target[r.idx] = (Block) r.reader.read(loaderBlockFactory, docs, offset);
-                operator.sanityCheckBlock(r.reader, docs.count(), target[r.idx], r.idx);
+                operator.sanityCheckBlock(r.reader, docs.count() - offset, target[r.idx], r.idx);
             }
             if (log.isDebugEnabled()) {
                 long total = 0;
diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java
@@ -9,13 +9,21 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 
+import org.elasticsearch.client.Request;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.test.TestClustersThreadFilter;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.esql.CsvSpecReader.CsvTestCase;
+import org.elasticsearch.xpack.esql.planner.PhysicalSettings;
 import org.elasticsearch.xpack.esql.plugin.ComputeService;
 import org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase;
+import org.junit.Before;
 import org.junit.ClassRule;
 
+import java.io.IOException;
+
 @ThreadLeakFilters(filters = TestClustersThreadFilter.class)
 public class EsqlSpecIT extends EsqlSpecTestCase {
     @ClassRule
@@ -50,4 +58,14 @@ protected boolean enableRoundingDoubleValuesOnAsserting() {
     protected boolean supportsSourceFieldMapping() {
         return cluster.getNumNodes() == 1;
     }
+
+    @Before
+    public void configureChunks() throws IOException {
+        boolean smallChunks = randomBoolean();
+        Request request = new Request("PUT", "/_cluster/settings");
+        XContentBuilder builder = JsonXContent.contentBuilder().startObject().startObject("persistent");
+        builder.field(PhysicalSettings.VALUES_LOADING_JUMBO_SIZE.getKey(), smallChunks ? "1kb" : null);
+        request.setJsonEntity(Strings.toString(builder.endObject().endObject()));
+        assertOK(client().performRequest(request));
+    }
 }
diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java
@@ -276,7 +276,7 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
             iw.close();
             try (DirectoryReader reader = DirectoryReader.open(directory)) {
                 TestBlock block = (TestBlock) loader.columnAtATimeReader(reader.leaves().get(0))
-                    .read(TestBlock.factory(reader.numDocs()), new BlockLoader.Docs() {
+                    .read(TestBlock.factory(), new BlockLoader.Docs() {
                         @Override
                         public int count() {
                             return 1;

Original file line number	Diff line number	Diff line change
`@@ -645,9 +645,9 @@ private BlockLoader.Block readSingleDoc(BlockFactory factory, int docId) throws`
`645`	`645`	`if (ordinals.advanceExact(docId)) {`
`646`	`646`	`BytesRef v = ordinals.lookupOrd(ordinals.ordValue());`
`647`	`647`	`// the returned BytesRef can be reused`
`648`		`- return factory.constantBytes(BytesRef.deepCopyOf(v));`
	`648`	`+ return factory.constantBytes(BytesRef.deepCopyOf(v), 1);`
`649`	`649`	`} else {`
`650`		`- return factory.constantNulls();`
	`650`	`+ return factory.constantNulls(1);`
`651`	`651`	`}`
`652`	`652`	`}`
`653`	`653`
Original file line number	Diff line number	Diff line change
`@@ -126,7 +126,7 @@ private static void testBoundsBlockLoaderAux(`
`126`	`126`	`expected.add(visitor.apply(geometries.get(j + currentIndex)).get());`
`127`	`127`	`}`
`128`	`128`	`try (`
`129`		`- var block = (TestBlock) loader.reader(leaf).read(TestBlock.factory(leafReader.numDocs()), TestBlock.docs(array), 0)`
	`129`	`+ var block = (TestBlock) loader.reader(leaf).read(TestBlock.factory(), TestBlock.docs(array), 0)`
`130`	`130`	`) {`
`131`	`131`	`for (int i = 0; i < block.size(); i++) {`
`132`	`132`	`intArrayResults.add(block.get(i));`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@`
`22`	`22`	`import java.util.Map;`
`23`	`23`	`import java.util.Objects;`
`24`	`24`
`25`		`-wpublic class DateFieldBlockLoaderTests extends BlockLoaderTestCase {`
	`25`	`+public class DateFieldBlockLoaderTests extends BlockLoaderTestCase {`
`26`	`26`	`public DateFieldBlockLoaderTests(Params params) {`
`27`	`27`	`super(FieldType.DATE.toString(), params);`
`28`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -113,7 +113,7 @@ private Object load(BlockLoader blockLoader, LeafReaderContext context, MapperSe`
`113`	`113`	`}`
`114`	`114`	`}`
`115`	`115`	`BlockLoader.Docs docs = TestBlock.docs(docArray);`
`116`		`- var block = (TestBlock) columnAtATimeReader.read(TestBlock.factory(docArray.length - offset), docs, offset);`
	`116`	`+ var block = (TestBlock) columnAtATimeReader.read(TestBlock.factory(), docs, offset);`
`117`	`117`	`assertThat(block.size(), equalTo(docArray.length - offset));`
`118`	`118`	`return block.get(0);`
`119`	`119`	`}`
`@@ -133,7 +133,7 @@ private Object load(BlockLoader blockLoader, LeafReaderContext context, MapperSe`
`133`	`133`	`);`
`134`	`134`	`storedFieldsLoader.advanceTo(1);`
`135`	`135`
`136`		`- BlockLoader.Builder builder = blockLoader.builder(TestBlock.factory(1), 1);`
	`136`	`+ BlockLoader.Builder builder = blockLoader.builder(TestBlock.factory(), 1);`
`137`	`137`	`blockLoader.rowStrideReader(context).read(1, storedFieldsLoader, builder);`
`138`	`138`	`var block = (TestBlock) builder.build();`
`139`	`139`	`assertThat(block.size(), equalTo(1));`
Original file line number	Diff line number	Diff line change
`@@ -14,18 +14,16 @@`
`14`	`14`	`import org.elasticsearch.core.Releasable;`
`15`	`15`
`16`	`16`	`class ComputeBlockLoaderFactory extends DelegatingBlockLoaderFactory implements Releasable {`
`17`		`- private final int pageSize;`
`18`	`17`	`private Block nullBlock;`
`19`	`18`
`20`		`- ComputeBlockLoaderFactory(BlockFactory factory, int pageSize) {`
	`19`	`+ ComputeBlockLoaderFactory(BlockFactory factory) {`
`21`	`20`	`super(factory);`
`22`		`- this.pageSize = pageSize;`
`23`	`21`	`}`
`24`	`22`
`25`	`23`	`@Override`
`26`		`- public Block constantNulls() {`
	`24`	`+ public Block constantNulls(int count) {`
`27`	`25`	`if (nullBlock == null) {`
`28`		`- nullBlock = factory.newConstantNullBlock(pageSize);`
	`26`	`+ nullBlock = factory.newConstantNullBlock(count);`
`29`	`27`	`}`
`30`	`28`	`nullBlock.incRef();`
`31`	`29`	`return nullBlock;`
`@@ -39,7 +37,7 @@ public void close() {`
`39`	`37`	`}`
`40`	`38`
`41`	`39`	`@Override`
`42`		`- public BytesRefBlock constantBytes(BytesRef value) {`
`43`		`- return factory.newConstantBytesRefBlockWith(value, pageSize);`
	`40`	`+ public BytesRefBlock constantBytes(BytesRef value, int count) {`
	`41`	`+ return factory.newConstantBytesRefBlockWith(value, count);`
`44`	`42`	`}`
`45`	`43`	`}`
Original file line number	Diff line number	Diff line change
`@@ -198,12 +198,12 @@ static class BlockLoaderFactory extends DelegatingBlockLoaderFactory {`
`198`	`198`	`}`
`199`	`199`
`200`	`200`	`@Override`
`201`		`- public BlockLoader.Block constantNulls() {`
	`201`	`+ public BlockLoader.Block constantNulls(int count) {`
`202`	`202`	`throw new UnsupportedOperationException("must not be used by column readers");`
`203`	`203`	`}`
`204`	`204`
`205`	`205`	`@Override`
`206`		`- public BlockLoader.Block constantBytes(BytesRef value) {`
	`206`	`+ public BlockLoader.Block constantBytes(BytesRef value, int count) {`
`207`	`207`	`throw new UnsupportedOperationException("must not be used by column readers");`
`208`	`208`	`}`
`209`	`209`