From 738976e34ec61459580a8909dd73f738dcd8fe36 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Oct 2025 12:50:55 -0400 Subject: [PATCH 1/9] Block loaders for MV_MIN and MV_MAX for keywords Adds special purpose `BlockLoader` implementations for the `MV_MIN` and `MV_MAX` functions for `keyword` fields with doc values. These are a noop for single valued keywords but should be *much* faster for multivalued keywords. These aren't plugged in yet. We can plug them in and performance test them in #137382. And they give us two more functions we can use to demonstrate #137382. --- .../AbstractBytesRefsFromOrdsBlockLoader.java | 206 ++++++++++++++++++ .../BytesRefsFromOrdsBlockLoader.java | 179 +-------------- .../MvMaxBytesRefsFromOrdsBlockLoader.java | 114 ++++++++++ .../MvMinBytesRefsFromOrdsBlockLoader.java | 104 +++++++++ .../Utf8CodePointsFromOrdsBlockLoader.java | 20 +- .../AbstractFromOrdsBlockLoaderTests.java | 100 +++++++++ ...vMaxBytesRefsFromOrdsBlockLoaderTests.java | 86 ++++++++ ...vMinBytesRefsFromOrdsBlockLoaderTests.java | 86 ++++++++ ...tf8CodePointsFromOrdsBlockLoaderTests.java | 143 ++++-------- 9 files changed, 758 insertions(+), 280 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractBytesRefsFromOrdsBlockLoader.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoader.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractFromOrdsBlockLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractBytesRefsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractBytesRefsFromOrdsBlockLoader.java new file mode 100644 index 0000000000000..00edc40002247 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractBytesRefsFromOrdsBlockLoader.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; + +/** + * Loads {@code keyword} style fields that are stored as a lookup table. + */ +public abstract class AbstractBytesRefsFromOrdsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { + protected final String fieldName; + + public AbstractBytesRefsFromOrdsBlockLoader(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public BytesRefBuilder builder(BlockFactory factory, int expectedCount) { + return factory.bytesRefs(expectedCount); + } + + @Override + public AllReader reader(LeafReaderContext context) throws IOException { + SortedSetDocValues docValues = context.reader().getSortedSetDocValues(fieldName); + if (docValues != null) { + SortedDocValues singleton = DocValues.unwrapSingleton(docValues); + if (singleton != null) { + return singletonReader(singleton); + } + return sortedSetReader(docValues); + } + SortedDocValues singleton = context.reader().getSortedDocValues(fieldName); + if (singleton != null) { + return singletonReader(singleton); + } + return new ConstantNullsReader(); + } + + protected abstract AllReader singletonReader(SortedDocValues docValues); + + protected abstract AllReader sortedSetReader(SortedSetDocValues docValues); + + protected static class Singleton extends BlockDocValuesReader { + private final SortedDocValues ordinals; + + Singleton(SortedDocValues ordinals) { + this.ordinals = ordinals; + } + + private Block readSingleDoc(BlockFactory factory, int docId) throws IOException { + if (ordinals.advanceExact(docId)) { + BytesRef v = ordinals.lookupOrd(ordinals.ordValue()); + // the returned BytesRef can be reused + return factory.constantBytes(BytesRef.deepCopyOf(v), 1); + } else { + return factory.constantNulls(1); + } + } + + @Override + public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { + if (docs.count() - offset == 1) { + return readSingleDoc(factory, docs.get(offset)); + } + if (ordinals instanceof OptionalColumnAtATimeReader direct) { + Block block = direct.tryRead(factory, docs, offset, nullsFiltered, null, false); + if (block != null) { + return block; + } + } + try (var builder = factory.singletonOrdinalsBuilder(ordinals, docs.count() - offset, false)) { + for (int i = offset; i < docs.count(); i++) { + int doc = docs.get(i); + if (ordinals.advanceExact(doc)) { + builder.appendOrd(ordinals.ordValue()); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + if (ordinals.advanceExact(docId)) { + ((BytesRefBuilder) builder).appendBytesRef(ordinals.lookupOrd(ordinals.ordValue())); + } else { + builder.appendNull(); + } + } + + @Override + public int docId() { + return ordinals.docID(); + } + + @Override + public String toString() { + return "BytesRefsFromOrds.Singleton"; + } + } + + protected static class SortedSet extends BlockDocValuesReader { + private final SortedSetDocValues ordinals; + + SortedSet(SortedSetDocValues ordinals) { + this.ordinals = ordinals; + } + + @Override + public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { + if (docs.count() - offset == 1) { + return readSingleDoc(factory, docs.get(offset)); + } + try (var builder = factory.sortedSetOrdinalsBuilder(ordinals, docs.count() - offset)) { + for (int i = offset; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < ordinals.docID()) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (ordinals.advanceExact(doc) == false) { + builder.appendNull(); + continue; + } + int count = ordinals.docValueCount(); + if (count == 1) { + builder.appendOrd(Math.toIntExact(ordinals.nextOrd())); + } else { + builder.beginPositionEntry(); + for (int c = 0; c < count; c++) { + builder.appendOrd(Math.toIntExact(ordinals.nextOrd())); + } + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + read(docId, (BytesRefBuilder) builder); + } + + private Block readSingleDoc(BlockFactory factory, int docId) throws IOException { + if (ordinals.advanceExact(docId) == false) { + return factory.constantNulls(1); + } + int count = ordinals.docValueCount(); + if (count == 1) { + BytesRef v = ordinals.lookupOrd(ordinals.nextOrd()); + return factory.constantBytes(BytesRef.deepCopyOf(v), 1); + } + try (var builder = factory.bytesRefsFromDocValues(count)) { + builder.beginPositionEntry(); + for (int c = 0; c < count; c++) { + BytesRef v = ordinals.lookupOrd(ordinals.nextOrd()); + builder.appendBytesRef(v); + } + builder.endPositionEntry(); + return builder.build(); + } + } + + private void read(int docId, BytesRefBuilder builder) throws IOException { + if (false == ordinals.advanceExact(docId)) { + builder.appendNull(); + return; + } + int count = ordinals.docValueCount(); + if (count == 1) { + builder.appendBytesRef(ordinals.lookupOrd(ordinals.nextOrd())); + return; + } + builder.beginPositionEntry(); + for (int v = 0; v < count; v++) { + builder.appendBytesRef(ordinals.lookupOrd(ordinals.nextOrd())); + } + builder.endPositionEntry(); + } + + @Override + public int docId() { + return ordinals.docID(); + } + + @Override + public String toString() { + return "BytesRefsFromOrds.SortedSet"; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromOrdsBlockLoader.java index f6b3daf536d8d..fc829a798f18f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromOrdsBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/BytesRefsFromOrdsBlockLoader.java @@ -13,41 +13,25 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.index.mapper.BlockLoader; import java.io.IOException; /** * Loads {@code keyword} style fields that are stored as a lookup table and ordinals. */ -public class BytesRefsFromOrdsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { - private final String fieldName; - +public class BytesRefsFromOrdsBlockLoader extends AbstractBytesRefsFromOrdsBlockLoader { public BytesRefsFromOrdsBlockLoader(String fieldName) { - this.fieldName = fieldName; + super(fieldName); } @Override - public BytesRefBuilder builder(BlockFactory factory, int expectedCount) { - return factory.bytesRefs(expectedCount); + protected AllReader singletonReader(SortedDocValues docValues) { + return new Singleton(docValues); } @Override - public AllReader reader(LeafReaderContext context) throws IOException { - SortedSetDocValues docValues = context.reader().getSortedSetDocValues(fieldName); - if (docValues != null) { - SortedDocValues singleton = DocValues.unwrapSingleton(docValues); - if (singleton != null) { - return new SingletonOrdinals(singleton); - } - return new Ordinals(docValues); - } - SortedDocValues singleton = context.reader().getSortedDocValues(fieldName); - if (singleton != null) { - return new SingletonOrdinals(singleton); - } - return new ConstantNullsReader(); + protected AllReader sortedSetReader(SortedSetDocValues docValues) { + return new SortedSet(docValues); } @Override @@ -64,155 +48,4 @@ public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException public String toString() { return "BytesRefsFromOrds[" + fieldName + "]"; } - - private static class SingletonOrdinals extends BlockDocValuesReader { - private final SortedDocValues ordinals; - - SingletonOrdinals(SortedDocValues ordinals) { - this.ordinals = ordinals; - } - - private BlockLoader.Block readSingleDoc(BlockFactory factory, int docId) throws IOException { - if (ordinals.advanceExact(docId)) { - BytesRef v = ordinals.lookupOrd(ordinals.ordValue()); - // the returned BytesRef can be reused - return factory.constantBytes(BytesRef.deepCopyOf(v), 1); - } else { - return factory.constantNulls(1); - } - } - - @Override - public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { - if (docs.count() - offset == 1) { - return readSingleDoc(factory, docs.get(offset)); - } - if (ordinals instanceof BlockLoader.OptionalColumnAtATimeReader direct) { - BlockLoader.Block block = direct.tryRead(factory, docs, offset, nullsFiltered, null, false); - if (block != null) { - return block; - } - } - try (var builder = factory.singletonOrdinalsBuilder(ordinals, docs.count() - offset, false)) { - for (int i = offset; i < docs.count(); i++) { - int doc = docs.get(i); - if (ordinals.advanceExact(doc)) { - builder.appendOrd(ordinals.ordValue()); - } else { - builder.appendNull(); - } - } - return builder.build(); - } - } - - @Override - public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { - if (ordinals.advanceExact(docId)) { - ((BytesRefBuilder) builder).appendBytesRef(ordinals.lookupOrd(ordinals.ordValue())); - } else { - builder.appendNull(); - } - } - - @Override - public int docId() { - return ordinals.docID(); - } - - @Override - public String toString() { - return "BlockDocValuesReader.SingletonOrdinals"; - } - } - - private static class Ordinals extends BlockDocValuesReader { - private final SortedSetDocValues ordinals; - - Ordinals(SortedSetDocValues ordinals) { - this.ordinals = ordinals; - } - - @Override - public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { - if (docs.count() - offset == 1) { - return readSingleDoc(factory, docs.get(offset)); - } - try (var builder = factory.sortedSetOrdinalsBuilder(ordinals, docs.count() - offset)) { - for (int i = offset; i < docs.count(); i++) { - int doc = docs.get(i); - if (doc < ordinals.docID()) { - throw new IllegalStateException("docs within same block must be in order"); - } - if (ordinals.advanceExact(doc) == false) { - builder.appendNull(); - continue; - } - int count = ordinals.docValueCount(); - if (count == 1) { - builder.appendOrd(Math.toIntExact(ordinals.nextOrd())); - } else { - builder.beginPositionEntry(); - for (int c = 0; c < count; c++) { - builder.appendOrd(Math.toIntExact(ordinals.nextOrd())); - } - builder.endPositionEntry(); - } - } - return builder.build(); - } - } - - @Override - public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { - read(docId, (BytesRefBuilder) builder); - } - - private BlockLoader.Block readSingleDoc(BlockFactory factory, int docId) throws IOException { - if (ordinals.advanceExact(docId) == false) { - return factory.constantNulls(1); - } - int count = ordinals.docValueCount(); - if (count == 1) { - BytesRef v = ordinals.lookupOrd(ordinals.nextOrd()); - return factory.constantBytes(BytesRef.deepCopyOf(v), 1); - } - try (var builder = factory.bytesRefsFromDocValues(count)) { - builder.beginPositionEntry(); - for (int c = 0; c < count; c++) { - BytesRef v = ordinals.lookupOrd(ordinals.nextOrd()); - builder.appendBytesRef(v); - } - builder.endPositionEntry(); - return builder.build(); - } - } - - private void read(int docId, BytesRefBuilder builder) throws IOException { - if (false == ordinals.advanceExact(docId)) { - builder.appendNull(); - return; - } - int count = ordinals.docValueCount(); - if (count == 1) { - builder.appendBytesRef(ordinals.lookupOrd(ordinals.nextOrd())); - return; - } - builder.beginPositionEntry(); - for (int v = 0; v < count; v++) { - builder.appendBytesRef(ordinals.lookupOrd(ordinals.nextOrd())); - } - builder.endPositionEntry(); - } - - @Override - public int docId() { - return ordinals.docID(); - } - - @Override - public String toString() { - return "BlockDocValuesReader.Ordinals"; - } - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java new file mode 100644 index 0000000000000..25f836d465930 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; + +/** + * Loads {@code MV_MIN} applied to {@code keyword} style fields that are stored as + * a lookup table and ordinals. + */ +public class MvMaxBytesRefsFromOrdsBlockLoader extends AbstractBytesRefsFromOrdsBlockLoader { + private final String fieldName; + + public MvMaxBytesRefsFromOrdsBlockLoader(String fieldName) { + super(fieldName); + this.fieldName = fieldName; + } + + @Override + protected AllReader singletonReader(SortedDocValues docValues) { + return new Singleton(docValues); + } + + @Override + protected AllReader sortedSetReader(SortedSetDocValues docValues) { + return new MvMaxSortedSet(docValues); + } + + @Override + public String toString() { + return "MvMaxBytesRefsFromOrds[" + fieldName + "]"; + } + + private static class MvMaxSortedSet extends BlockDocValuesReader { + private final SortedSetDocValues ordinals; + + MvMaxSortedSet(SortedSetDocValues ordinals) { + this.ordinals = ordinals; + } + + @Override + public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { + if (docs.count() - offset == 1) { + return readSingleDoc(factory, docs.get(offset)); + } + try (var builder = factory.sortedSetOrdinalsBuilder(ordinals, docs.count() - offset)) { + for (int i = offset; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < ordinals.docID()) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (ordinals.advanceExact(doc) == false) { + builder.appendNull(); + continue; + } + discardAllButLast(); + builder.appendOrd(Math.toIntExact(ordinals.nextOrd())); + } + return builder.build(); + } + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + read(docId, (BytesRefBuilder) builder); + } + + private Block readSingleDoc(BlockFactory factory, int docId) throws IOException { + if (ordinals.advanceExact(docId) == false) { + return factory.constantNulls(1); + } + discardAllButLast(); + BytesRef v = ordinals.lookupOrd(ordinals.nextOrd()); + return factory.constantBytes(BytesRef.deepCopyOf(v), 1); + } + + private void read(int docId, BytesRefBuilder builder) throws IOException { + if (false == ordinals.advanceExact(docId)) { + builder.appendNull(); + return; + } + discardAllButLast(); + builder.appendBytesRef(ordinals.lookupOrd(ordinals.nextOrd())); + } + + private void discardAllButLast() throws IOException { + int count = ordinals.docValueCount(); + for (int i = 0; i < count - 1; i++) { + ordinals.nextOrd(); + } + } + + @Override + public int docId() { + return ordinals.docID(); + } + + @Override + public String toString() { + return "MvMaxBytesRefsFromOrds.SortedSet"; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoader.java new file mode 100644 index 0000000000000..f7b48e92b7df0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoader.java @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; + +/** + * Loads {@code MV_MIN} applied to {@code keyword} style fields that are stored as + * a lookup table and ordinals. + */ +public class MvMinBytesRefsFromOrdsBlockLoader extends AbstractBytesRefsFromOrdsBlockLoader { + private final String fieldName; + + public MvMinBytesRefsFromOrdsBlockLoader(String fieldName) { + super(fieldName); + this.fieldName = fieldName; + } + + @Override + protected AllReader singletonReader(SortedDocValues docValues) { + return new Singleton(docValues); + } + + @Override + protected AllReader sortedSetReader(SortedSetDocValues docValues) { + return new MvMinSortedSet(docValues); + } + + @Override + public String toString() { + return "MvMinBytesRefsFromOrds[" + fieldName + "]"; + } + + private static class MvMinSortedSet extends BlockDocValuesReader { + private final SortedSetDocValues ordinals; + + MvMinSortedSet(SortedSetDocValues ordinals) { + this.ordinals = ordinals; + } + + @Override + public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { + if (docs.count() - offset == 1) { + return readSingleDoc(factory, docs.get(offset)); + } + try (var builder = factory.sortedSetOrdinalsBuilder(ordinals, docs.count() - offset)) { + for (int i = offset; i < docs.count(); i++) { + int doc = docs.get(i); + if (doc < ordinals.docID()) { + throw new IllegalStateException("docs within same block must be in order"); + } + if (ordinals.advanceExact(doc) == false) { + builder.appendNull(); + continue; + } + builder.appendOrd(Math.toIntExact(ordinals.nextOrd())); + } + return builder.build(); + } + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + read(docId, (BytesRefBuilder) builder); + } + + private Block readSingleDoc(BlockFactory factory, int docId) throws IOException { + if (ordinals.advanceExact(docId) == false) { + return factory.constantNulls(1); + } + BytesRef v = ordinals.lookupOrd(ordinals.nextOrd()); + return factory.constantBytes(BytesRef.deepCopyOf(v), 1); + } + + private void read(int docId, BytesRefBuilder builder) throws IOException { + if (false == ordinals.advanceExact(docId)) { + builder.appendNull(); + return; + } + builder.appendBytesRef(ordinals.lookupOrd(ordinals.nextOrd())); + } + + @Override + public int docId() { + return ordinals.docID(); + } + + @Override + public String toString() { + return "MvMinBytesRefsFromOrds.SortedSet"; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoader.java index 730ef2a58a6f3..b5f044e813e31 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoader.java @@ -56,16 +56,16 @@ public AllReader reader(LeafReaderContext context) throws IOException { } SortedDocValues singleton = DocValues.unwrapSingleton(docValues); if (singleton != null) { - return new SingletonOrdinals(singleton); + return new Singleton(singleton); } - return new Ordinals(warnings, docValues); + return new SortedSet(warnings, docValues); } SortedDocValues singleton = context.reader().getSortedDocValues(fieldName); if (singleton != null) { if (singleton.getValueCount() > LOW_CARDINALITY) { return new ImmediateOrdinals(warnings, DocValues.singleton(singleton)); } - return new SingletonOrdinals(singleton); + return new Singleton(singleton); } return new ConstantNullsReader(); } @@ -93,13 +93,13 @@ public String toString() { * ordinals and look them up in the cache immediately. *

*/ - private static class SingletonOrdinals extends BlockDocValuesReader { + private static class Singleton extends BlockDocValuesReader { private final SortedDocValues ordinals; private final int[] cache; private int cacheEntriesFilled; - SingletonOrdinals(SortedDocValues ordinals) { + Singleton(SortedDocValues ordinals) { this.ordinals = ordinals; // TODO track this memory. we can't yet because this isn't Closeable @@ -142,7 +142,7 @@ public int docId() { @Override public String toString() { - return "Utf8CodePointsFromOrds.SingletonOrdinals"; + return "Utf8CodePointsFromOrds.Singleton"; } private Block blockForSingleDoc(BlockFactory factory, int docId) throws IOException { @@ -225,16 +225,16 @@ private int codePointsAtOrd(int ord) throws IOException { /** * Loads low cardinality non-singleton ordinals in using a cache of code point counts. - * See {@link SingletonOrdinals} for the process + * See {@link Singleton} for the process */ - private static class Ordinals extends BlockDocValuesReader { + private static class SortedSet extends BlockDocValuesReader { private final Warnings warnings; private final SortedSetDocValues ordinals; private final int[] cache; private int cacheEntriesFilled; - Ordinals(Warnings warnings, SortedSetDocValues ordinals) { + SortedSet(Warnings warnings, SortedSetDocValues ordinals) { this.warnings = warnings; this.ordinals = ordinals; @@ -283,7 +283,7 @@ public int docId() { @Override public String toString() { - return "Utf8CodePointsFromOrds.Ordinals"; + return "Utf8CodePointsFromOrds.SortedSet"; } private Block blockForSingleDoc(BlockFactory factory, int docId) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractFromOrdsBlockLoaderTests.java new file mode 100644 index 0000000000000..2f0c1e618ef90 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractFromOrdsBlockLoaderTests.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.TestBlock; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.index.mapper.blockloader.docvalues.Utf8CodePointsFromOrdsBlockLoader.LOW_CARDINALITY; + +public abstract class AbstractFromOrdsBlockLoaderTests extends ESTestCase { + @ParametersFactory(argumentFormatting = "blockAtATime=%s, lowCardinality=%s, multiValues=%s, missingValues=%s") + public static List parameters() throws IOException { + List parameters = new ArrayList<>(); + for (boolean blockAtATime : new boolean[] { true, false }) { + for (boolean lowCardinality : new boolean[] { true, false }) { + for (boolean multiValues : new boolean[] { true, false }) { + for (boolean missingValues : new boolean[] { true, false }) { + parameters.add(new Object[] { blockAtATime, lowCardinality, multiValues, missingValues }); + } + } + } + } + return parameters; + } + + protected final boolean blockAtATime; + protected final boolean lowCardinality; + protected final boolean multiValues; + protected final boolean missingValues; + + public AbstractFromOrdsBlockLoaderTests(boolean blockAtATime, boolean highCardinality, boolean multiValues, boolean missingValues) { + this.blockAtATime = blockAtATime; + this.lowCardinality = highCardinality; + this.multiValues = multiValues; + this.missingValues = missingValues; + } + + protected abstract void innerTest(LeafReaderContext ctx) throws IOException; + + public void test() throws IOException { + try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) { + int docCount = 10_000; + int cardinality = lowCardinality ? between(1, LOW_CARDINALITY) : between(LOW_CARDINALITY + 1, LOW_CARDINALITY * 2); + for (int i = 0; i < docCount; i++) { + List doc = new ArrayList<>(2); + doc.add(field(i % cardinality)); + if (multiValues && i % cardinality == 0) { + doc.add(field((i % cardinality) + 1)); + } + iw.addDocument(doc); + } + if (missingValues) { + iw.addDocument(List.of()); + } + iw.forceMerge(1); + try (DirectoryReader dr = iw.getReader()) { + LeafReaderContext ctx = getOnlyLeafReader(dr).getContext(); + innerTest(ctx); + } + } + } + + protected final TestBlock read(BlockLoader loader, BlockLoader.AllReader reader, LeafReaderContext ctx, BlockLoader.Docs docs) + throws IOException { + BlockLoader.AllReader toUse = blockAtATime + ? reader + : new ForceDocAtATime(() -> loader.builder(TestBlock.factory(), docs.count()), reader); + + return (TestBlock) toUse.read(TestBlock.factory(), docs, 0, false); + } + + private static KeywordFieldMapper.KeywordField field(int codePointCount) { + return new KeywordFieldMapper.KeywordField( + "field", + new BytesRef("a".repeat(codePointCount)), + KeywordFieldMapper.Defaults.FIELD_TYPE + ); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java new file mode 100644 index 0000000000000..945ce8ac67ed5 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.TestBlock; +import org.hamcrest.Matcher; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.nullValue; + +public class MvMaxBytesRefsFromOrdsBlockLoaderTests extends AbstractFromOrdsBlockLoaderTests { + public MvMaxBytesRefsFromOrdsBlockLoaderTests( + boolean blockAtATime, + boolean highCardinality, + boolean multiValues, + boolean missingValues + ) { + super(blockAtATime, highCardinality, multiValues, missingValues); + } + + @Override + protected void innerTest(LeafReaderContext ctx) throws IOException { + var stringsLoader = new BytesRefsFromOrdsBlockLoader("field"); + var mvMaxLoader = new MvMaxBytesRefsFromOrdsBlockLoader("field"); + + var stringsReader = stringsLoader.reader(ctx); + var mvMaxReader = mvMaxLoader.reader(ctx); + assertThat(mvMaxReader, readerMatcher()); + BlockLoader.Docs docs = TestBlock.docs(ctx); + try ( + TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); + TestBlock codePoints = read(mvMaxLoader, mvMaxReader, ctx, docs); + ) { + checkBlocks(strings, codePoints); + } + + stringsReader = stringsLoader.reader(ctx); + mvMaxReader = mvMaxLoader.reader(ctx); + for (int i = 0; i < ctx.reader().numDocs(); i += 10) { + int[] docsArray = new int[Math.min(10, ctx.reader().numDocs() - i)]; + for (int d = 0; d < docsArray.length; d++) { + docsArray[d] = i + d; + } + docs = TestBlock.docs(docsArray); + try ( + TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); + TestBlock codePoints = read(mvMaxLoader, mvMaxReader, ctx, docs); + ) { + checkBlocks(strings, codePoints); + } + } + } + + private Matcher readerMatcher() { + if (multiValues) { + return hasToString("MvMaxBytesRefsFromOrds.SortedSet"); + } + return hasToString("BytesRefsFromOrds.Singleton"); + } + + private void checkBlocks(TestBlock strings, TestBlock mvMin) { + for (int i = 0; i < strings.size(); i++) { + Object str = strings.get(i); + if (str == null) { + assertThat(mvMin.get(i), nullValue()); + continue; + } + BytesRef bytes = (BytesRef) (str instanceof List l ? l.getLast() : str); + assertThat(mvMin.get(i), equalTo(bytes)); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java new file mode 100644 index 0000000000000..f10a2d9369181 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.TestBlock; +import org.hamcrest.Matcher; + +import java.io.IOException; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.nullValue; + +public class MvMinBytesRefsFromOrdsBlockLoaderTests extends AbstractFromOrdsBlockLoaderTests { + public MvMinBytesRefsFromOrdsBlockLoaderTests( + boolean blockAtATime, + boolean highCardinality, + boolean multiValues, + boolean missingValues + ) { + super(blockAtATime, highCardinality, multiValues, missingValues); + } + + @Override + protected void innerTest(LeafReaderContext ctx) throws IOException { + var stringsLoader = new BytesRefsFromOrdsBlockLoader("field"); + var mvMinLoader = new MvMinBytesRefsFromOrdsBlockLoader("field"); + + var stringsReader = stringsLoader.reader(ctx); + var mvMinReader = mvMinLoader.reader(ctx); + assertThat(mvMinReader, readerMatcher()); + BlockLoader.Docs docs = TestBlock.docs(ctx); + try ( + TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); + TestBlock codePoints = read(mvMinLoader, mvMinReader, ctx, docs); + ) { + checkBlocks(strings, codePoints); + } + + stringsReader = stringsLoader.reader(ctx); + mvMinReader = mvMinLoader.reader(ctx); + for (int i = 0; i < ctx.reader().numDocs(); i += 10) { + int[] docsArray = new int[Math.min(10, ctx.reader().numDocs() - i)]; + for (int d = 0; d < docsArray.length; d++) { + docsArray[d] = i + d; + } + docs = TestBlock.docs(docsArray); + try ( + TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); + TestBlock codePoints = read(mvMinLoader, mvMinReader, ctx, docs); + ) { + checkBlocks(strings, codePoints); + } + } + } + + private Matcher readerMatcher() { + if (multiValues) { + return hasToString("MvMinBytesRefsFromOrds.SortedSet"); + } + return hasToString("BytesRefsFromOrds.Singleton"); + } + + private void checkBlocks(TestBlock strings, TestBlock mvMin) { + for (int i = 0; i < strings.size(); i++) { + Object str = strings.get(i); + if (str == null) { + assertThat(mvMin.get(i), nullValue()); + continue; + } + BytesRef bytes = (BytesRef) (str instanceof List l ? l.getFirst() : str); + assertThat(mvMin.get(i), equalTo(bytes)); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java index 5613abd02f4e5..dd7bc42be1042 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java @@ -9,18 +9,11 @@ package org.elasticsearch.index.mapper.blockloader.docvalues; -import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.index.mapper.BlockLoader; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.TestBlock; -import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matcher; import java.io.IOException; @@ -32,106 +25,62 @@ import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.nullValue; -public class Utf8CodePointsFromOrdsBlockLoaderTests extends ESTestCase { - @ParametersFactory(argumentFormatting = "blockAtATime=%s, lowCardinality=%s, multiValues=%s, missingValues=%s") - public static List parameters() throws IOException { - List parameters = new ArrayList<>(); - for (boolean blockAtATime : new boolean[] { true, false }) { - for (boolean lowCardinality : new boolean[] { true, false }) { - for (boolean multiValues : new boolean[] { true, false }) { - for (boolean missingValues : new boolean[] { true, false }) { - parameters.add(new Object[] { blockAtATime, lowCardinality, multiValues, missingValues }); - } - } - } - } - return parameters; - } - - private final boolean blockAtATime; - private final boolean lowCardinality; - private final boolean multiValues; - private final boolean missingValues; - +public class Utf8CodePointsFromOrdsBlockLoaderTests extends AbstractFromOrdsBlockLoaderTests { public Utf8CodePointsFromOrdsBlockLoaderTests( boolean blockAtATime, boolean highCardinality, boolean multiValues, boolean missingValues ) { - this.blockAtATime = blockAtATime; - this.lowCardinality = highCardinality; - this.multiValues = multiValues; - this.missingValues = missingValues; + super(blockAtATime, highCardinality, multiValues, missingValues); } - public void test() throws IOException { + @Override + protected void innerTest(LeafReaderContext ctx) throws IOException { List expectedWarnings = new ArrayList<>(); - try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) { - int docCount = 10_000; - int cardinality = lowCardinality ? between(1, LOW_CARDINALITY) : between(LOW_CARDINALITY + 1, LOW_CARDINALITY * 2); - for (int i = 0; i < docCount; i++) { - List doc = new ArrayList<>(2); - doc.add(field(i % cardinality)); - if (multiValues && i % cardinality == 0) { - doc.add(field((i % cardinality) + 1)); - expectedWarnings.add( - new MockWarnings.MockWarning(IllegalArgumentException.class, "single-value function encountered multi-value") - ); - } - iw.addDocument(doc); + int docCount = 10_000; + int cardinality = lowCardinality ? between(1, LOW_CARDINALITY) : between(LOW_CARDINALITY + 1, LOW_CARDINALITY * 2); + for (int i = 0; i < docCount; i++) { + if (multiValues && i % cardinality == 0) { + expectedWarnings.add( + new MockWarnings.MockWarning(IllegalArgumentException.class, "single-value function encountered multi-value") + ); } - if (missingValues) { - iw.addDocument(List.of()); - } - iw.forceMerge(1); - try (DirectoryReader dr = iw.getReader()) { - LeafReaderContext ctx = getOnlyLeafReader(dr).getContext(); - - var warnings = new MockWarnings(); - var stringsLoader = new BytesRefsFromOrdsBlockLoader("field"); - var codePointsLoader = new Utf8CodePointsFromOrdsBlockLoader(warnings, "field"); - - var stringsReader = stringsLoader.reader(ctx); - var codePointsReader = codePointsLoader.reader(ctx); - assertThat(codePointsReader, readerMatcher()); - BlockLoader.Docs docs = TestBlock.docs(ctx); - try ( - TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); - TestBlock codePoints = read(codePointsLoader, codePointsReader, ctx, docs); - ) { - checkBlocks(strings, codePoints); - } - assertThat(warnings.warnings(), equalTo(expectedWarnings)); - warnings.warnings().clear(); + } - stringsReader = stringsLoader.reader(ctx); - codePointsReader = codePointsLoader.reader(ctx); - for (int i = 0; i < ctx.reader().numDocs(); i += 10) { - int[] docsArray = new int[Math.min(10, ctx.reader().numDocs() - i)]; - for (int d = 0; d < docsArray.length; d++) { - docsArray[d] = i + d; - } - docs = TestBlock.docs(docsArray); - try ( - TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); - TestBlock codePoints = read(codePointsLoader, codePointsReader, ctx, docs); - ) { - checkBlocks(strings, codePoints); - } - } - assertThat(warnings.warnings(), equalTo(expectedWarnings)); + var warnings = new MockWarnings(); + var stringsLoader = new BytesRefsFromOrdsBlockLoader("field"); + var codePointsLoader = new Utf8CodePointsFromOrdsBlockLoader(warnings, "field"); + + var stringsReader = stringsLoader.reader(ctx); + var codePointsReader = codePointsLoader.reader(ctx); + assertThat(codePointsReader, readerMatcher()); + BlockLoader.Docs docs = TestBlock.docs(ctx); + try ( + TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); + TestBlock codePoints = read(codePointsLoader, codePointsReader, ctx, docs); + ) { + checkBlocks(strings, codePoints); + } + assertThat(warnings.warnings(), equalTo(expectedWarnings)); + warnings.warnings().clear(); + + stringsReader = stringsLoader.reader(ctx); + codePointsReader = codePointsLoader.reader(ctx); + for (int i = 0; i < ctx.reader().numDocs(); i += 10) { + int[] docsArray = new int[Math.min(10, ctx.reader().numDocs() - i)]; + for (int d = 0; d < docsArray.length; d++) { + docsArray[d] = i + d; + } + docs = TestBlock.docs(docsArray); + try ( + TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); + TestBlock codePoints = read(codePointsLoader, codePointsReader, ctx, docs); + ) { + checkBlocks(strings, codePoints); } } - } - - private TestBlock read(BlockLoader loader, BlockLoader.AllReader reader, LeafReaderContext ctx, BlockLoader.Docs docs) - throws IOException { - BlockLoader.AllReader toUse = blockAtATime - ? reader - : new ForceDocAtATime(() -> loader.builder(TestBlock.factory(), docs.count()), reader); - - return (TestBlock) toUse.read(TestBlock.factory(), docs, 0, false); + assertThat(warnings.warnings(), equalTo(expectedWarnings)); } private Matcher readerMatcher() { @@ -139,9 +88,9 @@ private Matcher readerMatcher() { return hasToString("Utf8CodePointsFromOrds.Immediate"); } if (multiValues) { - return hasToString("Utf8CodePointsFromOrds.Ordinals"); + return hasToString("Utf8CodePointsFromOrds.SortedSet"); } - return hasToString("Utf8CodePointsFromOrds.SingletonOrdinals"); + return hasToString("Utf8CodePointsFromOrds.Singleton"); } private static KeywordFieldMapper.KeywordField field(int codePointCount) { @@ -159,7 +108,7 @@ private void checkBlocks(TestBlock strings, TestBlock codePoints) { assertThat(codePoints.get(i), nullValue()); continue; } - BytesRef bytes = (BytesRef) strings.get(i); + BytesRef bytes = (BytesRef) str; assertThat(codePoints.get(i), equalTo(bytes.length)); } } From 2bf926945a6effe9bb6adbb1382273231cc0e3e6 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Oct 2025 13:31:32 -0400 Subject: [PATCH 2/9] Fix comment --- .../docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java index 25f836d465930..cc9d62285bb08 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java @@ -16,7 +16,7 @@ import java.io.IOException; /** - * Loads {@code MV_MIN} applied to {@code keyword} style fields that are stored as + * Loads {@code MV_MAX} applied to {@code keyword} style fields that are stored as * a lookup table and ordinals. */ public class MvMaxBytesRefsFromOrdsBlockLoader extends AbstractBytesRefsFromOrdsBlockLoader { From 9872f904f6b687cab9c0bd4361a9f40d95f99b25 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Oct 2025 15:59:00 -0400 Subject: [PATCH 3/9] Fixup test --- .../AbstractFromOrdsBlockLoaderTests.java | 6 ++++-- .../MvMaxBytesRefsFromOrdsBlockLoaderTests.java | 2 +- .../MvMinBytesRefsFromOrdsBlockLoaderTests.java | 2 +- .../Utf8CodePointsFromOrdsBlockLoaderTests.java | 14 +++++--------- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractFromOrdsBlockLoaderTests.java index 2f0c1e618ef90..09a9079dd2023 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractFromOrdsBlockLoaderTests.java @@ -56,9 +56,10 @@ public AbstractFromOrdsBlockLoaderTests(boolean blockAtATime, boolean highCardin this.missingValues = missingValues; } - protected abstract void innerTest(LeafReaderContext ctx) throws IOException; + protected abstract void innerTest(LeafReaderContext ctx, int mvCount) throws IOException; public void test() throws IOException { + int mvCount = 0; try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) { int docCount = 10_000; int cardinality = lowCardinality ? between(1, LOW_CARDINALITY) : between(LOW_CARDINALITY + 1, LOW_CARDINALITY * 2); @@ -67,6 +68,7 @@ public void test() throws IOException { doc.add(field(i % cardinality)); if (multiValues && i % cardinality == 0) { doc.add(field((i % cardinality) + 1)); + mvCount++; } iw.addDocument(doc); } @@ -76,7 +78,7 @@ public void test() throws IOException { iw.forceMerge(1); try (DirectoryReader dr = iw.getReader()) { LeafReaderContext ctx = getOnlyLeafReader(dr).getContext(); - innerTest(ctx); + innerTest(ctx, mvCount); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java index 945ce8ac67ed5..fdf8ae5d1e10b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java @@ -33,7 +33,7 @@ public MvMaxBytesRefsFromOrdsBlockLoaderTests( } @Override - protected void innerTest(LeafReaderContext ctx) throws IOException { + protected void innerTest(LeafReaderContext ctx, int mvCount) throws IOException { var stringsLoader = new BytesRefsFromOrdsBlockLoader("field"); var mvMaxLoader = new MvMaxBytesRefsFromOrdsBlockLoader("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java index f10a2d9369181..472131f5778ca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java @@ -33,7 +33,7 @@ public MvMinBytesRefsFromOrdsBlockLoaderTests( } @Override - protected void innerTest(LeafReaderContext ctx) throws IOException { + protected void innerTest(LeafReaderContext ctx, int mvCount) throws IOException { var stringsLoader = new BytesRefsFromOrdsBlockLoader("field"); var mvMinLoader = new MvMinBytesRefsFromOrdsBlockLoader("field"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java index dd7bc42be1042..c93aedcabe740 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java @@ -36,16 +36,12 @@ public Utf8CodePointsFromOrdsBlockLoaderTests( } @Override - protected void innerTest(LeafReaderContext ctx) throws IOException { + protected void innerTest(LeafReaderContext ctx, int mvCount) throws IOException { List expectedWarnings = new ArrayList<>(); - int docCount = 10_000; - int cardinality = lowCardinality ? between(1, LOW_CARDINALITY) : between(LOW_CARDINALITY + 1, LOW_CARDINALITY * 2); - for (int i = 0; i < docCount; i++) { - if (multiValues && i % cardinality == 0) { - expectedWarnings.add( - new MockWarnings.MockWarning(IllegalArgumentException.class, "single-value function encountered multi-value") - ); - } + for (int i = 0; i < mvCount; i++) { + expectedWarnings.add( + new MockWarnings.MockWarning(IllegalArgumentException.class, "single-value function encountered multi-value") + ); } var warnings = new MockWarnings(); From 90e67e4fd874facb3be7d0aa4e506253a2722007 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Oct 2025 16:12:53 -0400 Subject: [PATCH 4/9] More tests --- .../ValueSourceReaderTypeConversionTests.java | 14 ++++---- .../read/ValuesSourceReaderOperatorTests.java | 34 ++++++++++--------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java index c4a3079d866ba..d1b5bd7eeacbe 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java @@ -118,6 +118,8 @@ import java.util.stream.IntStream; import java.util.stream.LongStream; +import static org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperatorTests.StatusChecks.multiName; +import static org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperatorTests.StatusChecks.singleName; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; @@ -1196,13 +1198,13 @@ private static void docValues( assertMap( "Expected segment count in " + readers + "\n", readers, - matchesMap().entry(name + ":row_stride:BlockDocValuesReader.Singleton" + type, lessThanOrEqualTo(segmentCount)) + matchesMap().entry(name + ":row_stride:" + singleName(type), lessThanOrEqualTo(segmentCount)) ); } else { assertMap( "Expected segment count in " + readers + "\n", readers, - matchesMap().entry(name + ":column_at_a_time:BlockDocValuesReader.Singleton" + type, lessThanOrEqualTo(pageCount)) + matchesMap().entry(name + ":column_at_a_time:" + singleName(type), lessThanOrEqualTo(pageCount)) ); } } @@ -1216,19 +1218,19 @@ private static void mvDocValues( Map readers ) { if (forcedRowByRow) { - Integer singletons = (Integer) readers.remove(name + ":row_stride:BlockDocValuesReader.Singleton" + type); + Integer singletons = (Integer) readers.remove(name + ":row_stride:" + singleName(type)); if (singletons != null) { segmentCount -= singletons; } - assertMap(readers, matchesMap().entry(name + ":row_stride:BlockDocValuesReader." + type, segmentCount)); + assertMap(readers, matchesMap().entry(name + ":row_stride:" + multiName(type), segmentCount)); } else { - Integer singletons = (Integer) readers.remove(name + ":column_at_a_time:BlockDocValuesReader.Singleton" + type); + Integer singletons = (Integer) readers.remove(name + ":column_at_a_time:" + singleName(type)); if (singletons != null) { pageCount -= singletons; } assertMap( readers, - matchesMap().entry(name + ":column_at_a_time:BlockDocValuesReader." + type, lessThanOrEqualTo(pageCount)) + matchesMap().entry(name + ":column_at_a_time:" + multiName(type), lessThanOrEqualTo(pageCount)) ); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java index 97379a9ac8480..a866e20c58700 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java @@ -1310,16 +1310,13 @@ static void textWithDelegate(boolean forcedRowByRow, int pageCount, int segmentC if (forcedRowByRow) { assertMap( readers, - matchesMap().entry( - "text_with_delegate:row_stride:Delegating[to=kwd, impl=BlockDocValuesReader.SingletonOrdinals]", - segmentCount - ) + matchesMap().entry("text_with_delegate:row_stride:Delegating[to=kwd, impl=BytesRefsFromOrds.Singleton]", segmentCount) ); } else { assertMap( readers, matchesMap().entry( - "text_with_delegate:column_at_a_time:Delegating[to=kwd, impl=BlockDocValuesReader.SingletonOrdinals]", + "text_with_delegate:column_at_a_time:Delegating[to=kwd, impl=BytesRefsFromOrds.Singleton]", lessThanOrEqualTo(pageCount) ) ); @@ -1331,7 +1328,7 @@ static void mvTextWithDelegate(boolean forcedRowByRow, int pageCount, int segmen assertMap( readers, matchesMap().entry( - "mv_text_with_delegate:row_stride:Delegating[to=mv_kwd, impl=BlockDocValuesReader.Ordinals]", + "mv_text_with_delegate:row_stride:Delegating[to=mv_kwd, impl=BytesRefsFromOrds.SortedSet]", equalTo(segmentCount) ) ); @@ -1339,7 +1336,7 @@ static void mvTextWithDelegate(boolean forcedRowByRow, int pageCount, int segmen assertMap( readers, matchesMap().entry( - "mv_text_with_delegate:column_at_a_time:Delegating[to=mv_kwd, impl=BlockDocValuesReader.Ordinals]", + "mv_text_with_delegate:column_at_a_time:Delegating[to=mv_kwd, impl=BytesRefsFromOrds.SortedSet]", lessThanOrEqualTo(pageCount) ) ); @@ -1377,12 +1374,12 @@ private static void docValues( if (forcedRowByRow) { assertMap( readers, - matchesMap().entry(name + ":row_stride:BlockDocValuesReader.Singleton" + type, lessThanOrEqualTo(segmentCount)) + matchesMap().entry(name + ":row_stride:" + singleName(type), lessThanOrEqualTo(segmentCount)) ); } else { assertMap( readers, - matchesMap().entry(name + ":column_at_a_time:BlockDocValuesReader.Singleton" + type, lessThanOrEqualTo(pageCount)) + matchesMap().entry(name + ":column_at_a_time:" + singleName(type), lessThanOrEqualTo(pageCount)) ); } } @@ -1396,23 +1393,28 @@ private static void mvDocValues( Map readers ) { if (forcedRowByRow) { - Integer singletons = (Integer) readers.remove(name + ":row_stride:BlockDocValuesReader.Singleton" + type); + Integer singletons = (Integer) readers.remove(name + ":row_stride:" + singleName(type)); if (singletons != null) { segmentCount -= singletons; } - assertMap(readers, matchesMap().entry(name + ":row_stride:BlockDocValuesReader." + type, segmentCount)); + assertMap(readers, matchesMap().entry(name + ":row_stride:" + multiName(type), segmentCount)); } else { - Integer singletons = (Integer) readers.remove(name + ":column_at_a_time:BlockDocValuesReader.Singleton" + type); + Integer singletons = (Integer) readers.remove(name + ":column_at_a_time:" + singleName(type)); if (singletons != null) { pageCount -= singletons; } - assertMap( - readers, - matchesMap().entry(name + ":column_at_a_time:BlockDocValuesReader." + type, lessThanOrEqualTo(pageCount)) - ); + assertMap(readers, matchesMap().entry(name + ":column_at_a_time:" + multiName(type), lessThanOrEqualTo(pageCount))); } } + static String singleName(String type) { + return type.equals("Ordinals") ? "BytesRefsFromOrds.Singleton" : "BlockDocValuesReader.Singleton" + type; + } + + static String multiName(String type) { + return type.equals("Ordinals") ? "BytesRefsFromOrds.SortedSet" : "BlockDocValuesReader." + type; + } + static void id(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { stored("_id", "Id", forcedRowByRow, pageCount, segmentCount, readers); } From 7b5ee4489bd2dece46f17d620c6bbb13555edafe Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Oct 2025 16:19:57 -0400 Subject: [PATCH 5/9] More tests --- .../Utf8CodePointsFromOrdsBlockLoaderTests.java | 1 - .../read/ValueSourceReaderTypeConversionTests.java | 5 +---- .../lucene/read/ValuesSourceReaderOperatorTests.java | 10 ++-------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java index c93aedcabe740..dac645d4c7557 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/Utf8CodePointsFromOrdsBlockLoaderTests.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.index.mapper.blockloader.docvalues.Utf8CodePointsFromOrdsBlockLoader.LOW_CARDINALITY; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.nullValue; diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java index d1b5bd7eeacbe..7c96633a7a602 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValueSourceReaderTypeConversionTests.java @@ -1228,10 +1228,7 @@ private static void mvDocValues( if (singletons != null) { pageCount -= singletons; } - assertMap( - readers, - matchesMap().entry(name + ":column_at_a_time:" + multiName(type), lessThanOrEqualTo(pageCount)) - ); + assertMap(readers, matchesMap().entry(name + ":column_at_a_time:" + multiName(type), lessThanOrEqualTo(pageCount))); } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java index a866e20c58700..1b16473d78351 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorTests.java @@ -1372,15 +1372,9 @@ private static void docValues( Map readers ) { if (forcedRowByRow) { - assertMap( - readers, - matchesMap().entry(name + ":row_stride:" + singleName(type), lessThanOrEqualTo(segmentCount)) - ); + assertMap(readers, matchesMap().entry(name + ":row_stride:" + singleName(type), lessThanOrEqualTo(segmentCount))); } else { - assertMap( - readers, - matchesMap().entry(name + ":column_at_a_time:" + singleName(type), lessThanOrEqualTo(pageCount)) - ); + assertMap(readers, matchesMap().entry(name + ":column_at_a_time:" + singleName(type), lessThanOrEqualTo(pageCount))); } } From 1b0d5890aee937eed256d744c74619b88ba8c788 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 31 Oct 2025 17:36:08 -0400 Subject: [PATCH 6/9] Test --- .../org/elasticsearch/xpack/esql/action/TimeSeriesIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java index 31f872d050d48..a501da545d9b6 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java @@ -589,7 +589,7 @@ public void testProfile() { readMetrics.readersBuilt().keySet(), equalTo( Set.of( - "_tsid:column_at_a_time:BlockDocValuesReader.SingletonOrdinals", + "_tsid:column_at_a_time:BytesRefsFromOrds.Singleton", "cpu:column_at_a_time:BlockDocValuesReader.SingletonDoubles" ) ) @@ -600,8 +600,8 @@ public void testProfile() { assertThat(readDimensions.readersBuilt(), aMapWithSize(1)); assertThat( Iterables.get(readDimensions.readersBuilt().keySet(), 0), - either(equalTo("cluster:row_stride:BlockDocValuesReader.SingletonOrdinals")).or( - equalTo("cluster:column_at_a_time:BlockDocValuesReader.SingletonOrdinals") + either(equalTo("cluster:row_stride:BytesRefsFromOrds.Singleton")).or( + equalTo("cluster:column_at_a_time:BytesRefsFromOrds.Singleton") ) ); assertThat(ops.get(5).operator(), containsString("EvalOperator")); From 4528fccc391ba909c23a8d089d27d7d2352a6286 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Sat, 1 Nov 2025 08:29:12 -0400 Subject: [PATCH 7/9] Compare instead of grab --- .../docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java | 5 ++++- .../docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java index fdf8ae5d1e10b..e1095245d7425 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java @@ -16,6 +16,7 @@ import org.hamcrest.Matcher; import java.io.IOException; +import java.util.Comparator; import java.util.List; import static org.hamcrest.Matchers.equalTo; @@ -79,7 +80,9 @@ private void checkBlocks(TestBlock strings, TestBlock mvMin) { assertThat(mvMin.get(i), nullValue()); continue; } - BytesRef bytes = (BytesRef) (str instanceof List l ? l.getLast() : str); + BytesRef bytes = (BytesRef) (str instanceof List l + ? l.stream().map(b -> (BytesRef) b).max(Comparator.naturalOrder()).get() + : str); assertThat(mvMin.get(i), equalTo(bytes)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java index 472131f5778ca..765ef5b155ddb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java @@ -16,6 +16,7 @@ import org.hamcrest.Matcher; import java.io.IOException; +import java.util.Comparator; import java.util.List; import static org.hamcrest.Matchers.equalTo; @@ -79,7 +80,9 @@ private void checkBlocks(TestBlock strings, TestBlock mvMin) { assertThat(mvMin.get(i), nullValue()); continue; } - BytesRef bytes = (BytesRef) (str instanceof List l ? l.getFirst() : str); + BytesRef bytes = (BytesRef) (str instanceof List l + ? l.stream().map(b -> (BytesRef) b).min(Comparator.naturalOrder()).get() + : str); assertThat(mvMin.get(i), equalTo(bytes)); } } From 795935dc6407067ccf9f8699fa9d5af7d168c4d4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 3 Nov 2025 10:26:06 -0500 Subject: [PATCH 8/9] Ints --- .../AbstractBytesRefsFromOrdsBlockLoader.java | 8 +- .../AbstractIntsFromDocValuesBlockLoader.java | 159 ++++++++++++++++++ .../docvalues/IntsBlockLoader.java | 140 ++------------- .../MvMaxBytesRefsFromOrdsBlockLoader.java | 3 +- .../MvMaxIntsFromDocValuesBlockLoader.java | 89 ++++++++++ .../MvMinBytesRefsFromOrdsBlockLoader.java | 3 +- .../MvMinIntsFromDocValuesBlockLoader.java | 81 +++++++++ ...ractIntsFromDocValuesBlockLoaderTests.java | 91 ++++++++++ ...vMaxBytesRefsFromOrdsBlockLoaderTests.java | 8 +- ...vMaxIntsFromDocValuesBlockLoaderTests.java | 81 +++++++++ ...vMinBytesRefsFromOrdsBlockLoaderTests.java | 8 +- ...vMinIntsFromDocValuesBlockLoaderTests.java | 81 +++++++++ .../index/mapper/MapperTestCase.java | 6 +- .../read/ValuesSourceReaderOperatorTests.java | 14 +- 14 files changed, 622 insertions(+), 150 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractIntsFromDocValuesBlockLoader.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoader.java create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinIntsFromDocValuesBlockLoader.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractIntsFromDocValuesBlockLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoaderTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinIntsFromDocValuesBlockLoaderTests.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractBytesRefsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractBytesRefsFromOrdsBlockLoader.java index 00edc40002247..bee53465ed100 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractBytesRefsFromOrdsBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractBytesRefsFromOrdsBlockLoader.java @@ -20,20 +20,20 @@ /** * Loads {@code keyword} style fields that are stored as a lookup table. */ -public abstract class AbstractBytesRefsFromOrdsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { +abstract class AbstractBytesRefsFromOrdsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { protected final String fieldName; - public AbstractBytesRefsFromOrdsBlockLoader(String fieldName) { + AbstractBytesRefsFromOrdsBlockLoader(String fieldName) { this.fieldName = fieldName; } @Override - public BytesRefBuilder builder(BlockFactory factory, int expectedCount) { + public final BytesRefBuilder builder(BlockFactory factory, int expectedCount) { return factory.bytesRefs(expectedCount); } @Override - public AllReader reader(LeafReaderContext context) throws IOException { + public final AllReader reader(LeafReaderContext context) throws IOException { SortedSetDocValues docValues = context.reader().getSortedSetDocValues(fieldName); if (docValues != null) { SortedDocValues singleton = DocValues.unwrapSingleton(docValues); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractIntsFromDocValuesBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractIntsFromDocValuesBlockLoader.java new file mode 100644 index 0000000000000..22b2ce7162d64 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractIntsFromDocValuesBlockLoader.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; + +import java.io.IOException; + +/** + * Loads {@code int}s from doc values. + */ +public abstract class AbstractIntsFromDocValuesBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { + protected final String fieldName; + + AbstractIntsFromDocValuesBlockLoader(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public final Builder builder(BlockFactory factory, int expectedCount) { + return factory.ints(expectedCount); + } + + @Override + public final AllReader reader(LeafReaderContext context) throws IOException { + SortedNumericDocValues docValues = context.reader().getSortedNumericDocValues(fieldName); + if (docValues != null) { + NumericDocValues singleton = DocValues.unwrapSingleton(docValues); + if (singleton != null) { + return singletonReader(singleton); + } + return sortedReader(docValues); + } + NumericDocValues singleton = context.reader().getNumericDocValues(fieldName); + if (singleton != null) { + return singletonReader(singleton); + } + return new ConstantNullsReader(); + } + + protected abstract AllReader singletonReader(NumericDocValues docValues); + + protected abstract AllReader sortedReader(SortedNumericDocValues docValues); + + public static class Singleton extends BlockDocValuesReader implements BlockDocValuesReader.NumericDocValuesAccessor { + private final NumericDocValues numericDocValues; + + Singleton(NumericDocValues numericDocValues) { + this.numericDocValues = numericDocValues; + } + + @Override + public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { + if (numericDocValues instanceof OptionalColumnAtATimeReader direct) { + Block result = direct.tryRead(factory, docs, offset, nullsFiltered, null, true); + if (result != null) { + return result; + } + } + try (IntBuilder builder = factory.intsFromDocValues(docs.count() - offset)) { + for (int i = offset; i < docs.count(); i++) { + int doc = docs.get(i); + if (numericDocValues.advanceExact(doc)) { + builder.appendInt(Math.toIntExact(numericDocValues.longValue())); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + IntBuilder blockBuilder = (IntBuilder) builder; + if (numericDocValues.advanceExact(docId)) { + blockBuilder.appendInt(Math.toIntExact(numericDocValues.longValue())); + } else { + blockBuilder.appendNull(); + } + } + + @Override + public int docId() { + return numericDocValues.docID(); + } + + @Override + public String toString() { + return "IntsFromDocValues.Singleton"; + } + + @Override + public NumericDocValues numericDocValues() { + return numericDocValues; + } + } + + public static class Sorted extends BlockDocValuesReader { + private final SortedNumericDocValues numericDocValues; + + Sorted(SortedNumericDocValues numericDocValues) { + this.numericDocValues = numericDocValues; + } + + @Override + public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { + try (IntBuilder builder = factory.intsFromDocValues(docs.count() - offset)) { + for (int i = offset; i < docs.count(); i++) { + int doc = docs.get(i); + read(doc, builder); + } + return builder.build(); + } + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + read(docId, (IntBuilder) builder); + } + + private void read(int doc, IntBuilder builder) throws IOException { + if (false == numericDocValues.advanceExact(doc)) { + builder.appendNull(); + return; + } + int count = numericDocValues.docValueCount(); + if (count == 1) { + builder.appendInt(Math.toIntExact(numericDocValues.nextValue())); + return; + } + builder.beginPositionEntry(); + for (int v = 0; v < count; v++) { + builder.appendInt(Math.toIntExact(numericDocValues.nextValue())); + } + builder.endPositionEntry(); + } + + @Override + public int docId() { + return numericDocValues.docID(); + } + + @Override + public String toString() { + return "IntsFromDocValues.Sorted"; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java index 9f0bc40f02845..029653c4c5047 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/IntsBlockLoader.java @@ -9,145 +9,29 @@ package org.elasticsearch.index.mapper.blockloader.docvalues; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.elasticsearch.index.mapper.BlockLoader; - -import java.io.IOException; - -public class IntsBlockLoader extends BlockDocValuesReader.DocValuesBlockLoader { - private final String fieldName; +/** + * Loads {@code int}s from doc values. + */ +public class IntsBlockLoader extends AbstractIntsFromDocValuesBlockLoader { public IntsBlockLoader(String fieldName) { - this.fieldName = fieldName; + super(fieldName); } @Override - public Builder builder(BlockFactory factory, int expectedCount) { - return factory.ints(expectedCount); + protected AllReader singletonReader(NumericDocValues docValues) { + return new Singleton(docValues); } @Override - public AllReader reader(LeafReaderContext context) throws IOException { - SortedNumericDocValues docValues = context.reader().getSortedNumericDocValues(fieldName); - if (docValues != null) { - NumericDocValues singleton = DocValues.unwrapSingleton(docValues); - if (singleton != null) { - return new SingletonInts(singleton); - } - return new Ints(docValues); - } - NumericDocValues singleton = context.reader().getNumericDocValues(fieldName); - if (singleton != null) { - return new SingletonInts(singleton); - } - return new ConstantNullsReader(); - } - - public static class SingletonInts extends BlockDocValuesReader implements BlockDocValuesReader.NumericDocValuesAccessor { - private final NumericDocValues numericDocValues; - - SingletonInts(NumericDocValues numericDocValues) { - this.numericDocValues = numericDocValues; - } - - @Override - public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { - if (numericDocValues instanceof BlockLoader.OptionalColumnAtATimeReader direct) { - BlockLoader.Block result = direct.tryRead(factory, docs, offset, nullsFiltered, null, true); - if (result != null) { - return result; - } - } - try (BlockLoader.IntBuilder builder = factory.intsFromDocValues(docs.count() - offset)) { - for (int i = offset; i < docs.count(); i++) { - int doc = docs.get(i); - if (numericDocValues.advanceExact(doc)) { - builder.appendInt(Math.toIntExact(numericDocValues.longValue())); - } else { - builder.appendNull(); - } - } - return builder.build(); - } - } - - @Override - public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { - IntBuilder blockBuilder = (IntBuilder) builder; - if (numericDocValues.advanceExact(docId)) { - blockBuilder.appendInt(Math.toIntExact(numericDocValues.longValue())); - } else { - blockBuilder.appendNull(); - } - } - - @Override - public int docId() { - return numericDocValues.docID(); - } - - @Override - public String toString() { - return "BlockDocValuesReader.SingletonInts"; - } - - @Override - public NumericDocValues numericDocValues() { - return numericDocValues; - } + protected AllReader sortedReader(SortedNumericDocValues docValues) { + return new Sorted(docValues); } - public static class Ints extends BlockDocValuesReader { - private final SortedNumericDocValues numericDocValues; - - Ints(SortedNumericDocValues numericDocValues) { - this.numericDocValues = numericDocValues; - } - - @Override - public BlockLoader.Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { - try (BlockLoader.IntBuilder builder = factory.intsFromDocValues(docs.count() - offset)) { - for (int i = offset; i < docs.count(); i++) { - int doc = docs.get(i); - read(doc, builder); - } - return builder.build(); - } - } - - @Override - public void read(int docId, BlockLoader.StoredFields storedFields, Builder builder) throws IOException { - read(docId, (IntBuilder) builder); - } - - private void read(int doc, IntBuilder builder) throws IOException { - if (false == numericDocValues.advanceExact(doc)) { - builder.appendNull(); - return; - } - int count = numericDocValues.docValueCount(); - if (count == 1) { - builder.appendInt(Math.toIntExact(numericDocValues.nextValue())); - return; - } - builder.beginPositionEntry(); - for (int v = 0; v < count; v++) { - builder.appendInt(Math.toIntExact(numericDocValues.nextValue())); - } - builder.endPositionEntry(); - } - - @Override - public int docId() { - return numericDocValues.docID(); - } - - @Override - public String toString() { - return "BlockDocValuesReader.Ints"; - } + @Override + public String toString() { + return "IntsFromDocValues[" + fieldName + "]"; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java index cc9d62285bb08..4b26d784115f5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoader.java @@ -16,8 +16,7 @@ import java.io.IOException; /** - * Loads {@code MV_MAX} applied to {@code keyword} style fields that are stored as - * a lookup table and ordinals. + * Loads the MAX {@code keyword} in each doc. */ public class MvMaxBytesRefsFromOrdsBlockLoader extends AbstractBytesRefsFromOrdsBlockLoader { private final String fieldName; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoader.java new file mode 100644 index 0000000000000..1130d265fbd74 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoader.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; + +import java.io.IOException; + +/** + * Loads the MAX {@code int} in each doc. + */ +public class MvMaxIntsFromDocValuesBlockLoader extends AbstractIntsFromDocValuesBlockLoader { + public MvMaxIntsFromDocValuesBlockLoader(String fieldName) { + super(fieldName); + } + + @Override + protected AllReader singletonReader(NumericDocValues docValues) { + return new Singleton(docValues); + } + + @Override + protected AllReader sortedReader(SortedNumericDocValues docValues) { + return new MvMinSorted(docValues); + } + + @Override + public String toString() { + return "IntsFromDocValues[" + fieldName + "]"; + } + + public static class MvMinSorted extends BlockDocValuesReader { + private final SortedNumericDocValues numericDocValues; + + MvMinSorted(SortedNumericDocValues numericDocValues) { + this.numericDocValues = numericDocValues; + } + + @Override + public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { + try (IntBuilder builder = factory.intsFromDocValues(docs.count() - offset)) { + for (int i = offset; i < docs.count(); i++) { + int doc = docs.get(i); + read(doc, builder); + } + return builder.build(); + } + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + read(docId, (IntBuilder) builder); + } + + private void read(int doc, IntBuilder builder) throws IOException { + if (false == numericDocValues.advanceExact(doc)) { + builder.appendNull(); + return; + } + discardAllButLast(); + builder.appendInt(Math.toIntExact(numericDocValues.nextValue())); + } + + @Override + public int docId() { + return numericDocValues.docID(); + } + + private void discardAllButLast() throws IOException { + int count = numericDocValues.docValueCount(); + for (int i = 0; i < count - 1; i++) { + numericDocValues.nextValue(); + } + } + + @Override + public String toString() { + return "MvMaxIntsFromDocValues.Sorted"; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoader.java index f7b48e92b7df0..f97e25170bdc0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoader.java @@ -16,8 +16,7 @@ import java.io.IOException; /** - * Loads {@code MV_MIN} applied to {@code keyword} style fields that are stored as - * a lookup table and ordinals. + * Loads the MIN {@code keyword} in each doc. */ public class MvMinBytesRefsFromOrdsBlockLoader extends AbstractBytesRefsFromOrdsBlockLoader { private final String fieldName; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinIntsFromDocValuesBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinIntsFromDocValuesBlockLoader.java new file mode 100644 index 0000000000000..23f3ad8fb34f4 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinIntsFromDocValuesBlockLoader.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; + +import java.io.IOException; + +/** + * Loads the MIN {@code int} in each doc. + */ +public class MvMinIntsFromDocValuesBlockLoader extends AbstractIntsFromDocValuesBlockLoader { + public MvMinIntsFromDocValuesBlockLoader(String fieldName) { + super(fieldName); + } + + @Override + protected AllReader singletonReader(NumericDocValues docValues) { + return new Singleton(docValues); + } + + @Override + protected AllReader sortedReader(SortedNumericDocValues docValues) { + return new MvMinSorted(docValues); + } + + @Override + public String toString() { + return "IntsFromDocValues[" + fieldName + "]"; + } + + public static class MvMinSorted extends BlockDocValuesReader { + private final SortedNumericDocValues numericDocValues; + + MvMinSorted(SortedNumericDocValues numericDocValues) { + this.numericDocValues = numericDocValues; + } + + @Override + public Block read(BlockFactory factory, Docs docs, int offset, boolean nullsFiltered) throws IOException { + try (IntBuilder builder = factory.intsFromDocValues(docs.count() - offset)) { + for (int i = offset; i < docs.count(); i++) { + int doc = docs.get(i); + read(doc, builder); + } + return builder.build(); + } + } + + @Override + public void read(int docId, StoredFields storedFields, Builder builder) throws IOException { + read(docId, (IntBuilder) builder); + } + + private void read(int doc, IntBuilder builder) throws IOException { + if (false == numericDocValues.advanceExact(doc)) { + builder.appendNull(); + return; + } + builder.appendInt(Math.toIntExact(numericDocValues.nextValue())); + } + + @Override + public int docId() { + return numericDocValues.docID(); + } + + @Override + public String toString() { + return "MvMinIntsFromDocValues.Sorted"; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractIntsFromDocValuesBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractIntsFromDocValuesBlockLoaderTests.java new file mode 100644 index 0000000000000..9bf41c8692572 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/AbstractIntsFromDocValuesBlockLoaderTests.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.TestBlock; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public abstract class AbstractIntsFromDocValuesBlockLoaderTests extends ESTestCase { + @ParametersFactory(argumentFormatting = "blockAtATime=%s, multiValues=%s, missingValues=%s") + public static List parameters() throws IOException { + List parameters = new ArrayList<>(); + for (boolean blockAtATime : new boolean[] { true, false }) { + for (boolean multiValues : new boolean[] { true, false }) { + for (boolean missingValues : new boolean[] { true, false }) { + parameters.add(new Object[] { blockAtATime, multiValues, missingValues }); + } + } + } + return parameters; + } + + protected final boolean blockAtATime; + protected final boolean multiValues; + protected final boolean missingValues; + + public AbstractIntsFromDocValuesBlockLoaderTests(boolean blockAtATime, boolean multiValues, boolean missingValues) { + this.blockAtATime = blockAtATime; + this.multiValues = multiValues; + this.missingValues = missingValues; + } + + protected abstract void innerTest(LeafReaderContext ctx, int mvCount) throws IOException; + + public void test() throws IOException { + int mvCount = 0; + try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir)) { + int docCount = 10_000; + for (int i = 0; i < docCount; i++) { + List doc = new ArrayList<>(2); + doc.add(field(i)); + if (multiValues && i % 100 == 0) { + doc.add(field((i % 100) + 1)); + mvCount++; + } + iw.addDocument(doc); + } + if (missingValues) { + iw.addDocument(List.of()); + } + iw.forceMerge(1); + try (DirectoryReader dr = iw.getReader()) { + LeafReaderContext ctx = getOnlyLeafReader(dr).getContext(); + innerTest(ctx, mvCount); + } + } + } + + protected final TestBlock read(BlockLoader loader, BlockLoader.AllReader reader, LeafReaderContext ctx, BlockLoader.Docs docs) + throws IOException { + BlockLoader.AllReader toUse = blockAtATime + ? reader + : new ForceDocAtATime(() -> loader.builder(TestBlock.factory(), docs.count()), reader); + + return (TestBlock) toUse.read(TestBlock.factory(), docs, 0, false); + } + + private static IntField field(int codePointCount) { + return new IntField("field", codePointCount, Field.Store.NO); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java index e1095245d7425..7b4e3772f3bd5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxBytesRefsFromOrdsBlockLoaderTests.java @@ -44,9 +44,9 @@ protected void innerTest(LeafReaderContext ctx, int mvCount) throws IOException BlockLoader.Docs docs = TestBlock.docs(ctx); try ( TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); - TestBlock codePoints = read(mvMaxLoader, mvMaxReader, ctx, docs); + TestBlock maxStrings = read(mvMaxLoader, mvMaxReader, ctx, docs); ) { - checkBlocks(strings, codePoints); + checkBlocks(strings, maxStrings); } stringsReader = stringsLoader.reader(ctx); @@ -59,9 +59,9 @@ protected void innerTest(LeafReaderContext ctx, int mvCount) throws IOException docs = TestBlock.docs(docsArray); try ( TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); - TestBlock codePoints = read(mvMaxLoader, mvMaxReader, ctx, docs); + TestBlock maxStrings = read(mvMaxLoader, mvMaxReader, ctx, docs); ) { - checkBlocks(strings, codePoints); + checkBlocks(strings, maxStrings); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoaderTests.java new file mode 100644 index 0000000000000..f60431ecbdafd --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoaderTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.TestBlock; +import org.hamcrest.Matcher; + +import java.io.IOException; +import java.util.Comparator; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.nullValue; + +public class MvMaxIntsFromDocValuesBlockLoaderTests extends AbstractIntsFromDocValuesBlockLoaderTests { + public MvMaxIntsFromDocValuesBlockLoaderTests(boolean blockAtATime, boolean multiValues, boolean missingValues) { + super(blockAtATime, multiValues, missingValues); + } + + @Override + protected void innerTest(LeafReaderContext ctx, int mvCount) throws IOException { + var intsLoader = new IntsBlockLoader("field"); + var mvMaxIntsLoader = new MvMaxIntsFromDocValuesBlockLoader("field"); + + var intsReader = intsLoader.reader(ctx); + var mvMaxIntsReader = mvMaxIntsLoader.reader(ctx); + assertThat(mvMaxIntsReader, readerMatcher()); + BlockLoader.Docs docs = TestBlock.docs(ctx); + try ( + TestBlock ints = read(intsLoader, intsReader, ctx, docs); + TestBlock minInts = read(mvMaxIntsLoader, mvMaxIntsReader, ctx, docs); + ) { + checkBlocks(ints, minInts); + } + + intsReader = intsLoader.reader(ctx); + mvMaxIntsReader = mvMaxIntsLoader.reader(ctx); + for (int i = 0; i < ctx.reader().numDocs(); i += 10) { + int[] docsArray = new int[Math.min(10, ctx.reader().numDocs() - i)]; + for (int d = 0; d < docsArray.length; d++) { + docsArray[d] = i + d; + } + docs = TestBlock.docs(docsArray); + try ( + TestBlock ints = read(intsLoader, intsReader, ctx, docs); + TestBlock maxInts = read(mvMaxIntsLoader, mvMaxIntsReader, ctx, docs); + ) { + checkBlocks(ints, maxInts); + } + } + } + + private Matcher readerMatcher() { + if (multiValues) { + return hasToString("MvMaxIntsFromDocValues.Sorted"); + } + return hasToString("IntsFromDocValues.Singleton"); + } + + private void checkBlocks(TestBlock ints, TestBlock mvMax) { + for (int i = 0; i < ints.size(); i++) { + Object v = ints.get(i); + if (v == null) { + assertThat(mvMax.get(i), nullValue()); + continue; + } + Integer max = (Integer) (v instanceof List l ? l.stream().map(b -> (Integer) b).max(Comparator.naturalOrder()).get() : v); + assertThat(mvMax.get(i), equalTo(max)); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java index 765ef5b155ddb..468bbdfa384c7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinBytesRefsFromOrdsBlockLoaderTests.java @@ -44,9 +44,9 @@ protected void innerTest(LeafReaderContext ctx, int mvCount) throws IOException BlockLoader.Docs docs = TestBlock.docs(ctx); try ( TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); - TestBlock codePoints = read(mvMinLoader, mvMinReader, ctx, docs); + TestBlock minStrings = read(mvMinLoader, mvMinReader, ctx, docs); ) { - checkBlocks(strings, codePoints); + checkBlocks(strings, minStrings); } stringsReader = stringsLoader.reader(ctx); @@ -59,9 +59,9 @@ protected void innerTest(LeafReaderContext ctx, int mvCount) throws IOException docs = TestBlock.docs(docsArray); try ( TestBlock strings = read(stringsLoader, stringsReader, ctx, docs); - TestBlock codePoints = read(mvMinLoader, mvMinReader, ctx, docs); + TestBlock minStrings = read(mvMinLoader, mvMinReader, ctx, docs); ) { - checkBlocks(strings, codePoints); + checkBlocks(strings, minStrings); } } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinIntsFromDocValuesBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinIntsFromDocValuesBlockLoaderTests.java new file mode 100644 index 0000000000000..c32010366448a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMinIntsFromDocValuesBlockLoaderTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.blockloader.docvalues; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.TestBlock; +import org.hamcrest.Matcher; + +import java.io.IOException; +import java.util.Comparator; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.nullValue; + +public class MvMinIntsFromDocValuesBlockLoaderTests extends AbstractIntsFromDocValuesBlockLoaderTests { + public MvMinIntsFromDocValuesBlockLoaderTests(boolean blockAtATime, boolean multiValues, boolean missingValues) { + super(blockAtATime, multiValues, missingValues); + } + + @Override + protected void innerTest(LeafReaderContext ctx, int mvCount) throws IOException { + var intsLoader = new IntsBlockLoader("field"); + var mvMinIntsLoader = new MvMinIntsFromDocValuesBlockLoader("field"); + + var intsReader = intsLoader.reader(ctx); + var mvMinIntsReader = mvMinIntsLoader.reader(ctx); + assertThat(mvMinIntsReader, readerMatcher()); + BlockLoader.Docs docs = TestBlock.docs(ctx); + try ( + TestBlock ints = read(intsLoader, intsReader, ctx, docs); + TestBlock minInts = read(mvMinIntsLoader, mvMinIntsReader, ctx, docs); + ) { + checkBlocks(ints, minInts); + } + + intsReader = intsLoader.reader(ctx); + mvMinIntsReader = mvMinIntsLoader.reader(ctx); + for (int i = 0; i < ctx.reader().numDocs(); i += 10) { + int[] docsArray = new int[Math.min(10, ctx.reader().numDocs() - i)]; + for (int d = 0; d < docsArray.length; d++) { + docsArray[d] = i + d; + } + docs = TestBlock.docs(docsArray); + try ( + TestBlock ints = read(intsLoader, intsReader, ctx, docs); + TestBlock minInts = read(mvMinIntsLoader, mvMinIntsReader, ctx, docs); + ) { + checkBlocks(ints, minInts); + } + } + } + + private Matcher readerMatcher() { + if (multiValues) { + return hasToString("MvMinIntsFromDocValues.Sorted"); + } + return hasToString("IntsFromDocValues.Singleton"); + } + + private void checkBlocks(TestBlock ints, TestBlock mvMin) { + for (int i = 0; i < ints.size(); i++) { + Object v = ints.get(i); + if (v == null) { + assertThat(mvMin.get(i), nullValue()); + continue; + } + Integer min = (Integer) (v instanceof List l ? l.stream().map(b -> (Integer) b).min(Comparator.naturalOrder()).get() : v); + assertThat(mvMin.get(i), equalTo(min)); + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 3ef66cebc1f14..912fd920f1db7 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -51,9 +51,9 @@ import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.index.fieldvisitor.StoredFieldLoader; +import org.elasticsearch.index.mapper.blockloader.docvalues.AbstractIntsFromDocValuesBlockLoader; import org.elasticsearch.index.mapper.blockloader.docvalues.BlockDocValuesReader; import org.elasticsearch.index.mapper.blockloader.docvalues.DoublesBlockLoader; -import org.elasticsearch.index.mapper.blockloader.docvalues.IntsBlockLoader; import org.elasticsearch.index.mapper.blockloader.docvalues.LongsBlockLoader; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.termvectors.TermVectorsService; @@ -1527,7 +1527,7 @@ protected Object[] getThreeEncodedSampleValues() { public void testSingletonIntBulkBlockReading() throws IOException { assumeTrue("field type supports bulk singleton int reading", supportsBulkIntBlockReading()); - testSingletonBulkBlockReading(columnAtATimeReader -> (IntsBlockLoader.SingletonInts) columnAtATimeReader); + testSingletonBulkBlockReading(columnAtATimeReader -> (AbstractIntsFromDocValuesBlockLoader.Singleton) columnAtATimeReader); } public void testSingletonLongBulkBlockReading() throws IOException { @@ -1643,7 +1643,7 @@ private void testSingletonBulkBlockReading(Function "BytesRefsFromOrds.Singleton"; + case "Ints" -> "IntsFromDocValues.Singleton"; + default -> "BlockDocValuesReader.Singleton" + type; + }; } static String multiName(String type) { - return type.equals("Ordinals") ? "BytesRefsFromOrds.SortedSet" : "BlockDocValuesReader." + type; + return switch (type) { + case "Ordinals" -> "BytesRefsFromOrds.SortedSet"; + case "Ints" -> "IntsFromDocValues.Sorted"; + default -> "BlockDocValuesReader." + type; + }; } static void id(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { @@ -1707,7 +1715,7 @@ private void testSequentialStoredFields(boolean sequential, int docCount) throws ValuesSourceReaderOperatorStatus status = (ValuesSourceReaderOperatorStatus) op.status(); assertMap( status.readersBuilt(), - matchesMap().entry("key:column_at_a_time:BlockDocValuesReader.SingletonInts", 1) + matchesMap().entry("key:column_at_a_time:IntsFromDocValues.Singleton", 1) .entry("stored_text:column_at_a_time:null", 1) .entry("stored_text:row_stride:BlockStoredFieldsReader.Bytes", 1) .entry("stored_fields[requires_source:false, fields:1, sequential: " + sequential + "]", 1) From 8fa3db25158b1fa27e22a0c97381d407bcafd0fc Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 3 Nov 2025 13:25:14 -0500 Subject: [PATCH 9/9] Rename --- .../docvalues/MvMaxIntsFromDocValuesBlockLoader.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoader.java index 1130d265fbd74..ed1571419effd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/blockloader/docvalues/MvMaxIntsFromDocValuesBlockLoader.java @@ -29,7 +29,7 @@ protected AllReader singletonReader(NumericDocValues docValues) { @Override protected AllReader sortedReader(SortedNumericDocValues docValues) { - return new MvMinSorted(docValues); + return new MvMaxSorted(docValues); } @Override @@ -37,10 +37,10 @@ public String toString() { return "IntsFromDocValues[" + fieldName + "]"; } - public static class MvMinSorted extends BlockDocValuesReader { + public static class MvMaxSorted extends BlockDocValuesReader { private final SortedNumericDocValues numericDocValues; - MvMinSorted(SortedNumericDocValues numericDocValues) { + MvMaxSorted(SortedNumericDocValues numericDocValues) { this.numericDocValues = numericDocValues; }