From 2e835bc0d5b9b1da188762a8ba85e4fea8d43041 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 25 Aug 2025 17:59:53 -0400 Subject: [PATCH 1/3] Adjust knn reader interfaces to use new AcceptDocs api --- .../codec/vectors/ES813FlatVectorFormat.java | 10 +++--- .../vectors/ES813Int8FlatVectorFormat.java | 10 +++--- .../index/codec/vectors/IVFVectorsReader.java | 16 ++++----- .../ES816BinaryQuantizedVectorsReader.java | 7 ++-- .../ES818BinaryQuantizedVectorsReader.java | 7 ++-- .../vectors/es818/MergeReaderWrapper.java | 6 ++-- .../index/engine/TranslogDirectoryReader.java | 5 +-- .../index/mapper/DocumentLeafReader.java | 5 +-- .../internal/ExitableDirectoryReader.java | 6 ++-- .../FieldUsageTrackingDirectoryReader.java | 6 ++-- .../vectors/AbstractIVFKnnVectorQuery.java | 32 ++++-------------- .../vectors/IVFKnnFloatVectorQuery.java | 4 +-- ...HnswScalarQuantizedVectorsFormatTests.java | 9 ++++- .../codec/vectors/IVFVectorsFormatTests.java | 25 ++++++++++++-- ...HnswBinaryQuantizedVectorsFormatTests.java | 9 ++++- ...HnswBinaryQuantizedVectorsFormatTests.java | 9 ++++- .../search/SearchCancellationTests.java | 9 ++++- .../CompositeValuesCollectorQueueTests.java | 6 ++-- .../frozen/RewriteCachingDirectoryReader.java | 5 +-- .../accesscontrol/FieldSubsetReader.java | 6 ++-- .../accesscontrol/FieldSubsetReaderTests.java | 33 ++++++++++++++++--- 21 files changed, 143 insertions(+), 82 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java index 325188624a2f4..ef043422a37be 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813FlatVectorFormat.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Sorter; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.util.Bits; import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; @@ -128,13 +129,14 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); } - private void collectAllMatchingDocs(KnnCollector knnCollector, Bits acceptDocs, RandomVectorScorer scorer) throws IOException { + private void collectAllMatchingDocs(KnnCollector knnCollector, AcceptDocs acceptDocs, RandomVectorScorer scorer) + throws IOException { OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); - Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs.bits()); for (int i = 0; i < scorer.maxOrd(); i++) { if (acceptedOrds == null || acceptedOrds.get(i)) { collector.collect(i, scorer.score(i)); @@ -145,7 +147,7 @@ private void collectAllMatchingDocs(KnnCollector knnCollector, Bits acceptDocs, } @Override - public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java index 4636a37d14f53..dd9c1dced748d 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Sorter; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.util.Bits; import org.apache.lucene.util.hnsw.OrdinalTranslatedKnnCollector; @@ -136,13 +137,14 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); } - private void collectAllMatchingDocs(KnnCollector knnCollector, Bits acceptDocs, RandomVectorScorer scorer) throws IOException { + private void collectAllMatchingDocs(KnnCollector knnCollector, AcceptDocs acceptDocs, RandomVectorScorer scorer) + throws IOException { OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); - Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs.bits()); for (int i = 0; i < scorer.maxOrd(); i++) { if (acceptedOrds == null || acceptedOrds.get(i)) { collector.collect(i, scorer.score(i)); @@ -153,7 +155,7 @@ private void collectAllMatchingDocs(KnnCollector knnCollector, Bits acceptDocs, } @Override - public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { collectAllMatchingDocs(knnCollector, acceptDocs, reader.getRandomVectorScorer(field, target)); } diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsReader.java index 8082d72482c0a..5fb7dd76ad3e5 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsReader.java @@ -22,12 +22,12 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.internal.hppc.IntObjectHashMap; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; import org.elasticsearch.core.IOUtils; import org.elasticsearch.search.vectors.IVFKnnSearchStrategy; @@ -212,7 +212,7 @@ public final ByteVectorValues getByteVectorValues(String field) throws IOExcepti } @Override - public final void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public final void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field); if (fieldInfo.getVectorEncoding().equals(VectorEncoding.FLOAT32) == false) { rawVectorsReader.search(field, target, knnCollector, acceptDocs); @@ -223,11 +223,8 @@ public final void search(String field, float[] target, KnnCollector knnCollector "vector query dimension: " + target.length + " differs from field dimension: " + fieldInfo.getVectorDimension() ); } - float percentFiltered = 1f; - if (acceptDocs instanceof BitSet bitSet) { - percentFiltered = Math.max(0f, Math.min(1f, (float) bitSet.approximateCardinality() / bitSet.length())); - } int numVectors = rawVectorsReader.getFloatVectorValues(field).size(); + float percentFiltered = Math.max(0f, Math.min(1f, (float) acceptDocs.cost() / numVectors)); float visitRatio = DYNAMIC_VISIT_RATIO; // Search strategy may be null if this is being called from checkIndex (e.g. from a test) if (knnCollector.getSearchStrategy() instanceof IVFKnnSearchStrategy ivfSearchStrategy) { @@ -255,7 +252,8 @@ public final void search(String field, float[] target, KnnCollector knnCollector target, postListSlice ); - PostingVisitor scorer = getPostingVisitor(fieldInfo, postListSlice, target, acceptDocs); + Bits acceptDocsBits = acceptDocs.bits(); + PostingVisitor scorer = getPostingVisitor(fieldInfo, postListSlice, target, acceptDocsBits); long expectedDocs = 0; long actualDocs = 0; // initially we visit only the "centroids to search" @@ -271,7 +269,7 @@ public final void search(String field, float[] target, KnnCollector knnCollector expectedDocs += scorer.resetPostingsScorer(offsetAndLength.offset()); actualDocs += scorer.visit(knnCollector); } - if (acceptDocs != null) { + if (acceptDocsBits != null) { float unfilteredRatioVisited = (float) expectedDocs / numVectors; int filteredVectors = (int) Math.ceil(numVectors * percentFiltered); float expectedScored = Math.min(2 * filteredVectors * unfilteredRatioVisited, expectedDocs / 2f); @@ -284,7 +282,7 @@ public final void search(String field, float[] target, KnnCollector knnCollector } @Override - public final void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public final void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field); final ByteVectorValues values = rawVectorsReader.getByteVectorValues(field); for (int i = 0; i < values.size(); i++) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java index 9d3b678c56d47..9dde74bad43ed 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsReader.java @@ -32,6 +32,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.search.VectorScorer; import org.apache.lucene.store.ChecksumIndexInput; @@ -226,17 +227,17 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { rawVectorsReader.search(field, target, knnCollector, acceptDocs); } @Override - public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { if (knnCollector.k() == 0) return; final RandomVectorScorer scorer = getRandomVectorScorer(field, target); if (scorer == null) return; OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); - Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs.bits()); for (int i = 0; i < scorer.maxOrd(); i++) { if (acceptedOrds == null || acceptedOrds.get(i)) { collector.collect(i, scorer.score(i)); diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java index 1feee9f642b32..1082faaca4256 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/ES818BinaryQuantizedVectorsReader.java @@ -32,6 +32,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.search.VectorScorer; import org.apache.lucene.store.ChecksumIndexInput; @@ -240,17 +241,17 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { rawVectorsReader.search(field, target, knnCollector, acceptDocs); } @Override - public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { if (knnCollector.k() == 0) return; final RandomVectorScorer scorer = getRandomVectorScorer(field, target); if (scorer == null) return; OrdinalTranslatedKnnCollector collector = new OrdinalTranslatedKnnCollector(knnCollector, scorer::ordToDoc); - Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs); + Bits acceptedOrds = scorer.getAcceptOrds(acceptDocs.bits()); for (int i = 0; i < scorer.maxOrd(); i++) { if (acceptedOrds == null || acceptedOrds.get(i)) { collector.collect(i, scorer.score(i)); diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java index e26784ecfdd96..0492d54ba57f3 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/es818/MergeReaderWrapper.java @@ -13,9 +13,9 @@ import org.apache.lucene.index.ByteVectorValues; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.util.Accountable; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.hnsw.RandomVectorScorer; import org.elasticsearch.core.IOUtils; @@ -60,12 +60,12 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void search(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { mainReader.search(field, target, knnCollector, acceptDocs); } @Override - public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void search(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { mainReader.search(field, target, knnCollector, acceptDocs); } diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index 598fb076ba222..24b0512d27598 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -40,6 +40,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.store.ByteBuffersDirectory; @@ -447,12 +448,12 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void searchNearestVectors(String field, float[] target, KnnCollector collector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, float[] target, KnnCollector collector, AcceptDocs acceptDocs) throws IOException { getDelegate().searchNearestVectors(field, target, collector, acceptDocs); } @Override - public void searchNearestVectors(String field, byte[] target, KnnCollector collector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, byte[] target, KnnCollector collector, AcceptDocs acceptDocs) throws IOException { getDelegate().searchNearestVectors(field, target, collector, acceptDocs); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java index d37f6c51d288d..5a540785bd9fc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentLeafReader.java @@ -34,6 +34,7 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.index.memory.MemoryIndex; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.util.Bits; @@ -210,7 +211,7 @@ public FloatVectorValues getFloatVectorValues(String field) throws IOException { } @Override - public void searchNearestVectors(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) { + public void searchNearestVectors(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) { throw new UnsupportedOperationException(); } @@ -255,7 +256,7 @@ public ByteVectorValues getByteVectorValues(String field) { } @Override - public void searchNearestVectors(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) { + public void searchNearestVectors(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java index d2adb248ad019..7ce82290e206d 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ExitableDirectoryReader.java @@ -23,11 +23,11 @@ import org.apache.lucene.index.QueryTimeout; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.search.VectorScorer; import org.apache.lucene.search.suggest.document.CompletionTerms; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; @@ -141,7 +141,7 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void searchNearestVectors(String field, byte[] target, KnnCollector collector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, byte[] target, KnnCollector collector, AcceptDocs acceptDocs) throws IOException { if (queryCancellation.isEnabled() == false) { in.searchNearestVectors(field, target, collector, acceptDocs); return; @@ -159,7 +159,7 @@ public FloatVectorValues getFloatVectorValues(String field) throws IOException { } @Override - public void searchNearestVectors(String field, float[] target, KnnCollector collector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, float[] target, KnnCollector collector, AcceptDocs acceptDocs) throws IOException { if (queryCancellation.isEnabled() == false) { in.searchNearestVectors(field, target, collector, acceptDocs); return; diff --git a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java index f03be3f09b7d2..8143e6ef28053 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java @@ -30,9 +30,9 @@ import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.search.suggest.document.CompletionTerms; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; @@ -221,7 +221,7 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void searchNearestVectors(String field, byte[] target, KnnCollector collector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, byte[] target, KnnCollector collector, AcceptDocs acceptDocs) throws IOException { super.searchNearestVectors(field, target, collector, acceptDocs); if (collector.visitedCount() > 0) { notifier.onKnnVectorsUsed(field); @@ -229,7 +229,7 @@ public void searchNearestVectors(String field, byte[] target, KnnCollector colle } @Override - public void searchNearestVectors(String field, float[] target, KnnCollector collector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, float[] target, KnnCollector collector, AcceptDocs acceptDocs) throws IOException { super.searchNearestVectors(field, target, collector, acceptDocs); if (collector.visitedCount() > 0) { notifier.onKnnVectorsUsed(field); diff --git a/server/src/main/java/org/elasticsearch/search/vectors/AbstractIVFKnnVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/AbstractIVFKnnVectorQuery.java index 50d94541fe666..e0b9be18d3980 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/AbstractIVFKnnVectorQuery.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/AbstractIVFKnnVectorQuery.java @@ -15,11 +15,10 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldExistsQuery; -import org.apache.lucene.search.FilteredDocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.search.MatchNoDocsQuery; @@ -35,9 +34,6 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.search.knn.KnnCollectorManager; import org.apache.lucene.search.knn.KnnSearchStrategy; -import org.apache.lucene.util.BitSet; -import org.apache.lucene.util.BitSetIterator; -import org.apache.lucene.util.Bits; import org.elasticsearch.search.profile.query.QueryProfiler; import java.io.IOException; @@ -185,10 +181,10 @@ private TopDocs searchLeaf(LeafReaderContext ctx, Weight filterWeight, KnnCollec TopDocs getLeafResults(LeafReaderContext ctx, Weight filterWeight, KnnCollectorManager knnCollectorManager, float visitRatio) throws IOException { final LeafReader reader = ctx.reader(); - final Bits liveDocs = reader.getLiveDocs(); if (filterWeight == null) { - return approximateSearch(ctx, liveDocs, Integer.MAX_VALUE, knnCollectorManager, visitRatio); + AcceptDocs acceptDocs = AcceptDocs.fromLiveDocs(reader.getLiveDocs(), reader.maxDoc()); + return approximateSearch(ctx, acceptDocs, Integer.MAX_VALUE, knnCollectorManager, visitRatio); } Scorer scorer = filterWeight.scorer(ctx); @@ -196,14 +192,14 @@ TopDocs getLeafResults(LeafReaderContext ctx, Weight filterWeight, KnnCollectorM return TopDocsCollector.EMPTY_TOPDOCS; } - BitSet acceptDocs = createBitSet(scorer.iterator(), liveDocs, reader.maxDoc()); - final int cost = acceptDocs.cardinality(); + AcceptDocs acceptDocs = AcceptDocs.fromIteratorSupplier(scorer::iterator, reader.getLiveDocs(), reader.maxDoc()); + final int cost = acceptDocs.cost(); return approximateSearch(ctx, acceptDocs, cost + 1, knnCollectorManager, visitRatio); } abstract TopDocs approximateSearch( LeafReaderContext context, - Bits acceptDocs, + AcceptDocs acceptDocs, int visitedLimit, KnnCollectorManager knnCollectorManager, float visitRatio @@ -218,22 +214,6 @@ public final void profile(QueryProfiler queryProfiler) { queryProfiler.addVectorOpsCount(vectorOpsCount); } - BitSet createBitSet(DocIdSetIterator iterator, Bits liveDocs, int maxDoc) throws IOException { - if (liveDocs == null && iterator instanceof BitSetIterator bitSetIterator) { - // If we already have a BitSet and no deletions, reuse the BitSet - return bitSetIterator.getBitSet(); - } else { - // Create a new BitSet from matching and live docs - FilteredDocIdSetIterator filterIterator = new FilteredDocIdSetIterator(iterator) { - @Override - protected boolean match(int doc) { - return liveDocs == null || liveDocs.get(doc); - } - }; - return BitSet.of(filterIterator, maxDoc); - } - } - static class IVFCollectorManager implements KnnCollectorManager { private final int k; diff --git a/server/src/main/java/org/elasticsearch/search/vectors/IVFKnnFloatVectorQuery.java b/server/src/main/java/org/elasticsearch/search/vectors/IVFKnnFloatVectorQuery.java index 30b37b11005b3..a711bb9b3c5ee 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/IVFKnnFloatVectorQuery.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/IVFKnnFloatVectorQuery.java @@ -11,12 +11,12 @@ import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.knn.KnnCollectorManager; import org.apache.lucene.search.knn.KnnSearchStrategy; -import org.apache.lucene.util.Bits; import java.io.IOException; import java.util.Arrays; @@ -76,7 +76,7 @@ public int hashCode() { @Override protected TopDocs approximateSearch( LeafReaderContext context, - Bits acceptDocs, + AcceptDocs acceptDocs, int visitedLimit, KnnCollectorManager knnCollectorManager, float visitRatio diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java index d9450b1332f7a..fdbf4679e6ab5 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormatTests.java @@ -24,6 +24,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MMapDirectory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; @@ -170,7 +171,13 @@ private void testSingleVectorPerSegment(VectorSimilarityFunction sim) throws Exc LeafReader leafReader = getOnlyLeafReader(reader); StoredFields storedFields = reader.storedFields(); float[] queryVector = new float[] { 0.6f, 0.8f }; - var hits = leafReader.searchNearestVectors("field", queryVector, 3, null, 100); + var hits = leafReader.searchNearestVectors( + "field", + queryVector, + 3, + AcceptDocs.fromLiveDocs(leafReader.getLiveDocs(), leafReader.maxDoc()), + 100 + ); assertEquals(hits.scoreDocs.length, 3); assertEquals("B", storedFields.document(hits.scoreDocs[0].doc).get("id")); assertEquals("A", storedFields.document(hits.scoreDocs[1].doc).get("id")); diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/IVFVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/IVFVectorsFormatTests.java index 7fdd2a7313e48..a4224c3d3e79a 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/IVFVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/IVFVectorsFormatTests.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; @@ -191,7 +192,13 @@ public void testFewVectorManyTimes() throws IOException { for (LeafReaderContext r : subReaders) { LeafReader leafReader = r.reader(); float[] vector = randomVector(dimensions); - TopDocs topDocs = leafReader.searchNearestVectors("f", vector, 10, leafReader.getLiveDocs(), Integer.MAX_VALUE); + TopDocs topDocs = leafReader.searchNearestVectors( + "f", + vector, + 10, + AcceptDocs.fromLiveDocs(leafReader.getLiveDocs(), leafReader.maxDoc()), + Integer.MAX_VALUE + ); assertEquals(Math.min(leafReader.maxDoc(), 10), topDocs.scoreDocs.length); } @@ -219,7 +226,13 @@ public void testOneRepeatedVector() throws IOException { for (LeafReaderContext r : subReaders) { LeafReader leafReader = r.reader(); float[] vector = randomVector(dimensions); - TopDocs topDocs = leafReader.searchNearestVectors("f", vector, 10, leafReader.getLiveDocs(), Integer.MAX_VALUE); + TopDocs topDocs = leafReader.searchNearestVectors( + "f", + vector, + 10, + AcceptDocs.fromLiveDocs(leafReader.getLiveDocs(), leafReader.maxDoc()), + Integer.MAX_VALUE + ); assertEquals(Math.min(leafReader.maxDoc(), 10), topDocs.scoreDocs.length); } @@ -250,7 +263,13 @@ public void testWithThreads() throws Exception { for (; totSearch < numSearches && failed.get() == false; totSearch++) { float[] vector = randomVector(dimensions); LeafReader leafReader = getOnlyLeafReader(reader); - leafReader.searchNearestVectors("f", vector, 10, leafReader.getLiveDocs(), Integer.MAX_VALUE); + leafReader.searchNearestVectors( + "f", + vector, + 10, + AcceptDocs.fromLiveDocs(leafReader.getLiveDocs(), leafReader.maxDoc()), + Integer.MAX_VALUE + ); } assertTrue(totSearch > 0); } catch (Exception exc) { diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java index c6573398d2db6..c10fa9428bc13 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816HnswBinaryQuantizedVectorsFormatTests.java @@ -35,6 +35,7 @@ import org.apache.lucene.index.KnnVectorValues; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; @@ -99,7 +100,13 @@ public void testSingleVectorCase() throws Exception { while (docIndexIterator.nextDoc() != NO_MORE_DOCS) { assertArrayEquals(vector, vectorValues.vectorValue(docIndexIterator.index()), 0.00001f); } - TopDocs td = r.searchNearestVectors("f", randomVector(vector.length), 1, null, Integer.MAX_VALUE); + TopDocs td = r.searchNearestVectors( + "f", + randomVector(vector.length), + 1, + AcceptDocs.fromLiveDocs(r.getLiveDocs(), r.maxDoc()), + Integer.MAX_VALUE + ); assertEquals(1, td.totalHits.value()); assertTrue(td.scoreDocs[0].score >= 0); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java index 35bac97013487..bd1ef28351981 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es818/ES818HnswBinaryQuantizedVectorsFormatTests.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.misc.store.DirectIODirectory; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; @@ -118,7 +119,13 @@ public void testSingleVectorCase() throws Exception { } float[] randomVector = randomVector(vector.length); float trueScore = similarityFunction.compare(vector, randomVector); - TopDocs td = r.searchNearestVectors("f", randomVector, 1, null, Integer.MAX_VALUE); + TopDocs td = r.searchNearestVectors( + "f", + randomVector, + 1, + AcceptDocs.fromLiveDocs(r.getLiveDocs(), r.maxDoc()), + Integer.MAX_VALUE + ); assertEquals(1, td.totalHits.value()); assertTrue(td.scoreDocs[0].score >= 0); // When it's the only vector in a segment, the score should be very close to the true score diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index aa2e76f512cc8..329f8806b552c 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.PointValues; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TotalHitCountCollectorManager; @@ -201,7 +202,13 @@ public void testExitableDirectoryReaderVectors() throws IOException { expectThrows(TaskCancelledException.class, () -> leaf.getFloatVectorValues(KNN_FIELD_NAME)); expectThrows( TaskCancelledException.class, - () -> leaf.searchNearestVectors(KNN_FIELD_NAME, new float[] { 1f, 1f, 1f }, 2, leaf.getLiveDocs(), Integer.MAX_VALUE) + () -> leaf.searchNearestVectors( + KNN_FIELD_NAME, + new float[] { 1f, 1f, 1f }, + 2, + AcceptDocs.fromLiveDocs(leaf.getLiveDocs(), leaf.maxDoc()), + Integer.MAX_VALUE + ) ); cancelled.set(false); // Avoid exception during construction of the wrapper objects diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java index 06600441b0a44..505e4c09aba1a 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java @@ -35,6 +35,7 @@ import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.KnnCollector; @@ -490,12 +491,13 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void searchNearestVectors(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) + throws IOException { } @Override - public void searchNearestVectors(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java index 12864dd66a857..65dff572f9037 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/RewriteCachingDirectoryReader.java @@ -26,6 +26,7 @@ import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; @@ -229,12 +230,12 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void searchNearestVectors(String field, float[] target, KnnCollector collector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, float[] target, KnnCollector collector, AcceptDocs acceptDocs) throws IOException { throw new UnsupportedOperationException(); } @Override - public void searchNearestVectors(String field, byte[] target, KnnCollector collector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, byte[] target, KnnCollector collector, AcceptDocs acceptDocs) throws IOException { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java index adf12490a7d90..d96dce01e25fe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReader.java @@ -28,8 +28,8 @@ import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FilterIterator; import org.apache.lucene.util.automaton.CharacterRunAutomaton; @@ -299,7 +299,7 @@ public FloatVectorValues getFloatVectorValues(String field) throws IOException { } @Override - public void searchNearestVectors(String field, float[] target, KnnCollector collector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, float[] target, KnnCollector collector, AcceptDocs acceptDocs) throws IOException { if (hasField(field)) { super.searchNearestVectors(field, target, collector, acceptDocs); } @@ -311,7 +311,7 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void searchNearestVectors(String field, byte[] target, KnnCollector collector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, byte[] target, KnnCollector collector, AcceptDocs acceptDocs) throws IOException { if (hasField(field)) { super.searchNearestVectors(field, target, collector, acceptDocs); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java index 0a072a685fd1a..b5918f0b388bd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/FieldSubsetReaderTests.java @@ -46,6 +46,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockAnalyzer; @@ -211,13 +212,25 @@ public void testKnnVectors() throws Exception { assertEquals(0, iterator.nextDoc()); assertNotNull(vectorValues.vectorValue(iterator.index())); - TopDocs topDocs = leafReader.searchNearestVectors("fieldA", new float[] { 1.0f, 1.0f, 1.0f }, 5, null, Integer.MAX_VALUE); + TopDocs topDocs = leafReader.searchNearestVectors( + "fieldA", + new float[] { 1.0f, 1.0f, 1.0f }, + 5, + AcceptDocs.fromLiveDocs(leafReader.getLiveDocs(), leafReader.maxDoc()), + Integer.MAX_VALUE + ); assertNotNull(topDocs); assertEquals(1, topDocs.scoreDocs.length); // Check that we can't see fieldB assertNull(leafReader.getFloatVectorValues("fieldB")); - topDocs = leafReader.searchNearestVectors("fieldB", new float[] { 1.0f, 1.0f, 1.0f }, 5, null, Integer.MAX_VALUE); + topDocs = leafReader.searchNearestVectors( + "fieldB", + new float[] { 1.0f, 1.0f, 1.0f }, + 5, + AcceptDocs.fromLiveDocs(leafReader.getLiveDocs(), leafReader.maxDoc()), + Integer.MAX_VALUE + ); assertEquals(0, topDocs.totalHits.value()); assertEquals(0, topDocs.scoreDocs.length); @@ -246,13 +259,25 @@ public void testKnnByteVectors() throws Exception { assertEquals(0, iterator.nextDoc()); assertNotNull(vectorValues.vectorValue(iterator.index())); - TopDocs topDocs = leafReader.searchNearestVectors("fieldA", new byte[] { 1, 1, 1 }, 5, null, Integer.MAX_VALUE); + TopDocs topDocs = leafReader.searchNearestVectors( + "fieldA", + new byte[] { 1, 1, 1 }, + 5, + AcceptDocs.fromLiveDocs(leafReader.getLiveDocs(), leafReader.maxDoc()), + Integer.MAX_VALUE + ); assertNotNull(topDocs); assertEquals(1, topDocs.scoreDocs.length); // Check that we can't see fieldB assertNull(leafReader.getByteVectorValues("fieldB")); - topDocs = leafReader.searchNearestVectors("fieldB", new byte[] { 1, 1, 1 }, 5, null, Integer.MAX_VALUE); + topDocs = leafReader.searchNearestVectors( + "fieldB", + new byte[] { 1, 1, 1 }, + 5, + AcceptDocs.fromLiveDocs(leafReader.getLiveDocs(), leafReader.maxDoc()), + Integer.MAX_VALUE + ); assertEquals(0, topDocs.totalHits.value()); assertEquals(0, topDocs.scoreDocs.length); From 157574c8173bda9e1ba5c74c21e7bbb71fb10a66 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 25 Aug 2025 18:58:49 -0400 Subject: [PATCH 2/3] Fix file usage in ivf --- .../elasticsearch/index/codec/vectors/IVFVectorsWriter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsWriter.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsWriter.java index 82f369662592f..661aefb3491fb 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsWriter.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsWriter.java @@ -24,11 +24,11 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.store.DataAccessHint; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RandomAccessInput; -import org.apache.lucene.store.ReadAdvice; import org.apache.lucene.util.LongValues; import org.apache.lucene.util.VectorUtil; import org.elasticsearch.core.IOUtils; @@ -302,11 +302,11 @@ private void mergeOneFieldIVF(FieldInfo fieldInfo, MergeState mergeState) throws try ( IndexInput vectors = mergeState.segmentInfo.dir.openInput( tempRawVectorsFileName, - IOContext.DEFAULT.withReadAdvice(ReadAdvice.SEQUENTIAL) + IOContext.DEFAULT.withHints(DataAccessHint.SEQUENTIAL) ); IndexInput docs = docsFileName == null ? null - : mergeState.segmentInfo.dir.openInput(docsFileName, IOContext.DEFAULT.withReadAdvice(ReadAdvice.SEQUENTIAL)) + : mergeState.segmentInfo.dir.openInput(docsFileName, IOContext.DEFAULT.withHints(DataAccessHint.SEQUENTIAL)) ) { final FloatVectorValues floatVectorValues = getFloatVectorValues(fieldInfo, docs, vectors, numVectors); From 33f70cdaace858f9400babbfcb7743d9083ce6bf Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 25 Aug 2025 19:17:42 -0400 Subject: [PATCH 3/3] fixing compilation --- .../elasticsearch/compute/lucene/LuceneSliceQueueTests.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSliceQueueTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSliceQueueTests.java index 6054401fb58d5..79f1fcc86e2c3 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSliceQueueTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneSliceQueueTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.TermVectors; import org.apache.lucene.index.Terms; +import org.apache.lucene.search.AcceptDocs; import org.apache.lucene.search.KnnCollector; import org.apache.lucene.util.Bits; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; @@ -303,12 +304,13 @@ public ByteVectorValues getByteVectorValues(String field) throws IOException { } @Override - public void searchNearestVectors(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, float[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) + throws IOException { throw new UnsupportedOperationException(); } @Override - public void searchNearestVectors(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { + public void searchNearestVectors(String field, byte[] target, KnnCollector knnCollector, AcceptDocs acceptDocs) throws IOException { throw new UnsupportedOperationException(); }