From f031019377ba032ec7251c45ec1a169489a5feb9 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Tue, 14 Oct 2025 21:52:08 -0700 Subject: [PATCH 1/2] Handle deleted documents for filter rewrite subaggregation optimization Signed-off-by: Ankit Jain --- .../FilterRewriteOptimizationContext.java | 4 +++- .../rangecollector/SubAggRangeCollector.java | 15 +++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java index 639f3477b7868..15889538d60c0 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/FilterRewriteOptimizationContext.java @@ -115,7 +115,9 @@ public boolean tryOptimize( return false; } - if (leafCtx.reader().hasDeletions()) return false; + // Since we explicitly create bitset of matching docIds for each bucket + // in case of sub-aggregations, deleted documents can be filtered out + if (leafCtx.reader().hasDeletions() && hasSubAgg == false) return false; PointValues values = leafCtx.reader().getPointValues(aggregatorBridge.fieldType.name()); if (values == null) return false; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/rangecollector/SubAggRangeCollector.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/rangecollector/SubAggRangeCollector.java index 5c1f21b22e646..cd6b5a22fd0f7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/rangecollector/SubAggRangeCollector.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/rangecollector/SubAggRangeCollector.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BitDocIdSet; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; import org.opensearch.search.aggregations.BucketCollector; import org.opensearch.search.aggregations.LeafBucketCollector; @@ -37,6 +38,7 @@ public class SubAggRangeCollector extends SimpleRangeCollector { private final BucketCollector collectableSubAggregators; private final LeafReaderContext leafCtx; + private final Bits liveDocs; private final FixedBitSet bitSet; private final BitDocIdSet bitDocIdSet; @@ -53,6 +55,7 @@ public SubAggRangeCollector( this.getBucketOrd = getBucketOrd; this.collectableSubAggregators = subAggCollectorParam.collectableSubAggregators(); this.leafCtx = subAggCollectorParam.leafCtx(); + this.liveDocs = leafCtx.reader().getLiveDocs(); int numDocs = leafCtx.reader().maxDoc(); bitSet = new FixedBitSet(numDocs); bitDocIdSet = new BitDocIdSet(bitSet); @@ -65,12 +68,20 @@ public boolean hasSubAgg() { @Override public void collectDocId(int docId) { - bitSet.set(docId); + if (liveDocs.get(docId)) { + bitSet.set(docId); + } } @Override public void collectDocIdSet(DocIdSetIterator iter) throws IOException { - bitSet.or(iter); + // Explicitly OR iter intoBitSet to filter out deleted docs + iter.nextDoc(); + for (int doc = iter.docID(); doc < DocIdSetIterator.NO_MORE_DOCS; doc = iter.nextDoc()) { + if (liveDocs.get(doc)) { + bitSet.set(doc); + } + } } @Override From ec999eabb5ddd8d3eaa65eeac1d8f1c5eb64439b Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 15 Oct 2025 13:59:36 -0700 Subject: [PATCH 2/2] Addressing test failures due to NPE Signed-off-by: Ankit Jain --- .../rangecollector/SubAggRangeCollector.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/rangecollector/SubAggRangeCollector.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/rangecollector/SubAggRangeCollector.java index cd6b5a22fd0f7..bedff03f8bc79 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/rangecollector/SubAggRangeCollector.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/rangecollector/SubAggRangeCollector.java @@ -66,9 +66,13 @@ public boolean hasSubAgg() { return true; } + private boolean isDocLive(int docId) { + return liveDocs == null || liveDocs.get(docId); + } + @Override public void collectDocId(int docId) { - if (liveDocs.get(docId)) { + if (isDocLive(docId)) { bitSet.set(docId); } } @@ -78,7 +82,7 @@ public void collectDocIdSet(DocIdSetIterator iter) throws IOException { // Explicitly OR iter intoBitSet to filter out deleted docs iter.nextDoc(); for (int doc = iter.docID(); doc < DocIdSetIterator.NO_MORE_DOCS; doc = iter.nextDoc()) { - if (liveDocs.get(doc)) { + if (isDocLive(doc)) { bitSet.set(doc); } }