Merge DocAndFreqBuffer and DocAndScoreBuffer. (#14748)

jpountz · romseygeek · jpountz · commit 42c79561ac6c · 2025-06-03T07:20:02.000+02:00
I initially kept those separated but I'm now coming to the conclusion that they
should be merged:
 - While frequencies are ints today, some queries produce float frequencies
   (`SloppyPhraseQuery`, `CombinedFieldQuery`).
 - `TermScorer` no longer needs to keep track of a separate buffer for
   frequencies.

Co-authored-by: Alan Woodward &lt;romseygeek@gmail.com&gt;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java
@@ -46,7 +46,7 @@
 import org.apache.lucene.index.SegmentReadState;
 import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
 import org.apache.lucene.internal.vectorization.VectorizationProvider;
-import org.apache.lucene.search.DocAndFreqBuffer;
+import org.apache.lucene.search.DocAndFloatFeatureBuffer;
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.ChecksumIndexInput;
 import org.apache.lucene.store.DataInput;
@@ -1043,7 +1043,7 @@ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOExcept
     }
 
     @Override
-    public void nextPostings(int upTo, DocAndFreqBuffer buffer) throws IOException {
+    public void nextPostings(int upTo, DocAndFloatFeatureBuffer buffer) throws IOException {
       assert needsRefilling == false;
 
       if (needsFreq == false) {
@@ -1080,7 +1080,9 @@ public void nextPostings(int upTo, DocAndFreqBuffer buffer) throws IOException {
       }
 
       assert buffer.size > 0;
-      System.arraycopy(freqBuffer, start, buffer.freqs, 0, buffer.size);
+      for (int i = 0; i < buffer.size; ++i) {
+        buffer.features[i] = freqBuffer[start + i];
+      }
 
       advance(upTo);
     }
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -58,7 +58,7 @@
 import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
 import org.apache.lucene.index.PointValues.IntersectVisitor;
 import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.search.DocAndFreqBuffer;
+import org.apache.lucene.search.DocAndFloatFeatureBuffer;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.KnnCollector;
@@ -1614,7 +1614,7 @@ private static Status.TermIndexStatus checkFields(
         postings = termsEnum.postings(postings, PostingsEnum.ALL);
         bulkPostings = termsEnum.postings(bulkPostings, PostingsEnum.ALL);
         bulkPostings.nextDoc();
-        DocAndFreqBuffer buffer = new DocAndFreqBuffer();
+        DocAndFloatFeatureBuffer buffer = new DocAndFloatFeatureBuffer();
         int bufferIndex = 0;
 
         if (hasFreqs == false) {
@@ -1680,10 +1680,10 @@ private static Status.TermIndexStatus checkFields(
                     + " as next doc while PostingsEnum#nextDoc returns "
                     + doc);
           }
-          if (freq != buffer.freqs[bufferIndex]) {
+          if (freq != buffer.features[bufferIndex]) {
             throw new CheckIndexException(
                 "PostingsEnum#nextPostings returns "
-                    + buffer.freqs[bufferIndex]
+                    + buffer.features[bufferIndex]
                     + " as term freq while PostingsEnum#freq returns "
                     + freq);
           }
diff --git a/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java b/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java
@@ -17,7 +17,7 @@
 package org.apache.lucene.index;
 
 import java.io.IOException;
-import org.apache.lucene.search.DocAndFreqBuffer;
+import org.apache.lucene.search.DocAndFloatFeatureBuffer;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.BytesRef;
 
@@ -126,18 +126,18 @@ protected PostingsEnum() {}
    * buffer.size = size;
    * </pre>
    *
-   * <p><b>NOTE</b>: The provided {@link DocAndFreqBuffer} should not hold references to internal
-   * data structures.
+   * <p><b>NOTE</b>: The provided {@link DocAndFloatFeatureBuffer} should not hold references to
+   * internal data structures.
    *
    * @lucene.internal
    */
-  public void nextPostings(int upTo, DocAndFreqBuffer buffer) throws IOException {
+  public void nextPostings(int upTo, DocAndFloatFeatureBuffer buffer) throws IOException {
     int batchSize = 16; // arbitrary
     buffer.growNoCopy(batchSize);
     int size = 0;
     for (int doc = docID(); doc < upTo && size < batchSize; doc = nextDoc()) {
       buffer.docs[size] = doc;
-      buffer.freqs[size] = freq();
+      buffer.features[size] = freq();
       ++size;
     }
     buffer.size = size;
diff --git a/lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.java
@@ -26,7 +26,7 @@
 class BatchScoreBulkScorer extends BulkScorer {
 
   private final SimpleScorable scorable = new SimpleScorable();
-  private final DocAndScoreBuffer buffer = new DocAndScoreBuffer();
+  private final DocAndFloatFeatureBuffer buffer = new DocAndFloatFeatureBuffer();
   private final Scorer scorer;
 
   BatchScoreBulkScorer(Scorer scorer) {
@@ -50,7 +50,7 @@ public int score(LeafCollector collector, Bits acceptDocs, int min, int max) thr
         buffer.size > 0;
         scorer.nextDocsAndScores(max, acceptDocs, buffer)) {
       for (int i = 0, size = buffer.size; i < size; i++) {
-        float score = scorable.score = buffer.scores[i];
+        float score = scorable.score = buffer.features[i];
         if (score >= scorable.minCompetitiveScore) {
           collector.collect(buffer.docs[i]);
         }
diff --git a/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/BlockMaxConjunctionBulkScorer.java
@@ -41,7 +41,7 @@ final class BlockMaxConjunctionBulkScorer extends BulkScorer {
   private final DocAndScore scorable = new DocAndScore();
   private final double[] sumOfOtherClauses;
   private final int maxDoc;
-  private final DocAndScoreBuffer docAndScoreBuffer = new DocAndScoreBuffer();
+  private final DocAndFloatFeatureBuffer docAndScoreBuffer = new DocAndFloatFeatureBuffer();
   private final DocAndScoreAccBuffer docAndScoreAccBuffer = new DocAndScoreAccBuffer();
 
   BlockMaxConjunctionBulkScorer(int maxDoc, List<Scorer> scorers) throws IOException {
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
@@ -81,7 +81,7 @@ public DisiWrapper get(int i) {
   final int minShouldMatch;
   final long cost;
   final boolean needsScores;
-  private final DocAndScoreBuffer docAndScoreBuffer = new DocAndScoreBuffer();
+  private final DocAndFloatFeatureBuffer docAndScoreBuffer = new DocAndFloatFeatureBuffer();
 
   BooleanScorer(Collection<Scorer> scorers, int minShouldMatch, boolean needsScores) {
     if (minShouldMatch < 1 || minShouldMatch > scorers.size()) {
@@ -148,7 +148,7 @@ private void scoreWindowIntoBitSetAndReplay(
             w.scorer.nextDocsAndScores(max, acceptDocs, docAndScoreBuffer)) {
           for (int index = 0; index < docAndScoreBuffer.size; ++index) {
             final int doc = docAndScoreBuffer.docs[index];
-            final float score = docAndScoreBuffer.scores[index];
+            final float score = docAndScoreBuffer.features[index];
             final int d = doc & MASK;
             matching.set(d);
             final Bucket bucket = buckets[d];
diff --git a/lucene/core/src/java/org/apache/lucene/search/DocAndFloatFeatureBuffer.java b/lucene/core/src/java/org/apache/lucene/search/DocAndFloatFeatureBuffer.java
@@ -21,29 +21,32 @@
 import org.apache.lucene.util.IntsRef;
 
 /**
- * Wrapper around parallel arrays storing doc IDs and their corresponding frequencies.
+ * Wrapper around parallel arrays storing doc IDs and their corresponding features, stored as Java
+ * floats. These features may be anything, but are typically a term frequency or a score.
  *
  * @lucene.internal
  */
-public final class DocAndFreqBuffer {
+public final class DocAndFloatFeatureBuffer {
+
+  private static final float[] EMPTY_FLOATS = new float[0];
 
   /** Doc IDs */
   public int[] docs = IntsRef.EMPTY_INTS;
 
-  /** Frequencies */
-  public int[] freqs = IntsRef.EMPTY_INTS;
+  /** Float-valued features */
+  public float[] features = EMPTY_FLOATS;
 
-  /** Number of valid entries in the doc ID and frequency arrays. */
+  /** Number of valid entries in the doc ID and float-valued feature arrays. */
   public int size;
 
   /** Sole constructor. */
-  public DocAndFreqBuffer() {}
+  public DocAndFloatFeatureBuffer() {}
 
   /** Grow both arrays to ensure that they can store at least the given number of entries. */
   public void growNoCopy(int minSize) {
     if (docs.length < minSize) {
       docs = ArrayUtil.growNoCopy(docs, minSize);
-      freqs = new int[docs.length];
+      features = new float[docs.length];
     }
   }
 
@@ -53,7 +56,7 @@ public void apply(Bits liveDocs) {
     for (int i = 0; i < size; ++i) {
       if (liveDocs.get(docs[i])) {
         docs[newSize] = docs[i];
-        freqs[newSize] = freqs[i];
+        features[newSize] = features[i];
         newSize++;
       }
     }
diff --git a/lucene/core/src/java/org/apache/lucene/search/DocAndScoreAccBuffer.java b/lucene/core/src/java/org/apache/lucene/search/DocAndScoreAccBuffer.java
@@ -62,12 +62,15 @@ public void grow(int minSize) {
     }
   }
 
-  /** Copy content from the given {@link DocAndScoreBuffer}, expanding float scores to doubles. */
-  public void copyFrom(DocAndScoreBuffer buffer) {
+  /**
+   * Copy content from the given {@link DocAndFloatFeatureBuffer}, expanding float scores to
+   * doubles.
+   */
+  public void copyFrom(DocAndFloatFeatureBuffer buffer) {
     growNoCopy(buffer.size);
     System.arraycopy(buffer.docs, 0, docs, 0, buffer.size);
     for (int i = 0; i < buffer.size; ++i) {
-      scores[i] = buffer.scores[i];
+      scores[i] = buffer.features[i];
     }
     this.size = buffer.size;
   }
diff --git a/lucene/core/src/java/org/apache/lucene/search/DocAndScoreBuffer.java b/lucene/core/src/java/org/apache/lucene/search/DocAndScoreBuffer.java
diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
@@ -51,7 +51,7 @@ final class MaxScoreBulkScorer extends BulkScorer {
   private final FixedBitSet windowMatches = new FixedBitSet(INNER_WINDOW_SIZE);
   private final double[] windowScores = new double[INNER_WINDOW_SIZE];
 
-  private final DocAndScoreBuffer docAndScoreBuffer = new DocAndScoreBuffer();
+  private final DocAndFloatFeatureBuffer docAndScoreBuffer = new DocAndFloatFeatureBuffer();
   private final DocAndScoreAccBuffer docAndScoreAccBuffer = new DocAndScoreAccBuffer();
 
   MaxScoreBulkScorer(int maxDoc, List<Scorer> scorers, Scorer filter) throws IOException {
@@ -287,7 +287,7 @@ private void scoreInnerWindowMultipleEssentialClauses(
           top.scorer.nextDocsAndScores(innerWindowMax, acceptDocs, docAndScoreBuffer)) {
         for (int index = 0; index < docAndScoreBuffer.size; ++index) {
           final int doc = docAndScoreBuffer.docs[index];
-          final float score = docAndScoreBuffer.scores[index];
+          final float score = docAndScoreBuffer.features[index];
           final int i = doc - innerWindowMin;
           windowMatches.set(i);
           windowScores[i] += score;
diff --git a/lucene/core/src/java/org/apache/lucene/search/Scorer.java b/lucene/core/src/java/org/apache/lucene/search/Scorer.java
@@ -107,15 +107,15 @@ public int advanceShallow(int target) throws IOException {
    * buffer.size = size;
    * </pre>
    *
-   * <p><b>NOTE</b>: The provided {@link DocAndScoreBuffer} should not hold references to internal
-   * data structures.
+   * <p><b>NOTE</b>: The provided {@link DocAndFloatFeatureBuffer} should not hold references to
+   * internal data structures.
    *
    * <p><b>NOTE</b>: In case this {@link Scorer} exposes a {@link #twoPhaseIterator()
    * TwoPhaseIterator}, it should be positioned on a matching document before this method is called.
    *
    * @lucene.internal
    */
-  public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndScoreBuffer buffer)
+  public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndFloatFeatureBuffer buffer)
       throws IOException {
     int batchSize = 16; // arbitrary
     buffer.growNoCopy(batchSize);
@@ -124,7 +124,7 @@ public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndScoreBuffer buffer)
     for (int doc = docID(); doc < upTo && size < batchSize; doc = iterator.nextDoc()) {
       if (liveDocs == null || liveDocs.get(doc)) {
         buffer.docs[size] = doc;
-        buffer.scores[size] = score();
+        buffer.features[size] = score();
         ++size;
       }
     }
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
@@ -39,7 +39,6 @@ public final class TermScorer extends Scorer {
   private final NumericDocValues norms;
   private final ImpactsDISI impactsDisi;
   private final MaxScoreCache maxScoreCache;
-  private DocAndFreqBuffer docAndFreqBuffer;
   private long[] normValues = LongsRef.EMPTY_LONGS;
 
   /** Construct a {@link TermScorer} that will iterate all documents. */
@@ -128,32 +127,28 @@ public void setMinCompetitiveScore(float minScore) {
   }
 
   @Override
-  public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndScoreBuffer buffer)
+  public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndFloatFeatureBuffer buffer)
       throws IOException {
-    if (docAndFreqBuffer == null) {
-      docAndFreqBuffer = new DocAndFreqBuffer();
-    }
-
     for (; ; ) {
       if (impactsDisi != null) {
         impactsDisi.ensureCompetitive();
       }
 
-      postingsEnum.nextPostings(upTo, docAndFreqBuffer);
-      if (liveDocs != null && docAndFreqBuffer.size != 0) {
+      postingsEnum.nextPostings(upTo, buffer);
+      if (liveDocs != null && buffer.size != 0) {
         // An empty return value indicates that there are no more docs before upTo. We may be
         // unlucky, and there are docs left, but all docs from the current batch happen to be marked
         // as deleted. So we need to iterate until we find a batch that has at least one non-deleted
         // doc.
-        docAndFreqBuffer.apply(liveDocs);
-        if (docAndFreqBuffer.size == 0) {
+        buffer.apply(liveDocs);
+        if (buffer.size == 0) {
           continue;
         }
       }
       break;
     }
 
-    int size = docAndFreqBuffer.size;
+    int size = buffer.size;
     if (normValues.length < size) {
       normValues = new long[ArrayUtil.oversize(size, Long.BYTES)];
       if (norms == null) {
@@ -162,21 +157,18 @@ public void nextDocsAndScores(int upTo, Bits liveDocs, DocAndScoreBuffer buffer)
     }
     if (norms != null) {
       for (int i = 0; i < size; ++i) {
-        if (norms.advanceExact(docAndFreqBuffer.docs[i])) {
+        if (norms.advanceExact(buffer.docs[i])) {
           normValues[i] = norms.longValue();
         } else {
           normValues[i] = 1L;
         }
       }
     }
 
-    buffer.growNoCopy(size);
-    buffer.size = size;
-    System.arraycopy(docAndFreqBuffer.docs, 0, buffer.docs, 0, size);
     for (int i = 0; i < size; ++i) {
       // Unless SimScorer#score is megamorphic, SimScorer#score should inline and (part of) score
       // computations should auto-vectorize.
-      buffer.scores[i] = scorer.score(docAndFreqBuffer.freqs[i], normValues[i]);
+      buffer.features[i] = scorer.score(buffer.features[i], normValues[i]);
     }
   }
 }
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java
@@ -281,7 +281,7 @@ public void testNextDocsAndScores() throws IOException {
     Scorer scorer2 = weight.scorer(context);
     scorer1.iterator().nextDoc();
     scorer2.iterator().nextDoc();
-    DocAndScoreBuffer buffer = new DocAndScoreBuffer();
+    DocAndFloatFeatureBuffer buffer = new DocAndFloatFeatureBuffer();
     while (true) {
       int curDoc = scorer2.iterator().docID();
       int upTo =
@@ -294,7 +294,7 @@ public void testNextDocsAndScores() throws IOException {
           scorer2.iterator().nextDoc();
         }
         assertEquals(scorer2.iterator().docID(), buffer.docs[i]);
-        assertEquals(scorer2.score(), buffer.scores[i], 0f);
+        assertEquals(scorer2.score(), buffer.features[i], 0f);
         scorer2.iterator().nextDoc();
       }
 
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/AssertingLeafReader.java
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/RandomPostingsTester.java
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/search/AssertingScorer.java b/lucene/test-framework/src/java/org/apache/lucene/tests/search/AssertingScorer.java

Original file line number	Diff line number	Diff line change
`@@ -62,12 +62,15 @@ public void grow(int minSize) {`
`62`	`62`	`}`
`63`	`63`	`}`
`64`	`64`
`65`		`- /** Copy content from the given {@link DocAndScoreBuffer}, expanding float scores to doubles. */`
`66`		`- public void copyFrom(DocAndScoreBuffer buffer) {`
	`65`	`+ /**`
	`66`	`+ * Copy content from the given {@link DocAndFloatFeatureBuffer}, expanding float scores to`
	`67`	`+ * doubles.`
	`68`	`+ */`
	`69`	`+ public void copyFrom(DocAndFloatFeatureBuffer buffer) {`
`67`	`70`	`growNoCopy(buffer.size);`
`68`	`71`	`System.arraycopy(buffer.docs, 0, docs, 0, buffer.size);`
`69`	`72`	`for (int i = 0; i < buffer.size; ++i) {`
`70`		`- scores[i] = buffer.scores[i];`
	`73`	`+ scores[i] = buffer.features[i];`
`71`	`74`	`}`
`72`	`75`	`this.size = buffer.size;`
`73`	`76`	`}`