Provide better impacts for fields indexed with IndexOptions.DOCS (#14511)

Postings always return impacts with freq=Integer.MAX_VALUE and norm=1 when frequencies are not indexed (IndexOptions.DOCS). This significantly overestimates the score upper bound of term queries, since the similarity scorer is effectively called with freq=1 all the time in this case (and either norm=1 if norms are not indexed, or the number of terms in the field otherwise).

This updates postings to always return impacts with freq=1 and norm=1 when frequencies are not indexed, which helps compute better score upper bounds, and in-turn makes dynamic pruning perform better.

Closes #14445

Author: expani

lucene/CHANGES.txt

@@ -114,6 +114,8 @@ Bug Fixes
 * GITHUB#14523, GITHUB#14530: Correct TermOrdValComparator competitive iterator so that it forces sparse
   field iteration to be at least scoring window baseline when doing intoBitSet. (Ben Trent, Adrien Grand)
 
+* GITHUB#14445: Provide better impacts for fields indexed with IndexOptions.DOCS GITHUB#14511 (Aniketh Jain)
+
 * GITHUB#14543: Fixed lead cost computations for bulk scorers of conjunctive
   queries that mix MUST and FILTER clauses, and disjunctive queries that
   configure a minimum number of matching SHOULD clauses.

lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsReader.java

@@ -70,6 +70,10 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
   private static final List<Impact> DUMMY_IMPACTS =
       Collections.singletonList(new Impact(Integer.MAX_VALUE, 1L));
 
+  // We stopped storing a placeholder impact with freq=1 for fields with DOCS after 9.12.0
+  private static final List<Impact> DUMMY_IMPACTS_NO_FREQS =
+      Collections.singletonList(new Impact(1, 1L));
+
   private final IndexInput docIn;
   private final IndexInput posIn;
   private final IndexInput payIn;
@@ -1325,13 +1329,14 @@ public int getDocIdUpTo(int level) {
 
           @Override
           public List<Impact> getImpacts(int level) {
-            if (indexHasFreq) {
-              if (level == 0 && level0LastDocID != NO_MORE_DOCS) {
-                return readImpacts(level0SerializedImpacts, level0Impacts);
-              }
-              if (level == 1) {
-                return readImpacts(level1SerializedImpacts, level1Impacts);
-              }
+            if (indexHasFreq == false) {
+              return DUMMY_IMPACTS_NO_FREQS;
+            }
+            if (level == 0 && level0LastDocID != NO_MORE_DOCS) {
+              return readImpacts(level0SerializedImpacts, level0Impacts);
+            }
+            if (level == 1) {
+              return readImpacts(level1SerializedImpacts, level1Impacts);
             }
             return DUMMY_IMPACTS;
           }

lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java

@@ -73,6 +73,10 @@ public final class Lucene103PostingsReader extends PostingsReaderBase {
   private static final List<Impact> DUMMY_IMPACTS =
       Collections.singletonList(new Impact(Integer.MAX_VALUE, 1L));
 
+  // We stopped storing a placeholder impact with freq=1 for fields with DOCS after 9.12.0
+  private static final List<Impact> DUMMY_IMPACTS_NO_FREQS =
+      Collections.singletonList(new Impact(1, 1L));
+
   private final IndexInput docIn;
   private final IndexInput posIn;
   private final IndexInput payIn;
@@ -1302,13 +1306,14 @@ public int getDocIdUpTo(int level) {
 
           @Override
           public List<Impact> getImpacts(int level) {
-            if (indexHasFreq) {
-              if (level == 0 && level0LastDocID != NO_MORE_DOCS) {
-                return readImpacts(level0SerializedImpacts, level0Impacts);
-              }
-              if (level == 1) {
-                return readImpacts(level1SerializedImpacts, level1Impacts);
-              }
+            if (indexHasFreq == false) {
+              return DUMMY_IMPACTS_NO_FREQS;
+            }
+            if (level == 0 && level0LastDocID != NO_MORE_DOCS) {
+              return readImpacts(level0SerializedImpacts, level0Impacts);
+            }
+            if (level == 1) {
+              return readImpacts(level1SerializedImpacts, level1Impacts);
             }
             return DUMMY_IMPACTS;
           }