Properly normalize scores – improves multiple-project search

Author: ahornace

suggester/src/main/java/org/opengrok/suggest/Suggester.java

@@ -96,6 +96,7 @@ public final class Suggester implements Closeable {
      * @param projectsEnabled specifies if the OpenGrok projects are enabled
      * @param allowedFields fields for which should the suggester be enabled,
      * if {@code null} then enabled for all fields
+     * @param timeThreshold time in milliseconds after which the suggestions requests should time out
      */
     public Suggester(
             final File suggesterDir,
@@ -585,7 +586,7 @@ public boolean isPartialResult() {
     }
 
     /**
-     * Model classes for holding project name and path to ist index directory.
+     * Model classes for holding project name and path to its index directory.
      */
     public static class NamedIndexDir {
 

suggester/src/main/java/org/opengrok/suggest/SuggesterSearcher.java

@@ -68,12 +68,15 @@ class SuggesterSearcher extends IndexSearcher {
 
     private boolean interrupted;
 
+    private final int numDocs;
+
     /**
      * @param reader reader of the index for which to provide suggestions
      * @param resultSize size of the results
      */
     SuggesterSearcher(final IndexReader reader, final int resultSize) {
         super(reader);
+        numDocs = reader.numDocs();
         this.resultSize = resultSize;
     }
 
@@ -180,7 +183,7 @@ private List<LookupResultItem> suggest(
 
             int score;
             if (!needsDocumentIds) {
-                score = termsEnum.docFreq();
+                score = normalizeDocumentFrequency(termsEnum.docFreq(), numDocs);
             } else if (needPositionsAndFrequencies) {
                 score = getPhraseScore(complexQueryData, leafReaderContext.docBase, postingsEnum);
             } else {
@@ -314,7 +317,7 @@ private int getDocumentFrequency(final IntsHolder documentIds, final int docBase
                 weight++;
             }
         }
-        return weight;
+        return normalizeDocumentFrequency(weight, documentIds.numberOfElements());
     }
 
     private boolean needPositionsAndFrequencies(final Query query) {
@@ -333,6 +336,10 @@ private boolean needPositionsAndFrequencies(final Query query) {
         return false;
     }
 
+    private static int normalizeDocumentFrequency(final int count, final int documents) {
+        return (int) (((double) count / documents) * SuggesterUtils.NORMALIZED_DOCUMENT_FREQUENCY_MULTIPLIER);
+    }
+
     private static class ComplexQueryData {
 
         private IntsHolder documentIds;

suggester/src/main/java/org/opengrok/suggest/SuggesterUtils.java

@@ -48,12 +48,12 @@
  */
 public class SuggesterUtils {
 
+    public static final int NORMALIZED_DOCUMENT_FREQUENCY_MULTIPLIER = 1000;
+
     private static final Logger logger = Logger.getLogger(SuggesterUtils.class.getName());
 
     private static final long DEFAULT_TERM_WEIGHT = 0;
 
-    private static final int NORMALIZED_DOCUMENT_FREQUENCY_MULTIPLIER = 1000;
-
     private SuggesterUtils() {
     }
 

suggester/src/main/java/org/opengrok/suggest/query/data/BitIntsHolder.java

@@ -42,4 +42,10 @@ public boolean has(final int i) {
         return get(i);
     }
 
+    /** {@inheritDoc} */
+    @Override
+    public int numberOfElements() {
+        return cardinality();
+    }
+
 }

suggester/src/main/java/org/opengrok/suggest/query/data/HashIntsHolder.java

@@ -35,4 +35,10 @@ public boolean has(final int i) {
         return contains(i);
     }
 
+    /** {@inheritDoc} */
+    @Override
+    public int numberOfElements() {
+        return size();
+    }
+
 }

suggester/src/main/java/org/opengrok/suggest/query/data/IntsHolder.java

@@ -34,4 +34,10 @@ public interface IntsHolder {
      */
     boolean has(int i);
 
+    /**
+     * Returns number of elements.
+     * @return number of elements.
+     */
+    int numberOfElements();
+
 }