assert exact prefixes are respected

Author: lofcz

src/Infidex.Tests/MovieSearchParityTests.cs

@@ -658,6 +658,123 @@ public void Search_MixedTerms_LongAndShort_ReturnsCorrectResults()
         }
     }
 
+    [TestMethod]
+    public void Search_TwoF_PrefersStrictPrefixMatch()
+    {
+        var engine = GetEngine();
+        
+        // Query: "two f"
+        // Expected: "Two for ..." titles should rank higher than partial matches like "Flat Two"
+        // because "two" is an exact token match and "f" is a clean prefix of "for/feet/face"
+        var result = engine.Search(new Query("two f", 10));
+        
+        Console.WriteLine($"Search 'two f' returned {result.Records.Length} results");
+        for (int i = 0; i < Math.Min(10, result.Records.Length); i++)
+        {
+            var doc = engine.GetDocument(result.Records[i].DocumentId);
+            Console.WriteLine($"  [{i + 1}] [{result.Records[i].Score}] {doc?.IndexedText}");
+        }
+        
+        Assert.IsTrue(result.Records.Length >= 2, "Should return at least 2 results for 'two f'");
+        
+        // First result MUST be a "Two [word starting with 'f']" title, NOT "Flat Two" or similar
+        var doc1 = engine.GetDocument(result.Records[0].DocumentId);
+        Assert.IsNotNull(doc1);
+        
+        // Valid strict prefix matches: "Two for ...", "Two Faces", "Happy Feet Two", etc.
+        // The key is that "Two" appears first and is followed by a token starting with "f"
+        bool isValidPrefixMatch = doc1!.IndexedText.StartsWith("Two ", StringComparison.OrdinalIgnoreCase) &&
+                                   System.Text.RegularExpressions.Regex.IsMatch(
+                                       doc1.IndexedText,
+                                       @"\bTwo\s+[Ff]",
+                                       System.Text.RegularExpressions.RegexOptions.IgnoreCase);
+        
+        Assert.IsTrue(isValidPrefixMatch,
+            $"Position #1 MUST be a strict prefix match like 'Two for/face/feet...', but was '{doc1.IndexedText}'");
+    }
+
+    [TestMethod]
+    public void Search_TwoFo_AllExactPrefixesBeforePartialMatches()
+    {
+        var engine = GetEngine();
+        
+        // Query: "two fo"
+        // GUARANTEE: ALL documents starting with "Two Fo..." MUST rank above documents where "Two" is not at the start
+        var result = engine.Search(new Query("two fo", 20));
+        
+        Console.WriteLine($"Search 'two fo' returned {result.Records.Length} results");
+        for (int i = 0; i < Math.Min(15, result.Records.Length); i++)
+        {
+            var doc = engine.GetDocument(result.Records[i].DocumentId);
+            Console.WriteLine($"  [{i + 1}] [{result.Records[i].Score}] {doc?.IndexedText}");
+        }
+        
+        Assert.IsTrue(result.Records.Length >= 5, "Should return at least 5 results for 'two fo'");
+        
+        // Find where exact prefix matches end
+        int firstNonPrefixIndex = -1;
+        for (int i = 0; i < result.Records.Length; i++)
+        {
+            var doc = engine.GetDocument(result.Records[i].DocumentId);
+            bool isExactPrefix = doc!.IndexedText.StartsWith("Two Fo", StringComparison.OrdinalIgnoreCase);
+            
+            if (!isExactPrefix)
+            {
+                firstNonPrefixIndex = i;
+                break;
+            }
+        }
+        
+        // Now verify ALL documents before firstNonPrefixIndex start with "Two Fo"
+        // and NO documents after that point have higher scores
+        if (firstNonPrefixIndex > 0)
+        {
+            var lastPrefixDoc = engine.GetDocument(result.Records[firstNonPrefixIndex - 1].DocumentId);
+            var firstNonPrefixDoc = engine.GetDocument(result.Records[firstNonPrefixIndex].DocumentId);
+            
+            Assert.IsTrue(lastPrefixDoc!.IndexedText.StartsWith("Two Fo", StringComparison.OrdinalIgnoreCase),
+                $"Document at index {firstNonPrefixIndex - 1} should be an exact prefix match");
+            
+            Assert.IsFalse(firstNonPrefixDoc!.IndexedText.StartsWith("Two Fo", StringComparison.OrdinalIgnoreCase),
+                $"Document at index {firstNonPrefixIndex} should NOT be an exact prefix match");
+            
+            // The key assertion: prefix match MUST score higher than non-prefix
+            Assert.IsTrue(result.Records[firstNonPrefixIndex - 1].Score > result.Records[firstNonPrefixIndex].Score,
+                $"Exact prefix '{lastPrefixDoc.IndexedText}' (score {result.Records[firstNonPrefixIndex - 1].Score}) " +
+                $"MUST score higher than non-prefix '{firstNonPrefixDoc.IndexedText}' (score {result.Records[firstNonPrefixIndex].Score})");
+        }
+        
+        // Additional check: "Tea for Two" should appear AFTER all "Two for..." variants
+        var teaForTwoIndex = -1;
+        for (int i = 0; i < result.Records.Length; i++)
+        {
+            var doc = engine.GetDocument(result.Records[i].DocumentId);
+            if (doc!.IndexedText.Equals("Tea for Two", StringComparison.OrdinalIgnoreCase))
+            {
+                teaForTwoIndex = i;
+                break;
+            }
+        }
+        
+        if (teaForTwoIndex >= 0)
+        {
+            // Count how many "Two for..." documents appear BEFORE "Tea for Two"
+            int twoForCount = 0;
+            for (int i = 0; i < teaForTwoIndex; i++)
+            {
+                var doc = engine.GetDocument(result.Records[i].DocumentId);
+                if (doc!.IndexedText.StartsWith("Two Fo", StringComparison.OrdinalIgnoreCase))
+                {
+                    twoForCount++;
+                }
+            }
+            
+            Assert.IsTrue(twoForCount > 0,
+                "'Tea for Two' should appear AFTER at least one 'Two for...' variant, " +
+                $"but it appeared at index {teaForTwoIndex} with no 'Two for...' variants before it");
+        }
+    }
+
     [TestMethod]
     public void FellowshipOfTheRing_PrefersCorrectLotrMovie()
     {

src/Infidex/Coverage/CoverageEngine.cs

@@ -33,7 +33,7 @@ public void SetCorpusStatistics(TermCollection termCollection, int totalDocument
     public byte CalculateCoverageScore(string query, string documentText, double lcsSum, out int wordHits)
     {
         CoverageResult result = CalculateCoverageInternal(query, documentText, lcsSum, out wordHits, 
-            out _, out _, out _, out _, out _, out _, out _, out _, out _, out _);
+            out _, out _, out _, out _, out _, out _, out _, out _, out _, out _, out _);
         return result.CoverageScore;
     }
 
@@ -49,7 +49,8 @@ public ushort CalculateRankedScore(string query, string documentText, double lcs
             out _,  // suffixPrefixRun
             out _,  // phraseSpan
             out _,  // precedingStrictCount
-            out _); // lastTokenHasPrefix
+            out _,  // lastTokenHasPrefix
+            out _); // fusionSignals
 
         return CoverageScorer.CalculateRankedScore(
             result,
@@ -78,7 +79,8 @@ public CoverageFeatures CalculateFeatures(string query, string documentText, dou
             out int suffixPrefixRun,
             out int phraseSpan,
             out int precedingStrictCount,
-            out bool lastTokenHasPrefix);
+            out bool lastTokenHasPrefix,
+            out var fusionSignals);
 
         return new CoverageFeatures(
             result.CoverageScore,
@@ -101,7 +103,8 @@ public CoverageFeatures CalculateFeatures(string query, string documentText, dou
             result.LastTermIsTypeAhead,
             result.IdfCoverage,
             result.TotalIdf,
-            result.MissingIdf);
+            result.MissingIdf,
+            fusionSignals);
     }
 
     private CoverageResult CalculateCoverageInternal(string query, string documentText, double lcsSum, 
@@ -115,7 +118,8 @@ private CoverageResult CalculateCoverageInternal(string query, string documentTe
         out int suffixPrefixRun,
         out int phraseSpan,
         out int precedingStrictCount,
-        out bool lastTokenHasPrefix)
+        out bool lastTokenHasPrefix,
+        out FusionSignals fusionSignals)
     {
         wordHits = 0;
         docTokenCount = 0;
@@ -128,6 +132,7 @@ private CoverageResult CalculateCoverageInternal(string query, string documentTe
         phraseSpan = 0;
         precedingStrictCount = 0;
         lastTokenHasPrefix = false;
+        fusionSignals = default;
 
         if (query.Length == 0) 
             return new CoverageResult(0, 0, -1, 0);
@@ -147,6 +152,7 @@ private CoverageResult CalculateCoverageInternal(string query, string documentTe
         if (qCountRaw == 0)
         {
             if (queryTokenArray != null) ArrayPool<StringSlice>.Shared.Return(queryTokenArray);
+            fusionSignals = default;
             return new CoverageResult(0, 0, -1, 0);
         }
 
@@ -306,7 +312,7 @@ private CoverageResult CalculateCoverageInternal(string query, string documentTe
 
         wordHits = state.WordHits;
 
-        return CoverageScorer.CalculateFinalScore(
+        CoverageResult coverageResult = CoverageScorer.CalculateFinalScore(
             ref state,
             queryLen,
             lcsSum,
@@ -320,6 +326,42 @@ private CoverageResult CalculateCoverageInternal(string query, string documentTe
             out phraseSpan,
             out precedingStrictCount,
             out lastTokenHasPrefix);
+        
+        // Fusion signals need all tokens (no MinWordSize filtering)
+        int maxFusionQueryTokens = queryLen / 2 + 1;
+        StringSlice[]? fusionQueryTokenArray = null;
+        Span<StringSlice> fusionQueryTokens = maxFusionQueryTokens <= CoverageTokenizer.MaxStackTerms
+            ? stackalloc StringSlice[maxFusionQueryTokens]
+            : (fusionQueryTokenArray = ArrayPool<StringSlice>.Shared.Rent(maxFusionQueryTokens));
+
+        int fusionQCount = CoverageTokenizer.TokenizeToSpan(query, fusionQueryTokens, minWordSize: 0, delimiters);
+
+        int maxFusionDocTokens = docLen / 2 + 1;
+        StringSlice[]? fusionDocTokenArray = null;
+        Span<StringSlice> fusionDocTokens = maxFusionDocTokens <= CoverageTokenizer.MaxStackTerms
+            ? stackalloc StringSlice[maxFusionDocTokens]
+            : fusionDocTokenArray = ArrayPool<StringSlice>.Shared.Rent(maxFusionDocTokens);
+
+        int fusionDCount = CoverageTokenizer.TokenizeToSpan(documentText, fusionDocTokens, minWordSize: 0, delimiters);
+
+        try
+        {
+            fusionSignals = FusionSignalComputer.ComputeSignals(
+                querySpan,
+                docSpan,
+                fusionQueryTokens[..fusionQCount],
+                fusionDocTokens[..fusionDCount],
+                fusionQCount,
+                fusionDCount,
+                _setup.MinWordSize);
+        }
+        finally
+        {
+            if (fusionQueryTokenArray != null) ArrayPool<StringSlice>.Shared.Return(fusionQueryTokenArray);
+            if (fusionDocTokenArray != null) ArrayPool<StringSlice>.Shared.Return(fusionDocTokenArray);
+        }
+
+        return coverageResult;
     }
 
     /// <summary>

src/Infidex/Coverage/CoverageFeatures.cs

@@ -23,6 +23,9 @@ public readonly struct CoverageFeatures
     public readonly float IdfCoverage;      // Information-weighted coverage (IDF-based)
     public readonly float TotalIdf;         // Total information content of query
     public readonly float MissingIdf;       // Information content of unmatched terms
+    
+    // Precomputed fusion signals (Lucene-style: no string ops in fusion layer)
+    public readonly FusionSignals FusionSignals;
 
     public CoverageFeatures(
         byte coverageScore,
@@ -45,7 +48,8 @@ public CoverageFeatures(
         bool lastTermIsTypeAhead = false,
         float idfCoverage = 0f,
         float totalIdf = 0f,
-        float missingIdf = 0f)
+        float missingIdf = 0f,
+        FusionSignals fusionSignals = default)
     {
         CoverageScore = coverageScore;
         TermsCount = termsCount;
@@ -68,5 +72,6 @@ public CoverageFeatures(
         IdfCoverage = idfCoverage;
         TotalIdf = totalIdf;
         MissingIdf = missingIdf;
+        FusionSignals = fusionSignals;
     }
 }