improve coverage perf

Author: lofcz

src/Infidex.Tests/BugReproductionTests.cs

@@ -0,0 +1,46 @@
+using Infidex.Core;
+using Infidex.Scoring;
+using Infidex.Coverage;
+using Infidex.Tokenization;
+
+namespace Infidex.Tests;
+
+[TestClass]
+public class BugReproductionTests
+{
+    [TestMethod]
+    public void PrefixPreference_MatrixRev_PreferRevisitedOverReloaded()
+    {
+        // "the matrix rev"
+        // "The Matrix Reloaded" vs "The Matrix Revisited"
+        // "Revisited" starts with "Rev". "Reloaded" does not.
+        
+        string query = "the matrix rev";
+        string docReloaded = "The Matrix Reloaded";
+        string docRevisited = "The Matrix Revisited";
+        
+        var tokenizer = new Tokenizer([3], 2, 0, TextNormalizer.CreateDefault(), TokenizerSetup.CreateDefault());
+        var setup = CoverageSetup.CreateDefault();
+        var engine = new CoverageEngine(tokenizer, setup);
+        
+        // Calculate features for Reloaded
+        var featsReloaded = engine.CalculateFeatures(query, docReloaded, 0, 1);
+        
+        // Calculate features for Revisited
+        var featsRevisited = engine.CalculateFeatures(query, docRevisited, 0, 2);
+        
+        // Score
+        var scoreReloaded = FusionScorer.Calculate(query, docReloaded, featsReloaded, 0.5f, 3, new[]{' '});
+        var scoreRevisited = FusionScorer.Calculate(query, docRevisited, featsRevisited, 0.5f, 3, new[]{' '});
+        
+        Console.WriteLine($"Reloaded: {scoreReloaded.score}");
+        Console.WriteLine($"Revisited: {scoreRevisited.score}");
+        
+        Assert.IsTrue(scoreRevisited.score > scoreReloaded.score, 
+            $"Revisited ({scoreRevisited.score}) should score higher than Reloaded ({scoreReloaded.score})");
+            
+        // Check why
+        Assert.IsTrue(featsRevisited.FusionSignals.LexicalPrefixLast, "Revisited should have LexicalPrefixLast");
+        Assert.IsFalse(featsReloaded.FusionSignals.LexicalPrefixLast, "Reloaded should NOT have LexicalPrefixLast");
+    }
+}

src/Infidex.Tests/WordMatcherTests.cs

@@ -23,7 +23,8 @@ public void Lookup_ExactMatch_FindsDocument()
 
         var results = matcher.Lookup("world");
 
-        Assert.AreEqual(2, results.Count);
+        Assert.IsNotNull(results);
+        Assert.AreEqual(2, results.Cardinality);
         Assert.IsTrue(results.Contains(0));
         Assert.IsTrue(results.Contains(1));
     }
@@ -45,7 +46,8 @@ public void Lookup_LD1Support_FindsFuzzyMatches()
         // "batmam" is 1 edit away from "batman"
         var results = matcher.Lookup("batmam");
 
-        Assert.IsTrue(results.Count > 0);
+        Assert.IsNotNull(results);
+        Assert.IsTrue(results.Cardinality > 0);
         Assert.IsTrue(results.Contains(0));
     }
 
@@ -65,9 +67,8 @@ public void LookupAffix_FindsPrefixMatches()
         // "bat" is a prefix of "batman"
         var results = matcher.LookupAffix("bat");
 
-        Assert.IsTrue(results.Count > 0);
+        Assert.IsNotNull(results);
+        Assert.IsTrue(results.Cardinality > 0);
         Assert.IsTrue(results.Contains(0));
     }
 }
-
-

src/Infidex/Coverage/CoverageBuffer.cs

@@ -0,0 +1,95 @@
+using System.Buffers;
+using Infidex.Tokenization;
+
+namespace Infidex.Coverage;
+
+/// <summary>
+/// Reusable buffer for coverage calculations to avoid repeated array allocations.
+/// Not thread-safe; should be created per query execution.
+/// </summary>
+internal sealed class CoverageBuffer : IDisposable
+{
+    // Arrays sized for max document tokens
+    public StringSlice[] DocTokenArray;
+    public StringSlice[] UniqueDocTokenArray;
+    public bool[] QActiveArray;
+    public bool[] DActiveArray;
+    public float[] TermMatchedCharsArray;
+    public bool[] TermHasWholeArray;
+    public bool[] TermHasJoinedArray;
+    public bool[] TermHasPrefixArray;
+    public int[] TermFirstPosArray;
+    public float[] TermIdfArray;
+    
+    // Fusion arrays
+    public StringSlice[] FusionQueryTokenArray;
+    public StringSlice[] FusionDocTokenArray;
+
+    private const int DefaultCapacity = 1024; // Reasonable start for doc tokens
+    private const int DefaultQueryCapacity = 64;
+
+    public CoverageBuffer()
+    {
+        DocTokenArray = ArrayPool<StringSlice>.Shared.Rent(DefaultCapacity);
+        UniqueDocTokenArray = ArrayPool<StringSlice>.Shared.Rent(DefaultCapacity);
+        QActiveArray = ArrayPool<bool>.Shared.Rent(DefaultQueryCapacity);
+        DActiveArray = ArrayPool<bool>.Shared.Rent(DefaultCapacity);
+        TermMatchedCharsArray = ArrayPool<float>.Shared.Rent(DefaultQueryCapacity);
+        TermHasWholeArray = ArrayPool<bool>.Shared.Rent(DefaultQueryCapacity);
+        TermHasJoinedArray = ArrayPool<bool>.Shared.Rent(DefaultQueryCapacity);
+        TermHasPrefixArray = ArrayPool<bool>.Shared.Rent(DefaultQueryCapacity);
+        TermFirstPosArray = ArrayPool<int>.Shared.Rent(DefaultQueryCapacity);
+        TermIdfArray = ArrayPool<float>.Shared.Rent(DefaultQueryCapacity);
+        
+        FusionQueryTokenArray = ArrayPool<StringSlice>.Shared.Rent(DefaultQueryCapacity);
+        FusionDocTokenArray = ArrayPool<StringSlice>.Shared.Rent(DefaultCapacity);
+    }
+
+    public void EnsureDocCapacity(int required)
+    {
+        if (DocTokenArray.Length < required)
+        {
+            Resize(ref DocTokenArray, required);
+            Resize(ref UniqueDocTokenArray, required);
+            Resize(ref DActiveArray, required);
+            Resize(ref FusionDocTokenArray, required);
+        }
+    }
+
+    public void EnsureQueryCapacity(int required)
+    {
+        if (QActiveArray.Length < required)
+        {
+            Resize(ref QActiveArray, required);
+            Resize(ref TermMatchedCharsArray, required);
+            Resize(ref TermHasWholeArray, required);
+            Resize(ref TermHasJoinedArray, required);
+            Resize(ref TermHasPrefixArray, required);
+            Resize(ref TermFirstPosArray, required);
+            Resize(ref TermIdfArray, required);
+            Resize(ref FusionQueryTokenArray, required);
+        }
+    }
+
+    private void Resize<T>(ref T[] array, int newSize)
+    {
+        ArrayPool<T>.Shared.Return(array);
+        array = ArrayPool<T>.Shared.Rent(newSize);
+    }
+
+    public void Dispose()
+    {
+        if (DocTokenArray != null) ArrayPool<StringSlice>.Shared.Return(DocTokenArray);
+        if (UniqueDocTokenArray != null) ArrayPool<StringSlice>.Shared.Return(UniqueDocTokenArray);
+        if (QActiveArray != null) ArrayPool<bool>.Shared.Return(QActiveArray);
+        if (DActiveArray != null) ArrayPool<bool>.Shared.Return(DActiveArray);
+        if (TermMatchedCharsArray != null) ArrayPool<float>.Shared.Return(TermMatchedCharsArray);
+        if (TermHasWholeArray != null) ArrayPool<bool>.Shared.Return(TermHasWholeArray);
+        if (TermHasJoinedArray != null) ArrayPool<bool>.Shared.Return(TermHasJoinedArray);
+        if (TermHasPrefixArray != null) ArrayPool<bool>.Shared.Return(TermHasPrefixArray);
+        if (TermFirstPosArray != null) ArrayPool<int>.Shared.Return(TermFirstPosArray);
+        if (TermIdfArray != null) ArrayPool<float>.Shared.Return(TermIdfArray);
+        if (FusionQueryTokenArray != null) ArrayPool<StringSlice>.Shared.Return(FusionQueryTokenArray);
+        if (FusionDocTokenArray != null) ArrayPool<StringSlice>.Shared.Return(FusionDocTokenArray);
+    }
+}