Track min/max in LengthBitArray to make consecutive check cheaper

Also fix a bug with word boundaries

Author: Genbox

Src/FastData/Internal/Analysis/Data/LengthBitArray.cs

@@ -1,5 +1,4 @@
 using System.Runtime.CompilerServices;
-using Genbox.FastData.Internal.Helpers;
 
 namespace Genbox.FastData.Internal.Analysis.Data;
 
@@ -8,20 +7,20 @@ internal sealed class LengthBitArray(int length = 64)
     private int _length = length;
     private ulong[] _values = new ulong[GetLength(length)];
 
+    public int Min { get; private set; } = int.MaxValue;
+    public int Max { get; private set; } = int.MinValue;
+
     internal ulong[] Values => _values;
     internal int BitCount { get; private set; }
 
     internal bool Consecutive
     {
         get
         {
-            foreach (ulong val in _values)
-            {
-                if (!BitHelper.AreBitsConsecutive(val))
-                    return false;
-            }
+            if (BitCount == 0)
+                return false;
 
-            return true;
+            return BitCount == Max - Min + 1;
         }
     }
 
@@ -31,7 +30,8 @@ internal bool Get(int index)
         if (unchecked((uint)index >= (uint)_length))
             throw new ArgumentException("Index out of range: " + index, nameof(index));
 
-        return (_values[index >> 6] & (1UL << ((index & 63) - 1))) != 0; //-1 because we want a length of 1 to set the 0th bit
+        GetPosition(index, out int wordIndex, out ulong mask);
+        return (_values[wordIndex] & mask) != 0;
     }
 
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -45,12 +45,16 @@ internal bool SetTrue(int index)
 
         unchecked
         {
-            ulong mask = 1UL << ((index & 63) - 1); //-1 because we want a length of 1 to set the 0th bit
-            ref ulong slot = ref _values[index >> 6];
+            GetPosition(index, out int wordIndex, out ulong mask);
+            ref ulong slot = ref _values[wordIndex];
             bool alreadySet = (slot & mask) != 0;
 
             if (!alreadySet)
+            {
                 BitCount++;
+                Min = Math.Min(Min, index);
+                Max = Math.Max(Max, index);
+            }
 
             slot |= mask;
             return alreadySet;
@@ -74,4 +78,13 @@ private static int GetLength(int n)
 
         return (n + 63) >> 6;
     }
+
+    private static void GetPosition(int index, out int wordIndex, out ulong mask)
+    {
+        int remainder = index & 63;
+        int bitIndex = remainder == 0 ? 63 : remainder - 1; //-1 because we want a length of 1 to set the 0th bit
+
+        wordIndex = index >> 6;
+        mask = 1UL << bitIndex;
+    }
 }

Src/FastData/Internal/Analysis/KeyAnalyzer.cs

@@ -29,7 +29,6 @@ internal static StringProperties GetStringProperties(string[] keys, bool enableT
 
         //We need to know the longest string for optimal mixing. Probably not 100% necessary.
         string maxStr = keys[0];
-        int minLength = int.MaxValue;
         int minUtf8ByteLength = int.MaxValue;
         int maxUtf8ByteLength = int.MinValue;
         int minUtf16ByteLength = int.MaxValue;
@@ -50,7 +49,6 @@ internal static StringProperties GetStringProperties(string[] keys, bool enableT
             minUtf16ByteLength = Math.Min(utf16ByteCount, minUtf16ByteLength);
             maxUtf16ByteLength = Math.Max(utf16ByteCount, maxUtf16ByteLength);
 
-            minLength = Math.Min(minLength, str.Length); //Track the smallest string. It might be more than what lengthmap supports
             uniq &= !lengthMap.SetTrue(str.Length);
 
             foreach (char c in str)
@@ -70,7 +68,7 @@ internal static StringProperties GetStringProperties(string[] keys, bool enableT
         int[]? right = null;
 
         // Prefix/suffix tracking only makes sense when there are multiple keys, and they are long enough
-        if (enableTrimming && keys.Length > 1 && minLength > 1)
+        if (enableTrimming && keys.Length > 1 && lengthMap.Min > 1)
         {
             // Special case: If all strings have the same length, we can build an entropy map in O(n) with O(1) memory
             // TODO: For now FastData only supports prefix/suffix
@@ -126,7 +124,7 @@ internal static StringProperties GetStringProperties(string[] keys, bool enableT
             }
 
             // Make sure that we handle the case where all characters in the inputs are the same
-            if (DeltaData.CountZero(left) == minLength || DeltaData.CountZero(right) == minLength)
+            if (DeltaData.CountZero(left) == lengthMap.Min || DeltaData.CountZero(right) == lengthMap.Min)
             {
                 left = null;
                 right = null;
@@ -135,7 +133,7 @@ internal static StringProperties GetStringProperties(string[] keys, bool enableT
             // }
         }
 
-        return new StringProperties(new LengthData((uint)minLength, (uint)maxStr.Length, (uint)minUtf8ByteLength, (uint)maxUtf8ByteLength, (uint)minUtf16ByteLength, (uint)maxUtf16ByteLength, uniq, lengthMap), new DeltaData(left, right), new CharacterData(allAscii));
+        return new StringProperties(new LengthData((uint)lengthMap.Min, (uint)maxStr.Length, (uint)minUtf8ByteLength, (uint)maxUtf8ByteLength, (uint)minUtf16ByteLength, (uint)maxUtf16ByteLength, uniq, lengthMap), new DeltaData(left, right), new CharacterData(allAscii));
     }
 
     private static KeyProperties<char> GetCharProperties(char[] keys)