fix: prevent stack overflow on large inputs and lazy-load encodings (#75, #48)

Fix #75: BytePairEncoding.FindParts used unbounded stackalloc for both
partsIndexes and partsRanks arrays. For pieces >512 bytes (e.g., large
regex matches on Russian text), this overflows the 1MB default stack.
Now uses heap allocation for large pieces, stackalloc for small ones.

Fix #48: ModelToEncoding eagerly created 5 duplicate Cl100KBase instances,
each loading the same 100K-entry .tiktoken file. Now uses Lazy<Encoding>
singletons so each encoding is loaded once on first access and shared
across all models that use the same encoding.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: HavenDV

src/libs/Tiktoken.Core/BytePairEncoding.cs

@@ -1,4 +1,4 @@
-#if NETSTANDARD2_1_OR_GREATER || NET6_0_OR_GREATER
+#if NETSTANDARD2_1_OR_GREATER || NET6_0_OR_GREATER
 using Bytes = System.ReadOnlyMemory<byte>;
 #else
 using Bytes = System.Collections.Generic.IReadOnlyCollection<byte>;
@@ -7,10 +7,14 @@
 namespace Tiktoken.Core;
 
 /// <summary>
-/// 
+///
 /// </summary>
 public static class BytePairEncoding
 {
+    // Maximum number of int elements to stackalloc (2 arrays × this size × 4 bytes each).
+    // 512 elements = 4KB per array = 8KB total on stack, well within safe limits.
+    private const int MaxStackAllocLength = 512;
+
     private static byte[] GetSlice(this Bytes bytes, int from, int to)
     {
 #if NETSTANDARD2_1_OR_GREATER || NET6_0_OR_GREATER
@@ -19,7 +23,7 @@ private static byte[] GetSlice(this Bytes bytes, int from, int to)
         return bytes.Skip(from).Take(to - from).ToArray();
 #endif
     }
-    
+
     private static int GetLength(this Bytes bytes)
     {
 #if NETSTANDARD2_1_OR_GREATER || NET6_0_OR_GREATER
@@ -28,7 +32,7 @@ private static int GetLength(this Bytes bytes)
         return bytes.Count;
 #endif
     }
-    
+
     private static unsafe bool TryFindMinRank(int* partsRanks, int count, out int result)
     {
         result = 0;
@@ -41,7 +45,7 @@ private static unsafe bool TryFindMinRank(int* partsRanks, int count, out int re
                 result = i;
             }
         }
-        
+
         return minRank != int.MaxValue;
     }
 
@@ -63,17 +67,17 @@ private static unsafe int GetRank(
                 return rank;
             }
         }
-        
+
         return int.MaxValue;
     }
-    
+
     private static unsafe int FindParts(
         Bytes piece,
         int* partsIndexes,
+        int* partsRanks,
+        int partsLength,
         IReadOnlyDictionary<byte[], int> ranks)
     {
-        var partsLength = piece.GetLength() + 1;
-        var partsRanks = stackalloc int [partsLength];
         for (var i = 0; i < partsLength; i++)
         {
             partsIndexes[i] = i;
@@ -83,15 +87,15 @@ private static unsafe int FindParts(
         {
             partsRanks[i] = GetRank(i, partsIndexes, partsLength, piece, ranks, length: 2);
         }
-        
+
         var count = partsLength - 1;
         while (true)
         {
             if (!TryFindMinRank(partsRanks, count, out var i))
             {
                 break;
             }
-            
+
             partsRanks[i] = GetRank(i, partsIndexes, count + 1, piece, ranks, length: 3);
             if (i > 0)
             {
@@ -104,70 +108,133 @@ private static unsafe int FindParts(
             }
             count--;
         }
-        
+
         return count;
     }
-    
+
     internal static unsafe void BytePairEncode(Bytes piece, IReadOnlyDictionary<byte[], int> ranks, List<int> outList)
     {
         var partsLength = piece.GetLength() + 1;
-        var partsIndexes = stackalloc int [partsLength];
-        var count = FindParts(piece, partsIndexes, ranks);
 
-        for (var i = 0; i < count; i++)
+        if (partsLength <= MaxStackAllocLength)
         {
-            var from = partsIndexes[i];
-            var to = partsIndexes[i + 1];
-            var slice = piece.GetSlice(from, to);
+            var partsIndexes = stackalloc int[partsLength];
+            var partsRanks = stackalloc int[partsLength];
+            var count = FindParts(piece, partsIndexes, partsRanks, partsLength, ranks);
+
+            for (var i = 0; i < count; i++)
+            {
+                outList.Add(ranks[piece.GetSlice(partsIndexes[i], partsIndexes[i + 1])]);
+            }
+        }
+        else
+        {
+            var heapIndexes = new int[partsLength];
+            var heapRanks = new int[partsLength];
+            fixed (int* partsIndexes = heapIndexes)
+            fixed (int* partsRanks = heapRanks)
+            {
+                var count = FindParts(piece, partsIndexes, partsRanks, partsLength, ranks);
 
-            outList.Add(ranks[slice]);
+                for (var i = 0; i < count; i++)
+                {
+                    outList.Add(ranks[piece.GetSlice(partsIndexes[i], partsIndexes[i + 1])]);
+                }
+            }
         }
     }
 
     internal static unsafe int[] BytePairEncodeToArray(Bytes piece, IReadOnlyDictionary<byte[], int> ranks)
     {
         var partsLength = piece.GetLength() + 1;
-        var partsIndexes = stackalloc int [partsLength];
-        var count = FindParts(piece, partsIndexes, ranks);
 
-        var result = new int[count];
-        for (var i = 0; i < count; i++)
+        if (partsLength <= MaxStackAllocLength)
         {
-            var from = partsIndexes[i];
-            var to = partsIndexes[i + 1];
-            var slice = piece.GetSlice(from, to);
+            var partsIndexes = stackalloc int[partsLength];
+            var partsRanks = stackalloc int[partsLength];
+            var count = FindParts(piece, partsIndexes, partsRanks, partsLength, ranks);
 
-            result[i] = ranks[slice];
+            var result = new int[count];
+            for (var i = 0; i < count; i++)
+            {
+                result[i] = ranks[piece.GetSlice(partsIndexes[i], partsIndexes[i + 1])];
+            }
+            return result;
         }
+        else
+        {
+            var heapIndexes = new int[partsLength];
+            var heapRanks = new int[partsLength];
+            fixed (int* partsIndexes = heapIndexes)
+            fixed (int* partsRanks = heapRanks)
+            {
+                var count = FindParts(piece, partsIndexes, partsRanks, partsLength, ranks);
 
-        return result;
+                var result = new int[count];
+                for (var i = 0; i < count; i++)
+                {
+                    result[i] = ranks[piece.GetSlice(partsIndexes[i], partsIndexes[i + 1])];
+                }
+                return result;
+            }
+        }
     }
-    
+
     internal static unsafe List<byte[]> BytePairExplore(Bytes piece, IReadOnlyDictionary<byte[], int> ranks)
     {
         var partsLength = piece.GetLength() + 1;
-        var partsIndexes = stackalloc int [partsLength];
-        var count = FindParts(piece, partsIndexes, ranks);
-        
-        var outList = new List<byte[]>(count);
-        for (var i = 0; i < count; i++)
+
+        if (partsLength <= MaxStackAllocLength)
         {
-            var from = partsIndexes[i];
-            var to = partsIndexes[i + 1];
-            var slice = piece.GetSlice(from, to);
-            
-            outList.Add(slice);
+            var partsIndexes = stackalloc int[partsLength];
+            var partsRanks = stackalloc int[partsLength];
+            var count = FindParts(piece, partsIndexes, partsRanks, partsLength, ranks);
+
+            var outList = new List<byte[]>(count);
+            for (var i = 0; i < count; i++)
+            {
+                outList.Add(piece.GetSlice(partsIndexes[i], partsIndexes[i + 1]));
+            }
+            return outList;
+        }
+        else
+        {
+            var heapIndexes = new int[partsLength];
+            var heapRanks = new int[partsLength];
+            fixed (int* partsIndexes = heapIndexes)
+            fixed (int* partsRanks = heapRanks)
+            {
+                var count = FindParts(piece, partsIndexes, partsRanks, partsLength, ranks);
+
+                var outList = new List<byte[]>(count);
+                for (var i = 0; i < count; i++)
+                {
+                    outList.Add(piece.GetSlice(partsIndexes[i], partsIndexes[i + 1]));
+                }
+                return outList;
+            }
         }
-        
-        return outList;
     }
-    
+
     internal static unsafe int BytePairEncodeCountTokens(Bytes piece, IReadOnlyDictionary<byte[], int> ranks)
     {
         var partsLength = piece.GetLength() + 1;
-        var partsIndexes = stackalloc int [partsLength];
-        var count = FindParts(piece, partsIndexes, ranks);
-        
-        return count;
+
+        if (partsLength <= MaxStackAllocLength)
+        {
+            var partsIndexes = stackalloc int[partsLength];
+            var partsRanks = stackalloc int[partsLength];
+            return FindParts(piece, partsIndexes, partsRanks, partsLength, ranks);
+        }
+        else
+        {
+            var heapIndexes = new int[partsLength];
+            var heapRanks = new int[partsLength];
+            fixed (int* partsIndexes = heapIndexes)
+            fixed (int* partsRanks = heapRanks)
+            {
+                return FindParts(piece, partsIndexes, partsRanks, partsLength, ranks);
+            }
+        }
     }
 }

src/libs/Tiktoken/ModelToEncoding.cs

@@ -1,24 +1,28 @@
-using Tiktoken.Encodings;
+using Tiktoken.Encodings;
 
 namespace Tiktoken;
 
 /// <summary>
-/// 
+///
 /// </summary>
 public static class ModelToEncoding
 {
-    private static Dictionary<string, Encoding> Dictionary { get; } = new()
+    // Lazy singletons — each encoding is loaded only once, on first access.
+    private static readonly Lazy<Encoding> Cl100K = new(static () => new Cl100KBase());
+    private static readonly Lazy<Encoding> O200K = new(static () => new O200KBase());
+
+    private static Dictionary<string, Lazy<Encoding>> Dictionary { get; } = new()
     {
         // chat
-        { "gpt-4o", new O200KBase() },
-        { "gpt-4", new Cl100KBase() },
-        { "gpt-3.5-turbo", new Cl100KBase() },
-        { "gpt-35-turbo", new Cl100KBase() }, // Azure deployment name
-        
+        { "gpt-4o", O200K },
+        { "gpt-4", Cl100K },
+        { "gpt-3.5-turbo", Cl100K },
+        { "gpt-35-turbo", Cl100K }, // Azure deployment name
+
         // embeddings
-        { "text-embedding-ada-002", new Cl100KBase() },
-        { "text-embedding-3-small", new Cl100KBase() },
-        { "text-embedding-3-large", new Cl100KBase() },
+        { "text-embedding-ada-002", Cl100K },
+        { "text-embedding-3-small", Cl100K },
+        { "text-embedding-3-large", Cl100K },
     };
 
     /// <summary>
@@ -29,8 +33,10 @@ public static class ModelToEncoding
     /// <returns></returns>
     public static Encoding? TryFor(string modelName)
     {
-        return Dictionary
+        var lazy = Dictionary
             .FirstOrDefault(a => modelName.StartsWith(a.Key, StringComparison.Ordinal)).Value;
+
+        return lazy?.Value;
     }
 
     /// <summary>
@@ -44,4 +50,4 @@ public static Encoding For(string modelName)
         return TryFor(modelName) ??
                throw new ArgumentException($"Model name {modelName} is not supported.");
     }
-}
+}