feat: add EncodeUtf8 zero-alloc API for encoding directly from UTF-8 bytes

Add EncodeUtf8(ReadOnlySpan<byte>, Span<int>) that encodes UTF-8 text
directly into a caller-provided token buffer. Pairs with existing
CountTokens(ReadOnlySpan<byte>) for buffer sizing. Uses stackalloc for
small inputs, ArrayPool for large ones. Add EncodeUtf8 benchmark category.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: HavenDV

src/benchmarks/Tiktoken.Benchmarks/Benchmarks.cs

@@ -202,4 +202,29 @@ public int Tiktoken_DecodeToUtf8()
     public int Tiktoken_CountTokens_FromUtf8() => _tiktoken.CountTokens(_dataUtf8.AsSpan());
 
 
+    [Benchmark(Baseline = true)]
+    [BenchmarkCategory("EncodeUtf8")]
+    public IReadOnlyCollection<int> Tiktoken_Encode_Baseline() => _tiktoken.Encode(Data);
+
+    [Benchmark]
+    [BenchmarkCategory("EncodeUtf8")]
+    public int Tiktoken_EncodeUtf8()
+    {
+        var tokenCount = _tiktoken.CountTokens(_dataUtf8.AsSpan());
+        if (tokenCount <= 1024)
+        {
+            Span<int> buffer = stackalloc int[tokenCount];
+            return _tiktoken.EncodeUtf8(_dataUtf8.AsSpan(), buffer);
+        }
+
+        var rented = System.Buffers.ArrayPool<int>.Shared.Rent(tokenCount);
+        try
+        {
+            return _tiktoken.EncodeUtf8(_dataUtf8.AsSpan(), rented.AsSpan(0, tokenCount));
+        }
+        finally
+        {
+            System.Buffers.ArrayPool<int>.Shared.Return(rented);
+        }
+    }
 }

src/libs/Tiktoken.Core/CoreBPE.cs

@@ -229,6 +229,49 @@ internal int CountTokensFromUtf8(ReadOnlySpan<byte> utf8Text)
             System.Buffers.ArrayPool<char>.Shared.Return(rented);
         }
     }
+
+    /// <summary>
+    /// Encodes UTF-8 bytes directly into a caller-provided token buffer.
+    /// Converts to chars internally using stackalloc/ArrayPool, then encodes.
+    /// </summary>
+    internal int EncodeFromUtf8(
+        ReadOnlySpan<byte> utf8Text,
+        Span<int> tokenDestination,
+        HashSet<string> disallowedSpecial)
+    {
+        var charCount = System.Text.Encoding.UTF8.GetCharCount(utf8Text);
+        char[]? rentedChars = null;
+        var chars = charCount <= 1024
+            ? stackalloc char[charCount]
+            : (rentedChars = System.Buffers.ArrayPool<char>.Shared.Rent(charCount)).AsSpan(0, charCount);
+
+        try
+        {
+            System.Text.Encoding.UTF8.GetChars(utf8Text, chars);
+            var tokens = EncodeNativeAllDisallowed(chars, disallowedSpecial);
+
+            if (tokens.Count > tokenDestination.Length)
+            {
+                throw new ArgumentException(
+                    "Destination buffer is too small. Use CountTokens to determine the required size.",
+                    nameof(tokenDestination));
+            }
+
+            var i = 0;
+            foreach (var token in tokens)
+            {
+                tokenDestination[i++] = token;
+            }
+            return tokens.Count;
+        }
+        finally
+        {
+            if (rentedChars != null)
+            {
+                System.Buffers.ArrayPool<char>.Shared.Return(rentedChars);
+            }
+        }
+    }
 #endif
 
     /// <summary>

src/libs/Tiktoken.Core/Encoder.cs

@@ -65,6 +65,19 @@ public int CountTokens(ReadOnlySpan<byte> utf8Text)
     {
         return _corePbe.CountTokensFromUtf8(utf8Text);
     }
+
+    /// <summary>
+    /// Encodes UTF-8 text directly into a caller-provided token buffer for zero-allocation encode.
+    /// Use <see cref="CountTokens(ReadOnlySpan{byte})"/> to determine the required buffer size.
+    /// </summary>
+    /// <param name="utf8Text">The UTF-8 encoded text to tokenize.</param>
+    /// <param name="tokenDestination">The destination buffer for token IDs.</param>
+    /// <returns>The number of tokens written to <paramref name="tokenDestination"/>.</returns>
+    /// <exception cref="ArgumentException">The destination buffer is too small.</exception>
+    public int EncodeUtf8(ReadOnlySpan<byte> utf8Text, Span<int> tokenDestination)
+    {
+        return _corePbe.EncodeFromUtf8(utf8Text, tokenDestination, _specialTokensSet);
+    }
 #endif
 
     /// <summary>