feat: add Tiktoken.Encodings.Tokenizer package for loading HuggingFace tokenizer.json files

Adds a new package that converts HuggingFace tokenizer.json files to Tiktoken encodings,
enabling tokenization for GPT-2, Llama 3, Qwen2, DeepSeek, and other BPE-based models.

Supports ByteLevel, Split, and Sequence[Split, ByteLevel] pre-tokenizer types with
auto-detected regex patterns. Includes FromFile, FromJson, FromStream, and FromUrlAsync APIs.

Closes #40

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: HavenDV

README.md

@@ -11,6 +11,7 @@
     <Project Path="src/libs/Tiktoken.Encodings.p50k/Tiktoken.Encodings.p50k.csproj" />
     <Project Path="src/libs/Tiktoken.Encodings.r50k/Tiktoken.Encodings.r50k.csproj" />
     <Project Path="src/libs/Tiktoken/Tiktoken.csproj" />
+    <Project Path="src/libs/Tiktoken.Encodings.Tokenizer/Tiktoken.Encodings.Tokenizer.csproj" />
   </Folder>
   <Folder Name="/misc/">
     <File Path=".gitattributes" />

Tiktoken.slnx

@@ -21,6 +21,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
     <PackageVersion Include="SharpToken" Version="2.0.4" />
+    <PackageVersion Include="System.Text.Json" Version="9.0.5" />
     <PackageVersion Include="System.ValueTuple" Version="4.6.2" />
     <PackageVersion Include="TiktokenSharp" Version="1.2.1" />
     <PackageVersion Include="Verify.MSTest" Version="31.13.2" />

benchmarks/2.2.0.0_encode.md

@@ -0,0 +1,6 @@
+using System.Text.Json.Serialization;
+
+namespace Tiktoken.Encodings;
+
+[JsonSerializable(typeof(TokenizerJson))]
+internal partial class SourceGenerationContext : JsonSerializerContext;

src/Directory.Packages.props

@@ -0,0 +1,32 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFrameworks>net4.6.2;netstandard2.0;netstandard2.1;net8.0;net9.0;net10.0</TargetFrameworks>
+        <RootNamespace>Tiktoken.Encodings</RootNamespace>
+    </PropertyGroup>
+
+    <PropertyGroup Label="NuGet">
+        <Description>Load tokenizer.json (HuggingFace format) as a Tiktoken encoding.</Description>
+        <PackageTags>chatgpt;openai;tiktoken;tokens;huggingface;tokenizer;bpe;llama;qwen;deepseek;gpt2</PackageTags>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Tiktoken.Encodings.Abstractions\Tiktoken.Encodings.Abstractions.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="System.Text.Json" />
+    </ItemGroup>
+
+    <ItemGroup Condition="'$(TargetFramework)' == 'net4.6.2'">
+      <PackageReference Include="System.ValueTuple" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <PackageReference Include="PolySharp">
+        <PrivateAssets>all</PrivateAssets>
+        <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      </PackageReference>
+    </ItemGroup>
+
+</Project>

src/libs/Tiktoken.Encodings.Tokenizer/SourceGenerationContext.cs

@@ -0,0 +1,115 @@
+using System.Text.Json.Serialization;
+
+namespace Tiktoken.Encodings;
+
+/// <summary>
+/// Represents a HuggingFace tokenizer.json file.
+/// </summary>
+public class TokenizerJson
+{
+    /// <summary>
+    /// </summary>
+    [JsonPropertyName("version")]
+    public string Version { get; set; } = string.Empty;
+
+    /// <summary>
+    /// </summary>
+    [JsonPropertyName("added_tokens")]
+    public IReadOnlyList<TokenizerAddedToken> AddedTokens { get; set; } = [];
+
+    /// <summary>
+    /// </summary>
+    [JsonPropertyName("pre_tokenizer")]
+    public TokenizerPreTokenizer? PreTokenizer { get; set; }
+
+    /// <summary>
+    /// </summary>
+    [JsonPropertyName("model")]
+    public TokenizerModel? Model { get; set; }
+}
+
+/// <summary>
+/// A token added to the tokenizer vocabulary.
+/// </summary>
+public class TokenizerAddedToken
+{
+    /// <summary>
+    /// </summary>
+    [JsonPropertyName("id")]
+    public int Id { get; set; }
+
+    /// <summary>
+    /// </summary>
+    [JsonPropertyName("special")]
+    public bool Special { get; set; }
+
+    /// <summary>
+    /// </summary>
+    [JsonPropertyName("content")]
+    public string Content { get; set; } = string.Empty;
+}
+
+/// <summary>
+/// The pre-tokenizer configuration. Supports ByteLevel, Split, Sequence, and other types.
+/// </summary>
+public class TokenizerPreTokenizer
+{
+    /// <summary>
+    /// The pre-tokenizer type (ByteLevel, Split, Sequence, Metaspace, etc.).
+    /// </summary>
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = string.Empty;
+
+    /// <summary>
+    /// For Split type: the pattern to split on.
+    /// </summary>
+    [JsonPropertyName("pattern")]
+    public TokenizerSplitPattern? Pattern { get; set; }
+
+    /// <summary>
+    /// For Sequence type: the list of nested pre-tokenizers.
+    /// </summary>
+    [JsonPropertyName("pretokenizers")]
+    public IReadOnlyList<TokenizerPreTokenizer>? PreTokenizers { get; set; }
+}
+
+/// <summary>
+/// A split pattern — either a Regex or a String.
+/// </summary>
+public class TokenizerSplitPattern
+{
+    /// <summary>
+    /// Regex pattern variant.
+    /// </summary>
+    [JsonPropertyName("Regex")]
+    public string? Regex { get; set; }
+
+    /// <summary>
+    /// String pattern variant.
+    /// </summary>
+    [JsonPropertyName("String")]
+    public string? String { get; set; }
+}
+
+/// <summary>
+/// The BPE model definition.
+/// </summary>
+public class TokenizerModel
+{
+    /// <summary>
+    /// </summary>
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = string.Empty;
+
+    /// <summary>
+    /// </summary>
+    [JsonPropertyName("vocab")]
+#pragma warning disable CA2227
+    public Dictionary<string, int>? Vocab { get; set; }
+#pragma warning restore CA2227
+
+    /// <summary>
+    /// </summary>
+    [JsonPropertyName("merges")]
+    public IReadOnlyList<string>? Merges { get; set; }
+}