chore: migrate FluentAssertions to AwesomeAssertions 9.4.0

HavenDV · claude · HavenDV · commit b877f2b19550 · 2026-03-19T04:17:01.000+04:00
FluentAssertions changed its license away from MIT.
AwesomeAssertions is an MIT-licensed fork with the same API.

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props
@@ -5,7 +5,7 @@
   <ItemGroup>
     <PackageVersion Include="BenchmarkDotNet" Version="0.15.8" />
     <PackageVersion Include="DotNet.ReproducibleBuilds" Version="2.0.2" />
-    <PackageVersion Include="FluentAssertions" Version="8.9.0" />
+    <PackageVersion Include="AwesomeAssertions" Version="9.4.0" />
     <PackageVersion Include="GitHubActionsTestLogger" Version="3.0.1" />
     <PackageVersion Include="H.Resources.Generator" Version="1.8.0">
       <PrivateAssets>all</PrivateAssets>
diff --git a/src/libs/Tiktoken.Core/ChatFunction.cs b/src/libs/Tiktoken.Core/ChatFunction.cs
@@ -79,6 +79,12 @@ public class FunctionParameter
     /// </summary>
     public string? ArrayItemType { get; set; }
 
+    /// <summary>
+    /// Union types for <c>anyOf</c> schemas (e.g., ["string", "number"] becomes "string | number").
+    /// When set, <see cref="Type"/> is ignored and this list is used instead.
+    /// </summary>
+    public IReadOnlyList<string>? AnyOf { get; set; }
+
     /// <summary>
     /// Creates a new function parameter.
     /// </summary>
@@ -96,7 +102,8 @@ public FunctionParameter(
         bool isRequired = false,
         IReadOnlyList<string>? enumValues = null,
         IReadOnlyList<FunctionParameter>? properties = null,
-        string? arrayItemType = null)
+        string? arrayItemType = null,
+        IReadOnlyList<string>? anyOf = null)
     {
         Name = name ?? throw new ArgumentNullException(nameof(name));
         Type = type ?? throw new ArgumentNullException(nameof(type));
@@ -105,5 +112,6 @@ public FunctionParameter(
         EnumValues = enumValues;
         Properties = properties;
         ArrayItemType = arrayItemType;
+        AnyOf = anyOf;
     }
 }
diff --git a/src/libs/Tiktoken.Core/Encoder.cs b/src/libs/Tiktoken.Core/Encoder.cs
@@ -334,13 +334,22 @@ private int CountParameterTokens(IReadOnlyList<FunctionParameter> parameters)
 
             // Tokenize "key:type:description" (trailing period stripped)
             var paramDesc = param.Description.TrimEnd('.');
-            var type = param.Type;
+            string type;
 
-            // For array types, append item type (e.g., "array:string[]")
-            if (type == "array" && param.ArrayItemType != null)
+            if (param.AnyOf != null && param.AnyOf.Count > 0)
             {
+                // anyOf union: "string | number"
+                type = string.Join(" | ", param.AnyOf);
+            }
+            else if (param.Type == "array" && param.ArrayItemType != null)
+            {
+                // Array with item type: "string[]"
                 type = param.ArrayItemType + "[]";
             }
+            else
+            {
+                type = param.Type;
+            }
 
             count += CountTokens(param.Name + ":" + type + ":" + paramDesc);
 
diff --git a/src/tests/Tiktoken.UnitTests/TestInitializer.cs b/src/tests/Tiktoken.UnitTests/TestInitializer.cs
@@ -0,0 +1,51 @@
+namespace Tiktoken.UnitTests;
+
+[TestClass]
+public static class TestInitializer
+{
+    [AssemblyInitialize]
+    public static void Initialize(TestContext _)
+    {
+        LoadEnvFile();
+    }
+
+    private static void LoadEnvFile()
+    {
+        // Walk up from the test output directory to find .env at repo root
+        var dir = AppContext.BaseDirectory;
+        while (dir != null)
+        {
+            var envPath = Path.Combine(dir, ".env");
+            if (File.Exists(envPath))
+            {
+                foreach (var line in File.ReadAllLines(envPath))
+                {
+                    var trimmed = line.Trim();
+                    if (trimmed.Length == 0 || trimmed.StartsWith('#'))
+                    {
+                        continue;
+                    }
+
+                    var eqIndex = trimmed.IndexOf('=');
+                    if (eqIndex <= 0)
+                    {
+                        continue;
+                    }
+
+                    var key = trimmed[..eqIndex].Trim();
+                    var value = trimmed[(eqIndex + 1)..].Trim();
+
+                    // Only set if not already set (env vars take precedence)
+                    if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(key)))
+                    {
+                        Environment.SetEnvironmentVariable(key, value);
+                    }
+                }
+
+                return;
+            }
+
+            dir = Path.GetDirectoryName(dir);
+        }
+    }
+}
diff --git a/src/tests/Tiktoken.UnitTests/Tests.Integration.cs b/src/tests/Tiktoken.UnitTests/Tests.Integration.cs
@@ -12,6 +12,25 @@ public partial class Tests
         return Environment.GetEnvironmentVariable("OPENAI_API_KEY");
     }
 
+    private static async Task<int> GetServerTokenCount(HttpClient httpClient, string requestJson)
+    {
+        var response = await httpClient.PostAsync(
+            "https://api.openai.com/v1/responses/input_tokens",
+            new StringContent(requestJson, Encoding.UTF8, "application/json"));
+
+        response.EnsureSuccessStatusCode();
+        var responseBody = await response.Content.ReadAsStringAsync();
+        using var doc = JsonDocument.Parse(responseBody);
+        return doc.RootElement.GetProperty("input_tokens").GetInt32();
+    }
+
+    private static HttpClient CreateOpenAiClient(string apiKey)
+    {
+        var httpClient = new HttpClient();
+        httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
+        return httpClient;
+    }
+
     [TestMethod]
     public async Task ValidateMessageTokensAgainstOpenAiApi()
     {
@@ -22,31 +41,20 @@ public async Task ValidateMessageTokensAgainstOpenAiApi()
             return;
         }
 
-        using var httpClient = new HttpClient();
-        httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
+        using var httpClient = CreateOpenAiClient(apiKey);
 
-        // Test: simple text input
         var requestJson = JsonSerializer.Serialize(new
         {
             model = "gpt-4o-mini",
             input = "Tell me a joke about programming.",
         });
 
-        var response = await httpClient.PostAsync(
-            "https://api.openai.com/v1/responses/input_tokens",
-            new StringContent(requestJson, Encoding.UTF8, "application/json"));
+        var serverTokens = await GetServerTokenCount(httpClient, requestJson);
 
-        response.EnsureSuccessStatusCode();
-        var responseBody = await response.Content.ReadAsStringAsync();
-        using var doc = JsonDocument.Parse(responseBody);
-        var serverTokens = doc.RootElement.GetProperty("input_tokens").GetInt32();
-
-        // Local count for the same text
         var encoder = ModelToEncoder.For("gpt-4o-mini");
         var localTokens = encoder.CountTokens("Tell me a joke about programming.");
 
         // The server count includes system prompt overhead, so server >= local
-        // We just verify our local count is reasonable (within the server count)
         localTokens.Should().BeGreaterThan(0);
         serverTokens.Should().BeGreaterThanOrEqualTo(localTokens,
             $"Server returned {serverTokens} tokens, local counted {localTokens}");
@@ -62,8 +70,7 @@ public async Task ValidateMessageTokensWithToolsAgainstOpenAiApi()
             return;
         }
 
-        using var httpClient = new HttpClient();
-        httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
+        using var httpClient = CreateOpenAiClient(apiKey);
 
         var requestJson = """
         {
@@ -95,16 +102,8 @@ public async Task ValidateMessageTokensWithToolsAgainstOpenAiApi()
         }
         """;
 
-        var response = await httpClient.PostAsync(
-            "https://api.openai.com/v1/responses/input_tokens",
-            new StringContent(requestJson, Encoding.UTF8, "application/json"));
+        var serverTokens = await GetServerTokenCount(httpClient, requestJson);
 
-        response.EnsureSuccessStatusCode();
-        var responseBody = await response.Content.ReadAsStringAsync();
-        using var doc = JsonDocument.Parse(responseBody);
-        var serverTokens = doc.RootElement.GetProperty("input_tokens").GetInt32();
-
-        // Local count
         var encoder = ModelToEncoder.For("gpt-4o-mini");
         var messages = new List<ChatMessage>
         {
@@ -121,12 +120,95 @@ public async Task ValidateMessageTokensWithToolsAgainstOpenAiApi()
 
         var localTokens = encoder.CountMessageTokens(messages, tools);
 
-        // Log both values for debugging
         Console.WriteLine($"Server tokens: {serverTokens}, Local tokens: {localTokens}, Diff: {serverTokens - localTokens}");
 
-        // Allow reasonable tolerance (±20%) since the formula is reverse-engineered
+        // Allow ±20% tolerance since the formula is reverse-engineered
         var tolerance = Math.Max(serverTokens * 0.2, 5);
         localTokens.Should().BeCloseTo(serverTokens, (uint)tolerance,
             $"Local estimate ({localTokens}) should be close to server count ({serverTokens})");
     }
+
+    [TestMethod]
+    public async Task ValidateMessagesOnlyAgainstOpenAiApi()
+    {
+        var apiKey = GetOpenAiApiKey();
+        if (string.IsNullOrEmpty(apiKey))
+        {
+            Assert.Inconclusive("OPENAI_API_KEY not set — skipping integration test.");
+            return;
+        }
+
+        using var httpClient = CreateOpenAiClient(apiKey);
+
+        // Messages-only test (no tools) to validate message counting
+        var requestJson = """
+        {
+            "model": "gpt-4o-mini",
+            "input": [
+                {"role": "developer", "content": "You are a helpful assistant."},
+                {"role": "user", "content": "hello world"}
+            ]
+        }
+        """;
+
+        var serverTokens = await GetServerTokenCount(httpClient, requestJson);
+
+        var encoder = ModelToEncoder.For("gpt-4o-mini");
+        var messages = new List<ChatMessage>
+        {
+            new("developer", "You are a helpful assistant."),
+            new("user", "hello world"),
+        };
+
+        var localTokens = encoder.CountMessageTokens(messages);
+
+        Console.WriteLine($"Messages-only: Server={serverTokens}, Local={localTokens}, Diff={serverTokens - localTokens}");
+
+        // Messages-only should be very close
+        localTokens.Should().BeCloseTo(serverTokens, 3,
+            $"Local ({localTokens}) should be very close to server ({serverTokens}) for messages-only");
+    }
+
+    [TestMethod]
+    public async Task ValidateToolsOnlyAgainstOpenAiApi()
+    {
+        var apiKey = GetOpenAiApiKey();
+        if (string.IsNullOrEmpty(apiKey))
+        {
+            Assert.Inconclusive("OPENAI_API_KEY not set — skipping integration test.");
+            return;
+        }
+
+        using var httpClient = CreateOpenAiClient(apiKey);
+
+        // Test no-params tool
+        var requestJson = """
+        {
+            "model": "gpt-4o-mini",
+            "input": [{"role": "user", "content": "hi"}],
+            "tools": [
+                {
+                    "type": "function",
+                    "name": "get_time",
+                    "description": "Get the current time"
+                }
+            ]
+        }
+        """;
+
+        var serverTokens = await GetServerTokenCount(httpClient, requestJson);
+
+        var encoder = ModelToEncoder.For("gpt-4o-mini");
+        var messages = new List<ChatMessage> { new("user", "hi") };
+        var tools = new List<ChatFunction> { new("get_time", "Get the current time") };
+
+        var localTokens = encoder.CountMessageTokens(messages, tools);
+        var toolTokensOnly = encoder.CountToolTokens(tools);
+
+        Console.WriteLine($"No-params tool: Server={serverTokens}, Local={localTokens}, ToolTokens={toolTokensOnly}, Diff={serverTokens - localTokens}");
+
+        var tolerance = Math.Max(serverTokens * 0.2, 5);
+        localTokens.Should().BeCloseTo(serverTokens, (uint)tolerance,
+            $"Local ({localTokens}) should be close to server ({serverTokens})");
+    }
 }
diff --git a/src/tests/Tiktoken.UnitTests/Tests.cs b/src/tests/Tiktoken.UnitTests/Tests.cs
@@ -633,4 +633,35 @@ public void CountToolTokensArrayType()
 
         count.Should().BeGreaterThan(19); // funcInit + funcEnd + propInit + propKey + tokens
     }
+
+    [TestMethod]
+    public void CountToolTokensAnyOf()
+    {
+        var encoder = ModelToEncoder.For("gpt-4o");
+        var tools = new List<ChatFunction>
+        {
+            new("update_field", "Update a field value", new List<FunctionParameter>
+            {
+                new("value", "", "The new value", isRequired: true,
+                    anyOf: new[] { "string", "number", "boolean" }),
+            }),
+        };
+
+        var count = encoder.CountToolTokens(tools);
+
+        // Should tokenize the type as "string | number | boolean"
+        count.Should().BeGreaterThan(19);
+
+        // Compare with a simple string type — anyOf should produce more tokens
+        var simpleTools = new List<ChatFunction>
+        {
+            new("update_field", "Update a field value", new List<FunctionParameter>
+            {
+                new("value", "string", "The new value", isRequired: true),
+            }),
+        };
+
+        var simpleCount = encoder.CountToolTokens(simpleTools);
+        count.Should().BeGreaterThan(simpleCount);
+    }
 }
diff --git a/src/tests/Tiktoken.UnitTests/Tiktoken.UnitTests.csproj b/src/tests/Tiktoken.UnitTests/Tiktoken.UnitTests.csproj
@@ -10,7 +10,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" />
+    <PackageReference Include="AwesomeAssertions" />
     <PackageReference Include="GitHubActionsTestLogger">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
