tokenize: accept chatrequest, responsesrequest

Author: lofcz

src/LlmTornado.Demo/ChatDemo.cs

@@ -959,6 +959,71 @@ await chat.StreamResponseRich(new ChatStreamEventHandler
         });
     }
 
+    [TornadoTest]
+    public static async Task AudioInGptAudioMp3()
+    {
+        Conversation chat = Program.Connect().Chat.CreateConversation(new ChatRequest
+        {
+            Model = ChatModel.OpenAi.Gpt5.Audio,
+            Modalities = [ ChatModelModalities.Text ],
+            MaxTokens = 2000
+        });
+
+        byte[] audioData = await File.ReadAllBytesAsync("Static/Audio/sample.mp3");
+        
+        chat.AppendUserInput([
+            new ChatMessagePart("What is being said in this audio?"),
+            new ChatMessagePart(audioData, ChatAudioFormats.Mp3)
+        ]);
+        
+        string? str = await chat.GetResponse();
+        Console.WriteLine(str);
+    }
+    
+    [TornadoTest]
+    public static async Task AudioInGptAudioWav()
+    {
+        Conversation chat = Program.Connect().Chat.CreateConversation(new ChatRequest
+        {
+            Model = ChatModel.OpenAi.Gpt5.Audio,
+            Modalities = [ ChatModelModalities.Text ],
+            MaxTokens = 2000
+        });
+
+        byte[] audioData = await File.ReadAllBytesAsync("Static/Audio/sample.wav");
+        
+        chat.AppendUserInput([
+            new ChatMessagePart(audioData, ChatAudioFormats.Wav)
+        ]);
+        
+        string? str = await chat.GetResponse();
+        Console.WriteLine(str);
+    }
+    
+    [TornadoTest]
+    public static async Task AudioInGptAudioDirect()
+    {
+        TornadoApi api = Program.Connect();
+        
+        byte[] audioData = await File.ReadAllBytesAsync("Static/Audio/sample.mp3");
+        
+        List<ChatMessagePart> parts =
+        [
+            new ChatMessagePart("What is being said in this audio?"),
+            new ChatMessagePart(audioData, ChatAudioFormats.Mp3)
+        ];
+
+        ChatResult? result = await api.Chat.CreateChatCompletion(new ChatRequest
+        {
+            Model = ChatModel.OpenAi.Gpt5.Audio,
+            Modalities = [ ChatModelModalities.Text ],
+            MaxTokens = 2000,
+            Messages = [new ChatMessage(ChatMessageRoles.User, parts)]
+        });
+        
+        Console.WriteLine(result?.Choices?.FirstOrDefault()?.Message?.Content);
+    }
+    
     [TornadoTest]
     public static async Task AudioInAudioOutMultiturn()
     {

src/LlmTornado.Demo/TokenizeDemo.cs

@@ -7,6 +7,7 @@
 using LlmTornado.ChatFunctions;
 using LlmTornado.Code;
 using LlmTornado.Common;
+using LlmTornado.Responses;
 using LlmTornado.Tokenize;
 using LlmTornado.Tokenize.Vendors;
 
@@ -165,5 +166,129 @@ public static async Task TokenizeCohereLongText()
 
         Console.WriteLine($"Cohere long text tokenization: {result.TotalTokens} tokens");
     }
+    
+    [TornadoTest]
+    public static async Task TokenizeOpenAiText()
+    {
+        TokenizeRequest request = new TokenizeRequest(ChatModel.OpenAi.Gpt5.V5Mini, "Hello, world! This is a test message.");
+        TokenizeResult? result = await Program.ConnectMulti().Tokenize.CountTokens(request);
+        
+        Assert.That(result, Is.NotNull);
+        Assert.That(result!.TotalTokens, Is.GreaterThan(0));
+        
+        Console.WriteLine($"OpenAI text tokenization: {result.TotalTokens} tokens");
+    }
+    
+    [TornadoTest]
+    public static async Task TokenizeOpenAiMessages()
+    {
+        TokenizeRequest request = new TokenizeRequest(ChatModel.OpenAi.Gpt5.V5Mini, [
+            new ChatMessage(ChatMessageRoles.System, "You are a helpful assistant."),
+            new ChatMessage(ChatMessageRoles.User, "Hello, how are you?"),
+            new ChatMessage(ChatMessageRoles.Assistant, "I'm doing well, thank you!")
+        ]);
+        
+        TokenizeResult? result = await Program.ConnectMulti().Tokenize.CountTokens(request);
+        
+        Assert.That(result, Is.NotNull);
+        Assert.That(result!.TotalTokens, Is.GreaterThan(0));
+        
+        Console.WriteLine($"OpenAI messages tokenization: {result.TotalTokens} tokens");
+    }
+    
+    [TornadoTest]
+    public static async Task TokenizeOpenAiWithTools()
+    {
+        List<Tool> tools =
+        [
+            new Tool(new ToolFunction("get_weather", "Get the weather for a location", new
+            {
+                type = "object",
+                properties = new
+                {
+                    location = new
+                    {
+                        type = "string",
+                        description = "The city and state, e.g. San Francisco, CA"
+                    }
+                },
+                required = new[] { "location" }
+            }))
+        ];
+        
+        TokenizeRequest request = new TokenizeRequest(
+            ChatModel.OpenAi.Gpt5.V5Mini,
+            [
+                new ChatMessage(ChatMessageRoles.System, "You are a helpful assistant."),
+                new ChatMessage(ChatMessageRoles.User, "What's the weather in San Francisco?")
+            ],
+            tools
+        );
+        
+        TokenizeResult? result = await Program.ConnectMulti().Tokenize.CountTokens(request);
+        
+        Assert.That(result, Is.NotNull);
+        Assert.That(result!.TotalTokens, Is.GreaterThan(0));
+        
+        Console.WriteLine($"OpenAI with tools tokenization: {result.TotalTokens} tokens");
+    }
+    
+    [TornadoTest]
+    [TornadoTestCase("gpt-5-mini")]
+    [TornadoTestCase("claude-sonnet-4-20250514")]
+    [TornadoTestCase("gemini-2.5-flash")]
+    public static async Task TokenizeChatRequestWithTools(string model)
+    {
+        ChatRequest chatRequest = new ChatRequest
+        {
+            Model = model,
+            Messages = [
+                new ChatMessage(ChatMessageRoles.System, "You are a helpful assistant with access to tools."),
+                new ChatMessage(ChatMessageRoles.User, "What's the weather in San Francisco and New York?")
+            ],
+            Tools = [
+                new Tool(new ToolFunction("get_weather", "Get the weather for a location", new
+                {
+                    type = "object",
+                    properties = new
+                    {
+                        location = new
+                        {
+                            type = "string",
+                            description = "The city and state, e.g. San Francisco, CA"
+                        }
+                    },
+                    required = new[] { "location" }
+                }))
+            ]
+        };
+        
+        TokenizeRequest request = new TokenizeRequest(chatRequest);
+        TokenizeResult? result = await Program.ConnectMulti().Tokenize.CountTokens(request);
+        
+        Assert.That(result, Is.NotNull);
+        Assert.That(result!.TotalTokens, Is.GreaterThan(0));
+        
+        Console.WriteLine($"[{model}] ChatRequest with tools tokenization: {result.TotalTokens} tokens");
+    }
+    
+    [TornadoTest]
+    public static async Task TokenizeOpenAiFromResponseRequest()
+    {
+        ResponseRequest responseRequest = new ResponseRequest
+        {
+            Model = ChatModel.OpenAi.Gpt5.V5Mini,
+            Instructions = "You are a helpful assistant.",
+            InputString = "What is the meaning of life?"
+        };
+        
+        TokenizeRequest request = new TokenizeRequest(responseRequest);
+        TokenizeResult? result = await Program.ConnectMulti().Tokenize.CountTokens(request);
+        
+        Assert.That(result, Is.NotNull);
+        Assert.That(result!.TotalTokens, Is.GreaterThan(0));
+        
+        Console.WriteLine($"OpenAI from ResponseRequest tokenization: {result.TotalTokens} tokens");
+    }
 }
 

src/LlmTornado/Chat/Models/OpenAi/ChatModelOpenAiGpt5.cs

@@ -132,7 +132,7 @@ public class ChatModelOpenAiGpt5 : IVendorModelClassProvider
     public static List<IModel> ModelsAll => LazyModelsAll.Value;
 
     private static readonly Lazy<List<IModel>> LazyModelsAll = new Lazy<List<IModel>>(() => [
-        ModelV5, ModelV5Mini, ModelV5Nano, ModelV5Codex, ModelV5Pro, ModelAudioMini, ModelAudio15, ModelRealtimeMini
+        ModelV5, ModelV5Mini, ModelV5Nano, ModelV5Codex, ModelV5Pro, ModelAudio, ModelAudioMini, ModelAudio15, ModelRealtimeMini
     ]);
 
     /// <summary>

src/LlmTornado/Responses/ResponseHelpers.cs

@@ -556,7 +556,7 @@ public static ChatChoice ToChatChoice(ResponseResult response, ResponseRequest r
         return null;
     }
 
-    private static List<ResponseTool> ConvertTools(List<Tool> tools)
+    internal static List<ResponseTool> ConvertTools(List<Tool> tools)
     {
         List<ResponseTool> result = new List<ResponseTool>(tools.Count);
 

src/LlmTornado/Responses/ResponseRequest.cs

@@ -290,4 +290,53 @@ public ResponseRequest(ChatModel model, List<ResponseInputItem> inputItems)
     public ResponseRequest()
     {
     }
+
+    /// <summary>
+    /// Creates a shallow copy of the given request, optionally stripping generation-only fields.
+    /// </summary>
+    internal ResponseRequest(ResponseRequest basedOn, bool forTokenization = false)
+    {
+        Background = basedOn.Background;
+        Conversation = basedOn.Conversation;
+        Include = basedOn.Include;
+        InputString = basedOn.InputString;
+        InputItems = basedOn.InputItems;
+        Instructions = basedOn.Instructions;
+        MaxOutputTokens = basedOn.MaxOutputTokens;
+        MaxToolCalls = basedOn.MaxToolCalls;
+        Metadata = basedOn.Metadata;
+        Model = basedOn.Model;
+        ParallelToolCalls = basedOn.ParallelToolCalls;
+        PreviousResponseId = basedOn.PreviousResponseId;
+        Prompt = basedOn.Prompt;
+        Reasoning = basedOn.Reasoning;
+        ServiceTier = basedOn.ServiceTier;
+        Store = basedOn.Store;
+        Temperature = basedOn.Temperature;
+        Text = basedOn.Text;
+        ToolChoice = basedOn.ToolChoice;
+        Tools = basedOn.Tools;
+        TopLogprobs = basedOn.TopLogprobs;
+        TopP = basedOn.TopP;
+        Truncation = basedOn.Truncation;
+        PromptCacheKey = basedOn.PromptCacheKey;
+        PromptCacheRetention = basedOn.PromptCacheRetention;
+        SafetyIdentifier = basedOn.SafetyIdentifier;
+        User = basedOn.User;
+        Verbosity = basedOn.Verbosity;
+        ContextManagement = basedOn.ContextManagement;
+
+        if (forTokenization)
+        {
+            Stream = null;
+            StreamOptions = null;
+            Store = null;
+            Background = null;
+        }
+        else
+        {
+            Stream = basedOn.Stream;
+            StreamOptions = basedOn.StreamOptions;
+        }
+    }
 }