Skip to content

Commit 546f019

Browse files
committed
Add simple benchmark test for GPT-5 reasoning efforts
1 parent 5f0fec8 commit 546f019

File tree

1 file changed

+48
-0
lines changed

1 file changed

+48
-0
lines changed

src/AI.Tests/OpenAITests.cs

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,54 @@ public async Task GPT5_ThinksFast()
119119
Assert.Equal("minimal", search["effort"]?.GetValue<string>());
120120
});
121121
}
122+
123+
[SecretsTheory("OPENAI_API_KEY")]
124+
[InlineData(ReasoningEffort.Minimal)]
125+
[InlineData(ReasoningEffort.Low)]
126+
[InlineData(ReasoningEffort.Medium)]
127+
[InlineData(ReasoningEffort.High)]
128+
public async Task GPT5_ThinkingTime(ReasoningEffort effort)
129+
{
130+
var messages = new Chat()
131+
{
132+
{ "system", "You are an intelligent AI assistant that's an expert on financial matters." },
133+
{ "user", "If you have a debt of 100k and accumulate a compounding 5% debt on top of it every year, how long before you are a negative millonaire? (round up to full integer value)" },
134+
};
135+
136+
var requests = new List<JsonNode>();
137+
138+
var chat = new OpenAIChatClient(Configuration["OPENAI_API_KEY"]!, "gpt-5-nano",
139+
OpenAIClientOptions.Observable(requests.Add).WriteTo(output));
140+
141+
var options = new ChatOptions
142+
{
143+
ModelId = "gpt-5",
144+
ReasoningEffort = effort
145+
};
146+
147+
var watch = System.Diagnostics.Stopwatch.StartNew();
148+
var response = await chat.GetResponseAsync(messages, options);
149+
watch.Stop();
150+
151+
var text = response.Text;
152+
153+
Assert.Contains("48 years", text);
154+
// NOTE: the chat client was requested as grok-3 but the chat options wanted a
155+
// different model and the grok client honors that choice.
156+
Assert.StartsWith("gpt-5", response.ModelId);
157+
Assert.DoesNotContain("nano", response.ModelId);
158+
159+
// Reasoning should have been set to medium
160+
Assert.All(requests, x =>
161+
{
162+
var search = Assert.IsType<JsonObject>(x["reasoning"]);
163+
Assert.Equal(effort.ToString().ToLowerInvariant(), search["effort"]?.GetValue<string>());
164+
});
165+
166+
output.WriteLine($"Effort: {effort}, Time: {watch.ElapsedMilliseconds}ms, Tokens: {response.Usage?.TotalTokenCount}");
167+
}
168+
169+
122170
[SecretsFact("OPENAI_API_KEY")]
123171
public async Task WebSearchCountryHighContext()
124172
{

0 commit comments

Comments
 (0)