Skip to content

Commit 3ad3952

Browse files
kzukzu
committed
Add tool-focused extension to inspect calls and results
It's sometimes useful to be able to inspect (wether in tests or in production) the invocations that were performed by the model, including typed results. The introduced `ToolJsonOptions` provide a mechanism to automatically inject a `$type` for result types so they can be later inspected by the `FindCall<TResult>` extension for `ChatResponse`. To simplify this scenario, we also provide a `ToolFactory` that automatically sets things up for this scenario, additionally making the tool name default to the naming convention for tools, rather than the .NET method name.
1 parent e42de34 commit 3ad3952

File tree

5 files changed

+280
-0
lines changed

5 files changed

+280
-0
lines changed

‎src/AI.Tests/ToolsTests.cs‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
using System.ComponentModel;
2+
using Microsoft.Extensions.AI;
3+
using static ConfigurationExtensions;
4+
5+
namespace Devlooped.Extensions.AI;
6+
7+
public class ToolsTests(ITestOutputHelper output)
8+
{
9+
public record ToolResult(string Name, string Description, string Content);
10+
11+
[SecretsFact("OPENAI_API_KEY")]
12+
public async Task RunToolResult()
13+
{
14+
var chat = new Chat()
15+
{
16+
{ "system", "You make up a tool run by making up a name, description and content based on whatever the user says." },
17+
{ "user", "I want to create an order for a dozen eggs" },
18+
};
19+
20+
var client = new OpenAIChatClient(Configuration["OPENAI_API_KEY"]!, "gpt-4.1",
21+
OpenAI.OpenAIClientOptions.WriteTo(output))
22+
.AsBuilder()
23+
.UseFunctionInvocation()
24+
.Build();
25+
26+
var tool = ToolFactory.Create(RunTool);
27+
var options = new ChatOptions
28+
{
29+
ToolMode = ChatToolMode.RequireSpecific(tool.Name),
30+
Tools = [tool]
31+
};
32+
33+
var response = await client.GetResponseAsync(chat, options);
34+
35+
var result = response.FindCall<ToolResult>(tool);
36+
37+
Assert.NotNull(result);
38+
Assert.NotNull(result.Call);
39+
Assert.Equal(tool.Name, result.Call.Name);
40+
Assert.NotNull(result.Outcome);
41+
Assert.Null(result.Outcome.Exception);
42+
}
43+
44+
[SecretsFact("OPENAI_API_KEY")]
45+
public async Task RunToolTerminateResult()
46+
{
47+
var chat = new Chat()
48+
{
49+
{ "system", "You make up a tool run by making up a name, description and content based on whatever the user says." },
50+
{ "user", "I want to create an order for a dozen eggs" },
51+
};
52+
53+
var client = new OpenAIChatClient(Configuration["OPENAI_API_KEY"]!, "gpt-4.1",
54+
OpenAI.OpenAIClientOptions.WriteTo(output))
55+
.AsBuilder()
56+
.UseFunctionInvocation()
57+
.Build();
58+
59+
var tool = ToolFactory.Create(RunToolTerminate);
60+
var options = new ChatOptions
61+
{
62+
ToolMode = ChatToolMode.RequireSpecific(tool.Name),
63+
Tools = [tool]
64+
};
65+
66+
var response = await client.GetResponseAsync(chat, options);
67+
68+
var result = response.FindCall<ToolResult>(tool);
69+
70+
Assert.NotNull(result);
71+
Assert.NotNull(result.Call);
72+
Assert.Equal(tool.Name, result.Call.Name);
73+
Assert.NotNull(result.Outcome);
74+
Assert.Null(result.Outcome.Exception);
75+
}
76+
77+
[Description("Runs a tool to provide a result based on user input.")]
78+
ToolResult RunTool(
79+
[Description("The name")] string name,
80+
[Description("The description")] string description,
81+
[Description("The content")] string content)
82+
{
83+
// Simulate running a tool and returning a result
84+
return new ToolResult(name, description, content);
85+
}
86+
87+
[Description("Runs a tool to provide a result based on user input.")]
88+
ToolResult RunToolTerminate(
89+
[Description("The name")] string name,
90+
[Description("The description")] string description,
91+
[Description("The content")] string content)
92+
{
93+
FunctionInvokingChatClient.CurrentContext?.Terminate = true;
94+
// Simulate running a tool and returning a result
95+
return new ToolResult(name, description, content);
96+
}
97+
}

‎src/AI/ToolExtensions.cs‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
using System.Text.Json;
2+
using Microsoft.Extensions.AI;
3+
4+
namespace Devlooped.Extensions.AI;
5+
6+
/// <summary>
7+
/// Represents a tool call made by the AI, including the function call content and the result of the function execution.
8+
/// </summary>
9+
public record ToolCall(FunctionCallContent Call, FunctionResultContent Outcome);
10+
11+
/// <summary>
12+
/// Represents a tool call made by the AI, including the function call content, the result of the function execution,
13+
/// and the deserialized result of type <typeparamref name="TResult"/>.
14+
/// </summary>
15+
public record ToolCall<TResult>(FunctionCallContent Call, FunctionResultContent Outcome, TResult Result) where TResult : notnull;
16+
17+
/// <summary>
18+
/// Extensions for inspecting chat messages and responses for tool
19+
/// usage and processing responses.
20+
/// </summary>
21+
public static class ToolExtensions
22+
{
23+
/// <summary>
24+
/// Looks for a user prompt in the chat response messages.
25+
/// </summary>
26+
/// <remarks>
27+
/// In order for this to work, the <see cref="AIFunctionFactory"/> must have been invoked using
28+
/// the <see cref="ToolJsonOptions.Default"/> or with a <see cref="JsonSerializerOptions"/> configured
29+
/// with <see cref="TypeInjectingResolverExtensions.WithTypeInjection(JsonSerializerOptions)"/> so
30+
/// that the tool result type can be properly inspected.
31+
/// </remarks>
32+
public static ToolCall<TResult>? FindCall<TResult>(this ChatResponse response, AIFunction tool) where TResult : notnull
33+
=> FindCall<TResult>(response.Messages, tool.Name);
34+
35+
/// <summary>
36+
/// Looks for a user prompt in the chat response messages.
37+
/// </summary>
38+
/// <remarks>
39+
/// In order for this to work, the <see cref="AIFunctionFactory"/> must have been invoked using
40+
/// the <see cref="ToolJsonOptions.Default"/> or with a <see cref="JsonSerializerOptions"/> configured
41+
/// with <see cref="TypeInjectingResolverExtensions.WithTypeInjection(JsonSerializerOptions)"/> so
42+
/// that the tool result type can be properly inspected.
43+
/// </remarks>
44+
public static ToolCall<TResult>? FindCall<TResult>(this IEnumerable<ChatMessage> messages, AIFunction tool) where TResult : notnull
45+
=> FindCall<TResult>(messages, tool.Name);
46+
47+
/// <summary>
48+
/// Looks for a user prompt in the chat response messages.
49+
/// </summary>
50+
/// <remarks>
51+
/// In order for this to work, the <see cref="AIFunctionFactory"/> must have been invoked using
52+
/// the <see cref="ToolJsonOptions.Default"/> or with a <see cref="JsonSerializerOptions"/> configured
53+
/// with <see cref="TypeInjectingResolverExtensions.WithTypeInjection(JsonSerializerOptions)"/> so
54+
/// that the tool result type can be properly inspected.
55+
/// </remarks>
56+
public static ToolCall<TResult>? FindCall<TResult>(this IEnumerable<ChatMessage> messages, string tool) where TResult : notnull
57+
{
58+
var calls = messages
59+
.Where(x => x.Role == ChatRole.Assistant)
60+
.SelectMany(x => x.Contents)
61+
.OfType<FunctionCallContent>()
62+
.Where(x => x.Name == tool)
63+
.ToDictionary(x => x.CallId);
64+
65+
var results = messages
66+
.Where(x => x.Role == ChatRole.Tool)
67+
.SelectMany(x => x.Contents)
68+
.OfType<FunctionResultContent>()
69+
.Where(x => x.Result is JsonElement element &&
70+
element.ValueKind == JsonValueKind.Object &&
71+
element.TryGetProperty("$type", out var type) &&
72+
type.GetString() == typeof(TResult).FullName)
73+
.Where(x => calls.TryGetValue(x.CallId, out var call) && call.Name == tool)
74+
.Select(x => new ToolCall<TResult>(
75+
Call: calls[x.CallId],
76+
Outcome: x,
77+
Result: JsonSerializer.Deserialize<TResult>((JsonElement)x.Result!, ToolJsonOptions.Default) ??
78+
throw new InvalidOperationException($"Failed to deserialize result for tool '{tool}' to {typeof(TResult).FullName}.")));
79+
80+
return results.FirstOrDefault();
81+
}
82+
}

‎src/AI/ToolFactory.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
using Microsoft.Extensions.AI;
2+
3+
namespace Devlooped.Extensions.AI;
4+
5+
/// <summary>
6+
/// Creates tools for function calling that can leverage the <see cref="ToolExtensions"/>
7+
/// extension methods for locating invocations and their results.
8+
/// </summary>
9+
public static class ToolFactory
10+
{
11+
/// <summary>
12+
/// Invokes <see cref="AIFunctionFactory.Create(Delegate, string?, string?, System.Text.Json.JsonSerializerOptions?)"/>
13+
/// using the method name following the naming convention and serialization options from <see cref="ToolJsonOptions.Default"/>.
14+
/// </summary>
15+
public static AIFunction Create(Delegate method)
16+
=> AIFunctionFactory.Create(method,
17+
ToolJsonOptions.Default.PropertyNamingPolicy!.ConvertName(method.Method.Name),
18+
serializerOptions: ToolJsonOptions.Default);
19+
}

‎src/AI/ToolJsonOptions.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
using System.Diagnostics;
2+
using System.Text.Json;
3+
using System.Text.Json.Serialization;
4+
using System.Text.Json.Serialization.Metadata;
5+
6+
namespace Devlooped.Extensions.AI;
7+
8+
/// <summary>
9+
/// Provides a <see cref="JsonSerializerOptions"/> optimized for use with
10+
/// function calling and tools.
11+
/// </summary>
12+
public static class ToolJsonOptions
13+
{
14+
static ToolJsonOptions() => Default.MakeReadOnly();
15+
16+
/// <summary>
17+
/// Default <see cref="JsonSerializerOptions"/> for function calling and tools.
18+
/// </summary>
19+
public static JsonSerializerOptions Default { get; } = new(JsonSerializerDefaults.Web)
20+
{
21+
Converters =
22+
{
23+
new AdditionalPropertiesDictionaryConverter(),
24+
new JsonStringEnumConverter(),
25+
},
26+
DefaultIgnoreCondition =
27+
JsonIgnoreCondition.WhenWritingDefault |
28+
JsonIgnoreCondition.WhenWritingNull,
29+
Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
30+
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
31+
WriteIndented = Debugger.IsAttached,
32+
TypeInfoResolver = new TypeInjectingResolver(new DefaultJsonTypeInfoResolver())
33+
};
34+
}

‎src/AI/TypeInjectingResolver.cs‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
using System.ComponentModel;
2+
using System.Text.Json;
3+
using System.Text.Json.Serialization.Metadata;
4+
5+
namespace Devlooped.Extensions.AI;
6+
7+
/// <summary>
8+
/// Extensions for <see cref="JsonSerializerOptions"/> to enable type injection for object types.
9+
/// </summary>
10+
[EditorBrowsable(EditorBrowsableState.Never)]
11+
public static class TypeInjectingResolverExtensions
12+
{
13+
/// <summary>
14+
/// Creates a new <see cref="TypeInjectingResolver"/> that injects a $type property into object types.
15+
/// </summary>
16+
public static JsonSerializerOptions WithTypeInjection(this JsonSerializerOptions options)
17+
{
18+
if (options.IsReadOnly)
19+
options = new(options);
20+
21+
options.TypeInfoResolver = new TypeInjectingResolver(
22+
JsonTypeInfoResolver.Combine([.. options.TypeInfoResolverChain]));
23+
24+
return options;
25+
}
26+
}
27+
28+
/// <summary>
29+
/// A custom <see cref="IJsonTypeInfoResolver"/> that injects a $type property into object types
30+
/// so they can be automatically distinguished during deserialization or inspection.
31+
/// </summary>
32+
public class TypeInjectingResolver(IJsonTypeInfoResolver inner) : IJsonTypeInfoResolver
33+
{
34+
/// <inheritdoc />
35+
public JsonTypeInfo? GetTypeInfo(Type type, JsonSerializerOptions options)
36+
{
37+
var info = inner.GetTypeInfo(type, options);
38+
// The $type would already be present for polymorphic serialization.
39+
if (info?.Kind == JsonTypeInfoKind.Object && !info.Properties.Any(x => x.Name == "$type"))
40+
{
41+
var prop = info.CreateJsonPropertyInfo(typeof(string), "$type");
42+
prop.Get = obj => obj.GetType().FullName;
43+
prop.Order = -1000; // Ensure it is serialized first
44+
info.Properties.Add(prop);
45+
}
46+
return info;
47+
}
48+
}

0 commit comments

Comments
 (0)