Merge remote-tracking branch 'upstream/main' into feature/postgres-memory

Author: lossyrob

dotnet/samples/Concepts/Agents/OpenAIAssistant_Streaming.cs

@@ -11,23 +11,24 @@ namespace Agents;
 /// </summary>
 public class OpenAIAssistant_Streaming(ITestOutputHelper output) : BaseAgentsTest(output)
 {
-    private const string ParrotName = "Parrot";
-    private const string ParrotInstructions = "Repeat the user message in the voice of a pirate and then end with a parrot sound.";
-
     [Fact]
     public async Task UseStreamingAssistantAgentAsync()
     {
+        const string AgentName = "Parrot";
+        const string AgentInstructions = "Repeat the user message in the voice of a pirate and then end with a parrot sound.";
+
         // Define the agent
         OpenAIAssistantAgent agent =
-            await OpenAIAssistantAgent.CreateAsync(
-                kernel: new(),
-                clientProvider: this.GetClientProvider(),
-                definition: new OpenAIAssistantDefinition(this.Model)
-                {
-                    Instructions = ParrotInstructions,
-                    Name = ParrotName,
-                    Metadata = AssistantSampleMetadata,
-                });
+                await OpenAIAssistantAgent.CreateAsync(
+                    kernel: new(),
+                    clientProvider: this.GetClientProvider(),
+                    definition: new OpenAIAssistantDefinition(this.Model)
+                    {
+                        Instructions = AgentInstructions,
+                        Name = AgentName,
+                        EnableCodeInterpreter = true,
+                        Metadata = AssistantSampleMetadata,
+                    });
 
         // Create a thread for the agent conversation.
         string threadId = await agent.CreateThreadAsync(new OpenAIThreadCreationOptions { Metadata = AssistantSampleMetadata });
@@ -44,7 +45,8 @@ await OpenAIAssistantAgent.CreateAsync(
     [Fact]
     public async Task UseStreamingAssistantAgentWithPluginAsync()
     {
-        const string MenuInstructions = "Answer questions about the menu.";
+        const string AgentName = "Host";
+        const string AgentInstructions = "Answer questions about the menu.";
 
         // Define the agent
         OpenAIAssistantAgent agent =
@@ -53,8 +55,8 @@ await OpenAIAssistantAgent.CreateAsync(
                 clientProvider: this.GetClientProvider(),
                 definition: new OpenAIAssistantDefinition(this.Model)
                 {
-                    Instructions = MenuInstructions,
-                    Name = "Host",
+                    Instructions = AgentInstructions,
+                    Name = AgentName,
                     Metadata = AssistantSampleMetadata,
                 });
 
@@ -73,6 +75,36 @@ await OpenAIAssistantAgent.CreateAsync(
         await DisplayChatHistoryAsync(agent, threadId);
     }
 
+    [Fact]
+    public async Task UseStreamingAssistantWithCodeInterpreterAsync()
+    {
+        const string AgentName = "MathGuy";
+        const string AgentInstructions = "Solve math problems with code.";
+
+        // Define the agent
+        OpenAIAssistantAgent agent =
+            await OpenAIAssistantAgent.CreateAsync(
+                kernel: new(),
+                clientProvider: this.GetClientProvider(),
+                definition: new OpenAIAssistantDefinition(this.Model)
+                {
+                    Instructions = AgentInstructions,
+                    Name = AgentName,
+                    EnableCodeInterpreter = true,
+                    Metadata = AssistantSampleMetadata,
+                });
+
+        // Create a thread for the agent conversation.
+        string threadId = await agent.CreateThreadAsync(new OpenAIThreadCreationOptions { Metadata = AssistantSampleMetadata });
+
+        // Respond to user input
+        await InvokeAgentAsync(agent, threadId, "Is 191 a prime number?");
+        await InvokeAgentAsync(agent, threadId, "Determine the values in the Fibonacci sequence that that are less then the value of 101");
+
+        // Output the entire chat history
+        await DisplayChatHistoryAsync(agent, threadId);
+    }
+
     // Local function to invoke agent and display the conversation messages.
     private async Task InvokeAgentAsync(OpenAIAssistantAgent agent, string threadId, string input)
     {
@@ -83,13 +115,21 @@ private async Task InvokeAgentAsync(OpenAIAssistantAgent agent, string threadId,
         ChatHistory history = [];
 
         bool isFirst = false;
+        bool isCode = false;
         await foreach (StreamingChatMessageContent response in agent.InvokeStreamingAsync(threadId, messages: history))
         {
             if (string.IsNullOrEmpty(response.Content))
             {
                 continue;
             }
 
+            // Differentiate between assistant and tool messages
+            if (isCode != (response.Metadata?.ContainsKey(OpenAIAssistantAgent.CodeInterpreterMetadataKey) ?? false))
+            {
+                isFirst = false;
+                isCode = !isCode;
+            }
+
             if (!isFirst)
             {
                 Console.WriteLine($"\n# {response.Role} - {response.AuthorName ?? "*"}:");

dotnet/src/Agents/OpenAI/Internal/AssistantThreadActions.cs

@@ -406,6 +406,14 @@ public static async IAsyncEnumerable<StreamingChatMessageContent> InvokeStreamin
                             break;
                     }
                 }
+                else if (update is RunStepDetailsUpdate detailsUpdate)
+                {
+                    StreamingChatMessageContent? toolContent = GenerateStreamingCodeInterpreterContent(agent.GetName(), detailsUpdate);
+                    if (toolContent != null)
+                    {
+                        yield return toolContent;
+                    }
+                }
                 else if (update is RunStepUpdate stepUpdate)
                 {
                     switch (stepUpdate.UpdateKind)
@@ -416,6 +424,8 @@ public static async IAsyncEnumerable<StreamingChatMessageContent> InvokeStreamin
                         case StreamingUpdateReason.RunStepCompleted:
                             currentStep = null;
                             break;
+                        default:
+                            break;
                     }
                 }
             }
@@ -571,6 +581,35 @@ private static StreamingChatMessageContent GenerateStreamingMessageContent(strin
         return content;
     }
 
+    private static StreamingChatMessageContent? GenerateStreamingCodeInterpreterContent(string? assistantName, RunStepDetailsUpdate update)
+    {
+        StreamingChatMessageContent content =
+            new(AuthorRole.Assistant, content: null)
+            {
+                AuthorName = assistantName,
+            };
+
+        // Process text content
+        if (update.CodeInterpreterInput != null)
+        {
+            content.Items.Add(new StreamingTextContent(update.CodeInterpreterInput));
+            content.Metadata = new Dictionary<string, object?> { { OpenAIAssistantAgent.CodeInterpreterMetadataKey, true } };
+        }
+
+        if ((update.CodeInterpreterOutputs?.Count ?? 0) > 0)
+        {
+            foreach (var output in update.CodeInterpreterOutputs!)
+            {
+                if (output.ImageFileId != null)
+                {
+                    content.Items.Add(new StreamingFileReferenceContent(output.ImageFileId));
+                }
+            }
+        }
+
+        return content.Items.Count > 0 ? content : null;
+    }
+
     private static AnnotationContent GenerateAnnotationContent(TextAnnotation annotation)
     {
         string? fileId = null;