From f3eaf5108ee55cc898cf948f95dd9cf79d356128 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Fri, 7 Mar 2025 10:52:08 -0800 Subject: [PATCH 1/2] Update flight flags and make corresponding changes --- .../Microsoft.Azure.Agent/AzureAgent.cs | 36 +++++++++++++++---- .../Microsoft.Azure.Agent/ChatSession.cs | 10 +++--- .../Microsoft.Azure.Agent/DataRetriever.cs | 13 +++++++ shell/agents/Microsoft.Azure.Agent/Schema.cs | 5 +-- 4 files changed, 51 insertions(+), 13 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index e70a0c30..b4150bea 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using System.Runtime.InteropServices; using System.Text; using AIShell.Abstraction; @@ -22,11 +23,11 @@ public sealed class AzureAgent : ILLMAgent private const string SettingFileName = "az.config.json"; private const string LoggingFileName = "log..txt"; private const string InstructionPrompt = """ - NOTE: follow the below instructions when generating responses that include Azure CLI commands with placeholders: - 1. User's OS is `{0}`. Make sure the generated commands are suitable for the specified OS. - 2. DO NOT include the command for creating a new resource group unless the query explicitly asks for it. Otherwise, assume a resource group already exists. - 3. DO NOT include an additional example with made-up values unless it provides additional context or value beyond the initial command. - 4. DO NOT use the line continuation operator (backslash `\` in Bash) in the generated commands. + NOTE: Follow the instructions below when generating Azure CLI or Azure PowerShell commands with placeholders: + 1. The targeting OS is '{0}'. + 2. Always assume the user has logged in Azure and a resource group already exists. + 3. DO NOT include any additional examples with made-up values. + 4. DO NOT use the line continuation operator (backslash `\`) in commands. 5. Always represent a placeholder in the form of `` and enclose it within double quotes. 6. Always use the consistent placeholder names across all your responses. For example, `` should be used for all the places where a resource group name value is needed. 7. When the commands contain placeholders, the placeholders should be summarized in markdown bullet points at the end of the response in the same order as they appear in the commands, following this format: @@ -56,7 +57,7 @@ public AzureAgent() _chatSession = new ChatSession(_httpClient); _valueStore = new Dictionary(StringComparer.OrdinalIgnoreCase); - _instructions = string.Format(InstructionPrompt, Environment.OSVersion.VersionString); + _instructions = string.Format(InstructionPrompt, RuntimeInformation.OSDescription); Name = "azure"; Company = "Microsoft"; @@ -326,6 +327,7 @@ public async Task ChatAsync(string input, IShell shell) } } + Log.Debug("[AzureAgent] TopicName: {0}", _copilotResponse.TopicName); Telemetry.Trace(AzTrace.Chat(_copilotResponse)); } catch (Exception ex) @@ -417,7 +419,7 @@ private ResponseData ParseCodeResponse(IShell shell) // - ``: const string pattern = "- `{0}`:"; int index = text.IndexOf(string.Format(pattern, first), begin); - if (index > 0 && text[index - 1] is '\n' && text[index - 2] is ':') + if (index > 0 && IsInPlaceholderSection(text, index)) { // Get the start index of the placeholder section. int n = index - 2; @@ -453,6 +455,26 @@ private ResponseData ParseCodeResponse(IShell shell) ReplaceKnownPlaceholders(data); return data; + + static bool IsInPlaceholderSection(string text, int index) + { + // This section should immediately follow "Placeholders:" on the next line. + // The "- ``" part mostly starts at the beginning of the new line, but + // sometimes starts after a few space characters. + int firstNonSpaceCharBackward = -1; + for (int i = index - 1; i >= 0; i--) + { + if (text[i] is not ' ') + { + firstNonSpaceCharBackward = i; + break; + } + } + + return firstNonSpaceCharBackward > 0 + && text[firstNonSpaceCharBackward] is '\n' + && text[firstNonSpaceCharBackward - 1] is ':'; + } } internal void ResetArgumentPlaceholder() diff --git a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs index e6702f93..be19aa07 100644 --- a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs +++ b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs @@ -36,7 +36,7 @@ internal ChatSession(HttpClient httpClient) _flights = new Dictionary() { ["openAIModel"] = "gpt4optum", - ["openAIEndpointName"] = "norwayeast,australiaeast,swedencentral", + ["openAIEndpointName"] = "australiaeast,norwayeast", ["docsHandlerEndpoint"] = "learnDocs", ["unifiedcopilotdebug"] = false, ["unifiedcopilottest"] = false, @@ -52,11 +52,10 @@ internal ChatSession(HttpClient httpClient) ["copilotmanageability"] = true, ["gpt4tcsprompt"] = true, ["copilotmanageabilityuimenu"] = true, - ["usenewchatinputcomponent"] = true, // not sure what this is for ["getformstate"] = true, - ["notificationcopilotbuttonallerror"] = false, ["chitchatprompt"] = true, ["azurepluginstore"] = true, + ["pipelineorchestration"] = true, // TODO: the streaming is slow and not sending chunks, very clumsy for now. // ["streamresponse"] = true, }; @@ -315,7 +314,10 @@ internal async Task GetChatResponseAsync(string input, IStatusC if (activity.IsTyping) { - context?.Status(activity.Text); + if (activity.TopicName is CopilotActivity.ProgressTopic) + { + context?.Status(activity.Text); + } continue; } diff --git a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs index 0809ca94..39b22a0a 100644 --- a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs +++ b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs @@ -885,6 +885,19 @@ private AzCLICommand QueryForMetadata(string azCommand) command = metadata.Deserialize(Utils.JsonOptions); } } + catch (TaskCanceledException) + { + Log.Error("[QueryForMetadata] Metadata query timed out (1200ms): {0}", azCommand); + if (Telemetry.Enabled) + { + Dictionary details = new() + { + ["Command"] = azCommand, + ["Message"] = "AzCLI metadata query timed out (1200ms)." + }; + Telemetry.Trace(AzTrace.Exception(details)); + } + } catch (Exception e) { Log.Error(e, "[QueryForMetadata] Exception while processing command: {0}", azCommand); diff --git a/shell/agents/Microsoft.Azure.Agent/Schema.cs b/shell/agents/Microsoft.Azure.Agent/Schema.cs index 9e33b436..072e6bec 100644 --- a/shell/agents/Microsoft.Azure.Agent/Schema.cs +++ b/shell/agents/Microsoft.Azure.Agent/Schema.cs @@ -120,8 +120,9 @@ internal class CopilotActivity { public const string ConversationStateName = "azurecopilot/conversationstate"; public const string SuggestedResponseName = "azurecopilot/suggesteduserresponses"; - public const string CLIHandlerTopic = "generate_azure_cli_scripts"; - public const string PSHandlerTopic = "generate_powershell_script"; + public const string CLIHandlerTopic = "generate_azure_cli_scripts,Orchestrator_Respond"; + public const string PSHandlerTopic = "generate_powershell_script,Orchestrator_Respond"; + public const string ProgressTopic = "InProgressActivity"; public string Type { get; set; } public string Id { get; set; } From 2183ffc1f3c92318e0b3545d97fdf29479c5bbd7 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Fri, 7 Mar 2025 14:12:54 -0800 Subject: [PATCH 2/2] Pass in the right OS information and fix the placeholder section detection --- .../Microsoft.Azure.Agent/AzureAgent.cs | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index b4150bea..8dc483ec 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -57,7 +57,9 @@ public AzureAgent() _chatSession = new ChatSession(_httpClient); _valueStore = new Dictionary(StringComparer.OrdinalIgnoreCase); - _instructions = string.Format(InstructionPrompt, RuntimeInformation.OSDescription); + _instructions = string.Format( + InstructionPrompt, + OperatingSystem.IsMacOS() ? $"Mac OS X {Environment.OSVersion.Version}" : RuntimeInformation.OSDescription); Name = "azure"; Company = "Microsoft"; @@ -419,13 +421,8 @@ private ResponseData ParseCodeResponse(IShell shell) // - ``: const string pattern = "- `{0}`:"; int index = text.IndexOf(string.Format(pattern, first), begin); - if (index > 0 && IsInPlaceholderSection(text, index)) + if (index > 0 && IsInPlaceholderSection(text, index, out begin)) { - // Get the start index of the placeholder section. - int n = index - 2; - for (; text[n] is not '\n'; n--); - begin = n + 1; - // For each placeholder, try to extract its description. foreach (var phItem in placeholders) { @@ -456,7 +453,7 @@ private ResponseData ParseCodeResponse(IShell shell) ReplaceKnownPlaceholders(data); return data; - static bool IsInPlaceholderSection(string text, int index) + static bool IsInPlaceholderSection(string text, int index, out int sectionStart) { // This section should immediately follow "Placeholders:" on the next line. // The "- ``" part mostly starts at the beginning of the new line, but @@ -471,9 +468,20 @@ static bool IsInPlaceholderSection(string text, int index) } } - return firstNonSpaceCharBackward > 0 + if (firstNonSpaceCharBackward > 0 && text[firstNonSpaceCharBackward] is '\n' - && text[firstNonSpaceCharBackward - 1] is ':'; + && text[firstNonSpaceCharBackward - 1] is ':') + { + // Get the start index of the placeholder section. + int n = firstNonSpaceCharBackward - 1; + for (; text[n] is not '\n'; n--); + + sectionStart = n + 1; + return true; + } + + sectionStart = -1; + return false; } }