Update flight flags and make corresponding changes (#355)

1. Add the flag `pipelineorchestration` per the Azure Copilot team's request.
2. Update flag values based on the values used by Azure Portal, and remove 2 flags that are no longer used by Azure Portal.
3. Update the topic names for AzCLI handler and AzPS handler. They changed after adopting the new flag values.
4. Simplify the instruction prompt to save words for the input. Also include Azure PowerShell in the instruction.
5. Use better OS description for the instruction prompt.
6. Log the topic name for every response, for diagnostic purpose. We already send this information in telemetry.
7. Fix a bug in response parsing in the validation of the beginning of placeholder section.
8. Fix the status report to only report `ProgressTopic` status, which are progress reports only (fix #352).
9. Special case `TaskCanceledException` in AzCLI metadata query for logging and telemetry collection. This exception simply means the query action timed out after 1200 ms. Given that the metadata endpoint is an Azure Function, this could easily happen when the query hits a cold start of the function app. So, we don't want to log the exception object here.

Author: daxian-dbw

shell/agents/Microsoft.Azure.Agent/AzureAgent.cs

@@ -1,4 +1,5 @@
 using System.Diagnostics;
+using System.Runtime.InteropServices;
 using System.Text;
 
 using AIShell.Abstraction;
@@ -22,11 +23,11 @@ public sealed class AzureAgent : ILLMAgent
     private const string SettingFileName = "az.config.json";
     private const string LoggingFileName = "log..txt";
     private const string InstructionPrompt = """
-        NOTE: follow the below instructions when generating responses that include Azure CLI commands with placeholders:
-        1. User's OS is `{0}`. Make sure the generated commands are suitable for the specified OS.
-        2. DO NOT include the command for creating a new resource group unless the query explicitly asks for it. Otherwise, assume a resource group already exists.
-        3. DO NOT include an additional example with made-up values unless it provides additional context or value beyond the initial command.
-        4. DO NOT use the line continuation operator (backslash `\` in Bash) in the generated commands.
+        NOTE: Follow the instructions below when generating Azure CLI or Azure PowerShell commands with placeholders:
+        1. The targeting OS is '{0}'.
+        2. Always assume the user has logged in Azure and a resource group already exists.
+        3. DO NOT include any additional examples with made-up values.
+        4. DO NOT use the line continuation operator (backslash `\`) in commands.
         5. Always represent a placeholder in the form of `<placeholder-name>` and enclose it within double quotes.
         6. Always use the consistent placeholder names across all your responses. For example, `<resourceGroupName>` should be used for all the places where a resource group name value is needed.
         7. When the commands contain placeholders, the placeholders should be summarized in markdown bullet points at the end of the response in the same order as they appear in the commands, following this format:
@@ -56,7 +57,9 @@ public AzureAgent()
 
         _chatSession = new ChatSession(_httpClient);
         _valueStore = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
-        _instructions = string.Format(InstructionPrompt, Environment.OSVersion.VersionString);
+        _instructions = string.Format(
+            InstructionPrompt,
+            OperatingSystem.IsMacOS() ? $"Mac OS X {Environment.OSVersion.Version}" : RuntimeInformation.OSDescription);
 
         Name = "azure";
         Company = "Microsoft";
@@ -326,6 +329,7 @@ public async Task<bool> ChatAsync(string input, IShell shell)
                 }
             }
 
+            Log.Debug("[AzureAgent] TopicName: {0}", _copilotResponse.TopicName);
             Telemetry.Trace(AzTrace.Chat(_copilotResponse));
         }
         catch (Exception ex)
@@ -417,13 +421,8 @@ private ResponseData ParseCodeResponse(IShell shell)
         //  - `<second-placeholder>`: <concise-description>
         const string pattern = "- `{0}`:";
         int index = text.IndexOf(string.Format(pattern, first), begin);
-        if (index > 0 && text[index - 1] is '\n' && text[index - 2] is ':')
+        if (index > 0 && IsInPlaceholderSection(text, index, out begin))
         {
-            // Get the start index of the placeholder section.
-            int n = index - 2;
-            for (; text[n] is not '\n'; n--);
-            begin = n + 1;
-
             // For each placeholder, try to extract its description.
             foreach (var phItem in placeholders)
             {
@@ -453,6 +452,37 @@ private ResponseData ParseCodeResponse(IShell shell)
 
         ReplaceKnownPlaceholders(data);
         return data;
+
+        static bool IsInPlaceholderSection(string text, int index, out int sectionStart)
+        {
+            // This section should immediately follow "Placeholders:" on the next line.
+            // The "- `<xxx>`" part mostly starts at the beginning of the new line, but
+            // sometimes starts after a few space characters.
+            int firstNonSpaceCharBackward = -1;
+            for (int i = index - 1; i >= 0; i--)
+            {
+                if (text[i] is not ' ')
+                {
+                    firstNonSpaceCharBackward = i;
+                    break;
+                }
+            }
+
+            if (firstNonSpaceCharBackward > 0
+                && text[firstNonSpaceCharBackward] is '\n'
+                && text[firstNonSpaceCharBackward - 1] is ':')
+            {
+                // Get the start index of the placeholder section.
+                int n = firstNonSpaceCharBackward - 1;
+                for (; text[n] is not '\n'; n--);
+
+                sectionStart = n + 1;
+                return true;
+            }
+
+            sectionStart = -1;
+            return false;
+        }
     }
 
     internal void ResetArgumentPlaceholder()

shell/agents/Microsoft.Azure.Agent/ChatSession.cs

@@ -36,7 +36,7 @@ internal ChatSession(HttpClient httpClient)
         _flights = new Dictionary<string, object>()
         {
             ["openAIModel"] = "gpt4optum",
-            ["openAIEndpointName"] = "norwayeast,australiaeast,swedencentral",
+            ["openAIEndpointName"] = "australiaeast,norwayeast",
             ["docsHandlerEndpoint"] = "learnDocs",
             ["unifiedcopilotdebug"] = false,
             ["unifiedcopilottest"] = false,
@@ -52,11 +52,10 @@ internal ChatSession(HttpClient httpClient)
             ["copilotmanageability"] = true,
             ["gpt4tcsprompt"] = true,
             ["copilotmanageabilityuimenu"] = true,
-            ["usenewchatinputcomponent"] = true, // not sure what this is for
             ["getformstate"] = true,
-            ["notificationcopilotbuttonallerror"] = false,
             ["chitchatprompt"] = true,
             ["azurepluginstore"] = true,
+            ["pipelineorchestration"] = true,
             // TODO: the streaming is slow and not sending chunks, very clumsy for now.
             // ["streamresponse"] = true,
         };
@@ -315,7 +314,10 @@ internal async Task<CopilotResponse> GetChatResponseAsync(string input, IStatusC
 
                 if (activity.IsTyping)
                 {
-                    context?.Status(activity.Text);
+                    if (activity.TopicName is CopilotActivity.ProgressTopic)
+                    {
+                        context?.Status(activity.Text);
+                    }
                     continue;
                 }
 

shell/agents/Microsoft.Azure.Agent/DataRetriever.cs

@@ -885,6 +885,19 @@ private AzCLICommand QueryForMetadata(string azCommand)
                 command = metadata.Deserialize<AzCLICommand>(Utils.JsonOptions);
             }
         }
+        catch (TaskCanceledException)
+        {
+            Log.Error("[QueryForMetadata] Metadata query timed out (1200ms): {0}", azCommand);
+            if (Telemetry.Enabled)
+            {
+                Dictionary<string, string> details = new()
+                {
+                    ["Command"] = azCommand,
+                    ["Message"] = "AzCLI metadata query timed out (1200ms)."
+                };
+                Telemetry.Trace(AzTrace.Exception(details));
+            }
+        }
         catch (Exception e)
         {
             Log.Error(e, "[QueryForMetadata] Exception while processing command: {0}", azCommand);

shell/agents/Microsoft.Azure.Agent/Schema.cs

@@ -120,8 +120,9 @@ internal class CopilotActivity
 {
     public const string ConversationStateName = "azurecopilot/conversationstate";
     public const string SuggestedResponseName = "azurecopilot/suggesteduserresponses";
-    public const string CLIHandlerTopic = "generate_azure_cli_scripts";
-    public const string PSHandlerTopic = "generate_powershell_script";
+    public const string CLIHandlerTopic = "generate_azure_cli_scripts,Orchestrator_Respond";
+    public const string PSHandlerTopic = "generate_powershell_script,Orchestrator_Respond";
+    public const string ProgressTopic = "InProgressActivity";
 
     public string Type { get; set; }
     public string Id { get; set; }