Refactor the telemetry work

Author: daxian-dbw

shell/agents/Microsoft.Azure.Agent/AzureAgent.cs

@@ -1,6 +1,6 @@
 using System.Diagnostics;
 using System.Text;
-using System.Text.Json;
+
 using AIShell.Abstraction;
 using Azure.Identity;
 using Serilog;
@@ -17,6 +17,7 @@ public sealed class AzureAgent : ILLMAgent
     public string SettingFile { private set; get; }
 
     internal ArgumentPlaceholder ArgPlaceholder { set; get; }
+    internal CopilotResponse CopilotResponse { set; get; }
 
     private const string SettingFileName = "az.config.json";
     private const string LoggingFileName = "log..txt";
@@ -37,13 +38,13 @@ 7. DO NOT include the placeholder summary when the commands contains no placehol
         """;
 
     private int _turnsLeft;
-    internal CopilotResponse _copilotResponse;
+    private CopilotResponse _copilotResponse;
     private AgentSetting _setting;
 
     private readonly string _instructions;
     private readonly StringBuilder _buffer;
     private readonly HttpClient _httpClient;
-    internal readonly ChatSession _chatSession;
+    private readonly ChatSession _chatSession;
     private readonly Dictionary<string, string> _valueStore;
     // private MetricHelper _metricHelper;
 
@@ -111,36 +112,33 @@ public void Initialize(AgentConfig config)
                 .CreateLogger();
             Log.Information("Azure agent initialized.");
         }
+
+        if (_setting.Telemetry)
+        {
+            Telemetry.Initialize();
+        }
     }
 
     public IEnumerable<CommandBase> GetCommands() => [new ReplaceCommand(this)];
-    public bool CanAcceptFeedback(UserAction action) => !MetricHelper.TelemetryOptOut;
+    public bool CanAcceptFeedback(UserAction action) => _setting.Telemetry;
     public void OnUserAction(UserActionPayload actionPayload) {
         // Send telemetry about the user action.
-        // DisLike Action
-        string DetailedMessage = null;
-        bool IsUserFeedback = false;
-        if (actionPayload.Action == UserAction.Dislike)
+        bool isUserFeedback = false;
+        string details = null;
+        UserAction action = actionPayload.Action;
+
+        if (action is UserAction.Dislike)
         {
-            IsUserFeedback = true;
-            DislikePayload dislikePayload = (DislikePayload)actionPayload;
-            DetailedMessage = string.Format("{0} | {1}", dislikePayload.ShortFeedback, dislikePayload.LongFeedback);
+            var dislike = (DislikePayload) actionPayload;
+            isUserFeedback = true;
+            details = string.Format("{0} | {1}", dislike.ShortFeedback, dislike.LongFeedback);
         }
-        else if (actionPayload.Action == UserAction.Like)
+        else if (action is UserAction.Like)
         {
-            IsUserFeedback = true;
+            isUserFeedback = true;
         }
 
-        MetricHelper.metricHelper.LogTelemetry(
-            new AzTrace()
-            {
-                Command = actionPayload.Action.ToString(),
-                ConversationId = _chatSession.ConversationId,
-                ActivityId = _copilotResponse.ReplyToId,
-                EventType = IsUserFeedback ? "Feedback" : "UserAction",
-                TopicName = _copilotResponse.TopicName,
-                DetailedMessage = DetailedMessage
-            });
+        Telemetry.Log(AzTrace.UserAction(action.ToString(), _copilotResponse, details, isUserFeedback));
     }
 
     public async Task RefreshChatAsync(IShell shell, bool force)
@@ -282,16 +280,9 @@ public async Task<bool> ChatAsync(string input, IShell shell)
                 }
             }
 
-            if (!MetricHelper.TelemetryOptOut)
+            if (Telemetry.Enabled)
             {
-                MetricHelper.metricHelper.LogTelemetry(
-                    new AzTrace()
-                    {
-                        ConversationId = _chatSession.ConversationId,
-                        EventType = "Chat",
-                        TopicName = _copilotResponse.TopicName,
-                        ActivityId = _copilotResponse.ReplyToId
-                    });
+                Telemetry.Log(AzTrace.Chat(_copilotResponse));
             }
         }
         catch (Exception ex) when (ex is TokenRequestException or ConnectionDroppedException)
@@ -401,18 +392,9 @@ private ResponseData ParseCLIHandlerResponse(IShell shell)
         else
         {
             // The placeholder section is not in the format as we've instructed ...
-            // TODO: send telemetry about this case.
-            if (!MetricHelper.TelemetryOptOut)
+            if (Telemetry.Enabled)
             {
-                MetricHelper.metricHelper.LogTelemetry(
-                    new AzTrace()
-                    {
-                        ConversationId = _chatSession.ConversationId,
-                        ActivityId = _copilotResponse.ReplyToId,
-                        EventType = "Exception",
-                        TopicName = _copilotResponse.TopicName,
-                        DetailedMessage = $"Placeholder section not in expected format:{text}"
-                    });
+                Telemetry.Log(AzTrace.Exception(_copilotResponse, $"Placeholder section not in expected format: {text}"));
             }
             Log.Error("Placeholder section not in expected format:\n{0}", text);
         }

shell/agents/Microsoft.Azure.Agent/Command.cs

@@ -1,8 +1,6 @@
 using System.CommandLine;
 using System.Text;
-using System.Text.Encodings.Web;
-using System.Text.Json;
-using System.Text.Unicode;
+
 using AIShell.Abstraction;
 
 namespace Microsoft.Azure.Agent;
@@ -70,8 +68,6 @@ private void ReplaceAction()
 
         try
         {
-            // Detailed Message recorded indicating whether each placeholder is replaced.
-            Dictionary<string, Boolean> DetailedMessage = new();
             for (int i = 0; i < items.Count; i++)
             {
                 var item = items[i];
@@ -122,43 +118,17 @@ private void ReplaceAction()
 
                     _values.Add(item.Name, value);
                     _agent.SaveUserValue(item.Name, value);
-                    DetailedMessage.Add(item.Name, true);
 
                     if (nameArgInfo is not null && nameArgInfo.NamingRule.TryMatchName(value, out string prodName, out string envName))
                     {
                         _productNames.Add(prodName.ToLower());
                         _environmentNames.Add(envName.ToLower());
                     }
                 }
-                else
-                {
-                    DetailedMessage.Add(item.Name, false);
-                }
 
                 // Write an extra new line.
                 host.WriteLine();
             }
-
-            // Customize the Json Serializer Options to avoid unnecessary encoding.
-            var options = new JsonSerializerOptions
-            {
-                Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
-            };
-
-            // Send Telemetry for Replace Action.
-            if (!MetricHelper.TelemetryOptOut)
-            {
-                MetricHelper.metricHelper.LogTelemetry(
-                    new AzTrace()
-                    {
-                        Command = "Replace",
-                        ConversationId = _agent._chatSession.ConversationId,
-                        ActivityId = _agent._copilotResponse.ReplyToId,
-                        EventType = "UserAction",
-                        TopicName = _agent._copilotResponse.TopicName,
-                        DetailedMessage = JsonSerializer.Serialize(DetailedMessage, options)
-                    });
-            }
         }
         catch (OperationCanceledException)
         {
@@ -189,6 +159,18 @@ private void ReplaceAction()
             host.RenderDivider("Regenerate", DividerAlignment.Left);
             host.MarkupLine($"\nQuery: [teal]{ap.Query}[/]");
 
+            if (Telemetry.Enabled)
+            {
+                Dictionary<string, bool> details = new(items.Count);
+                foreach (var item in items)
+                {
+                    string name = item.Name;
+                    details.Add(name, _values.ContainsKey(name));
+                }
+
+                Telemetry.Log(AzTrace.UserAction("Replace", _agent.CopilotResponse, details));
+            }
+
             try
             {
                 string answer = host.RunWithSpinnerAsync(RegenerateAsync).GetAwaiter().GetResult();

shell/agents/Microsoft.Azure.Agent/DataRetriever.cs

@@ -6,7 +6,6 @@
 using System.Text.RegularExpressions;
 
 using AIShell.Abstraction;
-using Azure;
 using Serilog;
 
 namespace Microsoft.Azure.Agent;
@@ -578,47 +577,31 @@ private AzCLICommand QueryForMetadata(string azCommand)
             }
             else
             {
-                if (!MetricHelper.TelemetryOptOut)
+                Log.Error("[QueryForMetadata] Received status code '{0}' for command '{1}'", response.StatusCode, azCommand);
+                if (Telemetry.Enabled)
                 {
-                    Dictionary<string, string> errorMessage = new Dictionary<string, string>
+                    Dictionary<string, string> details = new()
                     {
-                        { "StatusCode", response.StatusCode.ToString() },
-                        { "Command", azCommand },
-                        { "ErrorMessage", $"[QueryForMetadata] Received status code {response.StatusCode} for command {azCommand}" },
+                        ["StatusCode"] = response.StatusCode.ToString(),
+                        ["Command"] = azCommand,
+                        ["Message"] = "AzCLI metadata service returns unsuccessful status code for query."
                     };
-                    MetricHelper.metricHelper.LogTelemetry(
-                        new AzTrace()
-                        {
-                            // ConversationId = _agent._chatSession.ConversationId,
-                            // ActivityId = _agent._copilotResponse.ReplyToId,
-                            EventType = "Exception",
-                            // TopicName = _agent._copilotResponse.TopicName,
-                            DetailedMessage = JsonSerializer.Serialize(errorMessage)
-                        });
+                    Telemetry.Log(AzTrace.Exception(response: null, details));
                 }
-                Log.Error("[QueryForMetadata] Received status code '{0}' for command '{1}'", response.StatusCode, azCommand);
             }
         }
         catch (Exception e)
         {
-            if (!MetricHelper.TelemetryOptOut)
+            Log.Error(e, "[QueryForMetadata] Exception while processing command: {0}", azCommand);
+            if (Telemetry.Enabled)
             {
-                Dictionary<string, string> errorMessage = new Dictionary<string, string>
+                Dictionary<string, string> details = new()
                 {
-                    { "Command", azCommand },
-                    { "ErrorMessage", $"[QueryForMetadata] Exception while processing command: {azCommand}" },
+                    ["Command"] = azCommand,
+                    ["Message"] = "AzCLI metadata query and process raised an exception."
                 };
-                MetricHelper.metricHelper.LogTelemetry(
-                    new AzTrace()
-                    {
-                        // ConversationId = _agent._chatSession.ConversationId,
-                        // ActivityId = _agent._copilotResponse.ReplyToId,
-                        EventType = "Exception",
-                        // TopicName = _agent._copilotResponse.TopicName,
-                        DetailedMessage = JsonSerializer.Serialize(errorMessage)
-                    });
+                Telemetry.Log(AzTrace.Exception(response: null, details));
             }
-            Log.Error(e, "[QueryForMetadata] Exception while processing command: {0}", azCommand);
         }
 
         return command;