Update Telemetry for Public Preview in MSAZ

shell/agents/Microsoft.Azure.Agent/AzureAgent.cs

@@ -17,6 +17,7 @@ public sealed class AzureAgent : ILLMAgent
     public string SettingFile { private set; get; }
 
     internal ArgumentPlaceholder ArgPlaceholder { set; get; }
+    internal CopilotResponse CopilotResponse { set; get; }
 
     private const string SettingFileName = "az.config.json";
     private const string LoggingFileName = "log..txt";
@@ -110,11 +111,34 @@ public void Initialize(AgentConfig config)
                 .CreateLogger();
             Log.Information("Azure agent initialized.");
         }
+
+        if (_setting.Telemetry)
+        {
+            Telemetry.Initialize();
+        }
     }
 
     public IEnumerable<CommandBase> GetCommands() => [new ReplaceCommand(this)];
-    public bool CanAcceptFeedback(UserAction action) => false;
-    public void OnUserAction(UserActionPayload actionPayload) {}
+    public bool CanAcceptFeedback(UserAction action) => Telemetry.Enabled;
+    public void OnUserAction(UserActionPayload actionPayload) {
+        // Send telemetry about the user action.
+        bool isUserFeedback = false;
+        string details = null;
+        UserAction action = actionPayload.Action;
+
+        if (action is UserAction.Dislike)
+        {
+            var dislike = (DislikePayload) actionPayload;
+            isUserFeedback = true;
+            details = string.Format("{0} | {1}", dislike.ShortFeedback, dislike.LongFeedback);
+        }
+        else if (action is UserAction.Like)
+        {
+            isUserFeedback = true;
+        }
+
+        Telemetry.Trace(AzTrace.UserAction(action.ToString(), _copilotResponse, details, isUserFeedback));
+    }
 
     public async Task RefreshChatAsync(IShell shell, bool force)
     {
@@ -254,6 +278,8 @@ public async Task<bool> ChatAsync(string input, IShell shell)
                     host.WriteLine("\nYou've reached the maximum length of a conversation. To continue, please run '/refresh' to start a new conversation.\n");
                 }
             }
+
+            Telemetry.Trace(AzTrace.Chat(_copilotResponse));
         }
         catch (Exception ex) when (ex is TokenRequestException or ConnectionDroppedException)
         {
@@ -362,8 +388,8 @@ private ResponseData ParseCLIHandlerResponse(IShell shell)
         else
         {
             // The placeholder section is not in the format as we've instructed ...
-            // TODO: send telemetry about this case.
             Log.Error("Placeholder section not in expected format:\n{0}", text);
+            Telemetry.Trace(AzTrace.Exception(_copilotResponse, "Placeholder section not in expected format."));
         }
 
         ReplaceKnownPlaceholders(data);

shell/agents/Microsoft.Azure.Agent/ChatSession.cs

@@ -262,6 +262,8 @@ private HttpRequestMessage PrepareForChat(string input)
         var request = new HttpRequestMessage(HttpMethod.Post, _conversationUrl) { Content = content };
 
         request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _token);
+        // This header is for server side telemetry to identify where the request comes from.
+        request.Headers.Add("ClientType", "AIShell");
         return request;
     }
 

shell/agents/Microsoft.Azure.Agent/Command.cs

@@ -1,5 +1,6 @@
 using System.CommandLine;
 using System.Text;
+
 using AIShell.Abstraction;
 
 namespace Microsoft.Azure.Agent;
@@ -158,6 +159,18 @@ private void ReplaceAction()
             host.RenderDivider("Regenerate", DividerAlignment.Left);
             host.MarkupLine($"\nQuery: [teal]{ap.Query}[/]");
 
+            if (Telemetry.Enabled)
+            {
+                Dictionary<string, bool> details = new(items.Count);
+                foreach (var item in items)
+                {
+                    string name = item.Name;
+                    details.Add(name, _values.ContainsKey(name));
+                }
+
+                Telemetry.Trace(AzTrace.UserAction("Replace", _agent.CopilotResponse, details));
+            }
+
             try
             {
                 string answer = host.RunWithSpinnerAsync(RegenerateAsync).GetAwaiter().GetResult();

shell/agents/Microsoft.Azure.Agent/DataRetriever.cs

@@ -577,14 +577,31 @@ private AzCLICommand QueryForMetadata(string azCommand)
             }
             else
             {
-                // TODO: telemetry.
                 Log.Error("[QueryForMetadata] Received status code '{0}' for command '{1}'", response.StatusCode, azCommand);
+                if (Telemetry.Enabled)
+                {
+                    Dictionary<string, string> details = new()
+                    {
+                        ["StatusCode"] = response.StatusCode.ToString(),
+                        ["Command"] = azCommand,
+                        ["Message"] = "AzCLI metadata service returns unsuccessful status code for query."
+                    };
+                    Telemetry.Trace(AzTrace.Exception(response: null, details));
+                }
             }
         }
         catch (Exception e)
         {
-            // TODO: telemetry.
             Log.Error(e, "[QueryForMetadata] Exception while processing command: {0}", azCommand);
+            if (Telemetry.Enabled)
+            {
+                Dictionary<string, string> details = new()
+                {
+                    ["Command"] = azCommand,
+                    ["Message"] = $"AzCLI metadata query and process raised an exception"
+                };
+                Telemetry.Trace(AzTrace.Exception(response: null, details), e);
+            }
         }
 
         return command;

shell/agents/Microsoft.Azure.Agent/Telemetry.cs

@@ -0,0 +1,225 @@
+using System.Text.Json;
+
+using Microsoft.ApplicationInsights;
+using Microsoft.ApplicationInsights.WorkerService;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Microsoft.Azure.Agent;
+
+public class AzTrace
+{
+    private static readonly string s_installationId;
+    static AzTrace()
+    {
+        string azureConfigDir = Environment.GetEnvironmentVariable("AZURE_CONFIG_DIR");
+        string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
+        string userProfilePath = Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json");
+
+        JsonElement array;
+        s_installationId = null;
+
+        if (File.Exists(userProfilePath))
+        {
+            using var jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read);
+            array = JsonSerializer.Deserialize<JsonElement>(jsonStream);
+            s_installationId = array.GetProperty("installationId").GetString();
+        }
+        else
+        {
+            try
+            {
+                Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "AzureRmContextSettings.json");
+                using var jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read);
+                array = JsonSerializer.Deserialize<JsonElement>(jsonStream);
+                s_installationId = array.GetProperty("Settings").GetProperty("InstallationId").GetString();
+            }
+            catch
+            {
+                // If finally no installation id found, just return null.
+                s_installationId = null;
+            }
+        }
+    }
+
+    internal AzTrace()
+    {
+        InstallationId = s_installationId;
+    }
+
+    /// <summary>
+    /// Installation id from the Azure CLI installation.
+    /// </summary>
+    internal string InstallationId { get; }
+
+    /// <summary>
+    /// Topic name of the response from Azure Copilot.
+    /// </summary>
+    internal string TopicName { get; set; }
+
+    /// <summary>
+    /// Each chat has a unique conversation id. When the customer runs '/refresh',
+    /// a new chat will be initiated (i.e. a new conversation id will be created).
+    /// </summary>
+    internal string ConversationId { get; set; }
+
+    /// <summary>
+    /// The activity id of the user's query.
+    /// </summary>
+    internal string QueryId { get; set; }
+
+    /// <summary>
+    /// The event type of this telemetry.
+    /// </summary>
+    internal string EventType { get; set; }
+
+    /// <summary>
+    /// The shell command that triggered this telemetry.
+    /// </summary>
+    internal string ShellCommand { get; set; }
+
+    /// <summary>
+    /// Detailed information.
+    /// </summary>
+    internal object Details { get; set; }
+
+    internal static AzTrace UserAction(
+        string shellCommand,
+        CopilotResponse response,
+        object details,
+        bool isFeedback = false)
+    {
+        if (Telemetry.Enabled)
+        {
+            return new()
+            {
+                QueryId = response.ReplyToId,
+                TopicName = response.TopicName,
+                ConversationId = response.ConversationId,
+                ShellCommand = shellCommand,
+                EventType = isFeedback ? "Feedback" : "UserAction",
+                Details = details
+            };
+        }
+
+        // Don't create an object when telemetry is disabled.
+        return null;
+    }
+
+    internal static AzTrace Chat(CopilotResponse response)
+    {
+        if (Telemetry.Enabled)
+        {
+            return new()
+            {
+                EventType = "Chat",
+                QueryId = response.ReplyToId,
+                TopicName = response.TopicName,
+                ConversationId = response.ConversationId
+            };
+        }
+
+        // Don't create an object when telemetry is disabled.
+        return null;
+    }
+
+    internal static AzTrace Exception(CopilotResponse response, object details)
+    {
+        if (Telemetry.Enabled)
+        {
+            return new()
+            {
+                EventType = "Exception",
+                QueryId = response?.ReplyToId,
+                TopicName = response?.TopicName,
+                ConversationId = response?.ConversationId,
+                Details = details
+            };
+        }
+
+        // Don't create an object when telemetry is disabled.
+        return null;
+    }
+}
+
+internal class Telemetry
+{
+    private static Telemetry s_singleton;
+    private readonly TelemetryClient _telemetryClient;
+
+    private Telemetry()
+    {
+        // Being a regular console app, there is no appsettings.json or configuration providers enabled by default.
+        // Hence connection string must be specified here.
+        IServiceCollection services = new ServiceCollection()
+            .AddApplicationInsightsTelemetryWorkerService((ApplicationInsightsServiceOptions options) =>
+                {
+                    // Application insights in the temp environment.
+                    options.ConnectionString = "InstrumentationKey=eea660a1-d969-44f8-abe4-96666e7fb159";
+                    options.EnableHeartbeat = false;
+                    options.EnableDiagnosticsTelemetryModule = false;
+                });
+
+        // Obtain TelemetryClient instance from DI, for additional manual tracking or to flush.
+        _telemetryClient = services
+            .BuildServiceProvider()
+            .GetRequiredService<TelemetryClient>();
+
+        // Suppress the PII recorded by default to reduce risk.
+        _telemetryClient.Context.Cloud.RoleInstance = "Not Available";
+    }
+
+    private void LogTelemetry(AzTrace trace, Exception e = null)
+    {
+        Dictionary<string, string> telemetryEvent = new()
+        {
+            ["QueryId"] = trace.QueryId,
+            ["ConversationId"] = trace.ConversationId,
+            ["InstallationId"] = trace.InstallationId,
+            ["TopicName"] = trace.TopicName,
+            ["EventType"] = trace.EventType,
+            ["ShellCommand"] = trace.ShellCommand,
+            ["Details"] = GetDetailedMessage(trace.Details),
+        };
+
+        _telemetryClient.TrackTrace("AIShell-Test1022", telemetryEvent);
+        if (e != null) { _telemetryClient.TrackException(e); }
+        _telemetryClient.Flush();
+    }
+
+    private static string GetDetailedMessage(object details)
+    {
+        if (details is null)
+        {
+            return null;
+        }
+
+        if (details is string str)
+        {
+            return str;
+        }
+
+        return JsonSerializer.Serialize(details, Utils.RelaxedJsonEscapingOptions);
+    }
+
+    /// <summary>
+    /// Gets whether or not telemetry is enabled.
+    /// </summary>
+    internal static bool Enabled => s_singleton is not null;
+
+    /// <summary>
+    /// Initialize telemetry client.
+    /// </summary>
+    internal static void Initialize() => s_singleton ??= new Telemetry();
+
+    /// <summary>
+    /// Trace a telemetry metric.
+    /// The method does nothing when it's disabled.
+    /// </summary>
+    internal static void Trace(AzTrace trace) => s_singleton?.LogTelemetry(trace);
+
+    /// <summary>
+    /// Trace a telemetry metric and an Exception with it.
+    /// The method does nothing when it's disabled.
+    /// </summary>
+    internal static void Trace(AzTrace trace, Exception e) => s_singleton?.LogTelemetry(trace, e);
+}

shell/agents/Microsoft.Azure.Agent/Utils.cs

@@ -1,3 +1,4 @@
+using System.Text.Encodings.Web;
 using System.Text.Json;
 
 namespace Microsoft.Azure.Agent;
@@ -8,6 +9,7 @@ internal static class Utils
 
     private static readonly JsonSerializerOptions s_jsonOptions;
     private static readonly JsonSerializerOptions s_humanReadableOptions;
+    private static readonly JsonSerializerOptions s_relaxedJsonEscapingOptions;
 
     static Utils()
     {
@@ -22,10 +24,16 @@ static Utils()
             WriteIndented = true,
             PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
         };
+
+        s_relaxedJsonEscapingOptions = new JsonSerializerOptions
+        {
+            Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
+        };
     }
 
     internal static JsonSerializerOptions JsonOptions => s_jsonOptions;
     internal static JsonSerializerOptions JsonHumanReadableOptions => s_humanReadableOptions;
+    internal static JsonSerializerOptions RelaxedJsonEscapingOptions => s_relaxedJsonEscapingOptions;
 }
 
 internal class TokenRequestException : Exception