From d7a00ce8aaad9e102d836caafa72729fbd362020 Mon Sep 17 00:00:00 2001 From: Nori Zhang Date: Tue, 22 Oct 2024 18:44:37 +0800 Subject: [PATCH 01/14] telemetry for msaz telemetry for msaz Remove unneeded changes to `CopilotResponse` Address Comments Minor fix set json serializer option --- .../Microsoft.Azure.Agent/AzureAgent.cs | 61 +++++++++- .../Microsoft.Azure.Agent/ChatSession.cs | 2 + shell/agents/Microsoft.Azure.Agent/Command.cs | 31 ++++++ .../Microsoft.Azure.Agent/DataRetriever.cs | 38 ++++++- .../Telemetry/AzTrace.cs | 66 +++++++++++ .../Telemetry/MetricHelper.cs | 104 ++++++++++++++++++ 6 files changed, 295 insertions(+), 7 deletions(-) create mode 100644 shell/agents/Microsoft.Azure.Agent/Telemetry/AzTrace.cs create mode 100644 shell/agents/Microsoft.Azure.Agent/Telemetry/MetricHelper.cs diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index 7e5fb983..64a23297 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -1,6 +1,6 @@ using System.Diagnostics; using System.Text; - +using System.Text.Json; using AIShell.Abstraction; using Azure.Identity; using Serilog; @@ -37,14 +37,15 @@ 7. DO NOT include the placeholder summary when the commands contains no placehol """; private int _turnsLeft; - private CopilotResponse _copilotResponse; + internal CopilotResponse _copilotResponse; private AgentSetting _setting; private readonly string _instructions; private readonly StringBuilder _buffer; private readonly HttpClient _httpClient; - private readonly ChatSession _chatSession; + internal readonly ChatSession _chatSession; private readonly Dictionary _valueStore; + // private MetricHelper _metricHelper; public AzureAgent() { @@ -113,8 +114,34 @@ public void Initialize(AgentConfig config) } public IEnumerable GetCommands() => [new ReplaceCommand(this)]; - public bool CanAcceptFeedback(UserAction action) => false; - public void OnUserAction(UserActionPayload actionPayload) {} + public bool CanAcceptFeedback(UserAction action) => !MetricHelper.TelemetryOptOut; + public void OnUserAction(UserActionPayload actionPayload) { + // Send telemetry about the user action. + // DisLike Action + string DetailedMessage = null; + bool IsUserFeedback = false; + if (actionPayload.Action == UserAction.Dislike) + { + IsUserFeedback = true; + DislikePayload dislikePayload = (DislikePayload)actionPayload; + DetailedMessage = string.Format("{0} | {1}", dislikePayload.ShortFeedback, dislikePayload.LongFeedback); + } + else if (actionPayload.Action == UserAction.Like) + { + IsUserFeedback = true; + } + + MetricHelper.metricHelper.LogTelemetry( + new AzTrace() + { + Command = actionPayload.Action.ToString(), + ConversationId = _chatSession.ConversationId, + ActivityId = _copilotResponse.ReplyToId, + EventType = IsUserFeedback ? "Feedback" : "UserAction", + TopicName = _copilotResponse.TopicName, + DetailedMessage = DetailedMessage + }); + } public async Task RefreshChatAsync(IShell shell, bool force) { @@ -254,6 +281,18 @@ public async Task ChatAsync(string input, IShell shell) host.WriteLine("\nYou've reached the maximum length of a conversation. To continue, please run '/refresh' to start a new conversation.\n"); } } + + if (!MetricHelper.TelemetryOptOut) + { + MetricHelper.metricHelper.LogTelemetry( + new AzTrace() + { + ConversationId = _chatSession.ConversationId, + EventType = "Chat", + TopicName = _copilotResponse.TopicName, + ActivityId = _copilotResponse.ReplyToId + }); + } } catch (Exception ex) when (ex is TokenRequestException or ConnectionDroppedException) { @@ -363,6 +402,18 @@ private ResponseData ParseCLIHandlerResponse(IShell shell) { // The placeholder section is not in the format as we've instructed ... // TODO: send telemetry about this case. + if (!MetricHelper.TelemetryOptOut) + { + MetricHelper.metricHelper.LogTelemetry( + new AzTrace() + { + ConversationId = _chatSession.ConversationId, + ActivityId = _copilotResponse.ReplyToId, + EventType = "Exception", + TopicName = _copilotResponse.TopicName, + DetailedMessage = $"Placeholder section not in expected format:{text}" + }); + } Log.Error("Placeholder section not in expected format:\n{0}", text); } diff --git a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs index de11cd1d..04d6f704 100644 --- a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs +++ b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs @@ -262,6 +262,8 @@ private HttpRequestMessage PrepareForChat(string input) var request = new HttpRequestMessage(HttpMethod.Post, _conversationUrl) { Content = content }; request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _token); + // These header is for server side telemetry to identify where the request comes from. + request.Headers.Add("ClientType", "AIShell"); return request; } diff --git a/shell/agents/Microsoft.Azure.Agent/Command.cs b/shell/agents/Microsoft.Azure.Agent/Command.cs index e664417a..23c20d34 100644 --- a/shell/agents/Microsoft.Azure.Agent/Command.cs +++ b/shell/agents/Microsoft.Azure.Agent/Command.cs @@ -1,5 +1,8 @@ using System.CommandLine; using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Unicode; using AIShell.Abstraction; namespace Microsoft.Azure.Agent; @@ -67,6 +70,8 @@ private void ReplaceAction() try { + // Detailed Message recorded indicating whether each placeholder is replaced. + Dictionary DetailedMessage = new(); for (int i = 0; i < items.Count; i++) { var item = items[i]; @@ -117,6 +122,7 @@ private void ReplaceAction() _values.Add(item.Name, value); _agent.SaveUserValue(item.Name, value); + DetailedMessage.Add(item.Name, true); if (nameArgInfo is not null && nameArgInfo.NamingRule.TryMatchName(value, out string prodName, out string envName)) { @@ -124,10 +130,35 @@ private void ReplaceAction() _environmentNames.Add(envName.ToLower()); } } + else + { + DetailedMessage.Add(item.Name, false); + } // Write an extra new line. host.WriteLine(); } + + // Customize the Json Serializer Options to avoid unnecessary encoding. + var options = new JsonSerializerOptions + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + // Send Telemetry for Replace Action. + if (!MetricHelper.TelemetryOptOut) + { + MetricHelper.metricHelper.LogTelemetry( + new AzTrace() + { + Command = "Replace", + ConversationId = _agent._chatSession.ConversationId, + ActivityId = _agent._copilotResponse.ReplyToId, + EventType = "UserAction", + TopicName = _agent._copilotResponse.TopicName, + DetailedMessage = JsonSerializer.Serialize(DetailedMessage, options) + }); + } } catch (OperationCanceledException) { diff --git a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs index 5674f422..4ab01a56 100644 --- a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs +++ b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs @@ -6,6 +6,7 @@ using System.Text.RegularExpressions; using AIShell.Abstraction; +using Azure; using Serilog; namespace Microsoft.Azure.Agent; @@ -577,13 +578,46 @@ private AzCLICommand QueryForMetadata(string azCommand) } else { - // TODO: telemetry. + if (!MetricHelper.TelemetryOptOut) + { + Dictionary errorMessage = new Dictionary + { + { "StatusCode", response.StatusCode.ToString() }, + { "Command", azCommand }, + { "ErrorMessage", $"[QueryForMetadata] Received status code {response.StatusCode} for command {azCommand}" }, + }; + MetricHelper.metricHelper.LogTelemetry( + new AzTrace() + { + // ConversationId = _agent._chatSession.ConversationId, + // ActivityId = _agent._copilotResponse.ReplyToId, + EventType = "Exception", + // TopicName = _agent._copilotResponse.TopicName, + DetailedMessage = JsonSerializer.Serialize(errorMessage) + }); + } Log.Error("[QueryForMetadata] Received status code '{0}' for command '{1}'", response.StatusCode, azCommand); } } catch (Exception e) { - // TODO: telemetry. + if (!MetricHelper.TelemetryOptOut) + { + Dictionary errorMessage = new Dictionary + { + { "Command", azCommand }, + { "ErrorMessage", $"[QueryForMetadata] Exception while processing command: {azCommand}" }, + }; + MetricHelper.metricHelper.LogTelemetry( + new AzTrace() + { + // ConversationId = _agent._chatSession.ConversationId, + // ActivityId = _agent._copilotResponse.ReplyToId, + EventType = "Exception", + // TopicName = _agent._copilotResponse.TopicName, + DetailedMessage = JsonSerializer.Serialize(errorMessage) + }); + } Log.Error(e, "[QueryForMetadata] Exception while processing command: {0}", azCommand); } diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry/AzTrace.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry/AzTrace.cs new file mode 100644 index 00000000..b5f9b4ee --- /dev/null +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry/AzTrace.cs @@ -0,0 +1,66 @@ +using System.Text.Json; +using static Microsoft.ApplicationInsights.MetricDimensionNames.TelemetryContext; + +namespace Microsoft.Azure.Agent; + +public class AzTrace +{ + private static readonly string s_installationId; + private static string GetInstallationID() + { + string azureConfigDir = Environment.GetEnvironmentVariable("AZURE_CONFIG_DIR"); + string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + string userProfilePath = Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json"); + + FileStream jsonStream; + JsonElement array; + string installationId; + + if (File.Exists(userProfilePath)) + { + jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); + array = JsonSerializer.Deserialize(jsonStream); + installationId = array.GetProperty("installationId").GetString(); + } + else + { + try + { + Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json"); + jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); + array = JsonSerializer.Deserialize(jsonStream); + installationId = array.GetProperty("Settings").GetProperty("InstallationId").GetString(); + } + catch + { + // If finally no installation id found, just return null. + return null; + } + } + + return installationId; + } + + public string TopicName; + // Each chat has a unique conversationId. When the cx runs /refresh, + // a new chat is initiated(i.e.a new conversationId will be created). + public string ConversationId; + // The activity id of the user's query + public string ActivityId; + public string InstallationId = s_installationId; + public string EventType; + public string Command; + /// + /// Detailed information containing additional Information - may contain: + /// Reason of dislike + /// + public string DetailedMessage; + /// + /// Agent Information - may contain: + /// Handler Version + /// Product Version + /// .net/python Version + /// + public Dictionary ExtendedProperties; + static AzTrace() => s_installationId = GetInstallationID(); +} diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry/MetricHelper.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry/MetricHelper.cs new file mode 100644 index 00000000..425d5da9 --- /dev/null +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry/MetricHelper.cs @@ -0,0 +1,104 @@ +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights.WorkerService; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Azure.Agent; + +public sealed class MetricHelper +{ + public static readonly bool TelemetryOptOut = GetEnvironmentVariableAsBool("COPILOT_TELEMETRY_OPTOUT", false); + private const int CustomDomainMaximumSize = 8192; + private TelemetryClient _telemetryClient; + + private static readonly Lazy lazy = + new Lazy(() => new MetricHelper()); + + public static MetricHelper metricHelper { get { return lazy.Value; } } + + private MetricHelper() + { + InitializeTelemetryClient(); + } + + private void InitializeTelemetryClient() + { + // Create the DI container. + IServiceCollection services = new ServiceCollection(); + + // Add custom TelemetryInitializer + services.AddSingleton(typeof(ITelemetryInitializer), new MyCustomTelemetryInitializer()); + + // Being a regular console app, there is no appsettings.json or configuration providers enabled by default. + // Hence connection string must be specified here. + services.AddApplicationInsightsTelemetryWorkerService((ApplicationInsightsServiceOptions options) => + { + // Application insights in the temp environment. + options.ConnectionString = "InstrumentationKey=eea660a1-d969-44f8-abe4-96666e7fb159"; + options.EnableHeartbeat = false; + options.EnableDiagnosticsTelemetryModule = false; + } + ); + + // Build ServiceProvider. + IServiceProvider serviceProvider = services.BuildServiceProvider(); + + // Obtain TelemetryClient instance from DI, for additional manual tracking or to flush. + _telemetryClient = serviceProvider.GetRequiredService(); + + // Suppress the PII recorded by default to reduce risk. + _telemetryClient.Context.Cloud.RoleInstance = "Not Available"; + } + + public void LogTelemetry(AzTrace trace) + { + Dictionary eventProperties = new() + { + { "ActivityId", trace.ActivityId}, + { "CoversationId", trace.ConversationId }, + { "InstallationId", trace.InstallationId }, + { "Handler", trace.TopicName }, + { "EventType", trace.EventType }, + { "Command", trace.Command }, + { "DetailedMessage", trace.DetailedMessage } + }; + + _telemetryClient.TrackTrace("AIShell-Test1022", eventProperties); + _telemetryClient.Flush(); + } + + private static bool GetEnvironmentVariableAsBool(string name, bool defaultValue) + { + var str = Environment.GetEnvironmentVariable(name); + + if (string.IsNullOrWhiteSpace(str)) + { + return defaultValue; + } + + if (bool.TryParse(str, out bool result)) + { + return result; + } + + if (string.Equals(str, "yes", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (string.Equals(str, "no", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return defaultValue; + } +} + +internal class MyCustomTelemetryInitializer : ITelemetryInitializer +{ + public void Initialize(ITelemetry telemetry){} + + public MyCustomTelemetryInitializer() {} +} From 78efff5d07556bdb6a0eebdbfc1803e00720ad15 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Mon, 28 Oct 2024 17:35:51 -0700 Subject: [PATCH 02/14] Refactor the telemetry work --- .../Microsoft.Azure.Agent/AzureAgent.cs | 68 +++---- shell/agents/Microsoft.Azure.Agent/Command.cs | 44 ++-- .../Microsoft.Azure.Agent/DataRetriever.cs | 43 ++-- .../agents/Microsoft.Azure.Agent/Telemetry.cs | 190 ++++++++++++++++++ .../Telemetry/AzTrace.cs | 66 ------ .../Telemetry/MetricHelper.cs | 104 ---------- shell/agents/Microsoft.Azure.Agent/Utils.cs | 8 + 7 files changed, 249 insertions(+), 274 deletions(-) create mode 100644 shell/agents/Microsoft.Azure.Agent/Telemetry.cs delete mode 100644 shell/agents/Microsoft.Azure.Agent/Telemetry/AzTrace.cs delete mode 100644 shell/agents/Microsoft.Azure.Agent/Telemetry/MetricHelper.cs diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index 64a23297..5fe63e17 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -1,6 +1,6 @@ using System.Diagnostics; using System.Text; -using System.Text.Json; + using AIShell.Abstraction; using Azure.Identity; using Serilog; @@ -17,6 +17,7 @@ public sealed class AzureAgent : ILLMAgent public string SettingFile { private set; get; } internal ArgumentPlaceholder ArgPlaceholder { set; get; } + internal CopilotResponse CopilotResponse { set; get; } private const string SettingFileName = "az.config.json"; private const string LoggingFileName = "log..txt"; @@ -37,13 +38,13 @@ 7. DO NOT include the placeholder summary when the commands contains no placehol """; private int _turnsLeft; - internal CopilotResponse _copilotResponse; + private CopilotResponse _copilotResponse; private AgentSetting _setting; private readonly string _instructions; private readonly StringBuilder _buffer; private readonly HttpClient _httpClient; - internal readonly ChatSession _chatSession; + private readonly ChatSession _chatSession; private readonly Dictionary _valueStore; // private MetricHelper _metricHelper; @@ -111,36 +112,33 @@ public void Initialize(AgentConfig config) .CreateLogger(); Log.Information("Azure agent initialized."); } + + if (_setting.Telemetry) + { + Telemetry.Initialize(); + } } public IEnumerable GetCommands() => [new ReplaceCommand(this)]; - public bool CanAcceptFeedback(UserAction action) => !MetricHelper.TelemetryOptOut; + public bool CanAcceptFeedback(UserAction action) => _setting.Telemetry; public void OnUserAction(UserActionPayload actionPayload) { // Send telemetry about the user action. - // DisLike Action - string DetailedMessage = null; - bool IsUserFeedback = false; - if (actionPayload.Action == UserAction.Dislike) + bool isUserFeedback = false; + string details = null; + UserAction action = actionPayload.Action; + + if (action is UserAction.Dislike) { - IsUserFeedback = true; - DislikePayload dislikePayload = (DislikePayload)actionPayload; - DetailedMessage = string.Format("{0} | {1}", dislikePayload.ShortFeedback, dislikePayload.LongFeedback); + var dislike = (DislikePayload) actionPayload; + isUserFeedback = true; + details = string.Format("{0} | {1}", dislike.ShortFeedback, dislike.LongFeedback); } - else if (actionPayload.Action == UserAction.Like) + else if (action is UserAction.Like) { - IsUserFeedback = true; + isUserFeedback = true; } - MetricHelper.metricHelper.LogTelemetry( - new AzTrace() - { - Command = actionPayload.Action.ToString(), - ConversationId = _chatSession.ConversationId, - ActivityId = _copilotResponse.ReplyToId, - EventType = IsUserFeedback ? "Feedback" : "UserAction", - TopicName = _copilotResponse.TopicName, - DetailedMessage = DetailedMessage - }); + Telemetry.Log(AzTrace.UserAction(action.ToString(), _copilotResponse, details, isUserFeedback)); } public async Task RefreshChatAsync(IShell shell, bool force) @@ -282,16 +280,9 @@ public async Task ChatAsync(string input, IShell shell) } } - if (!MetricHelper.TelemetryOptOut) + if (Telemetry.Enabled) { - MetricHelper.metricHelper.LogTelemetry( - new AzTrace() - { - ConversationId = _chatSession.ConversationId, - EventType = "Chat", - TopicName = _copilotResponse.TopicName, - ActivityId = _copilotResponse.ReplyToId - }); + Telemetry.Log(AzTrace.Chat(_copilotResponse)); } } catch (Exception ex) when (ex is TokenRequestException or ConnectionDroppedException) @@ -401,18 +392,9 @@ private ResponseData ParseCLIHandlerResponse(IShell shell) else { // The placeholder section is not in the format as we've instructed ... - // TODO: send telemetry about this case. - if (!MetricHelper.TelemetryOptOut) + if (Telemetry.Enabled) { - MetricHelper.metricHelper.LogTelemetry( - new AzTrace() - { - ConversationId = _chatSession.ConversationId, - ActivityId = _copilotResponse.ReplyToId, - EventType = "Exception", - TopicName = _copilotResponse.TopicName, - DetailedMessage = $"Placeholder section not in expected format:{text}" - }); + Telemetry.Log(AzTrace.Exception(_copilotResponse, $"Placeholder section not in expected format: {text}")); } Log.Error("Placeholder section not in expected format:\n{0}", text); } diff --git a/shell/agents/Microsoft.Azure.Agent/Command.cs b/shell/agents/Microsoft.Azure.Agent/Command.cs index 23c20d34..4694c99b 100644 --- a/shell/agents/Microsoft.Azure.Agent/Command.cs +++ b/shell/agents/Microsoft.Azure.Agent/Command.cs @@ -1,8 +1,6 @@ using System.CommandLine; using System.Text; -using System.Text.Encodings.Web; -using System.Text.Json; -using System.Text.Unicode; + using AIShell.Abstraction; namespace Microsoft.Azure.Agent; @@ -70,8 +68,6 @@ private void ReplaceAction() try { - // Detailed Message recorded indicating whether each placeholder is replaced. - Dictionary DetailedMessage = new(); for (int i = 0; i < items.Count; i++) { var item = items[i]; @@ -122,7 +118,6 @@ private void ReplaceAction() _values.Add(item.Name, value); _agent.SaveUserValue(item.Name, value); - DetailedMessage.Add(item.Name, true); if (nameArgInfo is not null && nameArgInfo.NamingRule.TryMatchName(value, out string prodName, out string envName)) { @@ -130,35 +125,10 @@ private void ReplaceAction() _environmentNames.Add(envName.ToLower()); } } - else - { - DetailedMessage.Add(item.Name, false); - } // Write an extra new line. host.WriteLine(); } - - // Customize the Json Serializer Options to avoid unnecessary encoding. - var options = new JsonSerializerOptions - { - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping - }; - - // Send Telemetry for Replace Action. - if (!MetricHelper.TelemetryOptOut) - { - MetricHelper.metricHelper.LogTelemetry( - new AzTrace() - { - Command = "Replace", - ConversationId = _agent._chatSession.ConversationId, - ActivityId = _agent._copilotResponse.ReplyToId, - EventType = "UserAction", - TopicName = _agent._copilotResponse.TopicName, - DetailedMessage = JsonSerializer.Serialize(DetailedMessage, options) - }); - } } catch (OperationCanceledException) { @@ -189,6 +159,18 @@ private void ReplaceAction() host.RenderDivider("Regenerate", DividerAlignment.Left); host.MarkupLine($"\nQuery: [teal]{ap.Query}[/]"); + if (Telemetry.Enabled) + { + Dictionary details = new(items.Count); + foreach (var item in items) + { + string name = item.Name; + details.Add(name, _values.ContainsKey(name)); + } + + Telemetry.Log(AzTrace.UserAction("Replace", _agent.CopilotResponse, details)); + } + try { string answer = host.RunWithSpinnerAsync(RegenerateAsync).GetAwaiter().GetResult(); diff --git a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs index 4ab01a56..52ee4618 100644 --- a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs +++ b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs @@ -6,7 +6,6 @@ using System.Text.RegularExpressions; using AIShell.Abstraction; -using Azure; using Serilog; namespace Microsoft.Azure.Agent; @@ -578,47 +577,31 @@ private AzCLICommand QueryForMetadata(string azCommand) } else { - if (!MetricHelper.TelemetryOptOut) + Log.Error("[QueryForMetadata] Received status code '{0}' for command '{1}'", response.StatusCode, azCommand); + if (Telemetry.Enabled) { - Dictionary errorMessage = new Dictionary + Dictionary details = new() { - { "StatusCode", response.StatusCode.ToString() }, - { "Command", azCommand }, - { "ErrorMessage", $"[QueryForMetadata] Received status code {response.StatusCode} for command {azCommand}" }, + ["StatusCode"] = response.StatusCode.ToString(), + ["Command"] = azCommand, + ["Message"] = "AzCLI metadata service returns unsuccessful status code for query." }; - MetricHelper.metricHelper.LogTelemetry( - new AzTrace() - { - // ConversationId = _agent._chatSession.ConversationId, - // ActivityId = _agent._copilotResponse.ReplyToId, - EventType = "Exception", - // TopicName = _agent._copilotResponse.TopicName, - DetailedMessage = JsonSerializer.Serialize(errorMessage) - }); + Telemetry.Log(AzTrace.Exception(response: null, details)); } - Log.Error("[QueryForMetadata] Received status code '{0}' for command '{1}'", response.StatusCode, azCommand); } } catch (Exception e) { - if (!MetricHelper.TelemetryOptOut) + Log.Error(e, "[QueryForMetadata] Exception while processing command: {0}", azCommand); + if (Telemetry.Enabled) { - Dictionary errorMessage = new Dictionary + Dictionary details = new() { - { "Command", azCommand }, - { "ErrorMessage", $"[QueryForMetadata] Exception while processing command: {azCommand}" }, + ["Command"] = azCommand, + ["Message"] = "AzCLI metadata query and process raised an exception." }; - MetricHelper.metricHelper.LogTelemetry( - new AzTrace() - { - // ConversationId = _agent._chatSession.ConversationId, - // ActivityId = _agent._copilotResponse.ReplyToId, - EventType = "Exception", - // TopicName = _agent._copilotResponse.TopicName, - DetailedMessage = JsonSerializer.Serialize(errorMessage) - }); + Telemetry.Log(AzTrace.Exception(response: null, details)); } - Log.Error(e, "[QueryForMetadata] Exception while processing command: {0}", azCommand); } return command; diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs new file mode 100644 index 00000000..4ff98e64 --- /dev/null +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -0,0 +1,190 @@ +using System.Text.Json; + +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.WorkerService; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Azure.Agent; + +public class AzTrace +{ + private static readonly string s_installationId; + static AzTrace() + { + string azureConfigDir = Environment.GetEnvironmentVariable("AZURE_CONFIG_DIR"); + string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + string userProfilePath = Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json"); + + JsonElement array; + s_installationId = null; + + if (File.Exists(userProfilePath)) + { + using var jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); + array = JsonSerializer.Deserialize(jsonStream); + s_installationId = array.GetProperty("installationId").GetString(); + } + else + { + try + { + Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json"); + using var jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); + array = JsonSerializer.Deserialize(jsonStream); + s_installationId = array.GetProperty("Settings").GetProperty("InstallationId").GetString(); + } + catch + { + // If finally no installation id found, just return null. + s_installationId = null; + } + } + } + + internal AzTrace() + { + InstallationId = s_installationId; + } + + /// + /// Installation id from the Azure CLI installation. + /// + internal string InstallationId { get; } + + /// + /// Topic name of the response from Azure Copilot. + /// + internal string TopicName { get; set; } + + /// + /// Each chat has a unique conversation id. When the customer runs '/refresh', + /// a new chat will be initiated (i.e. a new conversation id will be created). + /// + internal string ConversationId { get; set; } + + /// + /// The activity id of the user's query. + /// + internal string QueryId { get; set; } + + /// + /// The event type of this telemetry. + /// + internal string EventType { get; set; } + + /// + /// The shell command that triggered this telemetry. + /// + internal string ShellCommand { get; set; } + + /// + /// Detailed information. + /// + internal object Details { get; set; } + + internal static AzTrace UserAction( + string shellCommand, + CopilotResponse response, + object details, + bool isFeedback = false) => new() + { + QueryId = response.ReplyToId, + TopicName = response.TopicName, + ConversationId = response.ConversationId, + ShellCommand = shellCommand, + EventType = isFeedback ? "Feedback" : "UserAction", + Details = details + }; + + internal static AzTrace Chat(CopilotResponse response) => new() + { + EventType = "Chat", + QueryId = response.ReplyToId, + TopicName = response.TopicName, + ConversationId = response.ConversationId + }; + + internal static AzTrace Exception(CopilotResponse response, object details) => new() + { + EventType = "Exception", + QueryId = response?.ReplyToId, + TopicName = response?.TopicName, + ConversationId = response?.ConversationId, + Details = details + }; +} + +internal class Telemetry +{ + private static Telemetry s_singleton; + private readonly TelemetryClient _telemetryClient; + + private Telemetry() + { + // Being a regular console app, there is no appsettings.json or configuration providers enabled by default. + // Hence connection string must be specified here. + IServiceCollection services = new ServiceCollection() + .AddApplicationInsightsTelemetryWorkerService((ApplicationInsightsServiceOptions options) => + { + // Application insights in the temp environment. + options.ConnectionString = "InstrumentationKey=eea660a1-d969-44f8-abe4-96666e7fb159"; + options.EnableHeartbeat = false; + options.EnableDiagnosticsTelemetryModule = false; + }); + + // Obtain TelemetryClient instance from DI, for additional manual tracking or to flush. + _telemetryClient = services + .BuildServiceProvider() + .GetRequiredService(); + + // Suppress the PII recorded by default to reduce risk. + _telemetryClient.Context.Cloud.RoleInstance = "Not Available"; + } + + private void LogTelemetry(AzTrace trace) + { + Dictionary telemetryEvent = new() + { + ["QueryId"] = trace.QueryId, + ["ConversationId"] = trace.ConversationId, + ["InstallationId"] = trace.InstallationId, + ["TopicName"] = trace.TopicName, + ["EventType"] = trace.EventType, + ["ShellCommand"] = trace.ShellCommand, + ["Details"] = GetDetailedMessage(trace.Details), + }; + + _telemetryClient.TrackTrace("AIShell-Test1022", telemetryEvent); + _telemetryClient.Flush(); + } + + private static string GetDetailedMessage(object details) + { + if (details is null) + { + return null; + } + + if (details is string str) + { + return str; + } + + return JsonSerializer.Serialize(details, Utils.RelaxedJsonEscapingOptions); + } + + /// + /// Gets whether or not telemetry is enabled. + /// + internal static bool Enabled => s_singleton is not null; + + /// + /// Initialize telemetry client. + /// + internal static void Initialize() => s_singleton ??= new Telemetry(); + + /// + /// Log a telemetry metric. + /// + internal static void Log(AzTrace trace) => s_singleton.LogTelemetry(trace); +} diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry/AzTrace.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry/AzTrace.cs deleted file mode 100644 index b5f9b4ee..00000000 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry/AzTrace.cs +++ /dev/null @@ -1,66 +0,0 @@ -using System.Text.Json; -using static Microsoft.ApplicationInsights.MetricDimensionNames.TelemetryContext; - -namespace Microsoft.Azure.Agent; - -public class AzTrace -{ - private static readonly string s_installationId; - private static string GetInstallationID() - { - string azureConfigDir = Environment.GetEnvironmentVariable("AZURE_CONFIG_DIR"); - string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); - string userProfilePath = Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json"); - - FileStream jsonStream; - JsonElement array; - string installationId; - - if (File.Exists(userProfilePath)) - { - jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); - array = JsonSerializer.Deserialize(jsonStream); - installationId = array.GetProperty("installationId").GetString(); - } - else - { - try - { - Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json"); - jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); - array = JsonSerializer.Deserialize(jsonStream); - installationId = array.GetProperty("Settings").GetProperty("InstallationId").GetString(); - } - catch - { - // If finally no installation id found, just return null. - return null; - } - } - - return installationId; - } - - public string TopicName; - // Each chat has a unique conversationId. When the cx runs /refresh, - // a new chat is initiated(i.e.a new conversationId will be created). - public string ConversationId; - // The activity id of the user's query - public string ActivityId; - public string InstallationId = s_installationId; - public string EventType; - public string Command; - /// - /// Detailed information containing additional Information - may contain: - /// Reason of dislike - /// - public string DetailedMessage; - /// - /// Agent Information - may contain: - /// Handler Version - /// Product Version - /// .net/python Version - /// - public Dictionary ExtendedProperties; - static AzTrace() => s_installationId = GetInstallationID(); -} diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry/MetricHelper.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry/MetricHelper.cs deleted file mode 100644 index 425d5da9..00000000 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry/MetricHelper.cs +++ /dev/null @@ -1,104 +0,0 @@ -using Microsoft.ApplicationInsights; -using Microsoft.ApplicationInsights.Channel; -using Microsoft.ApplicationInsights.Extensibility; -using Microsoft.ApplicationInsights.WorkerService; -using Microsoft.Extensions.DependencyInjection; - -namespace Microsoft.Azure.Agent; - -public sealed class MetricHelper -{ - public static readonly bool TelemetryOptOut = GetEnvironmentVariableAsBool("COPILOT_TELEMETRY_OPTOUT", false); - private const int CustomDomainMaximumSize = 8192; - private TelemetryClient _telemetryClient; - - private static readonly Lazy lazy = - new Lazy(() => new MetricHelper()); - - public static MetricHelper metricHelper { get { return lazy.Value; } } - - private MetricHelper() - { - InitializeTelemetryClient(); - } - - private void InitializeTelemetryClient() - { - // Create the DI container. - IServiceCollection services = new ServiceCollection(); - - // Add custom TelemetryInitializer - services.AddSingleton(typeof(ITelemetryInitializer), new MyCustomTelemetryInitializer()); - - // Being a regular console app, there is no appsettings.json or configuration providers enabled by default. - // Hence connection string must be specified here. - services.AddApplicationInsightsTelemetryWorkerService((ApplicationInsightsServiceOptions options) => - { - // Application insights in the temp environment. - options.ConnectionString = "InstrumentationKey=eea660a1-d969-44f8-abe4-96666e7fb159"; - options.EnableHeartbeat = false; - options.EnableDiagnosticsTelemetryModule = false; - } - ); - - // Build ServiceProvider. - IServiceProvider serviceProvider = services.BuildServiceProvider(); - - // Obtain TelemetryClient instance from DI, for additional manual tracking or to flush. - _telemetryClient = serviceProvider.GetRequiredService(); - - // Suppress the PII recorded by default to reduce risk. - _telemetryClient.Context.Cloud.RoleInstance = "Not Available"; - } - - public void LogTelemetry(AzTrace trace) - { - Dictionary eventProperties = new() - { - { "ActivityId", trace.ActivityId}, - { "CoversationId", trace.ConversationId }, - { "InstallationId", trace.InstallationId }, - { "Handler", trace.TopicName }, - { "EventType", trace.EventType }, - { "Command", trace.Command }, - { "DetailedMessage", trace.DetailedMessage } - }; - - _telemetryClient.TrackTrace("AIShell-Test1022", eventProperties); - _telemetryClient.Flush(); - } - - private static bool GetEnvironmentVariableAsBool(string name, bool defaultValue) - { - var str = Environment.GetEnvironmentVariable(name); - - if (string.IsNullOrWhiteSpace(str)) - { - return defaultValue; - } - - if (bool.TryParse(str, out bool result)) - { - return result; - } - - if (string.Equals(str, "yes", StringComparison.OrdinalIgnoreCase)) - { - return true; - } - - if (string.Equals(str, "no", StringComparison.OrdinalIgnoreCase)) - { - return false; - } - - return defaultValue; - } -} - -internal class MyCustomTelemetryInitializer : ITelemetryInitializer -{ - public void Initialize(ITelemetry telemetry){} - - public MyCustomTelemetryInitializer() {} -} diff --git a/shell/agents/Microsoft.Azure.Agent/Utils.cs b/shell/agents/Microsoft.Azure.Agent/Utils.cs index eafab63c..5d216358 100644 --- a/shell/agents/Microsoft.Azure.Agent/Utils.cs +++ b/shell/agents/Microsoft.Azure.Agent/Utils.cs @@ -1,3 +1,4 @@ +using System.Text.Encodings.Web; using System.Text.Json; namespace Microsoft.Azure.Agent; @@ -8,6 +9,7 @@ internal static class Utils private static readonly JsonSerializerOptions s_jsonOptions; private static readonly JsonSerializerOptions s_humanReadableOptions; + private static readonly JsonSerializerOptions s_relaxedJsonEscapingOptions; static Utils() { @@ -22,10 +24,16 @@ static Utils() WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, }; + + s_relaxedJsonEscapingOptions = new JsonSerializerOptions + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; } internal static JsonSerializerOptions JsonOptions => s_jsonOptions; internal static JsonSerializerOptions JsonHumanReadableOptions => s_humanReadableOptions; + internal static JsonSerializerOptions RelaxedJsonEscapingOptions => s_relaxedJsonEscapingOptions; } internal class TokenRequestException : Exception From 32cf41d618c25d1a3bc22f7ef93af27e59aef705 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Mon, 28 Oct 2024 18:24:30 -0700 Subject: [PATCH 03/14] More refactoring --- .../Microsoft.Azure.Agent/AzureAgent.cs | 15 +--- shell/agents/Microsoft.Azure.Agent/Command.cs | 2 +- .../Microsoft.Azure.Agent/DataRetriever.cs | 4 +- .../agents/Microsoft.Azure.Agent/Telemetry.cs | 76 +++++++++++++------ 4 files changed, 59 insertions(+), 38 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index 5fe63e17..bf1e1078 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -46,7 +46,6 @@ 7. DO NOT include the placeholder summary when the commands contains no placehol private readonly HttpClient _httpClient; private readonly ChatSession _chatSession; private readonly Dictionary _valueStore; - // private MetricHelper _metricHelper; public AzureAgent() { @@ -120,7 +119,7 @@ public void Initialize(AgentConfig config) } public IEnumerable GetCommands() => [new ReplaceCommand(this)]; - public bool CanAcceptFeedback(UserAction action) => _setting.Telemetry; + public bool CanAcceptFeedback(UserAction action) => Telemetry.Enabled; public void OnUserAction(UserActionPayload actionPayload) { // Send telemetry about the user action. bool isUserFeedback = false; @@ -138,7 +137,7 @@ public void OnUserAction(UserActionPayload actionPayload) { isUserFeedback = true; } - Telemetry.Log(AzTrace.UserAction(action.ToString(), _copilotResponse, details, isUserFeedback)); + Telemetry.Trace(AzTrace.UserAction(action.ToString(), _copilotResponse, details, isUserFeedback)); } public async Task RefreshChatAsync(IShell shell, bool force) @@ -280,10 +279,7 @@ public async Task ChatAsync(string input, IShell shell) } } - if (Telemetry.Enabled) - { - Telemetry.Log(AzTrace.Chat(_copilotResponse)); - } + Telemetry.Trace(AzTrace.Chat(_copilotResponse)); } catch (Exception ex) when (ex is TokenRequestException or ConnectionDroppedException) { @@ -392,11 +388,8 @@ private ResponseData ParseCLIHandlerResponse(IShell shell) else { // The placeholder section is not in the format as we've instructed ... - if (Telemetry.Enabled) - { - Telemetry.Log(AzTrace.Exception(_copilotResponse, $"Placeholder section not in expected format: {text}")); - } Log.Error("Placeholder section not in expected format:\n{0}", text); + Telemetry.Trace(AzTrace.Exception(_copilotResponse, "Placeholder section not in expected format.")); } ReplaceKnownPlaceholders(data); diff --git a/shell/agents/Microsoft.Azure.Agent/Command.cs b/shell/agents/Microsoft.Azure.Agent/Command.cs index 4694c99b..10522d0e 100644 --- a/shell/agents/Microsoft.Azure.Agent/Command.cs +++ b/shell/agents/Microsoft.Azure.Agent/Command.cs @@ -168,7 +168,7 @@ private void ReplaceAction() details.Add(name, _values.ContainsKey(name)); } - Telemetry.Log(AzTrace.UserAction("Replace", _agent.CopilotResponse, details)); + Telemetry.Trace(AzTrace.UserAction("Replace", _agent.CopilotResponse, details)); } try diff --git a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs index 52ee4618..2ace7751 100644 --- a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs +++ b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs @@ -586,7 +586,7 @@ private AzCLICommand QueryForMetadata(string azCommand) ["Command"] = azCommand, ["Message"] = "AzCLI metadata service returns unsuccessful status code for query." }; - Telemetry.Log(AzTrace.Exception(response: null, details)); + Telemetry.Trace(AzTrace.Exception(response: null, details)); } } } @@ -600,7 +600,7 @@ private AzCLICommand QueryForMetadata(string azCommand) ["Command"] = azCommand, ["Message"] = "AzCLI metadata query and process raised an exception." }; - Telemetry.Log(AzTrace.Exception(response: null, details)); + Telemetry.Trace(AzTrace.Exception(response: null, details)); } } diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index 4ff98e64..3fd3f8a7 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -86,32 +86,59 @@ internal static AzTrace UserAction( string shellCommand, CopilotResponse response, object details, - bool isFeedback = false) => new() + bool isFeedback = false) { - QueryId = response.ReplyToId, - TopicName = response.TopicName, - ConversationId = response.ConversationId, - ShellCommand = shellCommand, - EventType = isFeedback ? "Feedback" : "UserAction", - Details = details - }; - - internal static AzTrace Chat(CopilotResponse response) => new() + if (Telemetry.Enabled) + { + return new() + { + QueryId = response.ReplyToId, + TopicName = response.TopicName, + ConversationId = response.ConversationId, + ShellCommand = shellCommand, + EventType = isFeedback ? "Feedback" : "UserAction", + Details = details + }; + } + + // Don't create an object when telemetry is disabled. + return null; + } + + internal static AzTrace Chat(CopilotResponse response) { - EventType = "Chat", - QueryId = response.ReplyToId, - TopicName = response.TopicName, - ConversationId = response.ConversationId - }; + if (Telemetry.Enabled) + { + return new() + { + EventType = "Chat", + QueryId = response.ReplyToId, + TopicName = response.TopicName, + ConversationId = response.ConversationId + }; + } + + // Don't create an object when telemetry is disabled. + return null; + } - internal static AzTrace Exception(CopilotResponse response, object details) => new() + internal static AzTrace Exception(CopilotResponse response, object details) { - EventType = "Exception", - QueryId = response?.ReplyToId, - TopicName = response?.TopicName, - ConversationId = response?.ConversationId, - Details = details - }; + if (Telemetry.Enabled) + { + return new() + { + EventType = "Exception", + QueryId = response?.ReplyToId, + TopicName = response?.TopicName, + ConversationId = response?.ConversationId, + Details = details + }; + } + + // Don't create an object when telemetry is disabled. + return null; + } } internal class Telemetry @@ -184,7 +211,8 @@ private static string GetDetailedMessage(object details) internal static void Initialize() => s_singleton ??= new Telemetry(); /// - /// Log a telemetry metric. + /// Trace a telemetry metric. + /// The method does nothing when it's disabled. /// - internal static void Log(AzTrace trace) => s_singleton.LogTelemetry(trace); + internal static void Trace(AzTrace trace) => s_singleton?.LogTelemetry(trace); } From e68ca3d16a49a0f7ace772f36ada84fa4d8ff475 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Mon, 28 Oct 2024 18:31:28 -0700 Subject: [PATCH 04/14] Fix comment --- shell/agents/Microsoft.Azure.Agent/ChatSession.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs index 04d6f704..bc33854c 100644 --- a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs +++ b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs @@ -262,7 +262,7 @@ private HttpRequestMessage PrepareForChat(string input) var request = new HttpRequestMessage(HttpMethod.Post, _conversationUrl) { Content = content }; request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _token); - // These header is for server side telemetry to identify where the request comes from. + // This header is for server side telemetry to identify where the request comes from. request.Headers.Add("ClientType", "AIShell"); return request; } From 1182fbc52c5a5d419b05ddbd1fab143a12a9362c Mon Sep 17 00:00:00 2001 From: Nori Zhang Date: Tue, 29 Oct 2024 09:51:55 +0800 Subject: [PATCH 05/14] fix config file for azps --- shell/agents/Microsoft.Azure.Agent/Telemetry.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index 3fd3f8a7..59c212ee 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -28,7 +28,7 @@ static AzTrace() { try { - Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json"); + Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "AzureRmContextSettings.json"); using var jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); array = JsonSerializer.Deserialize(jsonStream); s_installationId = array.GetProperty("Settings").GetProperty("InstallationId").GetString(); From 15c81b0b9d0123406e6704a6ab7799d421c038e3 Mon Sep 17 00:00:00 2001 From: Nori Zhang Date: Tue, 29 Oct 2024 11:16:02 +0800 Subject: [PATCH 06/14] log exception --- shell/agents/Microsoft.Azure.Agent/DataRetriever.cs | 4 ++-- shell/agents/Microsoft.Azure.Agent/Telemetry.cs | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs index 2ace7751..ceced951 100644 --- a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs +++ b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs @@ -598,9 +598,9 @@ private AzCLICommand QueryForMetadata(string azCommand) Dictionary details = new() { ["Command"] = azCommand, - ["Message"] = "AzCLI metadata query and process raised an exception." + ["Message"] = $"AzCLI metadata query and process raised an exception" }; - Telemetry.Trace(AzTrace.Exception(response: null, details)); + Telemetry.Trace(AzTrace.Exception(response: null, details), e); } } diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index 59c212ee..26abad68 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -168,7 +168,7 @@ private Telemetry() _telemetryClient.Context.Cloud.RoleInstance = "Not Available"; } - private void LogTelemetry(AzTrace trace) + private void LogTelemetry(AzTrace trace, Exception e = null) { Dictionary telemetryEvent = new() { @@ -182,6 +182,7 @@ private void LogTelemetry(AzTrace trace) }; _telemetryClient.TrackTrace("AIShell-Test1022", telemetryEvent); + if (e != null) { _telemetryClient.TrackException(e); } _telemetryClient.Flush(); } @@ -215,4 +216,10 @@ private static string GetDetailedMessage(object details) /// The method does nothing when it's disabled. /// internal static void Trace(AzTrace trace) => s_singleton?.LogTelemetry(trace); + + /// + /// Trace a telemetry metric and an Exception with it. + /// The method does nothing when it's disabled. + /// + internal static void Trace(AzTrace trace, Exception e) => s_singleton?.LogTelemetry(trace, e); } From d8cd11b1e758efda9c1158eea80b9b445d6f16f2 Mon Sep 17 00:00:00 2001 From: Nori Zhang Date: Tue, 29 Oct 2024 13:38:02 +0800 Subject: [PATCH 07/14] installation id --- shell/agents/Microsoft.Azure.Agent/Telemetry.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index 26abad68..e299bbf5 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -13,7 +13,7 @@ static AzTrace() { string azureConfigDir = Environment.GetEnvironmentVariable("AZURE_CONFIG_DIR"); string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); - string userProfilePath = Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json"); + string userProfilePath = string.IsNullOrEmpty(azureConfigDir) ? Path.Combine(userProfile, ".Azure", "azureProfile.json") : azureConfigDir; JsonElement array; s_installationId = null; @@ -28,7 +28,7 @@ static AzTrace() { try { - Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "AzureRmContextSettings.json"); + userProfilePath = string.IsNullOrEmpty(azureConfigDir) ? Path.Combine(userProfile, ".Azure", "azureProfile.json") : azureConfigDir; // "AzureRmContextSettings.json"); using var jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); array = JsonSerializer.Deserialize(jsonStream); s_installationId = array.GetProperty("Settings").GetProperty("InstallationId").GetString(); From 85d9bf70e8b452fdb9a0d7e7986d0586b22a089e Mon Sep 17 00:00:00 2001 From: Nori Zhang Date: Tue, 29 Oct 2024 14:13:12 +0800 Subject: [PATCH 08/14] remove test flag --- shell/agents/Microsoft.Azure.Agent/Telemetry.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index e299bbf5..e1680427 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -28,7 +28,7 @@ static AzTrace() { try { - userProfilePath = string.IsNullOrEmpty(azureConfigDir) ? Path.Combine(userProfile, ".Azure", "azureProfile.json") : azureConfigDir; // "AzureRmContextSettings.json"); + userProfilePath = string.IsNullOrEmpty(azureConfigDir) ? Path.Combine(userProfile, ".Azure", "AzureRmContextSettings.json") : azureConfigDir; using var jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); array = JsonSerializer.Deserialize(jsonStream); s_installationId = array.GetProperty("Settings").GetProperty("InstallationId").GetString(); @@ -153,7 +153,7 @@ private Telemetry() IServiceCollection services = new ServiceCollection() .AddApplicationInsightsTelemetryWorkerService((ApplicationInsightsServiceOptions options) => { - // Application insights in the temp environment. + // Application insights in the test environment. options.ConnectionString = "InstrumentationKey=eea660a1-d969-44f8-abe4-96666e7fb159"; options.EnableHeartbeat = false; options.EnableDiagnosticsTelemetryModule = false; @@ -181,7 +181,7 @@ private void LogTelemetry(AzTrace trace, Exception e = null) ["Details"] = GetDetailedMessage(trace.Details), }; - _telemetryClient.TrackTrace("AIShell-Test1022", telemetryEvent); + _telemetryClient.TrackTrace("AIShell", telemetryEvent); if (e != null) { _telemetryClient.TrackException(e); } _telemetryClient.Flush(); } From 9263c030115243edd98dac89fc252ce35e6cde81 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Tue, 29 Oct 2024 11:37:49 -0700 Subject: [PATCH 09/14] Fix in how to get the installation id in AzTrace --- .../Microsoft.Azure.Agent/DataRetriever.cs | 2 +- .../agents/Microsoft.Azure.Agent/Telemetry.cs | 65 +++++++++---------- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs index ceced951..993da4ec 100644 --- a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs +++ b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs @@ -598,7 +598,7 @@ private AzCLICommand QueryForMetadata(string azCommand) Dictionary details = new() { ["Command"] = azCommand, - ["Message"] = $"AzCLI metadata query and process raised an exception" + ["Message"] = "AzCLI metadata query and process raised an exception." }; Telemetry.Trace(AzTrace.Exception(response: null, details), e); } diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index e1680427..0629d504 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -8,49 +8,42 @@ namespace Microsoft.Azure.Agent; public class AzTrace { - private static readonly string s_installationId; - static AzTrace() + /// + /// Installation id from the Azure CLI installation. + /// + internal static string InstallationId { get; private set; } + + internal static void Initialize() { - string azureConfigDir = Environment.GetEnvironmentVariable("AZURE_CONFIG_DIR"); - string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); - string userProfilePath = string.IsNullOrEmpty(azureConfigDir) ? Path.Combine(userProfile, ".Azure", "azureProfile.json") : azureConfigDir; + InstallationId = null; - JsonElement array; - s_installationId = null; + string azureConfigDir = Environment.GetEnvironmentVariable("AZURE_CONFIG_DIR") + ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".azure"); + string azCLIProfilePath = Path.Combine(azureConfigDir, "azureProfile.json"); + string azPSHProfilePath = Path.Combine(azureConfigDir, "AzureRmContextSettings.json"); - if (File.Exists(userProfilePath)) + try { - using var jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); - array = JsonSerializer.Deserialize(jsonStream); - s_installationId = array.GetProperty("installationId").GetString(); - } - else - { - try + if (File.Exists(azCLIProfilePath)) { - userProfilePath = string.IsNullOrEmpty(azureConfigDir) ? Path.Combine(userProfile, ".Azure", "AzureRmContextSettings.json") : azureConfigDir; - using var jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read); - array = JsonSerializer.Deserialize(jsonStream); - s_installationId = array.GetProperty("Settings").GetProperty("InstallationId").GetString(); + using var stream = File.OpenRead(azCLIProfilePath); + var jsonElement = JsonSerializer.Deserialize(stream); + InstallationId = jsonElement.GetProperty("installationId").GetString(); } - catch + else if (File.Exists(azPSHProfilePath)) { - // If finally no installation id found, just return null. - s_installationId = null; + using var stream = File.OpenRead(azPSHProfilePath); + var jsonElement = JsonSerializer.Deserialize(stream); + InstallationId = jsonElement.GetProperty("Settings").GetProperty(nameof(InstallationId)).GetString(); } } + catch + { + // Something wrong when reading the config file. + InstallationId = null; + } } - internal AzTrace() - { - InstallationId = s_installationId; - } - - /// - /// Installation id from the Azure CLI installation. - /// - internal string InstallationId { get; } - /// /// Topic name of the response from Azure Copilot. /// @@ -174,7 +167,7 @@ private void LogTelemetry(AzTrace trace, Exception e = null) { ["QueryId"] = trace.QueryId, ["ConversationId"] = trace.ConversationId, - ["InstallationId"] = trace.InstallationId, + ["InstallationId"] = AzTrace.InstallationId, ["TopicName"] = trace.TopicName, ["EventType"] = trace.EventType, ["ShellCommand"] = trace.ShellCommand, @@ -209,7 +202,11 @@ private static string GetDetailedMessage(object details) /// /// Initialize telemetry client. /// - internal static void Initialize() => s_singleton ??= new Telemetry(); + internal static void Initialize() + { + s_singleton ??= new Telemetry(); + AzTrace.Initialize(); + } /// /// Trace a telemetry metric. From 49a9885fce3d3973fc6036cdc32d92b0dd2af44b Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Tue, 29 Oct 2024 11:44:45 -0700 Subject: [PATCH 10/14] More update --- shell/agents/Microsoft.Azure.Agent/Telemetry.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index 0629d504..bc3ecceb 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -136,7 +136,9 @@ internal static AzTrace Exception(CopilotResponse response, object details) internal class Telemetry { + private static bool s_enabled; private static Telemetry s_singleton; + private readonly TelemetryClient _telemetryClient; private Telemetry() @@ -197,15 +199,19 @@ private static string GetDetailedMessage(object details) /// /// Gets whether or not telemetry is enabled. /// - internal static bool Enabled => s_singleton is not null; + internal static bool Enabled => s_enabled; /// /// Initialize telemetry client. /// internal static void Initialize() { - s_singleton ??= new Telemetry(); - AzTrace.Initialize(); + if (s_singleton is null) + { + s_singleton = new Telemetry(); + s_enabled = true; + AzTrace.Initialize(); + } } /// From adfe00d14b949c1307a92601409ca223f687b816 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Tue, 29 Oct 2024 12:11:00 -0700 Subject: [PATCH 11/14] Update tracking exception and call `Flush` when shutting down --- .../Microsoft.Azure.Agent/AzureAgent.cs | 1 + .../agents/Microsoft.Azure.Agent/Telemetry.cs | 32 ++++++++++++++----- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index bf1e1078..497b10b5 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -83,6 +83,7 @@ public void Dispose() _httpClient.Dispose(); Log.CloseAndFlush(); + Telemetry.CloseAndFlush(); } public void Initialize(AgentConfig config) diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index bc3ecceb..bade3a0b 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -163,9 +163,9 @@ private Telemetry() _telemetryClient.Context.Cloud.RoleInstance = "Not Available"; } - private void LogTelemetry(AzTrace trace, Exception e = null) + private void LogTelemetry(AzTrace trace, Exception exception) { - Dictionary telemetryEvent = new() + Dictionary properties = new() { ["QueryId"] = trace.QueryId, ["ConversationId"] = trace.ConversationId, @@ -176,8 +176,18 @@ private void LogTelemetry(AzTrace trace, Exception e = null) ["Details"] = GetDetailedMessage(trace.Details), }; - _telemetryClient.TrackTrace("AIShell", telemetryEvent); - if (e != null) { _telemetryClient.TrackException(e); } + if (exception is null) + { + _telemetryClient.TrackTrace("AIShell", properties); + } + else + { + _telemetryClient.TrackException(exception, properties); + } + } + + private void Flush() + { _telemetryClient.Flush(); } @@ -216,13 +226,19 @@ internal static void Initialize() /// /// Trace a telemetry metric. - /// The method does nothing when it's disabled. + /// The method does nothing when telemetry is disabled. /// - internal static void Trace(AzTrace trace) => s_singleton?.LogTelemetry(trace); + internal static void Trace(AzTrace trace) => s_singleton?.LogTelemetry(trace, exception: null); /// /// Trace a telemetry metric and an Exception with it. - /// The method does nothing when it's disabled. + /// The method does nothing when telemetry is disabled. + /// + internal static void Trace(AzTrace trace, Exception exception) => s_singleton?.LogTelemetry(trace, exception); + + /// + /// Flush and close the telemetry. + /// The method does nothing when telemetry is disabled. /// - internal static void Trace(AzTrace trace, Exception e) => s_singleton?.LogTelemetry(trace, e); + internal static void CloseAndFlush() => s_singleton?.Flush(); } From 97849df68fb7fff1bb2446b471c5730c0d98ce8d Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Tue, 29 Oct 2024 13:37:38 -0700 Subject: [PATCH 12/14] Send telemetry when it fails to generate token, open conversation, or refresh token --- .../Microsoft.Azure.Agent/ChatSession.cs | 115 ++++++++++-------- .../Microsoft.Azure.Agent/DataRetriever.cs | 4 +- .../agents/Microsoft.Azure.Agent/Telemetry.cs | 4 +- 3 files changed, 72 insertions(+), 51 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs index bc33854c..4934a884 100644 --- a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs +++ b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs @@ -87,16 +87,8 @@ internal async Task RefreshAsync(IStatusContext context, bool force, Can } } - try - { - _token = await GenerateTokenAsync(context, cancellationToken); - return await StartConversationAsync(context, cancellationToken); - } - catch (Exception) - { - Reset(); - throw; - } + _token = await GenerateTokenAsync(context, cancellationToken); + return await OpenConversationAsync(context, cancellationToken); } private void Reset() @@ -113,57 +105,83 @@ private void Reset() private async Task GenerateTokenAsync(IStatusContext context, CancellationToken cancellationToken) { - context.Status("Get Azure CLI login token ..."); - // Get an access token from the AzCLI login, using the specific audience guid. - AccessToken accessToken = await new AzureCliCredential() - .GetTokenAsync( - new TokenRequestContext(["7000789f-b583-4714-ab18-aef39213018a/.default"]), - cancellationToken); - - context.Status("Request for DirectLine token ..."); - StringContent content = new("{\"conversationType\": \"Chat\"}", Encoding.UTF8, Utils.JsonContentType); - HttpRequestMessage request = new(HttpMethod.Post, DL_TOKEN_URL) { Content = content }; - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken.Token); + try + { + context.Status("Get Azure CLI login token ..."); + // Get an access token from the AzCLI login, using the specific audience guid. + AccessToken accessToken = await new AzureCliCredential() + .GetTokenAsync( + new TokenRequestContext(["7000789f-b583-4714-ab18-aef39213018a/.default"]), + cancellationToken); + + context.Status("Request for DirectLine token ..."); + StringContent content = new("{\"conversationType\": \"Chat\"}", Encoding.UTF8, Utils.JsonContentType); + HttpRequestMessage request = new(HttpMethod.Post, DL_TOKEN_URL) { Content = content }; + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken.Token); + + HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - response.EnsureSuccessStatusCode(); + using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken); + var dlToken = JsonSerializer.Deserialize(stream, Utils.JsonOptions); + return dlToken.DirectLine.Token; + } + catch (Exception e) + { + if (e is not OperationCanceledException) + { + Telemetry.Trace(AzTrace.Exception("Failed to generate the initial DL token."), e); + } - using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken); - var dlToken = JsonSerializer.Deserialize(stream, Utils.JsonOptions); - return dlToken.DirectLine.Token; + Reset(); + throw; + } } - private async Task StartConversationAsync(IStatusContext context, CancellationToken cancellationToken) + private async Task OpenConversationAsync(IStatusContext context, CancellationToken cancellationToken) { - context.Status("Start a new chat session ..."); - HttpRequestMessage request = new(HttpMethod.Post, CONVERSATION_URL); - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _token); + try + { + context.Status("Start a new chat session ..."); + HttpRequestMessage request = new(HttpMethod.Post, CONVERSATION_URL); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _token); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - response.EnsureSuccessStatusCode(); + HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); - using Stream content = await response.Content.ReadAsStreamAsync(cancellationToken); - SessionPayload spl = JsonSerializer.Deserialize(content, Utils.JsonOptions); + using Stream content = await response.Content.ReadAsStreamAsync(cancellationToken); + SessionPayload spl = JsonSerializer.Deserialize(content, Utils.JsonOptions); - _token = spl.Token; - _conversationId = spl.ConversationId; - _conversationUrl = $"{CONVERSATION_URL}/{_conversationId}/activities"; - _streamUrl = spl.StreamUrl; - _expireOn = DateTime.UtcNow.AddSeconds(spl.ExpiresIn); - _copilotReceiver = await AzureCopilotReceiver.CreateAsync(_streamUrl); + _token = spl.Token; + _conversationId = spl.ConversationId; + _conversationUrl = $"{CONVERSATION_URL}/{_conversationId}/activities"; + _streamUrl = spl.StreamUrl; + _expireOn = DateTime.UtcNow.AddSeconds(spl.ExpiresIn); + _copilotReceiver = await AzureCopilotReceiver.CreateAsync(_streamUrl); - Log.Debug("[ChatSession] Conversation started. Id: {0}", _conversationId); + Log.Debug("[ChatSession] Conversation started. Id: {0}", _conversationId); - while (true) + while (true) + { + CopilotActivity activity = _copilotReceiver.Take(cancellationToken); + if (activity.IsMessage && activity.IsFromCopilot && _copilotReceiver.Watermark is 0) + { + activity.ExtractMetadata(out _, out ConversationState conversationState); + int chatNumber = conversationState.DailyConversationNumber; + int requestNumber = conversationState.TurnNumber; + return $"{activity.Text}\nThis is chat #{chatNumber}, request #{requestNumber}.\n"; + } + } + } + catch (Exception e) { - CopilotActivity activity = _copilotReceiver.Take(cancellationToken); - if (activity.IsMessage && activity.IsFromCopilot && _copilotReceiver.Watermark is 0) + if (e is not OperationCanceledException) { - activity.ExtractMetadata(out _, out ConversationState conversationState); - int chatNumber = conversationState.DailyConversationNumber; - int requestNumber = conversationState.TurnNumber; - return $"{activity.Text}\nThis is chat #{chatNumber}, request #{requestNumber}.\n"; + Telemetry.Trace(AzTrace.Exception("Failed to open conversation with the initial DL token."), e); } + + Reset(); + throw; } } @@ -216,6 +234,7 @@ private async Task RenewTokenAsync(CancellationToken cancellationToken) catch (Exception e) when (e is not OperationCanceledException) { Reset(); + Telemetry.Trace(AzTrace.Exception("Failed to refresh the DL token."), e); throw new TokenRequestException($"Failed to refresh the 'DirectLine' token: {e.Message}.", e); } } diff --git a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs index 993da4ec..7c85bd38 100644 --- a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs +++ b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs @@ -586,7 +586,7 @@ private AzCLICommand QueryForMetadata(string azCommand) ["Command"] = azCommand, ["Message"] = "AzCLI metadata service returns unsuccessful status code for query." }; - Telemetry.Trace(AzTrace.Exception(response: null, details)); + Telemetry.Trace(AzTrace.Exception(details)); } } } @@ -600,7 +600,7 @@ private AzCLICommand QueryForMetadata(string azCommand) ["Command"] = azCommand, ["Message"] = "AzCLI metadata query and process raised an exception." }; - Telemetry.Trace(AzTrace.Exception(response: null, details), e); + Telemetry.Trace(AzTrace.Exception(details), e); } } diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs index bade3a0b..cdde4686 100644 --- a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -132,6 +132,8 @@ internal static AzTrace Exception(CopilotResponse response, object details) // Don't create an object when telemetry is disabled. return null; } + + internal static AzTrace Exception(object details) => Exception(response: null, details); } internal class Telemetry @@ -231,7 +233,7 @@ internal static void Initialize() internal static void Trace(AzTrace trace) => s_singleton?.LogTelemetry(trace, exception: null); /// - /// Trace a telemetry metric and an Exception with it. + /// Trace a telemetry metric and an exception with it. /// The method does nothing when telemetry is disabled. /// internal static void Trace(AzTrace trace, Exception exception) => s_singleton?.LogTelemetry(trace, exception); From 5da719a13d99cf32dc33004468c3724f88fd9fb6 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Tue, 29 Oct 2024 14:04:23 -0700 Subject: [PATCH 13/14] Minor update --- .../Microsoft.Azure.Agent/DataRetriever.cs | 30 +++++-------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs index 7c85bd38..2e291b31 100644 --- a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs +++ b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs @@ -562,32 +562,16 @@ private AzCLICommand QueryForMetadata(string azCommand) { using var cts = new CancellationTokenSource(1200); var response = _httpClient.Send(request, HttpCompletionOption.ResponseHeadersRead, cts.Token); + response.EnsureSuccessStatusCode(); - if (response.IsSuccessStatusCode) - { - using Stream stream = response.Content.ReadAsStream(cts.Token); - using JsonDocument document = JsonDocument.Parse(stream); + using Stream stream = response.Content.ReadAsStream(cts.Token); + using JsonDocument document = JsonDocument.Parse(stream); - JsonElement root = document.RootElement; - if (root.TryGetProperty("data", out JsonElement data) && - data.TryGetProperty("metadata", out JsonElement metadata)) - { - command = metadata.Deserialize(Utils.JsonOptions); - } - } - else + JsonElement root = document.RootElement; + if (root.TryGetProperty("data", out JsonElement data) && + data.TryGetProperty("metadata", out JsonElement metadata)) { - Log.Error("[QueryForMetadata] Received status code '{0}' for command '{1}'", response.StatusCode, azCommand); - if (Telemetry.Enabled) - { - Dictionary details = new() - { - ["StatusCode"] = response.StatusCode.ToString(), - ["Command"] = azCommand, - ["Message"] = "AzCLI metadata service returns unsuccessful status code for query." - }; - Telemetry.Trace(AzTrace.Exception(details)); - } + command = metadata.Deserialize(Utils.JsonOptions); } } catch (Exception e) From 98d73d669f350a6b44c075fa2d5fe8d49b15de10 Mon Sep 17 00:00:00 2001 From: Dongbo Wang Date: Tue, 29 Oct 2024 14:36:14 -0700 Subject: [PATCH 14/14] Fix the `CopilotResponse` property --- shell/agents/Microsoft.Azure.Agent/AzureAgent.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index 497b10b5..ce694385 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -17,7 +17,7 @@ public sealed class AzureAgent : ILLMAgent public string SettingFile { private set; get; } internal ArgumentPlaceholder ArgPlaceholder { set; get; } - internal CopilotResponse CopilotResponse { set; get; } + internal CopilotResponse CopilotResponse => _copilotResponse; private const string SettingFileName = "az.config.json"; private const string LoggingFileName = "log..txt";