diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs index 7e5fb983..ce694385 100644 --- a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs +++ b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs @@ -17,6 +17,7 @@ public sealed class AzureAgent : ILLMAgent public string SettingFile { private set; get; } internal ArgumentPlaceholder ArgPlaceholder { set; get; } + internal CopilotResponse CopilotResponse => _copilotResponse; private const string SettingFileName = "az.config.json"; private const string LoggingFileName = "log..txt"; @@ -82,6 +83,7 @@ public void Dispose() _httpClient.Dispose(); Log.CloseAndFlush(); + Telemetry.CloseAndFlush(); } public void Initialize(AgentConfig config) @@ -110,11 +112,34 @@ public void Initialize(AgentConfig config) .CreateLogger(); Log.Information("Azure agent initialized."); } + + if (_setting.Telemetry) + { + Telemetry.Initialize(); + } } public IEnumerable GetCommands() => [new ReplaceCommand(this)]; - public bool CanAcceptFeedback(UserAction action) => false; - public void OnUserAction(UserActionPayload actionPayload) {} + public bool CanAcceptFeedback(UserAction action) => Telemetry.Enabled; + public void OnUserAction(UserActionPayload actionPayload) { + // Send telemetry about the user action. + bool isUserFeedback = false; + string details = null; + UserAction action = actionPayload.Action; + + if (action is UserAction.Dislike) + { + var dislike = (DislikePayload) actionPayload; + isUserFeedback = true; + details = string.Format("{0} | {1}", dislike.ShortFeedback, dislike.LongFeedback); + } + else if (action is UserAction.Like) + { + isUserFeedback = true; + } + + Telemetry.Trace(AzTrace.UserAction(action.ToString(), _copilotResponse, details, isUserFeedback)); + } public async Task RefreshChatAsync(IShell shell, bool force) { @@ -254,6 +279,8 @@ public async Task ChatAsync(string input, IShell shell) host.WriteLine("\nYou've reached the maximum length of a conversation. To continue, please run '/refresh' to start a new conversation.\n"); } } + + Telemetry.Trace(AzTrace.Chat(_copilotResponse)); } catch (Exception ex) when (ex is TokenRequestException or ConnectionDroppedException) { @@ -362,8 +389,8 @@ private ResponseData ParseCLIHandlerResponse(IShell shell) else { // The placeholder section is not in the format as we've instructed ... - // TODO: send telemetry about this case. Log.Error("Placeholder section not in expected format:\n{0}", text); + Telemetry.Trace(AzTrace.Exception(_copilotResponse, "Placeholder section not in expected format.")); } ReplaceKnownPlaceholders(data); diff --git a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs index de11cd1d..4934a884 100644 --- a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs +++ b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs @@ -87,16 +87,8 @@ internal async Task RefreshAsync(IStatusContext context, bool force, Can } } - try - { - _token = await GenerateTokenAsync(context, cancellationToken); - return await StartConversationAsync(context, cancellationToken); - } - catch (Exception) - { - Reset(); - throw; - } + _token = await GenerateTokenAsync(context, cancellationToken); + return await OpenConversationAsync(context, cancellationToken); } private void Reset() @@ -113,57 +105,83 @@ private void Reset() private async Task GenerateTokenAsync(IStatusContext context, CancellationToken cancellationToken) { - context.Status("Get Azure CLI login token ..."); - // Get an access token from the AzCLI login, using the specific audience guid. - AccessToken accessToken = await new AzureCliCredential() - .GetTokenAsync( - new TokenRequestContext(["7000789f-b583-4714-ab18-aef39213018a/.default"]), - cancellationToken); - - context.Status("Request for DirectLine token ..."); - StringContent content = new("{\"conversationType\": \"Chat\"}", Encoding.UTF8, Utils.JsonContentType); - HttpRequestMessage request = new(HttpMethod.Post, DL_TOKEN_URL) { Content = content }; - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken.Token); + try + { + context.Status("Get Azure CLI login token ..."); + // Get an access token from the AzCLI login, using the specific audience guid. + AccessToken accessToken = await new AzureCliCredential() + .GetTokenAsync( + new TokenRequestContext(["7000789f-b583-4714-ab18-aef39213018a/.default"]), + cancellationToken); + + context.Status("Request for DirectLine token ..."); + StringContent content = new("{\"conversationType\": \"Chat\"}", Encoding.UTF8, Utils.JsonContentType); + HttpRequestMessage request = new(HttpMethod.Post, DL_TOKEN_URL) { Content = content }; + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken.Token); + + HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - response.EnsureSuccessStatusCode(); + using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken); + var dlToken = JsonSerializer.Deserialize(stream, Utils.JsonOptions); + return dlToken.DirectLine.Token; + } + catch (Exception e) + { + if (e is not OperationCanceledException) + { + Telemetry.Trace(AzTrace.Exception("Failed to generate the initial DL token."), e); + } - using Stream stream = await response.Content.ReadAsStreamAsync(cancellationToken); - var dlToken = JsonSerializer.Deserialize(stream, Utils.JsonOptions); - return dlToken.DirectLine.Token; + Reset(); + throw; + } } - private async Task StartConversationAsync(IStatusContext context, CancellationToken cancellationToken) + private async Task OpenConversationAsync(IStatusContext context, CancellationToken cancellationToken) { - context.Status("Start a new chat session ..."); - HttpRequestMessage request = new(HttpMethod.Post, CONVERSATION_URL); - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _token); + try + { + context.Status("Start a new chat session ..."); + HttpRequestMessage request = new(HttpMethod.Post, CONVERSATION_URL); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _token); - HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); - response.EnsureSuccessStatusCode(); + HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); + response.EnsureSuccessStatusCode(); - using Stream content = await response.Content.ReadAsStreamAsync(cancellationToken); - SessionPayload spl = JsonSerializer.Deserialize(content, Utils.JsonOptions); + using Stream content = await response.Content.ReadAsStreamAsync(cancellationToken); + SessionPayload spl = JsonSerializer.Deserialize(content, Utils.JsonOptions); - _token = spl.Token; - _conversationId = spl.ConversationId; - _conversationUrl = $"{CONVERSATION_URL}/{_conversationId}/activities"; - _streamUrl = spl.StreamUrl; - _expireOn = DateTime.UtcNow.AddSeconds(spl.ExpiresIn); - _copilotReceiver = await AzureCopilotReceiver.CreateAsync(_streamUrl); + _token = spl.Token; + _conversationId = spl.ConversationId; + _conversationUrl = $"{CONVERSATION_URL}/{_conversationId}/activities"; + _streamUrl = spl.StreamUrl; + _expireOn = DateTime.UtcNow.AddSeconds(spl.ExpiresIn); + _copilotReceiver = await AzureCopilotReceiver.CreateAsync(_streamUrl); - Log.Debug("[ChatSession] Conversation started. Id: {0}", _conversationId); + Log.Debug("[ChatSession] Conversation started. Id: {0}", _conversationId); - while (true) + while (true) + { + CopilotActivity activity = _copilotReceiver.Take(cancellationToken); + if (activity.IsMessage && activity.IsFromCopilot && _copilotReceiver.Watermark is 0) + { + activity.ExtractMetadata(out _, out ConversationState conversationState); + int chatNumber = conversationState.DailyConversationNumber; + int requestNumber = conversationState.TurnNumber; + return $"{activity.Text}\nThis is chat #{chatNumber}, request #{requestNumber}.\n"; + } + } + } + catch (Exception e) { - CopilotActivity activity = _copilotReceiver.Take(cancellationToken); - if (activity.IsMessage && activity.IsFromCopilot && _copilotReceiver.Watermark is 0) + if (e is not OperationCanceledException) { - activity.ExtractMetadata(out _, out ConversationState conversationState); - int chatNumber = conversationState.DailyConversationNumber; - int requestNumber = conversationState.TurnNumber; - return $"{activity.Text}\nThis is chat #{chatNumber}, request #{requestNumber}.\n"; + Telemetry.Trace(AzTrace.Exception("Failed to open conversation with the initial DL token."), e); } + + Reset(); + throw; } } @@ -216,6 +234,7 @@ private async Task RenewTokenAsync(CancellationToken cancellationToken) catch (Exception e) when (e is not OperationCanceledException) { Reset(); + Telemetry.Trace(AzTrace.Exception("Failed to refresh the DL token."), e); throw new TokenRequestException($"Failed to refresh the 'DirectLine' token: {e.Message}.", e); } } @@ -262,6 +281,8 @@ private HttpRequestMessage PrepareForChat(string input) var request = new HttpRequestMessage(HttpMethod.Post, _conversationUrl) { Content = content }; request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _token); + // This header is for server side telemetry to identify where the request comes from. + request.Headers.Add("ClientType", "AIShell"); return request; } diff --git a/shell/agents/Microsoft.Azure.Agent/Command.cs b/shell/agents/Microsoft.Azure.Agent/Command.cs index e664417a..10522d0e 100644 --- a/shell/agents/Microsoft.Azure.Agent/Command.cs +++ b/shell/agents/Microsoft.Azure.Agent/Command.cs @@ -1,5 +1,6 @@ using System.CommandLine; using System.Text; + using AIShell.Abstraction; namespace Microsoft.Azure.Agent; @@ -158,6 +159,18 @@ private void ReplaceAction() host.RenderDivider("Regenerate", DividerAlignment.Left); host.MarkupLine($"\nQuery: [teal]{ap.Query}[/]"); + if (Telemetry.Enabled) + { + Dictionary details = new(items.Count); + foreach (var item in items) + { + string name = item.Name; + details.Add(name, _values.ContainsKey(name)); + } + + Telemetry.Trace(AzTrace.UserAction("Replace", _agent.CopilotResponse, details)); + } + try { string answer = host.RunWithSpinnerAsync(RegenerateAsync).GetAwaiter().GetResult(); diff --git a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs index 5674f422..2e291b31 100644 --- a/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs +++ b/shell/agents/Microsoft.Azure.Agent/DataRetriever.cs @@ -562,29 +562,30 @@ private AzCLICommand QueryForMetadata(string azCommand) { using var cts = new CancellationTokenSource(1200); var response = _httpClient.Send(request, HttpCompletionOption.ResponseHeadersRead, cts.Token); + response.EnsureSuccessStatusCode(); - if (response.IsSuccessStatusCode) - { - using Stream stream = response.Content.ReadAsStream(cts.Token); - using JsonDocument document = JsonDocument.Parse(stream); + using Stream stream = response.Content.ReadAsStream(cts.Token); + using JsonDocument document = JsonDocument.Parse(stream); - JsonElement root = document.RootElement; - if (root.TryGetProperty("data", out JsonElement data) && - data.TryGetProperty("metadata", out JsonElement metadata)) - { - command = metadata.Deserialize(Utils.JsonOptions); - } - } - else + JsonElement root = document.RootElement; + if (root.TryGetProperty("data", out JsonElement data) && + data.TryGetProperty("metadata", out JsonElement metadata)) { - // TODO: telemetry. - Log.Error("[QueryForMetadata] Received status code '{0}' for command '{1}'", response.StatusCode, azCommand); + command = metadata.Deserialize(Utils.JsonOptions); } } catch (Exception e) { - // TODO: telemetry. Log.Error(e, "[QueryForMetadata] Exception while processing command: {0}", azCommand); + if (Telemetry.Enabled) + { + Dictionary details = new() + { + ["Command"] = azCommand, + ["Message"] = "AzCLI metadata query and process raised an exception." + }; + Telemetry.Trace(AzTrace.Exception(details), e); + } } return command; diff --git a/shell/agents/Microsoft.Azure.Agent/Telemetry.cs b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs new file mode 100644 index 00000000..cdde4686 --- /dev/null +++ b/shell/agents/Microsoft.Azure.Agent/Telemetry.cs @@ -0,0 +1,246 @@ +using System.Text.Json; + +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.WorkerService; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Azure.Agent; + +public class AzTrace +{ + /// + /// Installation id from the Azure CLI installation. + /// + internal static string InstallationId { get; private set; } + + internal static void Initialize() + { + InstallationId = null; + + string azureConfigDir = Environment.GetEnvironmentVariable("AZURE_CONFIG_DIR") + ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".azure"); + string azCLIProfilePath = Path.Combine(azureConfigDir, "azureProfile.json"); + string azPSHProfilePath = Path.Combine(azureConfigDir, "AzureRmContextSettings.json"); + + try + { + if (File.Exists(azCLIProfilePath)) + { + using var stream = File.OpenRead(azCLIProfilePath); + var jsonElement = JsonSerializer.Deserialize(stream); + InstallationId = jsonElement.GetProperty("installationId").GetString(); + } + else if (File.Exists(azPSHProfilePath)) + { + using var stream = File.OpenRead(azPSHProfilePath); + var jsonElement = JsonSerializer.Deserialize(stream); + InstallationId = jsonElement.GetProperty("Settings").GetProperty(nameof(InstallationId)).GetString(); + } + } + catch + { + // Something wrong when reading the config file. + InstallationId = null; + } + } + + /// + /// Topic name of the response from Azure Copilot. + /// + internal string TopicName { get; set; } + + /// + /// Each chat has a unique conversation id. When the customer runs '/refresh', + /// a new chat will be initiated (i.e. a new conversation id will be created). + /// + internal string ConversationId { get; set; } + + /// + /// The activity id of the user's query. + /// + internal string QueryId { get; set; } + + /// + /// The event type of this telemetry. + /// + internal string EventType { get; set; } + + /// + /// The shell command that triggered this telemetry. + /// + internal string ShellCommand { get; set; } + + /// + /// Detailed information. + /// + internal object Details { get; set; } + + internal static AzTrace UserAction( + string shellCommand, + CopilotResponse response, + object details, + bool isFeedback = false) + { + if (Telemetry.Enabled) + { + return new() + { + QueryId = response.ReplyToId, + TopicName = response.TopicName, + ConversationId = response.ConversationId, + ShellCommand = shellCommand, + EventType = isFeedback ? "Feedback" : "UserAction", + Details = details + }; + } + + // Don't create an object when telemetry is disabled. + return null; + } + + internal static AzTrace Chat(CopilotResponse response) + { + if (Telemetry.Enabled) + { + return new() + { + EventType = "Chat", + QueryId = response.ReplyToId, + TopicName = response.TopicName, + ConversationId = response.ConversationId + }; + } + + // Don't create an object when telemetry is disabled. + return null; + } + + internal static AzTrace Exception(CopilotResponse response, object details) + { + if (Telemetry.Enabled) + { + return new() + { + EventType = "Exception", + QueryId = response?.ReplyToId, + TopicName = response?.TopicName, + ConversationId = response?.ConversationId, + Details = details + }; + } + + // Don't create an object when telemetry is disabled. + return null; + } + + internal static AzTrace Exception(object details) => Exception(response: null, details); +} + +internal class Telemetry +{ + private static bool s_enabled; + private static Telemetry s_singleton; + + private readonly TelemetryClient _telemetryClient; + + private Telemetry() + { + // Being a regular console app, there is no appsettings.json or configuration providers enabled by default. + // Hence connection string must be specified here. + IServiceCollection services = new ServiceCollection() + .AddApplicationInsightsTelemetryWorkerService((ApplicationInsightsServiceOptions options) => + { + // Application insights in the test environment. + options.ConnectionString = "InstrumentationKey=eea660a1-d969-44f8-abe4-96666e7fb159"; + options.EnableHeartbeat = false; + options.EnableDiagnosticsTelemetryModule = false; + }); + + // Obtain TelemetryClient instance from DI, for additional manual tracking or to flush. + _telemetryClient = services + .BuildServiceProvider() + .GetRequiredService(); + + // Suppress the PII recorded by default to reduce risk. + _telemetryClient.Context.Cloud.RoleInstance = "Not Available"; + } + + private void LogTelemetry(AzTrace trace, Exception exception) + { + Dictionary properties = new() + { + ["QueryId"] = trace.QueryId, + ["ConversationId"] = trace.ConversationId, + ["InstallationId"] = AzTrace.InstallationId, + ["TopicName"] = trace.TopicName, + ["EventType"] = trace.EventType, + ["ShellCommand"] = trace.ShellCommand, + ["Details"] = GetDetailedMessage(trace.Details), + }; + + if (exception is null) + { + _telemetryClient.TrackTrace("AIShell", properties); + } + else + { + _telemetryClient.TrackException(exception, properties); + } + } + + private void Flush() + { + _telemetryClient.Flush(); + } + + private static string GetDetailedMessage(object details) + { + if (details is null) + { + return null; + } + + if (details is string str) + { + return str; + } + + return JsonSerializer.Serialize(details, Utils.RelaxedJsonEscapingOptions); + } + + /// + /// Gets whether or not telemetry is enabled. + /// + internal static bool Enabled => s_enabled; + + /// + /// Initialize telemetry client. + /// + internal static void Initialize() + { + if (s_singleton is null) + { + s_singleton = new Telemetry(); + s_enabled = true; + AzTrace.Initialize(); + } + } + + /// + /// Trace a telemetry metric. + /// The method does nothing when telemetry is disabled. + /// + internal static void Trace(AzTrace trace) => s_singleton?.LogTelemetry(trace, exception: null); + + /// + /// Trace a telemetry metric and an exception with it. + /// The method does nothing when telemetry is disabled. + /// + internal static void Trace(AzTrace trace, Exception exception) => s_singleton?.LogTelemetry(trace, exception); + + /// + /// Flush and close the telemetry. + /// The method does nothing when telemetry is disabled. + /// + internal static void CloseAndFlush() => s_singleton?.Flush(); +} diff --git a/shell/agents/Microsoft.Azure.Agent/Utils.cs b/shell/agents/Microsoft.Azure.Agent/Utils.cs index eafab63c..5d216358 100644 --- a/shell/agents/Microsoft.Azure.Agent/Utils.cs +++ b/shell/agents/Microsoft.Azure.Agent/Utils.cs @@ -1,3 +1,4 @@ +using System.Text.Encodings.Web; using System.Text.Json; namespace Microsoft.Azure.Agent; @@ -8,6 +9,7 @@ internal static class Utils private static readonly JsonSerializerOptions s_jsonOptions; private static readonly JsonSerializerOptions s_humanReadableOptions; + private static readonly JsonSerializerOptions s_relaxedJsonEscapingOptions; static Utils() { @@ -22,10 +24,16 @@ static Utils() WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase, }; + + s_relaxedJsonEscapingOptions = new JsonSerializerOptions + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; } internal static JsonSerializerOptions JsonOptions => s_jsonOptions; internal static JsonSerializerOptions JsonHumanReadableOptions => s_humanReadableOptions; + internal static JsonSerializerOptions RelaxedJsonEscapingOptions => s_relaxedJsonEscapingOptions; } internal class TokenRequestException : Exception