diff --git a/shell/agents/AIShell.Azure.Agent/AzCLI/AzCLIAgent.cs b/shell/agents/AIShell.Azure.Agent/AzCLI/AzCLIAgent.cs index d185f523..22b613b8 100644 --- a/shell/agents/AIShell.Azure.Agent/AzCLI/AzCLIAgent.cs +++ b/shell/agents/AIShell.Azure.Agent/AzCLI/AzCLIAgent.cs @@ -22,12 +22,10 @@ public sealed class AzCLIAgent : ILLMAgent internal UserValueStore ValueStore { get; } = new(); private const string SettingFileName = "az-cli.agent.json"; - private readonly Stopwatch _watch = new(); - private AzCLIChatService _chatService; + internal AzCLIChatService _chatService; private StringBuilder _text; - private MetricHelper _metricHelper; - private LinkedList _historyForTelemetry; + internal MetricHelper _metricHelper; public void Dispose() { @@ -38,7 +36,6 @@ public void Initialize(AgentConfig config) { _text = new StringBuilder(); _chatService = new AzCLIChatService(); - _historyForTelemetry = []; _metricHelper = new MetricHelper(AzCLIChatService.Endpoint); LegalLinks = new(StringComparer.OrdinalIgnoreCase) @@ -61,32 +58,10 @@ public void OnUserAction(UserActionPayload actionPayload) // Send telemetry about the user action. // DisLike Action string DetailedMessage = null; - LinkedList history = null; if (actionPayload.Action == UserAction.Dislike) { DislikePayload dislikePayload = (DislikePayload)actionPayload; DetailedMessage = string.Format("{0} | {1}", dislikePayload.ShortFeedback, dislikePayload.LongFeedback); - if (dislikePayload.ShareConversation) - { - history = _historyForTelemetry; - } - else - { - _historyForTelemetry.Clear(); - } - } - // Like Action - else if (actionPayload.Action == UserAction.Like) - { - LikePayload likePayload = (LikePayload)actionPayload; - if (likePayload.ShareConversation) - { - history = _historyForTelemetry; - } - else - { - _historyForTelemetry.Clear(); - } } _metricHelper.LogTelemetry( @@ -96,8 +71,7 @@ public void OnUserAction(UserActionPayload actionPayload) CorrelationID = _chatService.CorrelationID, EventType = "Feedback", Handler = "Azure CLI", - DetailedMessage = DetailedMessage, - HistoryMessage = history + DetailedMessage = DetailedMessage }); } @@ -113,10 +87,6 @@ public Task RefreshChatAsync(IShell shell) public async Task ChatAsync(string input, IShell shell) { - // Measure time spent - _watch.Restart(); - var startTime = DateTime.Now; - IHost host = shell.Host; CancellationToken token = shell.CancellationToken; @@ -143,28 +113,16 @@ public async Task ChatAsync(string input, IShell shell) string answer = GenerateAnswer(input, data); host.RenderFullResponse(answer); - // Measure time spent - _watch.Stop(); - - if (!MetricHelper.TelemetryOptOut) + if (!MetricHelper.TelemetryOptOut) { // TODO: extract into RecordQuestionTelemetry() : RecordTelemetry() - var EndTime = DateTime.Now; - var Duration = TimeSpan.FromTicks(_watch.ElapsedTicks); - - // Append last Q&A history in HistoryMessage - _historyForTelemetry.AddLast(new HistoryMessage("user", input, _chatService.CorrelationID)); - _historyForTelemetry.AddLast(new HistoryMessage("assistant", answer, _chatService.CorrelationID)); _metricHelper.LogTelemetry( new AzTrace() { CorrelationID = _chatService.CorrelationID, - Duration = Duration, - EndTime = EndTime, - EventType = "Question", - Handler = "Azure CLI", - StartTime = startTime + EventType = "Chat", + Handler = "Azure CLI" }); } } @@ -186,8 +144,7 @@ public async Task ChatAsync(string input, IShell shell) } finally { - // Stop the watch in case of early return or exception. - _watch.Stop(); + } return true; @@ -221,7 +178,7 @@ internal string GenerateAnswer(string input, ResponseData data) CommandItem action = data.CommandSet[i]; // Replace the pseudo values with the real values. string script = ValueStore.ReplacePseudoValues(action.Script); - + _text.Append($"{i+1}. {action.Desc}") .Append("\n\n") .Append("```sh\n") diff --git a/shell/agents/AIShell.Azure.Agent/AzCLI/Command.cs b/shell/agents/AIShell.Azure.Agent/AzCLI/Command.cs index 2d8cbb6e..7711b537 100644 --- a/shell/agents/AIShell.Azure.Agent/AzCLI/Command.cs +++ b/shell/agents/AIShell.Azure.Agent/AzCLI/Command.cs @@ -1,6 +1,7 @@ using System.CommandLine; using System.Text; using System.Text.Json; +using System.Timers; using AIShell.Abstraction; namespace AIShell.Azure.CLI; @@ -72,6 +73,7 @@ private void ReplaceAction() try { + Dictionary DetailedMessage = new(); for (int i = 0; i < items.Count; i++) { var item = items[i]; @@ -123,11 +125,26 @@ private void ReplaceAction() _productNames.Add(prodName.ToLower()); _environmentNames.Add(envName.ToLower()); } + DetailedMessage.Add(item.Name, true); + } + else + { + DetailedMessage.Add(item.Name, false); } // Write an extra new line. host.WriteLine(); } + + _agent._metricHelper.LogTelemetry( + new AzTrace() + { + Command = "Replace", + CorrelationID = _agent._chatService.CorrelationID, + EventType = "Feedback", + Handler = "Azure CLI", + DetailedMessage = JsonSerializer.Serialize(DetailedMessage) + }); } catch (OperationCanceledException) { diff --git a/shell/agents/AIShell.Azure.Agent/AzPS/AzPSAgent.cs b/shell/agents/AIShell.Azure.Agent/AzPS/AzPSAgent.cs index 2b64d45d..fbd15381 100644 --- a/shell/agents/AIShell.Azure.Agent/AzPS/AzPSAgent.cs +++ b/shell/agents/AIShell.Azure.Agent/AzPS/AzPSAgent.cs @@ -18,13 +18,11 @@ public sealed class AzPSAgent : ILLMAgent public string SettingFile { private set; get; } = null; private const string SettingFileName = "az-ps.agent.json"; - private readonly Stopwatch _watch = new(); private string _configRoot; private RenderingStyle _renderingStyle; private AzPSChatService _chatService; private MetricHelper _metricHelper; - private LinkedList _historyForTelemetry; public void Dispose() { @@ -51,7 +49,6 @@ public void Initialize(AgentConfig config) ["Transparency"] = "https://aka.ms/CopilotAzCLIPSTransparency", }; - _historyForTelemetry = []; _metricHelper = new MetricHelper(AzPSChatService.Endpoint); _chatService = new AzPSChatService(config.IsInteractive, tenantId); } @@ -64,32 +61,10 @@ public void OnUserAction(UserActionPayload actionPayload) { // DisLike Action string DetailedMessage = null; - LinkedList history = null; if (actionPayload.Action == UserAction.Dislike) { DislikePayload dislikePayload = (DislikePayload)actionPayload; DetailedMessage = string.Format("{0} | {1}", dislikePayload.ShortFeedback, dislikePayload.LongFeedback); - if (dislikePayload.ShareConversation) - { - history = _historyForTelemetry; - } - else - { - _historyForTelemetry.Clear(); - } - } - // Like Action - else if (actionPayload.Action == UserAction.Like) - { - LikePayload likePayload = (LikePayload)actionPayload; - if (likePayload.ShareConversation) - { - history = _historyForTelemetry; - } - else - { - _historyForTelemetry.Clear(); - } } // TODO: Extract into RecrodActionTelemetry : RecordTelemetry() @@ -100,8 +75,7 @@ public void OnUserAction(UserActionPayload actionPayload) CorrelationID = _chatService.CorrelationID, EventType = "Feedback", Handler = "Azure PowerShell", - DetailedMessage = DetailedMessage, - HistoryMessage = history + DetailedMessage = DetailedMessage }); } @@ -115,10 +89,6 @@ public Task RefreshChatAsync(IShell shell) public async Task ChatAsync(string input, IShell shell) { - // Measure time spent - _watch.Restart(); - var startTime = DateTime.Now; - IHost host = shell.Host; CancellationToken token = shell.CancellationToken; @@ -160,28 +130,16 @@ public async Task ChatAsync(string input, IShell shell) string accumulatedContent = streamingRender.AccumulatedContent; _chatService.AddResponseToHistory(accumulatedContent); - // Measure time spent - _watch.Stop(); - if (!MetricHelper.TelemetryOptOut) { // TODO: extract into RecordQuestionTelemetry() : RecordTelemetry() - var EndTime = DateTime.Now; - var Duration = TimeSpan.FromTicks(_watch.ElapsedTicks); - - // Append last Q&A history in HistoryMessage - _historyForTelemetry.AddLast(new HistoryMessage("user", input, _chatService.CorrelationID)); - _historyForTelemetry.AddLast(new HistoryMessage("assistant", accumulatedContent, _chatService.CorrelationID)); _metricHelper.LogTelemetry( new AzTrace() { CorrelationID = _chatService.CorrelationID, - Duration = Duration, - EndTime = EndTime, - EventType = "Question", - Handler = "Azure PowerShell", - StartTime = startTime + EventType = "Chat", + Handler = "Azure PowerShell" }); } } @@ -202,8 +160,7 @@ public async Task ChatAsync(string input, IShell shell) } finally { - // Stop the watch in case of early return or exception. - _watch.Stop(); + } return true; diff --git a/shell/agents/AIShell.Azure.Agent/Telemetry/AzTrace.cs b/shell/agents/AIShell.Azure.Agent/Telemetry/AzTrace.cs index 35ba819b..907b55af 100644 --- a/shell/agents/AIShell.Azure.Agent/Telemetry/AzTrace.cs +++ b/shell/agents/AIShell.Azure.Agent/Telemetry/AzTrace.cs @@ -43,9 +43,9 @@ private static string GetInstallationID() // CorrelationId from client side. public string CorrelationID; // private bool _enableAzureDataCollection = null; - public TimeSpan? Duration; - public DateTime? StartTime; - public DateTime? EndTime; + // public TimeSpan? Duration; + // public DateTime? StartTime; + // public DateTime? EndTime; public string InstallationID = s_installationId; public string EventType; public string Command; @@ -54,7 +54,7 @@ private static string GetInstallationID() /// Reason of dislike /// public string DetailedMessage; - internal LinkedList HistoryMessage; + // internal LinkedList HistoryMessage; /// /// Agent Information - may contain: /// Handler Version diff --git a/shell/agents/AIShell.Azure.Agent/Telemetry/MetricHelper.cs b/shell/agents/AIShell.Azure.Agent/Telemetry/MetricHelper.cs index 027eaaa7..2c076a5a 100644 --- a/shell/agents/AIShell.Azure.Agent/Telemetry/MetricHelper.cs +++ b/shell/agents/AIShell.Azure.Agent/Telemetry/MetricHelper.cs @@ -42,7 +42,7 @@ private void InitializeTelemetryClient() services.AddApplicationInsightsTelemetryWorkerService((ApplicationInsightsServiceOptions options) => { // Application insights in the temp environment. - options.ConnectionString = "InstrumentationKey=7c753881-1ada-44a0-81be-94fb6a06dc34"; + options.ConnectionString = "InstrumentationKey=eea660a1-d969-44f8-abe4-96666e7fb159"; options.EnableHeartbeat = false; options.EnableDiagnosticsTelemetryModule = false; } @@ -63,6 +63,7 @@ private void InitializeTelemetryClient() public void LogTelemetry(AzTrace trace) { + /* string historyJson; while (true) @@ -79,7 +80,7 @@ public void LogTelemetry(AzTrace trace) { break; } - } + }*/ Dictionary eventProperties = new() { @@ -87,15 +88,11 @@ public void LogTelemetry(AzTrace trace) { "InstallationID", trace.InstallationID }, { "Handler", trace.Handler }, { "EventType", trace.EventType }, - { "Duration", trace.Duration?.ToString() }, { "Command", trace.Command }, - { "DetailedMessage", trace.DetailedMessage }, - { "HistoryMessage", historyJson }, - { "StartTime", trace.StartTime?.ToString() }, - { "EndTime", trace.EndTime?.ToString() }, + { "DetailedMessage", trace.DetailedMessage } }; - _telemetryClient.TrackTrace("AIShell", eventProperties); + _telemetryClient.TrackTrace("AIShell-Test1021", eventProperties); // Explicitly call Flush() followed by sleep is required in Console Apps. // This is to ensure that even if application terminates, telemetry is sent to the back-end.