Skip to content

Update Telemetry for Public Preview #249

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 5 commits into from
Closed

Update Telemetry for Public Preview #249

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
92c22b9
1017 test
NoriZC Oct 16, 2024
78a5789
Merge branch 'main' of https://github.com/PowerShell/ProjectMercury
NoriZC Oct 16, 2024
203ab27
azcli telemetry polish
NoriZC Oct 17, 2024
0ca23dd
telemetry polish
NoriZC Oct 21, 2024
5aefd0b
Merge branch 'PowerShell:main' into main
NoriZC Oct 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 8 additions & 51 deletions shell/agents/AIShell.Azure.Agent/AzCLI/AzCLIAgent.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,10 @@ public sealed class AzCLIAgent : ILLMAgent
internal UserValueStore ValueStore { get; } = new();

private const string SettingFileName = "az-cli.agent.json";
private readonly Stopwatch _watch = new();

private AzCLIChatService _chatService;
internal AzCLIChatService _chatService;
private StringBuilder _text;
private MetricHelper _metricHelper;
private LinkedList<HistoryMessage> _historyForTelemetry;
internal MetricHelper _metricHelper;

public void Dispose()
{
Expand All @@ -38,7 +36,6 @@ public void Initialize(AgentConfig config)
{
_text = new StringBuilder();
_chatService = new AzCLIChatService();
_historyForTelemetry = [];
_metricHelper = new MetricHelper(AzCLIChatService.Endpoint);

LegalLinks = new(StringComparer.OrdinalIgnoreCase)
Expand All @@ -61,32 +58,10 @@ public void OnUserAction(UserActionPayload actionPayload)
// Send telemetry about the user action.
// DisLike Action
string DetailedMessage = null;
LinkedList<HistoryMessage> history = null;
if (actionPayload.Action == UserAction.Dislike)
{
DislikePayload dislikePayload = (DislikePayload)actionPayload;
DetailedMessage = string.Format("{0} | {1}", dislikePayload.ShortFeedback, dislikePayload.LongFeedback);
if (dislikePayload.ShareConversation)
{
history = _historyForTelemetry;
}
else
{
_historyForTelemetry.Clear();
}
}
// Like Action
else if (actionPayload.Action == UserAction.Like)
{
LikePayload likePayload = (LikePayload)actionPayload;
if (likePayload.ShareConversation)
{
history = _historyForTelemetry;
}
else
{
_historyForTelemetry.Clear();
}
}

_metricHelper.LogTelemetry(
Expand All @@ -96,8 +71,7 @@ public void OnUserAction(UserActionPayload actionPayload)
CorrelationID = _chatService.CorrelationID,
EventType = "Feedback",
Handler = "Azure CLI",
DetailedMessage = DetailedMessage,
HistoryMessage = history
DetailedMessage = DetailedMessage
});
}

Expand All @@ -113,10 +87,6 @@ public Task RefreshChatAsync(IShell shell)

public async Task<bool> ChatAsync(string input, IShell shell)
{
// Measure time spent
_watch.Restart();
var startTime = DateTime.Now;

IHost host = shell.Host;
CancellationToken token = shell.CancellationToken;

Expand All @@ -143,28 +113,16 @@ public async Task<bool> ChatAsync(string input, IShell shell)
string answer = GenerateAnswer(input, data);
host.RenderFullResponse(answer);

// Measure time spent
_watch.Stop();

if (!MetricHelper.TelemetryOptOut)
if (!MetricHelper.TelemetryOptOut)
{
// TODO: extract into RecordQuestionTelemetry() : RecordTelemetry()
var EndTime = DateTime.Now;
var Duration = TimeSpan.FromTicks(_watch.ElapsedTicks);

// Append last Q&A history in HistoryMessage
_historyForTelemetry.AddLast(new HistoryMessage("user", input, _chatService.CorrelationID));
_historyForTelemetry.AddLast(new HistoryMessage("assistant", answer, _chatService.CorrelationID));

_metricHelper.LogTelemetry(
new AzTrace()
{
CorrelationID = _chatService.CorrelationID,
Duration = Duration,
EndTime = EndTime,
EventType = "Question",
Handler = "Azure CLI",
StartTime = startTime
EventType = "Chat",
Handler = "Azure CLI"
});
}
}
Expand All @@ -186,8 +144,7 @@ public async Task<bool> ChatAsync(string input, IShell shell)
}
finally
{
// Stop the watch in case of early return or exception.
_watch.Stop();

}

return true;
Expand Down Expand Up @@ -221,7 +178,7 @@ internal string GenerateAnswer(string input, ResponseData data)
CommandItem action = data.CommandSet[i];
// Replace the pseudo values with the real values.
string script = ValueStore.ReplacePseudoValues(action.Script);

_text.Append($"{i+1}. {action.Desc}")
.Append("\n\n")
.Append("```sh\n")
Expand Down
17 changes: 17 additions & 0 deletions shell/agents/AIShell.Azure.Agent/AzCLI/Command.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System.CommandLine;
using System.Text;
using System.Text.Json;
using System.Timers;
using AIShell.Abstraction;

namespace AIShell.Azure.CLI;
Expand Down Expand Up @@ -72,6 +73,7 @@ private void ReplaceAction()

try
{
Dictionary<string, Boolean> DetailedMessage = new();
for (int i = 0; i < items.Count; i++)
{
var item = items[i];
Expand Down Expand Up @@ -123,11 +125,26 @@ private void ReplaceAction()
_productNames.Add(prodName.ToLower());
_environmentNames.Add(envName.ToLower());
}
DetailedMessage.Add(item.Name, true);
}
else
{
DetailedMessage.Add(item.Name, false);
}

// Write an extra new line.
host.WriteLine();
}

_agent._metricHelper.LogTelemetry(
new AzTrace()
{
Command = "Replace",
CorrelationID = _agent._chatService.CorrelationID,
EventType = "Feedback",
Handler = "Azure CLI",
DetailedMessage = JsonSerializer.Serialize(DetailedMessage)
});
}
catch (OperationCanceledException)
{
Expand Down
51 changes: 4 additions & 47 deletions shell/agents/AIShell.Azure.Agent/AzPS/AzPSAgent.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,11 @@ public sealed class AzPSAgent : ILLMAgent
public string SettingFile { private set; get; } = null;

private const string SettingFileName = "az-ps.agent.json";
private readonly Stopwatch _watch = new();

private string _configRoot;
private RenderingStyle _renderingStyle;
private AzPSChatService _chatService;
private MetricHelper _metricHelper;
private LinkedList<HistoryMessage> _historyForTelemetry;

public void Dispose()
{
Expand All @@ -51,7 +49,6 @@ public void Initialize(AgentConfig config)
["Transparency"] = "https://aka.ms/CopilotAzCLIPSTransparency",
};

_historyForTelemetry = [];
_metricHelper = new MetricHelper(AzPSChatService.Endpoint);
_chatService = new AzPSChatService(config.IsInteractive, tenantId);
}
Expand All @@ -64,32 +61,10 @@ public void OnUserAction(UserActionPayload actionPayload)
{
// DisLike Action
string DetailedMessage = null;
LinkedList<HistoryMessage> history = null;
if (actionPayload.Action == UserAction.Dislike)
{
DislikePayload dislikePayload = (DislikePayload)actionPayload;
DetailedMessage = string.Format("{0} | {1}", dislikePayload.ShortFeedback, dislikePayload.LongFeedback);
if (dislikePayload.ShareConversation)
{
history = _historyForTelemetry;
}
else
{
_historyForTelemetry.Clear();
}
}
// Like Action
else if (actionPayload.Action == UserAction.Like)
{
LikePayload likePayload = (LikePayload)actionPayload;
if (likePayload.ShareConversation)
{
history = _historyForTelemetry;
}
else
{
_historyForTelemetry.Clear();
}
}

// TODO: Extract into RecrodActionTelemetry : RecordTelemetry()
Expand All @@ -100,8 +75,7 @@ public void OnUserAction(UserActionPayload actionPayload)
CorrelationID = _chatService.CorrelationID,
EventType = "Feedback",
Handler = "Azure PowerShell",
DetailedMessage = DetailedMessage,
HistoryMessage = history
DetailedMessage = DetailedMessage
});
}

Expand All @@ -115,10 +89,6 @@ public Task RefreshChatAsync(IShell shell)

public async Task<bool> ChatAsync(string input, IShell shell)
{
// Measure time spent
_watch.Restart();
var startTime = DateTime.Now;

IHost host = shell.Host;
CancellationToken token = shell.CancellationToken;

Expand Down Expand Up @@ -160,28 +130,16 @@ public async Task<bool> ChatAsync(string input, IShell shell)
string accumulatedContent = streamingRender.AccumulatedContent;
_chatService.AddResponseToHistory(accumulatedContent);

// Measure time spent
_watch.Stop();

if (!MetricHelper.TelemetryOptOut)
{
// TODO: extract into RecordQuestionTelemetry() : RecordTelemetry()
var EndTime = DateTime.Now;
var Duration = TimeSpan.FromTicks(_watch.ElapsedTicks);

// Append last Q&A history in HistoryMessage
_historyForTelemetry.AddLast(new HistoryMessage("user", input, _chatService.CorrelationID));
_historyForTelemetry.AddLast(new HistoryMessage("assistant", accumulatedContent, _chatService.CorrelationID));

_metricHelper.LogTelemetry(
new AzTrace()
{
CorrelationID = _chatService.CorrelationID,
Duration = Duration,
EndTime = EndTime,
EventType = "Question",
Handler = "Azure PowerShell",
StartTime = startTime
EventType = "Chat",
Handler = "Azure PowerShell"
});
}
}
Expand All @@ -202,8 +160,7 @@ public async Task<bool> ChatAsync(string input, IShell shell)
}
finally
{
// Stop the watch in case of early return or exception.
_watch.Stop();

}

return true;
Expand Down
8 changes: 4 additions & 4 deletions shell/agents/AIShell.Azure.Agent/Telemetry/AzTrace.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@ private static string GetInstallationID()
// CorrelationId from client side.
public string CorrelationID;
// private bool _enableAzureDataCollection = null;
public TimeSpan? Duration;
public DateTime? StartTime;
public DateTime? EndTime;
// public TimeSpan? Duration;
// public DateTime? StartTime;
// public DateTime? EndTime;
public string InstallationID = s_installationId;
public string EventType;
public string Command;
Expand All @@ -54,7 +54,7 @@ private static string GetInstallationID()
/// Reason of dislike
/// </summary>
public string DetailedMessage;
internal LinkedList<HistoryMessage> HistoryMessage;
// internal LinkedList<HistoryMessage> HistoryMessage;
/// <summary>
/// Agent Information - may contain:
/// Handler Version
Expand Down
13 changes: 5 additions & 8 deletions shell/agents/AIShell.Azure.Agent/Telemetry/MetricHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ private void InitializeTelemetryClient()
services.AddApplicationInsightsTelemetryWorkerService((ApplicationInsightsServiceOptions options) =>
{
// Application insights in the temp environment.
options.ConnectionString = "InstrumentationKey=7c753881-1ada-44a0-81be-94fb6a06dc34";
options.ConnectionString = "InstrumentationKey=eea660a1-d969-44f8-abe4-96666e7fb159";
options.EnableHeartbeat = false;
options.EnableDiagnosticsTelemetryModule = false;
}
Expand All @@ -63,6 +63,7 @@ private void InitializeTelemetryClient()

public void LogTelemetry(AzTrace trace)
{
/*
string historyJson;

while (true)
Expand All @@ -79,23 +80,19 @@ public void LogTelemetry(AzTrace trace)
{
break;
}
}
}*/

Dictionary<string, string> eventProperties = new()
{
{ "CorrelationID", trace.CorrelationID },
{ "InstallationID", trace.InstallationID },
{ "Handler", trace.Handler },
{ "EventType", trace.EventType },
{ "Duration", trace.Duration?.ToString() },
{ "Command", trace.Command },
{ "DetailedMessage", trace.DetailedMessage },
{ "HistoryMessage", historyJson },
{ "StartTime", trace.StartTime?.ToString() },
{ "EndTime", trace.EndTime?.ToString() },
{ "DetailedMessage", trace.DetailedMessage }
};

_telemetryClient.TrackTrace("AIShell", eventProperties);
_telemetryClient.TrackTrace("AIShell-Test1021", eventProperties);

// Explicitly call Flush() followed by sleep is required in Console Apps.
// This is to ensure that even if application terminates, telemetry is sent to the back-end.
Expand Down