Skip to content

Commit d7a00ce

Browse files
NoriZCdaxian-dbw
NoriZC
authored and
daxian-dbw
committed
telemetry for msaz
telemetry for msaz Remove unneeded changes to `CopilotResponse` Address Comments Minor fix set json serializer option
1 parent 16e4c40 commit d7a00ce

File tree

6 files changed

+295
-7
lines changed

6 files changed

+295
-7
lines changed

‎shell/agents/Microsoft.Azure.Agent/AzureAgent.cs‎

Lines changed: 56 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
using System.Diagnostics;
22
using System.Text;
3-
3+
using System.Text.Json;
44
using AIShell.Abstraction;
55
using Azure.Identity;
66
using Serilog;
@@ -37,14 +37,15 @@ 7. DO NOT include the placeholder summary when the commands contains no placehol
3737
""";
3838

3939
private int _turnsLeft;
40-
private CopilotResponse _copilotResponse;
40+
internal CopilotResponse _copilotResponse;
4141
private AgentSetting _setting;
4242

4343
private readonly string _instructions;
4444
private readonly StringBuilder _buffer;
4545
private readonly HttpClient _httpClient;
46-
private readonly ChatSession _chatSession;
46+
internal readonly ChatSession _chatSession;
4747
private readonly Dictionary<string, string> _valueStore;
48+
// private MetricHelper _metricHelper;
4849

4950
public AzureAgent()
5051
{
@@ -113,8 +114,34 @@ public void Initialize(AgentConfig config)
113114
}
114115

115116
public IEnumerable<CommandBase> GetCommands() => [new ReplaceCommand(this)];
116-
public bool CanAcceptFeedback(UserAction action) => false;
117-
public void OnUserAction(UserActionPayload actionPayload) {}
117+
public bool CanAcceptFeedback(UserAction action) => !MetricHelper.TelemetryOptOut;
118+
public void OnUserAction(UserActionPayload actionPayload) {
119+
// Send telemetry about the user action.
120+
// DisLike Action
121+
string DetailedMessage = null;
122+
bool IsUserFeedback = false;
123+
if (actionPayload.Action == UserAction.Dislike)
124+
{
125+
IsUserFeedback = true;
126+
DislikePayload dislikePayload = (DislikePayload)actionPayload;
127+
DetailedMessage = string.Format("{0} | {1}", dislikePayload.ShortFeedback, dislikePayload.LongFeedback);
128+
}
129+
else if (actionPayload.Action == UserAction.Like)
130+
{
131+
IsUserFeedback = true;
132+
}
133+
134+
MetricHelper.metricHelper.LogTelemetry(
135+
new AzTrace()
136+
{
137+
Command = actionPayload.Action.ToString(),
138+
ConversationId = _chatSession.ConversationId,
139+
ActivityId = _copilotResponse.ReplyToId,
140+
EventType = IsUserFeedback ? "Feedback" : "UserAction",
141+
TopicName = _copilotResponse.TopicName,
142+
DetailedMessage = DetailedMessage
143+
});
144+
}
118145

119146
public async Task RefreshChatAsync(IShell shell, bool force)
120147
{
@@ -254,6 +281,18 @@ public async Task<bool> ChatAsync(string input, IShell shell)
254281
host.WriteLine("\nYou've reached the maximum length of a conversation. To continue, please run '/refresh' to start a new conversation.\n");
255282
}
256283
}
284+
285+
if (!MetricHelper.TelemetryOptOut)
286+
{
287+
MetricHelper.metricHelper.LogTelemetry(
288+
new AzTrace()
289+
{
290+
ConversationId = _chatSession.ConversationId,
291+
EventType = "Chat",
292+
TopicName = _copilotResponse.TopicName,
293+
ActivityId = _copilotResponse.ReplyToId
294+
});
295+
}
257296
}
258297
catch (Exception ex) when (ex is TokenRequestException or ConnectionDroppedException)
259298
{
@@ -363,6 +402,18 @@ private ResponseData ParseCLIHandlerResponse(IShell shell)
363402
{
364403
// The placeholder section is not in the format as we've instructed ...
365404
// TODO: send telemetry about this case.
405+
if (!MetricHelper.TelemetryOptOut)
406+
{
407+
MetricHelper.metricHelper.LogTelemetry(
408+
new AzTrace()
409+
{
410+
ConversationId = _chatSession.ConversationId,
411+
ActivityId = _copilotResponse.ReplyToId,
412+
EventType = "Exception",
413+
TopicName = _copilotResponse.TopicName,
414+
DetailedMessage = $"Placeholder section not in expected format:{text}"
415+
});
416+
}
366417
Log.Error("Placeholder section not in expected format:\n{0}", text);
367418
}
368419

‎shell/agents/Microsoft.Azure.Agent/ChatSession.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,8 @@ private HttpRequestMessage PrepareForChat(string input)
262262
var request = new HttpRequestMessage(HttpMethod.Post, _conversationUrl) { Content = content };
263263

264264
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _token);
265+
// These header is for server side telemetry to identify where the request comes from.
266+
request.Headers.Add("ClientType", "AIShell");
265267
return request;
266268
}
267269

‎shell/agents/Microsoft.Azure.Agent/Command.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
using System.CommandLine;
22
using System.Text;
3+
using System.Text.Encodings.Web;
4+
using System.Text.Json;
5+
using System.Text.Unicode;
36
using AIShell.Abstraction;
47

58
namespace Microsoft.Azure.Agent;
@@ -67,6 +70,8 @@ private void ReplaceAction()
6770

6871
try
6972
{
73+
// Detailed Message recorded indicating whether each placeholder is replaced.
74+
Dictionary<string, Boolean> DetailedMessage = new();
7075
for (int i = 0; i < items.Count; i++)
7176
{
7277
var item = items[i];
@@ -117,17 +122,43 @@ private void ReplaceAction()
117122

118123
_values.Add(item.Name, value);
119124
_agent.SaveUserValue(item.Name, value);
125+
DetailedMessage.Add(item.Name, true);
120126

121127
if (nameArgInfo is not null && nameArgInfo.NamingRule.TryMatchName(value, out string prodName, out string envName))
122128
{
123129
_productNames.Add(prodName.ToLower());
124130
_environmentNames.Add(envName.ToLower());
125131
}
126132
}
133+
else
134+
{
135+
DetailedMessage.Add(item.Name, false);
136+
}
127137

128138
// Write an extra new line.
129139
host.WriteLine();
130140
}
141+
142+
// Customize the Json Serializer Options to avoid unnecessary encoding.
143+
var options = new JsonSerializerOptions
144+
{
145+
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping
146+
};
147+
148+
// Send Telemetry for Replace Action.
149+
if (!MetricHelper.TelemetryOptOut)
150+
{
151+
MetricHelper.metricHelper.LogTelemetry(
152+
new AzTrace()
153+
{
154+
Command = "Replace",
155+
ConversationId = _agent._chatSession.ConversationId,
156+
ActivityId = _agent._copilotResponse.ReplyToId,
157+
EventType = "UserAction",
158+
TopicName = _agent._copilotResponse.TopicName,
159+
DetailedMessage = JsonSerializer.Serialize(DetailedMessage, options)
160+
});
161+
}
131162
}
132163
catch (OperationCanceledException)
133164
{

‎shell/agents/Microsoft.Azure.Agent/DataRetriever.cs‎

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System.Text.RegularExpressions;
77

88
using AIShell.Abstraction;
9+
using Azure;
910
using Serilog;
1011

1112
namespace Microsoft.Azure.Agent;
@@ -577,13 +578,46 @@ private AzCLICommand QueryForMetadata(string azCommand)
577578
}
578579
else
579580
{
580-
// TODO: telemetry.
581+
if (!MetricHelper.TelemetryOptOut)
582+
{
583+
Dictionary<string, string> errorMessage = new Dictionary<string, string>
584+
{
585+
{ "StatusCode", response.StatusCode.ToString() },
586+
{ "Command", azCommand },
587+
{ "ErrorMessage", $"[QueryForMetadata] Received status code {response.StatusCode} for command {azCommand}" },
588+
};
589+
MetricHelper.metricHelper.LogTelemetry(
590+
new AzTrace()
591+
{
592+
// ConversationId = _agent._chatSession.ConversationId,
593+
// ActivityId = _agent._copilotResponse.ReplyToId,
594+
EventType = "Exception",
595+
// TopicName = _agent._copilotResponse.TopicName,
596+
DetailedMessage = JsonSerializer.Serialize(errorMessage)
597+
});
598+
}
581599
Log.Error("[QueryForMetadata] Received status code '{0}' for command '{1}'", response.StatusCode, azCommand);
582600
}
583601
}
584602
catch (Exception e)
585603
{
586-
// TODO: telemetry.
604+
if (!MetricHelper.TelemetryOptOut)
605+
{
606+
Dictionary<string, string> errorMessage = new Dictionary<string, string>
607+
{
608+
{ "Command", azCommand },
609+
{ "ErrorMessage", $"[QueryForMetadata] Exception while processing command: {azCommand}" },
610+
};
611+
MetricHelper.metricHelper.LogTelemetry(
612+
new AzTrace()
613+
{
614+
// ConversationId = _agent._chatSession.ConversationId,
615+
// ActivityId = _agent._copilotResponse.ReplyToId,
616+
EventType = "Exception",
617+
// TopicName = _agent._copilotResponse.TopicName,
618+
DetailedMessage = JsonSerializer.Serialize(errorMessage)
619+
});
620+
}
587621
Log.Error(e, "[QueryForMetadata] Exception while processing command: {0}", azCommand);
588622
}
589623

‎shell/agents/Microsoft.Azure.Agent/Telemetry/AzTrace.cs‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
using System.Text.Json;
2+
using static Microsoft.ApplicationInsights.MetricDimensionNames.TelemetryContext;
3+
4+
namespace Microsoft.Azure.Agent;
5+
6+
public class AzTrace
7+
{
8+
private static readonly string s_installationId;
9+
private static string GetInstallationID()
10+
{
11+
string azureConfigDir = Environment.GetEnvironmentVariable("AZURE_CONFIG_DIR");
12+
string userProfile = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
13+
string userProfilePath = Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json");
14+
15+
FileStream jsonStream;
16+
JsonElement array;
17+
string installationId;
18+
19+
if (File.Exists(userProfilePath))
20+
{
21+
jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read);
22+
array = JsonSerializer.Deserialize<JsonElement>(jsonStream);
23+
installationId = array.GetProperty("installationId").GetString();
24+
}
25+
else
26+
{
27+
try
28+
{
29+
Path.Combine(string.IsNullOrEmpty(azureConfigDir) ? userProfile : azureConfigDir, "azureProfile.json");
30+
jsonStream = new FileStream(userProfilePath, FileMode.Open, FileAccess.Read);
31+
array = JsonSerializer.Deserialize<JsonElement>(jsonStream);
32+
installationId = array.GetProperty("Settings").GetProperty("InstallationId").GetString();
33+
}
34+
catch
35+
{
36+
// If finally no installation id found, just return null.
37+
return null;
38+
}
39+
}
40+
41+
return installationId;
42+
}
43+
44+
public string TopicName;
45+
// Each chat has a unique conversationId. When the cx runs /refresh,
46+
// a new chat is initiated(i.e.a new conversationId will be created).
47+
public string ConversationId;
48+
// The activity id of the user's query
49+
public string ActivityId;
50+
public string InstallationId = s_installationId;
51+
public string EventType;
52+
public string Command;
53+
/// <summary>
54+
/// Detailed information containing additional Information - may contain:
55+
/// Reason of dislike
56+
/// </summary>
57+
public string DetailedMessage;
58+
/// <summary>
59+
/// Agent Information - may contain:
60+
/// Handler Version
61+
/// Product Version
62+
/// .net/python Version
63+
/// </summary>
64+
public Dictionary<string, string> ExtendedProperties;
65+
static AzTrace() => s_installationId = GetInstallationID();
66+
}

0 commit comments

Comments
 (0)