Expose more states to prepare for telemetry collection (#253)

daxian-dbw · web-flow · commit 4670645c25c6 · 2024-10-22T12:57:14.000-07:00
- `CopilotResponse` received from a chat is now saved in a private member of the agent, so its states could be used by telemetry.
- Offer the `ConversationId` property in `ChatSession` for getting the current conversation id.
- Expose `ReplyToId` in `CopilotResponse`, which is the activity id of the user's query that the current response replies to.
diff --git a/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs b/shell/agents/Microsoft.Azure.Agent/AzureAgent.cs
@@ -36,6 +36,7 @@ 7. DO NOT include the placeholder summary when the commands contains no placehol
         """;
 
     private int _turnsLeft;
+    private CopilotResponse _copilotResponse;
 
     private readonly string _instructions;
     private readonly StringBuilder _buffer;
@@ -120,36 +121,36 @@ public async Task<bool> ChatAsync(string input, IShell shell)
         try
         {
             string query = $"{input}\n\n---\n\n{_instructions}";
-            CopilotResponse copilotResponse = await host.RunWithSpinnerAsync(
+            _copilotResponse = await host.RunWithSpinnerAsync(
                 status: "Thinking ...",
                 spinnerKind: SpinnerKind.Processing,
                 func: async context => await _chatSession.GetChatResponseAsync(query, context, token)
             ).ConfigureAwait(false);
 
-            if (copilotResponse is null)
+            if (_copilotResponse is null)
             {
                 // User cancelled the operation.
                 return true;
             }
 
-            if (copilotResponse.ChunkReader is null)
+            if (_copilotResponse.ChunkReader is null)
             {
                 ArgPlaceholder?.DataRetriever?.Dispose();
                 ArgPlaceholder = null;
 
                 // Process CLI handler response specially to support parameter injection.
                 ResponseData data = null;
-                if (copilotResponse.TopicName == CopilotActivity.CLIHandlerTopic)
+                if (_copilotResponse.TopicName == CopilotActivity.CLIHandlerTopic)
                 {
-                    data = ParseCLIHandlerResponse(copilotResponse, shell);
+                    data = ParseCLIHandlerResponse(shell);
                 }
 
                 if (data?.PlaceholderSet is not null)
                 {
                     ArgPlaceholder = new ArgumentPlaceholder(input, data, _httpClient);
                 }
 
-                string answer = data is null ? copilotResponse.Text : GenerateAnswer(data);
+                string answer = data is null ? _copilotResponse.Text : GenerateAnswer(data);
                 host.RenderFullResponse(answer);
             }
             else
@@ -161,12 +162,12 @@ public async Task<bool> ChatAsync(string input, IShell shell)
 
                     while (true)
                     {
-                        CopilotActivity activity = copilotResponse.ChunkReader.ReadChunk(token);
+                        CopilotActivity activity = _copilotResponse.ChunkReader.ReadChunk(token);
                         if (activity is null)
                         {
                             prevActivity.ExtractMetadata(out string[] suggestion, out ConversationState state);
-                            copilotResponse.SuggestedUserResponses = suggestion;
-                            copilotResponse.ConversationState = state;
+                            _copilotResponse.SuggestedUserResponses = suggestion;
+                            _copilotResponse.ConversationState = state;
                             break;
                         }
 
@@ -182,7 +183,7 @@ public async Task<bool> ChatAsync(string input, IShell shell)
                 }
             }
 
-            var conversationState = copilotResponse.ConversationState;
+            var conversationState = _copilotResponse.ConversationState;
             _turnsLeft = conversationState.TurnLimit - conversationState.TurnNumber;
             if (_turnsLeft <= 5)
             {
@@ -210,9 +211,9 @@ public async Task<bool> ChatAsync(string input, IShell shell)
         return true;
     }
 
-    private ResponseData ParseCLIHandlerResponse(CopilotResponse copilotResponse, IShell shell)
+    private ResponseData ParseCLIHandlerResponse(IShell shell)
     {
-        string text = copilotResponse.Text;
+        string text = _copilotResponse.Text;
         List<CodeBlock> codeBlocks = shell.ExtractCodeBlocks(text, out List<SourceInfo> sourceInfos);
         if (codeBlocks is null || codeBlocks.Count is 0)
         {
@@ -264,7 +265,7 @@ private ResponseData ParseCLIHandlerResponse(CopilotResponse copilotResponse, IS
             Text = text,
             CommandSet = commands,
             PlaceholderSet = placeholders,
-            Locale = copilotResponse.Locale,
+            Locale = _copilotResponse.Locale,
         };
 
         string first = placeholders[0].Name;
diff --git a/shell/agents/Microsoft.Azure.Agent/ChatSession.cs b/shell/agents/Microsoft.Azure.Agent/ChatSession.cs
@@ -16,6 +16,7 @@ internal class ChatSession : IDisposable
 
     private string _token;
     private string _streamUrl;
+    private string _conversationId;
     private string _conversationUrl;
     private DateTime _expireOn;
     private AzureCopilotReceiver _copilotReceiver;
@@ -24,6 +25,8 @@ internal class ChatSession : IDisposable
     private readonly HttpClient _httpClient;
     private readonly Dictionary<string, object> _flights;
 
+    internal string ConversationId => _conversationId;
+
     internal ChatSession(HttpClient httpClient)
     {
         _dl_secret = Environment.GetEnvironmentVariable("DL_SECRET");
@@ -88,6 +91,7 @@ private void Reset()
     {
         _token = null;
         _streamUrl = null;
+        _conversationId = null;
         _conversationUrl = null;
         _expireOn = DateTime.MinValue;
 
@@ -135,7 +139,8 @@ private async Task StartConversationAsync(IHost host, CancellationToken cancella
         SessionPayload spl = JsonSerializer.Deserialize<SessionPayload>(content, Utils.JsonOptions);
 
         _token = spl.Token;
-        _conversationUrl = $"{CONVERSATION_URL}/{spl.ConversationId}/activities";
+        _conversationId = spl.ConversationId;
+        _conversationUrl = $"{CONVERSATION_URL}/{_conversationId}/activities";
         _streamUrl = spl.StreamUrl;
         _expireOn = DateTime.UtcNow.AddSeconds(spl.ExpiresIn);
         _copilotReceiver = await AzureCopilotReceiver.CreateAsync(_streamUrl);
@@ -278,8 +283,8 @@ internal async Task<CopilotResponse> GetChatResponseAsync(string input, IStatusC
                 {
                     CopilotResponse ret = activity.InputHint switch
                     {
-                        "typing"         => new CopilotResponse(new ChunkReader(_copilotReceiver, activity), activity.Locale, activity.TopicName),
-                        "acceptingInput" => new CopilotResponse(activity.Text, activity.Locale, activity.TopicName),
+                        "typing"         => new CopilotResponse(activity, new ChunkReader(_copilotReceiver, activity)),
+                        "acceptingInput" => new CopilotResponse(activity),
                         _ => throw CorruptDataException.Create($"The 'inputHint' is {activity.InputHint}.", activity)
                     };
 
diff --git a/shell/agents/Microsoft.Azure.Agent/Schema.cs b/shell/agents/Microsoft.Azure.Agent/Schema.cs
@@ -102,30 +102,28 @@ internal class ConversationState
 
 internal class CopilotResponse
 {
-    internal CopilotResponse(ChunkReader chunkReader, string locale, string topicName)
+    internal CopilotResponse(CopilotActivity activity, ChunkReader chunkReader)
+        : this(activity)
     {
         ArgumentNullException.ThrowIfNull(chunkReader);
-        ArgumentException.ThrowIfNullOrEmpty(topicName);
-
         ChunkReader = chunkReader;
-        Locale = locale;
-        TopicName = topicName;
     }
 
-    internal CopilotResponse(string text, string locale, string topicName)
+    internal CopilotResponse(CopilotActivity activity)
     {
-        ArgumentException.ThrowIfNullOrEmpty(text);
-        ArgumentException.ThrowIfNullOrEmpty(topicName);
+        ArgumentNullException.ThrowIfNull(activity);
 
-        Text = text;
-        Locale = locale;
-        TopicName = topicName;
+        Text = activity.Text;
+        Locale = activity.Locale;
+        TopicName = activity.TopicName;
+        ReplyToId = activity.ReplyToId;
     }
 
     internal ChunkReader ChunkReader { get; }
     internal string Text { get; }
     internal string Locale { get; }
     internal string TopicName { get; }
+    internal string ReplyToId { get; }
     internal string[] SuggestedUserResponses { get; set; }
     internal ConversationState ConversationState { get; set; }
 }

Original file line number	Diff line number	Diff line change
`@@ -36,6 +36,7 @@ 7. DO NOT include the placeholder summary when the commands contains no placehol`
`36`	`36`	`""";`
`37`	`37`
`38`	`38`	`private int _turnsLeft;`
	`39`	`+ private CopilotResponse _copilotResponse;`
`39`	`40`
`40`	`41`	`private readonly string _instructions;`
`41`	`42`	`private readonly StringBuilder _buffer;`
`@@ -120,36 +121,36 @@ public async Task<bool> ChatAsync(string input, IShell shell)`
`120`	`121`	`try`
`121`	`122`	`{`
`122`	`123`	`string query = $"{input}\n\n---\n\n{_instructions}";`
`123`		`- CopilotResponse copilotResponse = await host.RunWithSpinnerAsync(`
	`124`	`+ _copilotResponse = await host.RunWithSpinnerAsync(`
`124`	`125`	`status: "Thinking ...",`
`125`	`126`	`spinnerKind: SpinnerKind.Processing,`
`126`	`127`	`func: async context => await _chatSession.GetChatResponseAsync(query, context, token)`
`127`	`128`	`).ConfigureAwait(false);`
`128`	`129`
`129`		`- if (copilotResponse is null)`
	`130`	`+ if (_copilotResponse is null)`
`130`	`131`	`{`
`131`	`132`	`// User cancelled the operation.`
`132`	`133`	`return true;`
`133`	`134`	`}`
`134`	`135`
`135`		`- if (copilotResponse.ChunkReader is null)`
	`136`	`+ if (_copilotResponse.ChunkReader is null)`
`136`	`137`	`{`
`137`	`138`	`ArgPlaceholder?.DataRetriever?.Dispose();`
`138`	`139`	`ArgPlaceholder = null;`
`139`	`140`
`140`	`141`	`// Process CLI handler response specially to support parameter injection.`
`141`	`142`	`ResponseData data = null;`
`142`		`- if (copilotResponse.TopicName == CopilotActivity.CLIHandlerTopic)`
	`143`	`+ if (_copilotResponse.TopicName == CopilotActivity.CLIHandlerTopic)`
`143`	`144`	`{`
`144`		`- data = ParseCLIHandlerResponse(copilotResponse, shell);`
	`145`	`+ data = ParseCLIHandlerResponse(shell);`
`145`	`146`	`}`
`146`	`147`
`147`	`148`	`if (data?.PlaceholderSet is not null)`
`148`	`149`	`{`
`149`	`150`	`ArgPlaceholder = new ArgumentPlaceholder(input, data, _httpClient);`
`150`	`151`	`}`
`151`	`152`
`152`		`- string answer = data is null ? copilotResponse.Text : GenerateAnswer(data);`
	`153`	`+ string answer = data is null ? _copilotResponse.Text : GenerateAnswer(data);`
`153`	`154`	`host.RenderFullResponse(answer);`
`154`	`155`	`}`
`155`	`156`	`else`
`@@ -161,12 +162,12 @@ public async Task<bool> ChatAsync(string input, IShell shell)`
`161`	`162`
`162`	`163`	`while (true)`
`163`	`164`	`{`
`164`		`- CopilotActivity activity = copilotResponse.ChunkReader.ReadChunk(token);`
	`165`	`+ CopilotActivity activity = _copilotResponse.ChunkReader.ReadChunk(token);`
`165`	`166`	`if (activity is null)`
`166`	`167`	`{`
`167`	`168`	`prevActivity.ExtractMetadata(out string[] suggestion, out ConversationState state);`
`168`		`- copilotResponse.SuggestedUserResponses = suggestion;`
`169`		`- copilotResponse.ConversationState = state;`
	`169`	`+ _copilotResponse.SuggestedUserResponses = suggestion;`
	`170`	`+ _copilotResponse.ConversationState = state;`
`170`	`171`	`break;`
`171`	`172`	`}`
`172`	`173`
`@@ -182,7 +183,7 @@ public async Task<bool> ChatAsync(string input, IShell shell)`
`182`	`183`	`}`
`183`	`184`	`}`
`184`	`185`
`185`		`- var conversationState = copilotResponse.ConversationState;`
	`186`	`+ var conversationState = _copilotResponse.ConversationState;`
`186`	`187`	`_turnsLeft = conversationState.TurnLimit - conversationState.TurnNumber;`
`187`	`188`	`if (_turnsLeft <= 5)`
`188`	`189`	`{`
`@@ -210,9 +211,9 @@ public async Task<bool> ChatAsync(string input, IShell shell)`
`210`	`211`	`return true;`
`211`	`212`	`}`
`212`	`213`
`213`		`- private ResponseData ParseCLIHandlerResponse(CopilotResponse copilotResponse, IShell shell)`
	`214`	`+ private ResponseData ParseCLIHandlerResponse(IShell shell)`
`214`	`215`	`{`
`215`		`- string text = copilotResponse.Text;`
	`216`	`+ string text = _copilotResponse.Text;`
`216`	`217`	`List<CodeBlock> codeBlocks = shell.ExtractCodeBlocks(text, out List<SourceInfo> sourceInfos);`
`217`	`218`	`if (codeBlocks is null \|\| codeBlocks.Count is 0)`
`218`	`219`	`{`
`@@ -264,7 +265,7 @@ private ResponseData ParseCLIHandlerResponse(CopilotResponse copilotResponse, IS`
`264`	`265`	`Text = text,`
`265`	`266`	`CommandSet = commands,`
`266`	`267`	`PlaceholderSet = placeholders,`
`267`		`- Locale = copilotResponse.Locale,`
	`268`	`+ Locale = _copilotResponse.Locale,`
`268`	`269`	`};`
`269`	`270`
`270`	`271`	`string first = placeholders[0].Name;`