chore(compass-assistant): keep track of confirmation and entry point sources (#7344)

Author: gagik

packages/compass-assistant/src/compass-assistant-provider.tsx

@@ -41,6 +41,8 @@ export type AssistantMessage = UIMessage & {
      *  Used for warning messages in cases like using non-genuine MongoDB.
      */
     isPermanent?: boolean;
+    /** The source of the message (i.e. the entry point used) */
+    source?: 'explain plan' | 'performance insights' | 'connection error';
     /** Information for confirmation messages. */
     confirmation?: {
       description: string;
@@ -184,7 +186,10 @@ export const AssistantProvider: React.FunctionComponent<
       void assistantActionsContext.current.ensureOptInAndSend(
         {
           text: prompt,
-          metadata,
+          metadata: {
+            ...metadata,
+            source: entryPointName,
+          },
         },
         {},
         () => {

packages/compass-assistant/src/components/assistant-chat.spec.tsx

@@ -39,6 +39,9 @@ describe('AssistantChat', function () {
           sourceId: '1',
         },
       ],
+      metadata: {
+        source: 'performance insights',
+      },
     },
   ];
 
@@ -427,6 +430,7 @@ describe('AssistantChat', function () {
           feedback: 'positive',
           text: undefined,
           request_id: null,
+          source: 'performance insights',
         });
       });
     });
@@ -453,6 +457,7 @@ describe('AssistantChat', function () {
           feedback: 'negative',
           text: undefined,
           request_id: null,
+          source: 'performance insights',
         });
       });
     });
@@ -489,12 +494,42 @@ describe('AssistantChat', function () {
           feedback: 'negative',
           text: undefined,
           request_id: null,
+          source: 'performance insights',
         });
 
         expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
           feedback: 'negative',
           text: 'This response was not helpful',
           request_id: null,
+          source: 'performance insights',
+        });
+      });
+    });
+
+    it('tracks it as "chat response" when source is not present', async function () {
+      const { result } = renderWithChat([
+        {
+          ...mockMessages[1],
+          metadata: {
+            ...mockMessages[1].metadata,
+            source: undefined,
+          },
+        },
+      ]);
+      const { track } = result;
+
+      const thumbsDownButton = within(
+        screen.getByTestId('assistant-message-assistant')
+      ).getByLabelText('Dislike this message');
+
+      userEvent.click(thumbsDownButton);
+
+      await waitFor(() => {
+        expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
+          feedback: 'negative',
+          text: undefined,
+          request_id: null,
+          source: 'chat response',
         });
       });
     });
@@ -534,6 +569,7 @@ describe('AssistantChat', function () {
             state: 'pending',
             description: 'Are you sure you want to proceed with this action?',
           },
+          source: 'performance insights',
         },
       };
     });
@@ -750,6 +786,65 @@ describe('AssistantChat', function () {
       expect(screen.queryByText('Cancel')).to.not.exist;
       expect(screen.getByText('This is a regular message')).to.exist;
     });
+
+    it('tracks confirmation submitted when confirm button is clicked', async function () {
+      const { result } = renderWithChat([mockConfirmationMessage]);
+      const { track } = result;
+
+      const confirmButton = screen.getByText('Confirm');
+      userEvent.click(confirmButton);
+
+      await waitFor(() => {
+        expect(track).to.have.been.calledWith(
+          'Assistant Confirmation Submitted',
+          {
+            status: 'confirmed',
+            source: 'performance insights',
+          }
+        );
+      });
+    });
+
+    it('tracks confirmation submitted when cancel button is clicked', async function () {
+      const { result } = renderWithChat([mockConfirmationMessage]);
+      const { track } = result;
+
+      const cancelButton = screen.getByText('Cancel');
+      userEvent.click(cancelButton);
+
+      await waitFor(() => {
+        expect(track).to.have.been.calledWith(
+          'Assistant Confirmation Submitted',
+          {
+            status: 'rejected',
+            source: 'performance insights',
+          }
+        );
+      });
+    });
+
+    it('tracks it as "chat response" when source is not present', async function () {
+      const { result } = renderWithChat([
+        {
+          ...mockConfirmationMessage,
+          metadata: {
+            ...mockConfirmationMessage.metadata,
+            source: undefined,
+          },
+        },
+      ]);
+      const { track } = result;
+
+      const confirmButton = screen.getByText('Confirm');
+      userEvent.click(confirmButton);
+
+      await waitFor(() => {
+        expect(track).to.have.been.calledWith(
+          'Assistant Confirmation Submitted',
+          { status: 'confirmed', source: 'chat response' }
+        );
+      });
+    });
   });
 
   describe('related sources', function () {

packages/compass-assistant/src/components/assistant-chat.tsx

@@ -203,8 +203,11 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
   );
 
   const handleFeedback = useCallback(
-    (
-      event,
+    ({
+      state,
+      message,
+    }: {
+      message: AssistantMessage;
       state:
         | {
             feedback: string;
@@ -213,8 +216,8 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
         | {
             rating: string;
           }
-        | undefined
-    ) => {
+        | undefined;
+    }) => {
       if (!state) {
         return;
       }
@@ -227,6 +230,7 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
         feedback,
         text: textFeedback,
         request_id: null,
+        source: message.metadata?.source ?? 'chat response',
       });
     },
     [track]
@@ -270,6 +274,10 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
         }
         return newMessages;
       });
+      track('Assistant Confirmation Submitted', {
+        status: newState,
+        source: confirmedMessage.metadata?.source ?? 'chat response',
+      });
       if (newState === 'confirmed') {
         // Force the new message request to be sent
         void ensureOptInAndSend?.(undefined, {}, () => {});
@@ -344,8 +352,12 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
                   >
                     {isSender === false && (
                       <Message.Actions
-                        onRatingChange={handleFeedback}
-                        onSubmitFeedback={handleFeedback}
+                        onRatingChange={(event, state) =>
+                          handleFeedback({ message, state })
+                        }
+                        onSubmitFeedback={(event, state) =>
+                          handleFeedback({ message, state })
+                        }
                       />
                     )}
                     {sources.length > 0 && <Message.Links links={sources} />}

packages/compass-telemetry/src/telemetry-events.ts

@@ -1471,6 +1471,18 @@ type AssistantPromptSubmittedEvent = CommonEvent<{
   };
 }>;
 
+/**
+ * This event is fired when a user uses an assistant entry point.
+ *
+ * @category Gen AI
+ */
+type AssistantEntryPointUsedEvent = CommonEvent<{
+  name: 'Assistant Entry Point Used';
+  payload: {
+    source: 'explain plan' | 'performance insights' | 'connection error';
+  };
+}>;
+
 /**
  * This event is fired when a user submits feedback for the assistant.
  *
@@ -1482,18 +1494,20 @@ type AssistantFeedbackSubmittedEvent = CommonEvent<{
     feedback: 'positive' | 'negative';
     text: string | undefined;
     request_id: string | null;
+    source: AssistantEntryPointUsedEvent['payload']['source'] | 'chat response';
   };
 }>;
 
 /**
- * This event is fired when a user uses an assistant entry point.
+ * This event is fired when a user confirms a confirmation message in the assistant chat.
  *
  * @category Gen AI
  */
-type AssistantEntryPointUsedEvent = CommonEvent<{
-  name: 'Assistant Entry Point Used';
+type AssistantConfirmationSubmittedEvent = CommonEvent<{
+  name: 'Assistant Confirmation Submitted';
   payload: {
-    source: 'explain plan' | 'performance insights' | 'connection error';
+    status: 'confirmed' | 'rejected';
+    source: AssistantEntryPointUsedEvent['payload']['source'] | 'chat response';
   };
 }>;
 
@@ -3032,6 +3046,7 @@ export type TelemetryEvent =
   | AssistantResponseFailedEvent
   | AssistantFeedbackSubmittedEvent
   | AssistantEntryPointUsedEvent
+  | AssistantConfirmationSubmittedEvent
   | AiOptInModalShownEvent
   | AiOptInModalDismissedEvent
   | AiGenerateQueryClickedEvent