chore(compass-assistant): keep track of confirmation and entry point sources

Author: gagik

packages/compass-assistant/src/compass-assistant-provider.tsx

@@ -41,6 +41,8 @@ export type AssistantMessage = UIMessage & {
      *  Used for warning messages in cases like using non-genuine MongoDB.
      */
     isPermanent?: boolean;
+    /** The source of the message (i.e. the entry point used) */
+    source?: 'explain plan' | 'performance insights' | 'connection error';
     /** Information for confirmation messages. */
     confirmation?: {
       description: string;
@@ -182,7 +184,10 @@ export const AssistantProvider: React.FunctionComponent<
       void assistantActionsContext.current.ensureOptInAndSend(
         {
           text: prompt,
-          metadata,
+          metadata: {
+            ...metadata,
+            source: entryPointName,
+          },
         },
         {},
         () => {

packages/compass-assistant/src/components/assistant-chat.spec.tsx

@@ -39,6 +39,9 @@ describe('AssistantChat', function () {
           sourceId: '1',
         },
       ],
+      metadata: {
+        source: 'performance insights',
+      },
     },
   ];
 
@@ -440,6 +443,7 @@ describe('AssistantChat', function () {
           feedback: 'positive',
           text: undefined,
           request_id: null,
+          source: 'performance insights',
         });
       });
     });
@@ -466,6 +470,7 @@ describe('AssistantChat', function () {
           feedback: 'negative',
           text: undefined,
           request_id: null,
+          source: 'performance insights',
         });
       });
     });
@@ -502,12 +507,14 @@ describe('AssistantChat', function () {
           feedback: 'negative',
           text: undefined,
           request_id: null,
+          source: 'performance insights',
         });
 
         expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
           feedback: 'negative',
           text: 'This response was not helpful',
           request_id: null,
+          source: 'performance insights',
         });
       });
     });
@@ -547,6 +554,7 @@ describe('AssistantChat', function () {
             state: 'pending',
             description: 'Are you sure you want to proceed with this action?',
           },
+          source: 'performance insights',
         },
       };
     });
@@ -763,6 +771,42 @@ describe('AssistantChat', function () {
       expect(screen.queryByText('Cancel')).to.not.exist;
       expect(screen.getByText('This is a regular message')).to.exist;
     });
+
+    it('tracks confirmation submitted when confirm button is clicked', async function () {
+      const { result } = renderWithChat([mockConfirmationMessage]);
+      const { track } = result;
+
+      const confirmButton = screen.getByText('Confirm');
+      userEvent.click(confirmButton);
+
+      await waitFor(() => {
+        expect(track).to.have.been.calledWith(
+          'Assistant Confirmation Submitted',
+          {
+            status: 'confirmed',
+            source: 'performance insights',
+          }
+        );
+      });
+    });
+
+    it('tracks confirmation submitted when cancel button is clicked', async function () {
+      const { result } = renderWithChat([mockConfirmationMessage]);
+      const { track } = result;
+
+      const cancelButton = screen.getByText('Cancel');
+      userEvent.click(cancelButton);
+
+      await waitFor(() => {
+        expect(track).to.have.been.calledWith(
+          'Assistant Confirmation Submitted',
+          {
+            status: 'rejected',
+            source: 'performance insights',
+          }
+        );
+      });
+    });
   });
 
   describe('related sources', function () {

packages/compass-assistant/src/components/assistant-chat.tsx

@@ -195,8 +195,11 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
   );
 
   const handleFeedback = useCallback(
-    (
-      event,
+    ({
+      state,
+      message,
+    }: {
+      message: AssistantMessage;
       state:
         | {
             feedback: string;
@@ -205,8 +208,8 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
         | {
             rating: string;
           }
-        | undefined
-    ) => {
+        | undefined;
+    }) => {
       if (!state) {
         return;
       }
@@ -219,6 +222,7 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
         feedback,
         text: textFeedback,
         request_id: null,
+        source: message.metadata?.source,
       });
     },
     [track]
@@ -262,6 +266,10 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
         }
         return newMessages;
       });
+      track('Assistant Confirmation Submitted', {
+        status: newState,
+        source: confirmedMessage.metadata?.source,
+      });
       if (newState === 'confirmed') {
         // Force the new message request to be sent
         void ensureOptInAndSend?.(undefined, {}, () => {});
@@ -336,8 +344,12 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
                   >
                     {isSender === false && (
                       <Message.Actions
-                        onRatingChange={handleFeedback}
-                        onSubmitFeedback={handleFeedback}
+                        onRatingChange={(event, state) =>
+                          handleFeedback({ message, state })
+                        }
+                        onSubmitFeedback={(event, state) =>
+                          handleFeedback({ message, state })
+                        }
                       />
                     )}
                     {sources.length > 0 && <Message.Links links={sources} />}

packages/compass-telemetry/src/telemetry-events.ts

@@ -1471,6 +1471,18 @@ type AssistantPromptSubmittedEvent = CommonEvent<{
   };
 }>;
 
+/**
+ * This event is fired when a user uses an assistant entry point.
+ *
+ * @category Gen AI
+ */
+type AssistantEntryPointUsedEvent = CommonEvent<{
+  name: 'Assistant Entry Point Used';
+  payload: {
+    source: 'explain plan' | 'performance insights' | 'connection error';
+  };
+}>;
+
 /**
  * This event is fired when a user submits feedback for the assistant.
  *
@@ -1482,18 +1494,20 @@ type AssistantFeedbackSubmittedEvent = CommonEvent<{
     feedback: 'positive' | 'negative';
     text: string | undefined;
     request_id: string | null;
+    source: AssistantEntryPointUsedEvent['payload']['source'] | undefined;
   };
 }>;
 
 /**
- * This event is fired when a user uses an assistant entry point.
+ * This event is fired when a user confirms a confirmation message in the assistant chat.
  *
  * @category Gen AI
  */
-type AssistantEntryPointUsedEvent = CommonEvent<{
-  name: 'Assistant Entry Point Used';
+type AssistantConfirmationSubmittedEvent = CommonEvent<{
+  name: 'Assistant Confirmation Submitted';
   payload: {
-    source: 'explain plan' | 'performance insights' | 'connection error';
+    status: 'confirmed' | 'rejected';
+    source: AssistantEntryPointUsedEvent['payload']['source'] | undefined;
   };
 }>;
 
@@ -3032,6 +3046,7 @@ export type TelemetryEvent =
   | AssistantResponseFailedEvent
   | AssistantFeedbackSubmittedEvent
   | AssistantEntryPointUsedEvent
+  | AssistantConfirmationSubmittedEvent
   | AiOptInModalShownEvent
   | AiOptInModalDismissedEvent
   | AiGenerateQueryClickedEvent