diff --git a/packages/compass-assistant/src/compass-assistant-provider.tsx b/packages/compass-assistant/src/compass-assistant-provider.tsx index 8bffc1a99c7..ffc38afb77f 100644 --- a/packages/compass-assistant/src/compass-assistant-provider.tsx +++ b/packages/compass-assistant/src/compass-assistant-provider.tsx @@ -41,6 +41,8 @@ export type AssistantMessage = UIMessage & { * Used for warning messages in cases like using non-genuine MongoDB. */ isPermanent?: boolean; + /** The source of the message (i.e. the entry point used) */ + source?: 'explain plan' | 'performance insights' | 'connection error'; /** Information for confirmation messages. */ confirmation?: { description: string; @@ -182,7 +184,10 @@ export const AssistantProvider: React.FunctionComponent< void assistantActionsContext.current.ensureOptInAndSend( { text: prompt, - metadata, + metadata: { + ...metadata, + source: entryPointName, + }, }, {}, () => { diff --git a/packages/compass-assistant/src/components/assistant-chat.spec.tsx b/packages/compass-assistant/src/components/assistant-chat.spec.tsx index ad33c41f369..5079f358548 100644 --- a/packages/compass-assistant/src/components/assistant-chat.spec.tsx +++ b/packages/compass-assistant/src/components/assistant-chat.spec.tsx @@ -39,6 +39,9 @@ describe('AssistantChat', function () { sourceId: '1', }, ], + metadata: { + source: 'performance insights', + }, }, ]; @@ -440,6 +443,7 @@ describe('AssistantChat', function () { feedback: 'positive', text: undefined, request_id: null, + source: 'performance insights', }); }); }); @@ -466,6 +470,7 @@ describe('AssistantChat', function () { feedback: 'negative', text: undefined, request_id: null, + source: 'performance insights', }); }); }); @@ -502,12 +507,42 @@ describe('AssistantChat', function () { feedback: 'negative', text: undefined, request_id: null, + source: 'performance insights', }); expect(track).to.have.been.calledWith('Assistant Feedback Submitted', { feedback: 'negative', text: 'This response was not helpful', request_id: null, + source: 'performance insights', + }); + }); + }); + + it('tracks it as "chat response" when source is not present', async function () { + const { result } = renderWithChat([ + { + ...mockMessages[1], + metadata: { + ...mockMessages[1].metadata, + source: undefined, + }, + }, + ]); + const { track } = result; + + const thumbsDownButton = within( + screen.getByTestId('assistant-message-assistant') + ).getByLabelText('Dislike this message'); + + userEvent.click(thumbsDownButton); + + await waitFor(() => { + expect(track).to.have.been.calledWith('Assistant Feedback Submitted', { + feedback: 'negative', + text: undefined, + request_id: null, + source: 'chat response', }); }); }); @@ -547,6 +582,7 @@ describe('AssistantChat', function () { state: 'pending', description: 'Are you sure you want to proceed with this action?', }, + source: 'performance insights', }, }; }); @@ -763,6 +799,65 @@ describe('AssistantChat', function () { expect(screen.queryByText('Cancel')).to.not.exist; expect(screen.getByText('This is a regular message')).to.exist; }); + + it('tracks confirmation submitted when confirm button is clicked', async function () { + const { result } = renderWithChat([mockConfirmationMessage]); + const { track } = result; + + const confirmButton = screen.getByText('Confirm'); + userEvent.click(confirmButton); + + await waitFor(() => { + expect(track).to.have.been.calledWith( + 'Assistant Confirmation Submitted', + { + status: 'confirmed', + source: 'performance insights', + } + ); + }); + }); + + it('tracks confirmation submitted when cancel button is clicked', async function () { + const { result } = renderWithChat([mockConfirmationMessage]); + const { track } = result; + + const cancelButton = screen.getByText('Cancel'); + userEvent.click(cancelButton); + + await waitFor(() => { + expect(track).to.have.been.calledWith( + 'Assistant Confirmation Submitted', + { + status: 'rejected', + source: 'performance insights', + } + ); + }); + }); + + it('tracks it as "chat response" when source is not present', async function () { + const { result } = renderWithChat([ + { + ...mockConfirmationMessage, + metadata: { + ...mockConfirmationMessage.metadata, + source: undefined, + }, + }, + ]); + const { track } = result; + + const confirmButton = screen.getByText('Confirm'); + userEvent.click(confirmButton); + + await waitFor(() => { + expect(track).to.have.been.calledWith( + 'Assistant Confirmation Submitted', + { status: 'confirmed', source: 'chat response' } + ); + }); + }); }); describe('related sources', function () { diff --git a/packages/compass-assistant/src/components/assistant-chat.tsx b/packages/compass-assistant/src/components/assistant-chat.tsx index 4d9d32ccef9..037b64f8604 100644 --- a/packages/compass-assistant/src/components/assistant-chat.tsx +++ b/packages/compass-assistant/src/components/assistant-chat.tsx @@ -195,8 +195,11 @@ export const AssistantChat: React.FunctionComponent = ({ ); const handleFeedback = useCallback( - ( - event, + ({ + state, + message, + }: { + message: AssistantMessage; state: | { feedback: string; @@ -205,8 +208,8 @@ export const AssistantChat: React.FunctionComponent = ({ | { rating: string; } - | undefined - ) => { + | undefined; + }) => { if (!state) { return; } @@ -219,6 +222,7 @@ export const AssistantChat: React.FunctionComponent = ({ feedback, text: textFeedback, request_id: null, + source: message.metadata?.source ?? 'chat response', }); }, [track] @@ -262,6 +266,10 @@ export const AssistantChat: React.FunctionComponent = ({ } return newMessages; }); + track('Assistant Confirmation Submitted', { + status: newState, + source: confirmedMessage.metadata?.source ?? 'chat response', + }); if (newState === 'confirmed') { // Force the new message request to be sent void ensureOptInAndSend?.(undefined, {}, () => {}); @@ -336,8 +344,12 @@ export const AssistantChat: React.FunctionComponent = ({ > {isSender === false && ( + handleFeedback({ message, state }) + } + onSubmitFeedback={(event, state) => + handleFeedback({ message, state }) + } /> )} {sources.length > 0 && } diff --git a/packages/compass-telemetry/src/telemetry-events.ts b/packages/compass-telemetry/src/telemetry-events.ts index 3962d1272f7..8d1ea607798 100644 --- a/packages/compass-telemetry/src/telemetry-events.ts +++ b/packages/compass-telemetry/src/telemetry-events.ts @@ -1471,6 +1471,18 @@ type AssistantPromptSubmittedEvent = CommonEvent<{ }; }>; +/** + * This event is fired when a user uses an assistant entry point. + * + * @category Gen AI + */ +type AssistantEntryPointUsedEvent = CommonEvent<{ + name: 'Assistant Entry Point Used'; + payload: { + source: 'explain plan' | 'performance insights' | 'connection error'; + }; +}>; + /** * This event is fired when a user submits feedback for the assistant. * @@ -1482,18 +1494,20 @@ type AssistantFeedbackSubmittedEvent = CommonEvent<{ feedback: 'positive' | 'negative'; text: string | undefined; request_id: string | null; + source: AssistantEntryPointUsedEvent['payload']['source'] | 'chat response'; }; }>; /** - * This event is fired when a user uses an assistant entry point. + * This event is fired when a user confirms a confirmation message in the assistant chat. * * @category Gen AI */ -type AssistantEntryPointUsedEvent = CommonEvent<{ - name: 'Assistant Entry Point Used'; +type AssistantConfirmationSubmittedEvent = CommonEvent<{ + name: 'Assistant Confirmation Submitted'; payload: { - source: 'explain plan' | 'performance insights' | 'connection error'; + status: 'confirmed' | 'rejected'; + source: AssistantEntryPointUsedEvent['payload']['source'] | 'chat response'; }; }>; @@ -3032,6 +3046,7 @@ export type TelemetryEvent = | AssistantResponseFailedEvent | AssistantFeedbackSubmittedEvent | AssistantEntryPointUsedEvent + | AssistantConfirmationSubmittedEvent | AiOptInModalShownEvent | AiOptInModalDismissedEvent | AiGenerateQueryClickedEvent