chore(compass-assistant): add feedback submission and telemetry COMPASS-9608 (#7243)

Author: gagik

package-lock.json

@@ -53,6 +53,7 @@
     "@mongodb-js/atlas-service": "^0.56.0",
     "@mongodb-js/compass-app-registry": "^9.4.20",
     "@mongodb-js/compass-components": "^1.49.0",
+    "@mongodb-js/compass-telemetry": "^1.14.0",
     "@mongodb-js/connection-info": "^0.17.1",
     "@mongodb-js/compass-logging": "^1.7.12",
     "mongodb-connection-string-url": "^3.0.1",

packages/compass-assistant/package.json

@@ -11,16 +11,6 @@ import { createMockChat } from '../test/utils';
 import type { AssistantMessage } from './compass-assistant-provider';
 
 describe('AssistantChat', function () {
-  let originalScrollTo: typeof Element.prototype.scrollTo;
-  // Mock scrollTo method for DOM elements to prevent test failures
-  before(function () {
-    originalScrollTo = Element.prototype.scrollTo.bind(Element.prototype);
-    Element.prototype.scrollTo = () => {};
-  });
-  after(function () {
-    Element.prototype.scrollTo = originalScrollTo;
-  });
-
   const mockMessages: AssistantMessage[] = [
     {
       id: 'user',
@@ -41,8 +31,9 @@ describe('AssistantChat', function () {
 
   function renderWithChat(messages: AssistantMessage[]) {
     const chat = createMockChat({ messages });
+    const result = render(<AssistantChat chat={chat} />);
     return {
-      result: render(<AssistantChat chat={chat} />),
+      result,
       chat,
     };
   }
@@ -130,8 +121,9 @@ describe('AssistantChat', function () {
     );
   });
 
-  it('calls sendMessage when form is submitted', function () {
-    const { chat } = renderWithChat([]);
+  it('calls sendMessage when form is submitted', async function () {
+    const { chat, result } = renderWithChat([]);
+    const { track } = result;
     const inputField = screen.getByPlaceholderText(
       'Ask MongoDB Assistant a question'
     );
@@ -142,6 +134,12 @@ describe('AssistantChat', function () {
 
     expect(chat.sendMessage.calledWith({ text: 'What is aggregation?' })).to.be
       .true;
+
+    await waitFor(() => {
+      expect(track).to.have.been.calledWith('Assistant Prompt Submitted', {
+        user_input_length: 'What is aggregation?'.length,
+      });
+    });
   });
 
   it('clears input field after successful submission', function () {
@@ -159,8 +157,9 @@ describe('AssistantChat', function () {
     expect(inputField.value).to.equal('');
   });
 
-  it('trims whitespace from input before sending', function () {
-    const { chat } = renderWithChat([]);
+  it('trims whitespace from input before sending', async function () {
+    const { chat, result } = renderWithChat([]);
+    const { track } = result;
 
     const inputField = screen.getByPlaceholderText(
       'Ask MongoDB Assistant a question'
@@ -171,6 +170,12 @@ describe('AssistantChat', function () {
 
     expect(chat.sendMessage.calledWith({ text: 'What is sharding?' })).to.be
       .true;
+
+    await waitFor(() => {
+      expect(track).to.have.been.calledWith('Assistant Prompt Submitted', {
+        user_input_length: 'What is sharding?'.length,
+      });
+    });
   });
 
   it('does not call sendMessage when input is empty or whitespace-only', function () {
@@ -271,4 +276,143 @@ describe('AssistantChat', function () {
     expect(screen.queryByText('Another part that should not display.')).to.not
       .exist;
   });
+
+  describe('feedback buttons', function () {
+    it('shows feedback buttons only for assistant messages', function () {
+      renderWithChat(mockMessages);
+
+      const userMessage = screen.getByTestId('assistant-message-user');
+      const assistantMessage = screen.getByTestId(
+        'assistant-message-assistant'
+      );
+
+      // User messages should not have feedback buttons
+      expect(userMessage.querySelector('[aria-label="Thumbs Up Icon"]')).to.not
+        .exist;
+      expect(userMessage.querySelector('[aria-label="Thumbs Down Icon"]')).to
+        .not.exist;
+
+      // Assistant messages should have feedback buttons
+      expect(assistantMessage.querySelector('[aria-label="Thumbs Up Icon"]')).to
+        .exist;
+      expect(assistantMessage.querySelector('[aria-label="Thumbs Down Icon"]'))
+        .to.exist;
+    });
+
+    it('tracks positive feedback when thumbs up is clicked', async function () {
+      const { result } = renderWithChat(mockMessages);
+      const { track } = result;
+
+      const assistantMessage = screen.getByTestId(
+        'assistant-message-assistant'
+      );
+
+      // Find and click the thumbs up button
+      const thumbsUpButton = assistantMessage.querySelector(
+        '[aria-label="Thumbs Up Icon"]'
+      ) as HTMLElement;
+
+      userEvent.click(thumbsUpButton);
+
+      await waitFor(() => {
+        expect(track).to.have.callCount(1);
+        expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
+          feedback: 'positive',
+          text: undefined,
+          request_id: null,
+        });
+      });
+    });
+
+    it('tracks negative feedback when thumbs down is clicked', async function () {
+      const { result } = renderWithChat(mockMessages);
+      const { track } = result;
+
+      const assistantMessage = screen.getByTestId(
+        'assistant-message-assistant'
+      );
+
+      // Find and click the thumbs down button
+      const thumbsDownButton = assistantMessage.querySelector(
+        '[aria-label="Thumbs Down Icon"]'
+      ) as HTMLElement;
+
+      userEvent.click(thumbsDownButton);
+
+      await waitFor(() => {
+        expect(track).to.have.callCount(1);
+
+        expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
+          feedback: 'negative',
+          text: undefined,
+          request_id: null,
+        });
+      });
+    });
+
+    it('tracks detailed feedback when feedback text is submitted', async function () {
+      const { result } = renderWithChat(mockMessages);
+      const { track } = result;
+
+      const assistantMessage = screen.getByTestId(
+        'assistant-message-assistant'
+      );
+
+      // First click thumbs down to potentially open feedback form
+      const thumbsDownButton = assistantMessage.querySelector(
+        '[aria-label="Thumbs Down Icon"]'
+      ) as HTMLElement;
+
+      userEvent.click(thumbsDownButton);
+
+      // Look for feedback text area (the exact implementation depends on LeafyGreen)
+      const feedbackTextArea = screen.getByTestId(
+        'lg-chat-message_actions-feedback_textarea'
+      );
+
+      userEvent.type(feedbackTextArea, 'This response was not helpful');
+
+      // Look for submit button
+      const submitButton = screen.getByText('Submit');
+
+      userEvent.click(submitButton);
+
+      await waitFor(() => {
+        expect(track).to.have.callCount(2);
+
+        expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
+          feedback: 'negative',
+          text: undefined,
+          request_id: null,
+        });
+
+        expect(track).to.have.been.calledWith('Assistant Feedback Submitted', {
+          feedback: 'negative',
+          text: 'This response was not helpful',
+          request_id: null,
+        });
+      });
+    });
+
+    it('does not show feedback buttons when there are no assistant messages', function () {
+      const userOnlyMessages: AssistantMessage[] = [
+        {
+          id: 'user1',
+          role: 'user',
+          parts: [{ type: 'text', text: 'Hello!' }],
+        },
+        {
+          id: 'user2',
+          role: 'user',
+          parts: [{ type: 'text', text: 'How are you?' }],
+        },
+      ];
+
+      renderWithChat(userOnlyMessages);
+
+      // Should not find any feedback buttons in the entire component
+      expect(screen.queryByLabelText('Thumbs Up Icon')).to.not.exist;
+      expect(screen.queryByLabelText('Thumbs Down Icon')).to.not.exist;
+    });
+  });
 });

packages/compass-assistant/src/assistant-chat.spec.tsx

@@ -7,6 +7,7 @@ import {
   LgChatLeafygreenChatProvider,
   LgChatMessage,
   LgChatMessageFeed,
+  LgChatMessageActions,
   LgChatInputBar,
   spacing,
   css,
@@ -16,11 +17,13 @@ import {
   palette,
   useDarkMode,
 } from '@mongodb-js/compass-components';
+import { useTelemetry } from '@mongodb-js/compass-telemetry/provider';
 
 const { ChatWindow } = LgChatChatWindow;
 const { LeafyGreenChatProvider, Variant } = LgChatLeafygreenChatProvider;
 const { Message } = LgChatMessage;
 const { MessageFeed } = LgChatMessageFeed;
+const { MessageActions } = LgChatMessageActions;
 const { InputBar } = LgChatInputBar;
 
 interface AssistantChatProps {
@@ -105,9 +108,15 @@ const errorBannerWrapperStyles = css({
 export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
   chat,
 }) => {
+  const track = useTelemetry();
   const darkMode = useDarkMode();
   const { messages, sendMessage, status, error, clearError } = useChat({
     chat,
+    onError: (error) => {
+      track('Assistant Response Failed', () => ({
+        error_name: error.name,
+      }));
+    },
   });
 
   // Transform AI SDK messages to LeafyGreen chat format
@@ -127,10 +136,43 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
     (messageBody: string) => {
       const trimmedMessageBody = messageBody.trim();
       if (trimmedMessageBody) {
+        track('Assistant Prompt Submitted', {
+          user_input_length: trimmedMessageBody.length,
+        });
         void sendMessage({ text: trimmedMessageBody });
       }
     },
-    [sendMessage]
+    [sendMessage, track]
+  );
+
+  const handleFeedback = useCallback(
+    (
+      event,
+      state:
+        | {
+            feedback: string;
+            rating: string;
+          }
+        | {
+            rating: string;
+          }
+        | undefined
+    ) => {
+      if (!state) {
+        return;
+      }
+      const { rating } = state;
+      const textFeedback = 'feedback' in state ? state.feedback : undefined;
+      const feedback: 'positive' | 'negative' =
+        rating === 'liked' ? 'positive' : 'negative';
+
+      track('Assistant Feedback Submitted', {
+        feedback,
+        text: textFeedback,
+        request_id: null,
+      });
+    },
+    [track]
   );
 
   return (
@@ -155,7 +197,14 @@ export const AssistantChat: React.FunctionComponent<AssistantChatProps> = ({
                 sourceType="markdown"
                 {...messageFields}
                 data-testid={`assistant-message-${messageFields.id}`}
-              />
+              >
+                {messageFields.isSender === false && (
+                  <MessageActions
+                    onRatingChange={handleFeedback}
+                    onSubmitFeedback={handleFeedback}
+                  />
+                )}
+              </Message>
             ))}
             {status === 'submitted' && (
               <Message

packages/compass-assistant/src/assistant-chat.tsx

@@ -14,6 +14,7 @@ import { usePreference } from 'compass-preferences-model/provider';
 import { createLoggerLocator } from '@mongodb-js/compass-logging/provider';
 import type { ConnectionInfo } from '@mongodb-js/connection-info';
 import { redactConnectionString } from 'mongodb-connection-string-url';
+import { useTelemetry } from '@mongodb-js/compass-telemetry/provider';
 
 export const ASSISTANT_DRAWER_ID = 'compass-assistant-drawer';
 
@@ -88,6 +89,7 @@ export const AssistantProvider: React.FunctionComponent<
     chat: Chat<AssistantMessage>;
   }>
 > = ({ chat, children }) => {
+  const track = useTelemetry();
   const assistantActionsContext = useRef<AssistantActionsContextType>({
     interpretExplainPlan: ({ explainPlan }) => {
       openDrawer(ASSISTANT_DRAWER_ID);
@@ -103,6 +105,9 @@ export const AssistantProvider: React.FunctionComponent<
         },
         {}
       );
+      track('Assistant Entry Point Used', {
+        source: 'explain plan',
+      });
     },
     interpretConnectionError: ({ connectionInfo, error }) => {
       openDrawer(ASSISTANT_DRAWER_ID);
@@ -122,6 +127,9 @@ export const AssistantProvider: React.FunctionComponent<
         },
         {}
       );
+      track('Assistant Entry Point Used', {
+        source: 'connection error',
+      });
     },
     clearChat: () => {
       chat.messages = [];