chore(compass-query-bar, compass-aggregations): add telemetry events for failed ai and success ai COMPASS-7137 (#4752)

Author: Anemy

packages/compass-aggregations/src/modules/pipeline-builder/pipeline-ai.ts

@@ -13,7 +13,7 @@ import type Stage from './stage';
 import { updatePipelinePreview } from './builder-helpers';
 import type { AtlasServiceNetworkError } from '@mongodb-js/atlas-service/renderer';
 
-const { log, mongoLogId } = createLoggerAndTelemetry('AI-PIPELINE-UI');
+const { log, mongoLogId, track } = createLoggerAndTelemetry('AI-PIPELINE-UI');
 
 const emptyPipelineError =
   'No pipeline was returned. Please try again with a different prompt.';
@@ -112,15 +112,34 @@ export type AIPipelineSucceededAction = {
   type: AIPipelineActionTypes.AIPipelineSucceeded;
 };
 
-function logFailed(errorMessage: string) {
-  log.info(
+type FailedResponseTrackMessage = {
+  editor_view_type: 'stages' | 'text';
+  errorCode?: number;
+  errorMessage: string;
+  errorName: string;
+};
+
+function trackAndLogFailed({
+  editor_view_type,
+  errorCode,
+  errorMessage,
+  errorName,
+}: FailedResponseTrackMessage) {
+  log.warn(
     mongoLogId(1_001_000_230),
     'AIPipeline',
     'AI pipeline request failed',
     {
+      errorCode,
       errorMessage,
+      errorName,
     }
   );
+  track('AI Response Failed', () => ({
+    editor_view_type,
+    error_code: errorCode,
+    error_name: errorName,
+  }));
 }
 
 export const runAIPipelineGeneration = (
@@ -136,11 +155,18 @@ export const runAIPipelineGeneration = (
     const {
       pipelineBuilder: {
         aiPipeline: { aiPipelineFetchId: existingFetchId },
+        pipelineMode,
       },
       namespace,
       dataService: { dataService },
     } = getState();
 
+    const editor_view_type = pipelineMode === 'builder-ui' ? 'stages' : 'text';
+    track('AI Prompt Submitted', () => ({
+      editor_view_type,
+      user_input_length: userInput.length,
+    }));
+
     if (aiPipelineFetchId !== -1) {
       // Cancel the active request as this one will override.
       abort(existingFetchId);
@@ -188,7 +214,12 @@ export const runAIPipelineGeneration = (
         // If we already aborted so we ignore the error.
         return;
       }
-      logFailed((err as AtlasServiceNetworkError).message);
+      trackAndLogFailed({
+        editor_view_type,
+        errorCode: (err as AtlasServiceNetworkError).statusCode,
+        errorMessage: (err as AtlasServiceNetworkError).message,
+        errorName: 'request_error',
+      });
       // We're going to reset input state with this error, show the error in the
       // toast instead
       if ((err as AtlasServiceNetworkError).statusCode === 401) {
@@ -226,7 +257,12 @@ export const runAIPipelineGeneration = (
         throw new Error(emptyPipelineError);
       }
     } catch (err) {
-      logFailed((err as Error).message);
+      trackAndLogFailed({
+        editor_view_type,
+        errorCode: (err as AtlasServiceNetworkError).statusCode,
+        errorMessage: (err as Error).message,
+        errorName: 'empty_pipeline_error',
+      });
       dispatch({
         type: AIPipelineActionTypes.AIPipelineFailed,
         errorMessage: (err as Error).message,
@@ -249,6 +285,12 @@ export const runAIPipelineGeneration = (
 
     pipelineBuilder.reset(pipelineText);
 
+    track('AI Prompt Generated', () => ({
+      editor_view_type,
+      syntax_errors: true,
+      query_shape: pipelineBuilder.stages.map((stage) => stage.operator),
+    }));
+
     dispatch({
       type: AIPipelineActionTypes.LoadGeneratedPipeline,
       stages: pipelineBuilder.stages,

packages/compass-query-bar/src/stores/ai-query-reducer.spec.ts

@@ -37,7 +37,7 @@ describe('aiQueryReducer', function () {
         const mockAtlasService = {
           getQueryFromUserInput: sandbox
             .stub()
-            .resolves({ content: { query: { _id: 1 } } }),
+            .resolves({ content: { query: { filter: '{_id: 1}' } } }),
         };
 
         const mockDataService = {

packages/compass-query-bar/src/stores/ai-query-reducer.ts

@@ -103,10 +103,27 @@ export type AIQuerySucceededAction = {
   fields: QueryFormFields;
 };
 
-function logFailed(errorMessage: string) {
-  log.info(mongoLogId(1_001_000_198), 'AIQuery', 'AI query request failed', {
+type FailedResponseTrackMessage = {
+  errorCode?: number;
+  errorName: string;
+  errorMessage: string;
+};
+
+function trackAndLogFailed({
+  errorCode,
+  errorName,
+  errorMessage,
+}: FailedResponseTrackMessage) {
+  log.warn(mongoLogId(1_001_000_198), 'AIQuery', 'AI query request failed', {
+    errorCode,
     errorMessage,
+    errorName,
   });
+  track('AI Response Failed', () => ({
+    editor_view_type: 'find',
+    error_name: errorName,
+    error_code: errorCode,
+  }));
 }
 
 export const runAIQuery = (
@@ -172,7 +189,11 @@ export const runAIQuery = (
         // If we already aborted so we ignore the error.
         return;
       }
-      logFailed((err as AtlasServiceNetworkError).message);
+      trackAndLogFailed({
+        errorName: 'request_error',
+        errorCode: (err as AtlasServiceNetworkError).statusCode,
+        errorMessage: (err as AtlasServiceNetworkError).message,
+      });
       // We're going to reset input state with this error, show the error in the
       // toast instead
       if ((err as AtlasServiceNetworkError).statusCode === 401) {
@@ -204,14 +225,17 @@ export const runAIQuery = (
       return;
     }
 
-    let fields;
+    let generatedFields: QueryFormFields;
     try {
-      fields = {
-        ...mapQueryToFormFields(DEFAULT_FIELD_VALUES),
-        ...parseQueryAttributesToFormFields(jsonResponse.content.query),
-      };
+      generatedFields = parseQueryAttributesToFormFields(
+        jsonResponse.content.query
+      );
     } catch (err: any) {
-      logFailed(err?.message);
+      trackAndLogFailed({
+        errorName: 'could_not_parse_fields',
+        errorCode: (err as AtlasServiceNetworkError).statusCode,
+        errorMessage: err?.message,
+      });
       dispatch({
         type: AIQueryActionTypes.AIQueryFailed,
         errorMessage: err?.message,
@@ -220,31 +244,43 @@ export const runAIQuery = (
     }
 
     // Error when the response is empty or there is nothing to map.
-    if (!fields || Object.keys(fields).length === 0) {
+    if (!generatedFields || Object.keys(generatedFields).length === 0) {
       const msg =
         'No query was returned from the ai. Consider re-wording your prompt.';
-      logFailed(msg);
+      trackAndLogFailed({
+        errorName: 'no_usable_query_from_ai',
+        errorMessage: msg,
+      });
       dispatch({
         type: AIQueryActionTypes.AIQueryFailed,
         errorMessage: msg,
       });
       return;
     }
 
+    const queryFields = {
+      ...mapQueryToFormFields(DEFAULT_FIELD_VALUES),
+      ...generatedFields,
+    };
+
     log.info(
       mongoLogId(1_001_000_199),
       'AIQuery',
       'AI query request succeeded',
       {
         query: {
-          ...fields,
+          ...queryFields,
         },
       }
     );
+    track('AI Prompt Generated', () => ({
+      editor_view_type: 'find',
+      query_shape: Object.keys(generatedFields),
+    }));
 
     dispatch({
       type: AIQueryActionTypes.AIQuerySucceeded,
-      fields,
+      fields: queryFields,
     });
   };
 };