feat(compass-collection): Output Validation Followup - Mock Data Generator LLM CLOUDP-347048 (#7365)

* feat(compass-collection): CLOUDP-347048 LLM Output Validation Followup - Mock Data Generator

* call faker method without args if invalid

* update faker argument handling and improve validation logging

* resolve TypeScript error by casting initialState to any

* add faker argument validation for large numbers and add sample function call display

* increase max number size, allow faker.helpers.arrayElements, parse args before call, update ui label

* add/fix faker call preview test

* replace parseFakerArgs with prepareFakerArgs for better argument handling

* Address comment

* Refactor util

* Address comment

* Comments

---------

Co-authored-by: Jacob Samuel Lu <[email protected]>
Co-authored-by: Jacob Lu <[email protected]>

Author: ncarbon

packages/compass-collection/src/components/mock-data-generator-modal/faker-mapping-selector.tsx

@@ -2,7 +2,9 @@ import {
   Banner,
   BannerVariant,
   Body,
+  Code,
   css,
+  Label,
   Option,
   palette,
   Select,
@@ -11,6 +13,7 @@ import {
 import React from 'react';
 import { UNRECOGNIZED_FAKER_METHOD } from '../../modules/collection-tab';
 import type { MongoDBFieldType } from '@mongodb-js/compass-generative-ai';
+import type { FakerArg } from './script-generation-utils';
 
 const fieldMappingSelectorsStyles = css({
   width: '50%',
@@ -24,16 +27,48 @@ const labelStyles = css({
   fontWeight: 600,
 });
 
+const stringifyFakerArg = (arg: FakerArg): string => {
+  if (typeof arg === 'object' && arg !== null && 'json' in arg) {
+    try {
+      return JSON.stringify(JSON.parse(arg.json));
+    } catch {
+      return '';
+    }
+  }
+
+  // Handle arrays recursively
+  if (Array.isArray(arg)) {
+    return `[${arg.map(stringifyFakerArg).join(', ')}]`;
+  }
+
+  if (typeof arg === 'string') {
+    return JSON.stringify(arg);
+  }
+
+  // Numbers and booleans
+  return String(arg);
+};
+
+const formatFakerFunctionCallWithArgs = (
+  fakerFunction: string,
+  fakerArgs: FakerArg[]
+) => {
+  const parsedFakerArgs = fakerArgs.map(stringifyFakerArg);
+  return `faker.${fakerFunction}(${parsedFakerArgs.join(', ')})`;
+};
+
 interface Props {
   activeJsonType: string;
   activeFakerFunction: string;
   onJsonTypeSelect: (jsonType: MongoDBFieldType) => void;
+  activeFakerArgs: FakerArg[];
   onFakerFunctionSelect: (fakerFunction: string) => void;
 }
 
 const FakerMappingSelector = ({
   activeJsonType,
   activeFakerFunction,
+  activeFakerArgs,
   onJsonTypeSelect,
   onFakerFunctionSelect,
 }: Props) => {
@@ -66,13 +101,29 @@ const FakerMappingSelector = ({
           </Option>
         ))}
       </Select>
-      {activeFakerFunction === UNRECOGNIZED_FAKER_METHOD && (
+      {activeFakerFunction === UNRECOGNIZED_FAKER_METHOD ? (
         <Banner variant={BannerVariant.Warning}>
           Please select a function or we will default fill this field with the
           string &quot;Unrecognized&quot;
         </Banner>
+      ) : (
+        <>
+          <Label htmlFor="faker-function-call-preview">
+            Preview Faker Function Call
+          </Label>
+          <Code
+            id="faker-function-call-preview"
+            data-testid="faker-function-call-preview"
+            language="javascript"
+            copyable={false}
+          >
+            {formatFakerFunctionCallWithArgs(
+              activeFakerFunction,
+              activeFakerArgs
+            )}
+          </Code>
+        </>
       )}
-      {/* TODO(CLOUDP-344400): Render faker function parameters once we have a way to validate them. */}
     </div>
   );
 };

packages/compass-collection/src/components/mock-data-generator-modal/faker-schema-editor-screen.tsx

@@ -64,6 +64,7 @@ const FakerSchemaEditorContent = ({
 
   const activeJsonType = fakerSchemaFormValues[activeField]?.mongoType;
   const activeFakerFunction = fakerSchemaFormValues[activeField]?.fakerMethod;
+  const activeFakerArgs = fakerSchemaFormValues[activeField]?.fakerArgs;
 
   const resetIsSchemaConfirmed = () => {
     onSchemaConfirmed(false);
@@ -109,6 +110,7 @@ const FakerSchemaEditorContent = ({
           <FakerMappingSelector
             activeJsonType={activeJsonType}
             activeFakerFunction={activeFakerFunction}
+            activeFakerArgs={activeFakerArgs}
             onJsonTypeSelect={onJsonTypeSelect}
             onFakerFunctionSelect={onFakerFunctionSelect}
           />

packages/compass-collection/src/components/mock-data-generator-modal/mock-data-generator-modal.spec.tsx

@@ -71,7 +71,7 @@ describe('MockDataGeneratorModal', () => {
 
     const store = createStore(
       collectionTabReducer,
-      initialState,
+      initialState as any,
       applyMiddleware(thunk.withExtraArgument(mockServices))
     );
 
@@ -545,6 +545,111 @@ describe('MockDataGeneratorModal', () => {
       );
     });
 
+    it('displays preview of the faker call without args when the args are invalid', async () => {
+      const largeLengthArgs = Array.from({ length: 11 }, () => 'testArg');
+      const mockServices = createMockServices();
+      mockServices.atlasAiService.getMockDataSchema = () =>
+        Promise.resolve({
+          fields: [
+            {
+              fieldPath: 'name',
+              mongoType: 'String',
+              fakerMethod: 'person.firstName',
+              fakerArgs: largeLengthArgs,
+              isArray: false,
+              probability: 1.0,
+            },
+            {
+              fieldPath: 'age',
+              mongoType: 'Int32',
+              fakerMethod: 'number.int',
+              fakerArgs: [
+                {
+                  json: JSON.stringify({
+                    a: largeLengthArgs,
+                  }),
+                },
+              ],
+              isArray: false,
+              probability: 1.0,
+            },
+            {
+              fieldPath: 'username',
+              mongoType: 'String',
+              fakerMethod: 'string.alpha',
+              // large string
+              fakerArgs: ['a'.repeat(1001)],
+              isArray: false,
+              probability: 1.0,
+            },
+            {
+              fieldPath: 'avatar',
+              mongoType: 'String',
+              fakerMethod: 'image.url',
+              fakerArgs: [
+                {
+                  json: JSON.stringify({
+                    width: 100_000,
+                    height: 100_000,
+                  }),
+                },
+              ],
+              isArray: false,
+              probability: 1.0,
+            },
+          ],
+        });
+
+      await renderModal({
+        mockServices,
+        schemaAnalysis: {
+          ...defaultSchemaAnalysisState,
+          processedSchema: {
+            name: {
+              type: 'String',
+              probability: 1.0,
+            },
+            age: {
+              type: 'Int32',
+              probability: 1.0,
+            },
+            username: {
+              type: 'String',
+              probability: 1.0,
+            },
+            avatar: {
+              type: 'String',
+              probability: 1.0,
+            },
+          },
+        },
+      });
+
+      // advance to the schema editor step
+      userEvent.click(screen.getByText('Confirm'));
+      await waitFor(() => {
+        expect(screen.getByTestId('faker-schema-editor')).to.exist;
+      });
+
+      userEvent.click(screen.getByText('name'));
+      expect(screen.getByTestId('faker-function-call-preview')).to.exist;
+      expect(screen.queryByText(/testArg/)).to.not.exist;
+
+      userEvent.click(screen.getByText('age'));
+      expect(screen.getByTestId('faker-function-call-preview')).to.exist;
+      expect(screen.queryByText(/testArg/)).to.not.exist;
+
+      userEvent.click(screen.getByText('username'));
+      expect(screen.queryByText(/aaaaaaa/)).to.not.exist;
+      expect(screen.getByTestId('faker-function-call-preview')).to.exist;
+
+      userEvent.click(screen.getByText('avatar'));
+      expect(screen.getByTestId('faker-function-call-preview')).to.exist;
+      expect(screen.queryByText(/width/)).to.not.exist;
+      expect(screen.queryByText(/height/)).to.not.exist;
+      expect(screen.queryByText(/100000/)).to.not.exist;
+    });
+
     it('disables the Next button when the faker schema mapping is not confirmed', async () => {
       await renderModal({
         mockServices: mockServicesWithMockDataResponse,

packages/compass-collection/src/components/mock-data-generator-modal/script-generation-utils.ts

@@ -2,7 +2,12 @@ import type { MongoDBFieldType } from '@mongodb-js/compass-generative-ai';
 import type { FakerFieldMapping } from './types';
 import { prettify } from '@mongodb-js/compass-editor';
 
-export type FakerArg = string | number | boolean | { json: string };
+export type FakerArg =
+  | string
+  | number
+  | boolean
+  | { json: string }
+  | FakerArg[];
 
 const DEFAULT_ARRAY_LENGTH = 3;