feat(import): analyse CSV fields and auto-select the correct type COMPASS-6536 (#4155)

* use analyzeCSVFields()

* test auto-detecting number

* add/fix tests

* file type select weirdness

* tooltips for mixed & number

* obvs clickVisible is not available on an element..

* feedback

* more cleanup

* Update packages/compass-import-export/src/components/import-preview.tsx

Co-authored-by: Rhys <[email protected]>

* Update packages/compass-import-export/src/components/select-field-type.tsx

Co-authored-by: Rhys <[email protected]>

* merge select-field-type into import-preview

* async/await

* css tweaks

* yellow column hilight for mixed fields

* tweaks

---------

Co-authored-by: Rhys <[email protected]>

Author: lerouxb

packages/compass-components/src/components/placeholder.tsx

@@ -57,6 +57,7 @@ const Placeholder: React.FunctionComponent<
     maxChar?: number;
     width?: CSSProperties['width'];
     height?: CSSProperties['height'];
+    'data-testid'?: string;
   }
 > = ({
   className,
@@ -76,7 +77,7 @@ const Placeholder: React.FunctionComponent<
     <div
       {...props}
       role="presentation"
-      data-testid="placeholder"
+      data-testid={props['data-testid'] ?? 'placeholder'}
       className={cx(placeholder, className, darkMode && placeholderDarkMode)}
       style={{ width, height: propsHeight }}
     ></div>

packages/compass-e2e-tests/helpers/selectors.ts

@@ -561,6 +561,9 @@ export const ImportDone =
   '[data-testid="import-modal"] [data-testid="done-button"]';
 export const ImportErrorBox = '[data-testid="import-error-box"]';
 
+export const importPreviewFieldHeaderField = (fieldName: string): string => {
+  return `[data-testid="import-preview-field-type-select-menu-${fieldName}"]`;
+};
 export const importPreviewFieldHeaderSelect = (fieldName: string): string => {
   return `[data-testid="preview-field-header-${fieldName}"] button`;
 };

packages/compass-e2e-tests/tests/collection-import.test.ts

@@ -54,6 +54,8 @@ async function selectFieldType(
 
   const fieldTypeSelectSpan = await fieldTypeSelectMenu.$(`span=${fieldType}`);
   await fieldTypeSelectSpan.waitForDisplayed();
+  await fieldTypeSelectSpan.scrollIntoView();
+  await browser.pause(1000);
   await fieldTypeSelectSpan.click();
 
   // Wait so that the menu animation can complete.
@@ -454,12 +456,21 @@ describe('Collection import', function () {
     await browser.clickVisible(Selectors.ImportFileOption);
 
     // Select the file.
+    // Unfortunately this opens a second open dialog and the one that got
+    // automatically opened when clicking on import file sticks around on top of
+    // everything :(
     await browser.selectFile(Selectors.ImportFileInput, csvPath);
 
     // Wait for the modal to appear.
     const importModal = await browser.$(Selectors.ImportModal);
     await importModal.waitForDisplayed();
 
+    // wait for it to finish analyzing
+    await browser.$(Selectors.ImportConfirm).waitForDisplayed();
+    await browser
+      .$(Selectors.importPreviewFieldHeaderField('id'))
+      .waitForDisplayed();
+
     // pick some types
     const typeMapping = {
       id: 'Number',
@@ -560,23 +571,11 @@ describe('Collection import', function () {
     );
     expect(await importDelimiterSelectButton.getText()).to.equal('semicolon');
 
-    /*
-    const importDelimiterSelectButton = await browser.$(
-      Selectors.ImportDelimiterSelect
-    );
-    await importDelimiterSelectButton.waitForDisplayed();
-    await importDelimiterSelectButton.click();
-
-    const importDelimiterSelectMenu = await browser.$(
-      Selectors.ImportDelimiterMenu
-    );
-    await importDelimiterSelectMenu.waitForDisplayed();
-    const delimiterSelectSpan = await importDelimiterSelectMenu.$(
-      'span=semicolon'
-    );
-    await delimiterSelectSpan.waitForDisplayed();
-    await delimiterSelectSpan.click();
-    */
+    // wait for it to finish analyzing
+    await browser.$(Selectors.ImportConfirm).waitForDisplayed();
+    await browser
+      .$(Selectors.importPreviewFieldHeaderField('amount'))
+      .waitForDisplayed();
 
     // pick some types
     const typeMapping = {
@@ -648,6 +647,12 @@ describe('Collection import', function () {
     const importModal = await browser.$(Selectors.ImportModal);
     await importModal.waitForDisplayed();
 
+    // wait for it to finish analyzing
+    await browser.$(Selectors.ImportConfirm).waitForDisplayed();
+    await browser
+      .$(Selectors.importPreviewFieldHeaderField('id'))
+      .waitForDisplayed();
+
     // pick an incompatible type
     await selectFieldType(browser, 'id', 'ObjectId');
 

packages/compass-import-export/src/components/import-file-input.tsx

@@ -20,7 +20,7 @@ function ImportFileInput({
   const handleChooseFile = useCallback(
     (files: string[]) => {
       if (files.length > 0) {
-        selectImportFileName(files[0]);
+        void selectImportFileName(files[0]);
       } else if (typeof onCancel === 'function') {
         onCancel();
       }

packages/compass-import-export/src/components/import-modal.tsx

@@ -43,6 +43,7 @@ import { ImportErrorList } from './import-error-list';
 import type { RootImportState } from '../stores/import-store';
 import type { CSVDelimiter, FieldFromCSV } from '../modules/import';
 import { ImportFileInput } from './import-file-input';
+import type { CSVParsableFieldType } from '../utils/csv';
 
 /**
  * Progress messages.
@@ -96,13 +97,14 @@ type ImportModalProps = {
    */
   fields: {
     path: string;
-    checked: boolean;
-    type?: string; // Only on csv imports.
+    checked?: boolean; // CSV placeholder fields don't have checked
+    type?: CSVParsableFieldType | 'placeholder'; // Only on csv imports.
   }[];
   values: string[][];
   toggleIncludeField: (path: string) => void;
   setFieldType: (path: string, bsonType: string) => void;
   previewLoaded: boolean;
+  csvAnalyzed: boolean;
 };
 
 function ImportModal({
@@ -136,6 +138,7 @@ function ImportModal({
   toggleIncludeField,
   setFieldType,
   previewLoaded,
+  csvAnalyzed,
 }: ImportModalProps) {
   const modalBodyRef = useRef<HTMLDivElement>(null);
   const handleCancel = useCallback(() => {
@@ -194,7 +197,12 @@ function ImportModal({
   }
 
   return (
-    <Modal open={isOpen} setOpen={handleClose} data-testid="import-modal">
+    <Modal
+      open={isOpen}
+      setOpen={handleClose}
+      data-testid="import-modal"
+      size="large"
+    >
       <ModalHeader title="Import" subtitle={`To Collection ${ns}`} />
       <ModalBody ref={modalBodyRef}>
         <ImportOptions
@@ -212,6 +220,7 @@ function ImportModal({
           <FormFieldContainer>
             <ImportPreview
               loaded={previewLoaded}
+              analyzed={csvAnalyzed}
               onFieldCheckedChanged={toggleIncludeField}
               setFieldType={setFieldType}
               values={values}
@@ -261,7 +270,11 @@ function ImportModal({
             <Button
               data-testid="import-button"
               onClick={handleImportBtnClicked}
-              disabled={!fileName || status === STARTED}
+              disabled={
+                !fileName ||
+                status === STARTED ||
+                (fileType === 'csv' && !csvAnalyzed)
+              }
               variant="primary"
             >
               {status === STARTED ? 'Importing\u2026' : 'Import'}
@@ -301,6 +314,7 @@ const mapStateToProps = (state: RootImportState) => ({
   fields: state.importData.fields,
   values: state.importData.values,
   previewLoaded: state.importData.previewLoaded,
+  csvAnalyzed: state.importData.analyzeStatus === 'COMPLETED',
 });
 
 /**

packages/compass-import-export/src/components/import-options.tsx

@@ -90,7 +90,7 @@ function ImportOptions({
             id="import-delimiter-select"
             data-testid="import-delimiter-select"
             onChange={(delimiter: string) =>
-              setDelimiter(delimiter as CSVDelimiter)
+              void setDelimiter(delimiter as CSVDelimiter)
             }
             value={delimiter}
             allowDeselect={false}

packages/compass-import-export/src/components/import-preview.spec.tsx

@@ -4,13 +4,14 @@ import sinon from 'sinon';
 import { expect } from 'chai';
 
 import { ImportPreview } from './import-preview';
+import type { CSVParsableFieldType } from '../utils/csv';
 
 const testText = 'Specify Fields and Types';
 
 const testField = {
   path: '_id',
   checked: true,
-  type: 'String',
+  type: 'string' as CSVParsableFieldType,
 };
 
 describe('ImportPreview [Component]', function () {
@@ -31,6 +32,7 @@ describe('ImportPreview [Component]', function () {
           loaded={false}
           onFieldCheckedChanged={onFieldCheckedChangedSpy}
           setFieldType={setFieldTypeSpy}
+          analyzed={false}
         />
       );
     });
@@ -49,6 +51,7 @@ describe('ImportPreview [Component]', function () {
           loaded
           onFieldCheckedChanged={onFieldCheckedChangedSpy}
           setFieldType={setFieldTypeSpy}
+          analyzed={true}
         />
       );
     });
@@ -67,6 +70,7 @@ describe('ImportPreview [Component]', function () {
           loaded
           onFieldCheckedChanged={onFieldCheckedChangedSpy}
           setFieldType={setFieldTypeSpy}
+          analyzed={true}
         />
       );
     });
@@ -91,6 +95,7 @@ describe('ImportPreview [Component]', function () {
           loaded
           onFieldCheckedChanged={onFieldCheckedChangedSpy}
           setFieldType={setFieldTypeSpy}
+          analyzed={true}
         />
       );
     });
@@ -99,4 +104,51 @@ describe('ImportPreview [Component]', function () {
       expect(screen.queryByText(testText)).to.be.visible;
     });
   });
+
+  it('renders placeholders when not analyzed', function () {
+    render(
+      <ImportPreview
+        fields={[testField]}
+        values={
+          [
+            {
+              _id: 25,
+            },
+          ] as any
+        }
+        loaded
+        onFieldCheckedChanged={onFieldCheckedChangedSpy}
+        setFieldType={setFieldTypeSpy}
+        analyzed={false}
+      />
+    );
+
+    expect(screen.queryByTestId('import-preview-placeholder-_id')).to.be
+      .visible;
+    expect(screen.queryByTestId('import-preview-field-type-select-menu-_id')).to
+      .not.exist;
+  });
+
+  it('renders field type selects once analyzed', function () {
+    render(
+      <ImportPreview
+        fields={[testField]}
+        values={
+          [
+            {
+              _id: 25,
+            },
+          ] as any
+        }
+        loaded
+        onFieldCheckedChanged={onFieldCheckedChangedSpy}
+        setFieldType={setFieldTypeSpy}
+        analyzed={true}
+      />
+    );
+
+    expect(screen.queryByTestId('import-preview-placeholder-_id')).to.not.exist;
+    expect(screen.queryByTestId('import-preview-field-type-select-menu-_id')).to
+      .be.visible;
+  });
 });