fix(compass-import-export): stream import errors to the log file with proper back pressure COMPASS-7820 (#6151)

* stream import errors to file with back pressure

* port import tests

* update expectation to two errors

* one callback per error now

* rm comment

* fix some bugs in our error transformation

* ImportWriter unit tests

Author: lerouxb

packages/compass-e2e-tests/tests/collection-import.test.ts

@@ -1210,12 +1210,13 @@ describe('Collection import', function () {
       .$(Selectors.closeToastButton(Selectors.ImportToast))
       .waitForDisplayed();
 
-    // Displays first error in the toast and view log.
+    // Displays first two errors in the toast and view log.
+    // (It tries to display two, but it also limits the text)
     const toastText = await toastElement.getText();
     expect(toastText).to.include('Import completed 0/3 with errors:');
     expect(
       (toastText.match(/E11000 duplicate key error collection/g) || []).length
-    ).to.equal(1);
+    ).to.equal(2);
     expect(toastText).to.include('VIEW LOG');
 
     const logFilePath = path.resolve(

packages/compass-import-export/src/components/import-modal.tsx

@@ -274,7 +274,7 @@ function ImportModal({
 const mapStateToProps = (state: RootImportState) => ({
   ns: state.import.namespace,
   isOpen: state.import.isOpen,
-  errors: state.import.errors,
+  errors: state.import.firstErrors,
   fileType: state.import.fileType,
   fileName: state.import.fileName,
   status: state.import.status,

packages/compass-import-export/src/import/import-csv.spec.ts

@@ -113,19 +113,9 @@ describe('importCSV', function () {
         });
 
         expect(omit(result, 'biggestDocSize')).to.deep.equal({
+          docsErrored: 0,
           docsProcessed: totalRows,
           docsWritten: totalRows,
-          dbErrors: [],
-          dbStats: {
-            insertedCount: totalRows,
-            matchedCount: 0,
-            modifiedCount: 0,
-            deletedCount: 0,
-            upsertedCount: 0,
-            ok: Math.ceil(totalRows / 1000),
-            writeConcernErrors: [],
-            writeErrors: [],
-          },
           hasUnboundArray: false,
         });
 
@@ -264,19 +254,9 @@ describe('importCSV', function () {
         });
 
         expect(omit(result, 'biggestDocSize')).to.deep.equal({
+          docsErrored: 0,
           docsProcessed: totalRows,
           docsWritten: totalRows,
-          dbErrors: [],
-          dbStats: {
-            insertedCount: totalRows,
-            matchedCount: 0,
-            modifiedCount: 0,
-            deletedCount: 0,
-            upsertedCount: 0,
-            ok: Math.ceil(totalRows / 1000),
-            writeConcernErrors: [],
-            writeErrors: [],
-          },
           hasUnboundArray: false,
         });
 
@@ -356,19 +336,9 @@ describe('importCSV', function () {
     expect(errorLog).to.equal('');
 
     expect(omit(result, 'biggestDocSize')).to.deep.equal({
+      docsErrored: 0,
       docsProcessed: totalRows,
       docsWritten: totalRows,
-      dbErrors: [],
-      dbStats: {
-        insertedCount: totalRows,
-        matchedCount: 0,
-        modifiedCount: 0,
-        deletedCount: 0,
-        upsertedCount: 0,
-        ok: Math.ceil(totalRows / 1000),
-        writeConcernErrors: [],
-        writeErrors: [],
-      },
       hasUnboundArray: false,
     });
 
@@ -429,19 +399,9 @@ describe('importCSV', function () {
     expect(errorLog).to.equal('');
 
     expect(omit(result, 'biggestDocSize')).to.deep.equal({
+      docsErrored: 0,
       docsProcessed: 2000,
       docsWritten: 2000,
-      dbErrors: [],
-      dbStats: {
-        insertedCount: 2000,
-        matchedCount: 0,
-        modifiedCount: 0,
-        deletedCount: 0,
-        upsertedCount: 0,
-        ok: 2, // expected two batches
-        writeConcernErrors: [],
-        writeErrors: [],
-      },
       hasUnboundArray: false,
     });
 
@@ -670,7 +630,12 @@ describe('importCSV', function () {
       errorCallback,
     });
 
-    expect(result.dbStats.insertedCount).to.equal(1);
+    expect(omit(result, 'biggestDocSize')).to.deep.equal({
+      docsErrored: 2,
+      docsProcessed: 3,
+      docsWritten: 1,
+      hasUnboundArray: false,
+    });
 
     expect(progressCallback.callCount).to.equal(3);
     expect(errorCallback.callCount).to.equal(2);
@@ -778,43 +743,45 @@ describe('importCSV', function () {
       errorCallback,
     });
 
-    expect(result.dbStats.insertedCount).to.equal(0);
+    expect(omit(result, 'biggestDocSize')).to.deep.equal({
+      docsErrored: 3,
+      docsProcessed: 3,
+      docsWritten: 0,
+      hasUnboundArray: false,
+    });
 
     expect(progressCallback.callCount).to.equal(3);
-    expect(errorCallback.callCount).to.equal(1); // once for the batch
+    expect(errorCallback.callCount).to.equal(3);
 
     const expectedErrors: ErrorJSON[] = [
       {
-        name: 'MongoBulkWriteError',
+        name: 'WriteError',
         message: 'Document failed validation',
+        index: 0,
+        code: 121,
+      },
+      {
+        name: 'WriteError',
+        message: 'Document failed validation',
+        index: 1,
+        code: 121,
+      },
+      {
+        name: 'WriteError',
+        message: 'Document failed validation',
+        index: 2,
         code: 121,
-        numErrors: 3,
       },
     ];
 
     const errors = errorCallback.args.map((args) => args[0]);
+    for (const [index, error] of errors.entries()) {
+      expect(error.op).to.exist;
+      // cheat and copy them over because it is big and with buffers
+      expectedErrors[index].op = error.op;
+    }
     expect(errors).to.deep.equal(expectedErrors);
 
-    // the log file has one for each error in the bulk write too
-    expectedErrors.push({
-      name: 'WriteConcernError',
-      message: 'Document failed validation',
-      index: 0,
-      code: 121,
-    });
-    expectedErrors.push({
-      name: 'WriteConcernError',
-      message: 'Document failed validation',
-      index: 1,
-      code: 121,
-    });
-    expectedErrors.push({
-      name: 'WriteConcernError',
-      message: 'Document failed validation',
-      index: 2,
-      code: 121,
-    });
-
     const errorsText = await fs.promises.readFile(output.path, 'utf8');
     expect(errorsText).to.equal(formatErrorLines(expectedErrors));
   });
@@ -842,19 +809,9 @@ describe('importCSV', function () {
     // only looked at the first row because we aborted before even starting
     expect(omit(result, 'biggestDocSize')).to.deep.equal({
       aborted: true,
+      docsErrored: 0,
       docsProcessed: 0,
       docsWritten: 0,
-      dbErrors: [],
-      dbStats: {
-        insertedCount: 0,
-        matchedCount: 0,
-        modifiedCount: 0,
-        deletedCount: 0,
-        upsertedCount: 0,
-        ok: 0,
-        writeConcernErrors: [],
-        writeErrors: [],
-      },
       hasUnboundArray: false,
     });
   });