feat(import): Function to guess the import file type COMPASS-6421 (#3983)

* guessFileType

* unnecessary

* Update packages/compass-import-export/src/import/guess-filetype.ts

Co-authored-by: Anna Henningsen <[email protected]>

* Update packages/compass-import-export/src/import/guess-filetype.ts

Co-authored-by: Anna Henningsen <[email protected]>

* once

* csvDelimiter required for csv

* fixes for return type

* lint

* ugh

Co-authored-by: Anna Henningsen <[email protected]>

Author: lerouxb

package-lock.json

@@ -89,10 +89,11 @@
     "@types/chai-dom": "^0.0.10",
     "@types/mime-types": "^2.1.1",
     "@types/mocha": "^9.0.0",
+    "@types/papaparse": "^5.3.7",
     "@types/react": "^17.0.5",
     "@types/react-dom": "^17.0.10",
     "@types/sinon-chai": "^3.2.5",
-    "JSONStream": "^1.3.5",
+    "@types/stream-json": "^1.7.3",
     "ansi-to-html": "^0.6.11",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
@@ -105,6 +106,7 @@
     "fast-csv": "^3.4.0",
     "flat": "cipacda/flat",
     "hadron-app-registry": "^9.0.2",
+    "JSONStream": "^1.3.5",
     "lodash": "^4.17.21",
     "lodash.throttle": "^4.1.1",
     "marky": "^1.2.1",
@@ -116,6 +118,7 @@
     "mongodb-query-parser": "^2.4.6",
     "nyc": "^15.1.0",
     "object-sizeof": "^1.5.1",
+    "papaparse": "^5.3.2",
     "parse-json": "^5.0.0",
     "peek-stream": "^1.1.3",
     "prettier": "^2.7.1",
@@ -128,6 +131,7 @@
     "redux-thunk": "^2.3.0",
     "rimraf": "^3.0.2",
     "sinon": "^9.2.3",
+    "stream-json": "^1.7.5",
     "strip-bom-stream": "^4.0.0",
     "xvfb-maybe": "^0.2.1"
   }

packages/compass-import-export/package.json

@@ -0,0 +1,75 @@
+import path from 'path';
+import { expect } from 'chai';
+import fs from 'fs';
+import { guessFileType } from './guess-filetype';
+import { fixtures } from '../../test/fixtures';
+
+const expectedDelimiters = {
+  'bad.csv': ',',
+  'good-commas.csv': ',',
+  'good-tabs.csv': '\t',
+  'number-transform.csv': ',',
+  'sparse.csv': ',',
+  'semicolons.csv': ';',
+  'spaces.csv': ' ',
+} as const;
+
+describe('guessFileType', function () {
+  for (const filepath of Object.values(fixtures.json)) {
+    const basename = path.basename(filepath);
+    it(`detects ${basename} as json`, async function () {
+      const input = fs.createReadStream(filepath);
+      const { type } = await guessFileType({ input });
+      expect(type).to.equal('json');
+    });
+  }
+
+  for (const filepath of Object.values(fixtures.jsonl)) {
+    const basename = path.basename(filepath);
+    it(`detects ${basename} as jsonl`, async function () {
+      const input = fs.createReadStream(filepath);
+      const { type } = await guessFileType({ input });
+      expect(type).to.equal('jsonl');
+    });
+  }
+
+  for (const filepath of Object.values(fixtures.csv)) {
+    const basename = path.basename(filepath);
+    it(`detects ${basename} as csv`, async function () {
+      const input = fs.createReadStream(filepath);
+      const result = await guessFileType({ input });
+      expect(result.type).to.equal('csv');
+      const expectedDelimiter =
+        expectedDelimiters[basename as keyof typeof expectedDelimiters];
+      if (expectedDelimiter) {
+        expect(result.type === 'csv' && result.csvDelimiter).to.equal(
+          expectedDelimiter
+        );
+      } else {
+        expect(result.type === 'csv' && result.csvDelimiter).to.equal(
+          `add an entry for ${basename} to expectedDelimiters`
+        );
+      }
+    });
+  }
+
+  // strip out known false positives
+  const unknownFiles = Object.values(fixtures.other).filter(
+    (filepath) => !filepath.endsWith('javascript')
+  );
+
+  for (const filepath of unknownFiles) {
+    const basename = path.basename(filepath);
+    it(`detects ${basename} as unknown`, async function () {
+      const input = fs.createReadStream(filepath);
+      const { type } = await guessFileType({ input });
+      expect(type).to.equal('unknown');
+    });
+  }
+
+  it('can treat javascript as a json false positive', async function () {
+    const input = fs.createReadStream(fixtures.other.javascript);
+    const { type } = await guessFileType({ input });
+    expect(type).to.equal('json');
+  });
+});