Skip to content

Commit 2c7ca44

Browse files
lerouxblerouxb
authored
chore(import/export): Add and restructure import/export test fixtures COMPASS-6420 (#3981)
* structure test fixtures, add some more from investigations * examples of CSV files containing values that should be parsed/detected as each bson type * restructure fixtures * bring back write locations
1 parent afc511e commit 2c7ca44

40 files changed

+6975
-65
lines changed

‎packages/compass-import-export/src/utils/detect-import-file.spec.js‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
import detectImportFile from './detect-import-file';
22
import { expect } from 'chai';
3-
import { FIXTURES } from '../../test/fixtures';
3+
import { fixtures } from '../../test/fixtures';
44

55
describe('detectImportFile', function () {
66
it('should detect a JSON array', function (done) {
7-
detectImportFile(FIXTURES.JSON_ARRAY, function (err, res) {
7+
detectImportFile(fixtures.json.good, function (err, res) {
88
if (err) return done(err);
99
expect(res.fileType).to.equal('json');
1010
expect(res.fileIsMultilineJSON).to.be.false;
@@ -13,23 +13,23 @@ describe('detectImportFile', function () {
1313
expect(true).to.equal(true);
1414
});
1515
it('should detect new line delimited JSON', function (done) {
16-
detectImportFile(FIXTURES.NDJSON, function (err, res) {
16+
detectImportFile(fixtures.jsonl.good, function (err, res) {
1717
if (err) return done(err);
1818
expect(res.fileType).to.equal('json');
1919
expect(res.fileIsMultilineJSON).to.be.true;
2020
done();
2121
});
2222
});
2323
it('should detect new line delimited JSON even with an empty last line', function (done) {
24-
detectImportFile(FIXTURES.NDJSON_EXTRA_LINE, function (err, res) {
24+
detectImportFile(fixtures.jsonl.extra_line, function (err, res) {
2525
if (err) return done(err);
2626
expect(res.fileType).to.equal('json');
2727
expect(res.fileIsMultilineJSON).to.be.true;
2828
done();
2929
});
3030
});
3131
it('should detect with a preference toward peek NOT just file extension', function (done) {
32-
detectImportFile(FIXTURES.JSON_WITH_CSV_FILEEXT, function (err, res) {
32+
detectImportFile(fixtures.json.json_with_csv_fileext, function (err, res) {
3333
if (err) return done(err);
3434
expect(res.fileType).to.equal('json');
3535
expect(res.fileIsMultilineJSON).to.be.true;

‎packages/compass-import-export/src/utils/formatters.spec.js‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import fs from 'fs';
66
import { promisify } from 'util';
77
import { EOL } from 'os';
88
import { expect } from 'chai';
9-
import { FIXTURES } from '../../test/fixtures';
9+
import { fixtures } from '../../test/fixtures';
1010

1111
const pipeline = promisify(stream.pipeline);
1212
const readFile = promisify(fs.readFile);
@@ -26,17 +26,17 @@ describe('formatters', function () {
2626
{ _id: new ObjectID('5e5ea7558d35931a05eafec0') },
2727
]);
2828
const formatter = createJSONFormatter({ brackets: true });
29-
const dest = fs.createWriteStream(FIXTURES.JSON_SINGLE_DOC);
29+
const dest = fs.createWriteStream(fixtures.JSON_SINGLE_DOC);
3030

3131
return pipeline(source, formatter, dest)
32-
.then(() => readFile(FIXTURES.JSON_SINGLE_DOC))
32+
.then(() => readFile(fixtures.JSON_SINGLE_DOC))
3333
.then((contents) => {
3434
const parsed = EJSON.parse(contents);
3535
expect(parsed).to.deep.equal([
3636
{ _id: new ObjectID('5e5ea7558d35931a05eafec0') },
3737
]);
3838
})
39-
.then(() => rm(FIXTURES.JSON_SINGLE_DOC));
39+
.then(() => rm(fixtures.JSON_SINGLE_DOC));
4040
});
4141
it('should format more than 2 documents in an array', function () {
4242
const docs = [
@@ -46,20 +46,20 @@ describe('formatters', function () {
4646
];
4747
const source = stream.Readable.from(docs);
4848
const formatter = createJSONFormatter({ brackets: true });
49-
const dest = fs.createWriteStream(FIXTURES.JSON_MULTI_SMALL_DOCS);
49+
const dest = fs.createWriteStream(fixtures.JSON_MULTI_SMALL_DOCS);
5050

5151
return pipeline(source, formatter, dest)
52-
.then(() => readFile(FIXTURES.JSON_MULTI_SMALL_DOCS))
52+
.then(() => readFile(fixtures.JSON_MULTI_SMALL_DOCS))
5353
.then((contents) => {
5454
const parsed = EJSON.parse(contents);
5555
expect(parsed).to.deep.equal(docs);
5656
})
57-
.then(() => rm(FIXTURES.JSON_MULTI_SMALL_DOCS));
57+
.then(() => rm(fixtures.JSON_MULTI_SMALL_DOCS));
5858
});
5959
describe('should format binary data correctly', function () {
6060
afterEach(async function () {
6161
try {
62-
await rm(FIXTURES.JSON_MULTI_SMALL_DOCS);
62+
await rm(fixtures.JSON_MULTI_SMALL_DOCS);
6363
} catch (e) {
6464
// noop
6565
}
@@ -79,11 +79,11 @@ describe('formatters', function () {
7979
];
8080
const source = stream.Readable.from(docs);
8181
const formatter = createJSONFormatter({ brackets: true });
82-
const dest = fs.createWriteStream(FIXTURES.JSON_MULTI_SMALL_DOCS);
82+
const dest = fs.createWriteStream(fixtures.JSON_MULTI_SMALL_DOCS);
8383

8484
await pipeline(source, formatter, dest);
8585

86-
const contents = await readFile(FIXTURES.JSON_MULTI_SMALL_DOCS);
86+
const contents = await readFile(fixtures.JSON_MULTI_SMALL_DOCS);
8787
const parsed = EJSON.parse(contents);
8888

8989
expect(parsed).to.deep.equal(docs);
@@ -93,7 +93,7 @@ describe('formatters', function () {
9393
describe('should format Longs correctly', function () {
9494
afterEach(async function () {
9595
try {
96-
await rm(FIXTURES.JSON_MULTI_SMALL_DOCS);
96+
await rm(fixtures.JSON_MULTI_SMALL_DOCS);
9797
} catch (e) {
9898
// noop
9999
}
@@ -110,11 +110,11 @@ describe('formatters', function () {
110110
];
111111
const source = stream.Readable.from(docs);
112112
const formatter = createJSONFormatter({ brackets: true });
113-
const dest = fs.createWriteStream(FIXTURES.JSON_MULTI_SMALL_DOCS);
113+
const dest = fs.createWriteStream(fixtures.JSON_MULTI_SMALL_DOCS);
114114

115115
await pipeline(source, formatter, dest);
116116

117-
const contents = await readFile(FIXTURES.JSON_MULTI_SMALL_DOCS);
117+
const contents = await readFile(fixtures.JSON_MULTI_SMALL_DOCS);
118118
const parsed = EJSON.parse(contents, { relaxed: false });
119119

120120
expect(parsed).to.deep.equal(docs);
@@ -129,17 +129,17 @@ describe('formatters', function () {
129129
];
130130
const source = stream.Readable.from(docs);
131131
const formatter = createJSONFormatter({ brackets: false });
132-
const dest = fs.createWriteStream(FIXTURES.JSONL);
132+
const dest = fs.createWriteStream(fixtures.JSONL);
133133

134134
return pipeline(source, formatter, dest)
135-
.then(() => readFile(FIXTURES.JSONL))
135+
.then(() => readFile(fixtures.JSONL))
136136
.then((buf) => {
137137
const sources = buf.toString('utf-8').split(EOL);
138138
expect(EJSON.parse(sources[0])).to.deep.equal(docs[0]);
139139
expect(EJSON.parse(sources[1])).to.deep.equal(docs[1]);
140140
expect(EJSON.parse(sources[2])).to.deep.equal(docs[2]);
141141
})
142-
.then(() => rm(FIXTURES.JSONL));
142+
.then(() => rm(fixtures.JSONL));
143143
});
144144
});
145145
describe('csv', function () {
@@ -152,13 +152,13 @@ describe('formatters', function () {
152152
const formatter = createCSVFormatter({
153153
columns: ['_id.foo'],
154154
});
155-
const dest = fs.createWriteStream(FIXTURES.CSV_FLAT_HEADERS);
155+
const dest = fs.createWriteStream(fixtures.CSV_FLAT_HEADERS);
156156

157157
return pipeline(source, formatter, dest)
158-
.then(() => readFile(FIXTURES.CSV_FLAT_HEADERS))
158+
.then(() => readFile(fixtures.CSV_FLAT_HEADERS))
159159
.then(() => {
160160
return pipeline(
161-
fs.createReadStream(FIXTURES.CSV_FLAT_HEADERS),
161+
fs.createReadStream(fixtures.CSV_FLAT_HEADERS),
162162
createCSVParser(),
163163
new stream.Writable({
164164
objectMode: true,
@@ -169,7 +169,7 @@ describe('formatters', function () {
169169
})
170170
);
171171
})
172-
.then(() => rm(FIXTURES.CSV_FLAT_HEADERS));
172+
.then(() => rm(fixtures.CSV_FLAT_HEADERS));
173173
});
174174

175175
/**
@@ -179,13 +179,13 @@ describe('formatters', function () {
179179
const docs = [{ _id: new ObjectID('5e5ea7558d35931a05eafec0') }];
180180
const source = stream.Readable.from(docs);
181181
const formatter = createCSVFormatter({ columns: ['_id'] });
182-
const dest = fs.createWriteStream(FIXTURES.CSV_FLAT_HEADERS);
182+
const dest = fs.createWriteStream(fixtures.CSV_FLAT_HEADERS);
183183

184184
return pipeline(source, formatter, dest)
185-
.then(() => readFile(FIXTURES.CSV_FLAT_HEADERS))
185+
.then(() => readFile(fixtures.CSV_FLAT_HEADERS))
186186
.then(() => {
187187
return pipeline(
188-
fs.createReadStream(FIXTURES.CSV_FLAT_HEADERS),
188+
fs.createReadStream(fixtures.CSV_FLAT_HEADERS),
189189
createCSVParser(),
190190
new stream.Writable({
191191
objectMode: true,
@@ -198,7 +198,7 @@ describe('formatters', function () {
198198
})
199199
);
200200
})
201-
.then(() => rm(FIXTURES.CSV_FLAT_HEADERS));
201+
.then(() => rm(fixtures.CSV_FLAT_HEADERS));
202202
});
203203
});
204204
});

‎packages/compass-import-export/src/utils/import-parser.spec.js‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import chaiAsPromised from 'chai-as-promised';
66
chai.use(chaiAsPromised);
77

88
import createParser from './import-parser';
9-
import { FIXTURES } from '../../test/fixtures';
9+
import { fixtures } from '../../test/fixtures';
1010
function runParser(src, parser) {
1111
const docs = [];
1212
const source = fs.createReadStream(src);
@@ -30,19 +30,19 @@ function runParser(src, parser) {
3030
describe('import-parser', function () {
3131
describe('json', function () {
3232
it('should parse a file', function () {
33-
return runParser(FIXTURES.GOOD_JSON, createParser()).then((docs) => {
33+
return runParser(fixtures.json.good, createParser()).then((docs) => {
3434
expect(docs).to.have.length(3);
3535
});
3636
});
3737
it('should parse a line-delimited file', function () {
3838
return runParser(
39-
FIXTURES.LINE_DELIMITED_JSON,
39+
fixtures.jsonl.good,
4040
createParser({ fileType: 'json', fileIsMultilineJSON: true })
4141
).then((docs) => expect(docs).to.have.length(3));
4242
});
4343
it('should parse a line-delimited file with an extra empty line', function () {
4444
return runParser(
45-
FIXTURES.LINE_DELIMITED_JSON_EXTRA_LINE,
45+
fixtures.jsonl.extra_line,
4646
createParser({ fileIsMultilineJSON: true })
4747
).then((docs) => {
4848
expect(docs).to.have.length(3);
@@ -51,7 +51,7 @@ describe('import-parser', function () {
5151
describe('deserialize', function () {
5252
const BSON_DOCS = [];
5353
before(function () {
54-
const src = FIXTURES.GOOD_JSON;
54+
const src = fixtures.json.good;
5555
return runParser(src, createParser()).then(function (docs) {
5656
BSON_DOCS.push.apply(BSON_DOCS, docs);
5757
});
@@ -63,7 +63,7 @@ describe('import-parser', function () {
6363
describe('errors', function () {
6464
let parseError;
6565
before(function (done) {
66-
const p = runParser(FIXTURES.JS_I_THINK_IS_JSON, createParser());
66+
const p = runParser(fixtures.other.javascript, createParser());
6767
p.catch((err) => (parseError = err));
6868
expect(p).to.be.rejected.and.notify(done);
6969
});
@@ -80,8 +80,8 @@ describe('import-parser', function () {
8080
describe('csv', function () {
8181
it('should work with commas', function () {
8282
return runParser(
83-
FIXTURES.GOOD_COMMAS_CSV,
84-
createParser({ fileType: 'csv', fileName: FIXTURES.GOOD_COMMAS_CSV })
83+
fixtures.csv.good_commas,
84+
createParser({ fileType: 'csv', fileName: fixtures.csv.good_commas })
8585
).then((docs) => {
8686
expect(docs).to.deep.equal([
8787
{ _id: '1', value: 'some string' },
@@ -92,10 +92,10 @@ describe('import-parser', function () {
9292
});
9393
it('should work with hard tabs', function () {
9494
return runParser(
95-
FIXTURES.GOOD_TABS_CSV,
95+
fixtures.csv.good_tabs,
9696
createParser({
9797
fileType: 'csv',
98-
fileName: FIXTURES.GOOD_TABS_CSV,
98+
fileName: fixtures.csv.good_tabs,
9999
delimiter: '\t',
100100
})
101101
).then((docs) => {
@@ -108,7 +108,7 @@ describe('import-parser', function () {
108108
});
109109
it('should parse number-transform', function () {
110110
return runParser(
111-
FIXTURES.NUMBER_TRANSFORM_CSV,
111+
fixtures.csv.number_transform,
112112
createParser({ fileType: 'csv' })
113113
).then((docs) => {
114114
expect(docs).to.have.length(1);
@@ -185,7 +185,7 @@ describe('import-parser', function () {
185185
let parseError;
186186
before(function (done) {
187187
const p = runParser(
188-
FIXTURES.BAD_CSV,
188+
fixtures.csv.bad,
189189
createParser({ fileType: 'csv', delimiter: '\n' })
190190
);
191191
p.catch((err) => (parseError = err));

‎packages/compass-import-export/src/utils/import-preview.spec.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import { createPreviewWritable, createPeekStream } from './import-preview';
22
import { Readable, pipeline } from 'stream';
33
import fs from 'fs';
44
import { expect } from 'chai';
5-
import { FIXTURES } from '../../test/fixtures';
5+
import { fixtures } from '../../test/fixtures';
66

77
describe('import-preview', function () {
88
describe('createPreviewWritable', function () {
@@ -71,7 +71,7 @@ describe('import-preview', function () {
7171

7272
describe('func', function () {
7373
it('should return 2 docs for a csv containing 3 docs', function (done) {
74-
const src = fs.createReadStream(FIXTURES.GOOD_COMMAS_CSV);
74+
const src = fs.createReadStream(fixtures.csv.good_commas);
7575
const dest = createPreviewWritable({ MAX_SIZE: 2 });
7676

7777
pipeline(src, createPeekStream('csv'), dest, () => {

packages/compass-import-export/test/test_bad.csv renamed to packages/compass-import-export/test/csv/bad.csv

File renamed without changes.

packages/compass-import-export/test/good-commas.csv renamed to packages/compass-import-export/test/csv/good-commas.csv

File renamed without changes.

packages/compass-import-export/test/good-tabs.csv renamed to packages/compass-import-export/test/csv/good-tabs.csv

File renamed without changes.

packages/compass-import-export/test/number-transform.csv renamed to packages/compass-import-export/test/csv/number-transform.csv

File renamed without changes.

‎packages/compass-import-export/test/csv/sparse.csv‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
foo,bar,baz
2+
1,
3+
1,2
4+
1,2,3
5+
,,3
6+
7+
,2,
8+
,,3
9+
,2
10+
1,2,3,4

‎packages/compass-import-export/test/csv/types/array.csv‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
something,numbers[0],numbers[1],numbers[3],something_else,notes
2+
foo,1,2,3,bar,longest array
3+
a,1,,,b,just one element
4+
a,,,,b,empty array

0 commit comments

Comments
 (0)