diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index b418aab94588..cd0a22bf8625 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -1244,7 +1244,9 @@ export class BasicCrawler {} +export interface DatasetExportOptions extends Omit { + /** + * If true, includes all unique keys from all dataset items in the CSV export header. + * If omitted or false, only keys from the first item are used. + */ + collectAllKeys?: boolean; +} export interface DatasetIteratorOptions extends Omit { @@ -170,11 +176,6 @@ export interface DatasetIteratorOptions export interface DatasetExportToOptions extends DatasetExportOptions { fromDataset?: string; toKVS?: string; - /** - * If true, includes all unique keys from all dataset items in the CSV export. - * If omitted or false, only keys from the first item are used. - */ - collectAllKeys?: boolean; } /** diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts index 4068cdc5f965..bce9bbd9e31f 100644 --- a/test/core/crawlers/basic_crawler.test.ts +++ b/test/core/crawlers/basic_crawler.test.ts @@ -1745,6 +1745,19 @@ describe('BasicCrawler', () => { expect((await crawler.getData()).items).toEqual(payload); }); + test('crawler.exportData works with `collectAllKeys`', async () => { + const crawler = new BasicCrawler(); + await crawler.pushData([{ foo: 'bar', baz: 123 }]); + await crawler.pushData([{ foo: 'baz', qux: 456 }]); + + await crawler.exportData(`${tmpDir}/result.csv`, 'csv', { collectAllKeys: true }); + + const csv = await readFile(`${tmpDir}/result.csv`); + expect(csv.toString()).toBe('foo,baz,qux\nbar,123,\nbaz,,456\n'); + + await rm(`${tmpDir}/result.csv`); + }); + test("Crawlers with different Configurations don't share Datasets", async () => { const crawlerA = new BasicCrawler({}, new Configuration({ persistStorage: false })); const crawlerB = new BasicCrawler({}, new Configuration({ persistStorage: false }));