Skip to content

Commit 52e9d2d

Browse files
committed
fixes
1 parent 4fa8dd3 commit 52e9d2d

File tree

2 files changed

+24
-3
lines changed

2 files changed

+24
-3
lines changed

js/botasaurus-server-js/package-lock.json

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

js/botasaurus-server-js/src/ndjson.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import * as fs from 'fs';
22
import readline from 'readline'
33
import { _readJsonFiles, _has,_remove, _deleteItems, _hash } from 'botasaurus/cache';
4+
import { appendOutputIfNeeded } from 'botasaurus/output';
45
import { isNotNullish } from './null-utils'
56
import { stringify } from 'csv-stringify/sync'
67

@@ -244,4 +245,24 @@ export async function readNdJsonCallback(taskPath: string, onData: (item: any, i
244245
processedItems = limit
245246
}
246247
return processedItems
248+
}
249+
250+
251+
function fixNdjsonFilename(filename: string): string {
252+
filename = appendOutputIfNeeded(filename)
253+
254+
if (!filename.endsWith('.ndjson')) {
255+
filename = filename + '.ndjson'
256+
}
257+
return filename
258+
}
259+
260+
export function readNdjson<T = any>(
261+
filename: string,
262+
onData: (item: any, index: number) => T,
263+
limit?: number | null
264+
): Promise<number>{
265+
filename = fixNdjsonFilename(filename)
266+
267+
return readNdJsonCallback(filename, onData, limit)
247268
}

0 commit comments

Comments
 (0)