feat: implement streaming JSON write in safeWriteJson

Refactor safeWriteJson to use stream-json for memory-efficient JSON serialization:
- Replace in-memory string creation with streaming pipeline
- Add Disassembler and Stringer from stream-json library
- Extract streaming logic to a dedicated helper function
- Add proper-lockfile and stream-json dependencies

This implementation reduces memory usage when writing large JSON objects.

Signed-off-by: Eric Wheeler <[email protected]>

Author: Eric Wheeler

src/package.json

@@ -410,6 +410,7 @@
 		"pdf-parse": "^1.1.1",
 		"pkce-challenge": "^5.0.0",
 		"pretty-bytes": "^7.0.0",
+		"proper-lockfile": "^4.1.2",
 		"ps-tree": "^1.2.0",
 		"puppeteer-chromium-resolver": "^24.0.0",
 		"puppeteer-core": "^23.4.0",
@@ -419,6 +420,7 @@
 		"serialize-error": "^12.0.0",
 		"simple-git": "^3.27.0",
 		"sound-play": "^1.1.0",
+		"stream-json": "^1.8.0",
 		"string-similarity": "^4.0.4",
 		"strip-ansi": "^7.1.0",
 		"strip-bom": "^5.0.0",
@@ -446,7 +448,9 @@
 		"@types/node": "20.x",
 		"@types/node-cache": "^4.1.3",
 		"@types/node-ipc": "^9.2.3",
+		"@types/proper-lockfile": "^4.1.4",
 		"@types/ps-tree": "^1.1.6",
+		"@types/stream-json": "^1.7.8",
 		"@types/string-similarity": "^4.0.2",
 		"@types/tmp": "^0.2.6",
 		"@types/turndown": "^5.0.5",

src/utils/safeWriteJson.ts

@@ -1,6 +1,9 @@
 import * as fs from "fs/promises"
+import * as fsSync from "fs"
 import * as path from "path"
 import * as lockfile from "proper-lockfile"
+import Disassembler from "stream-json/Disassembler"
+import Stringer from "stream-json/Stringer"
 
 /**
  * Safely writes JSON data to a file.
@@ -13,12 +16,8 @@ import * as lockfile from "proper-lockfile"
  * @param {any} data - The data to serialize to JSON and write.
  * @returns {Promise<void>}
  */
-async function safeWriteJson(
-	filePath: string,
-	data: any,
-	replacer?: (key: string, value: any) => any,
-	space: string | number = 2,
-): Promise<void> {
+
+async function safeWriteJson(filePath: string, data: any): Promise<void> {
 	const absoluteFilePath = path.resolve(filePath)
 	const lockPath = `${absoluteFilePath}.lock`
 	let releaseLock = async () => {} // Initialized to a no-op
@@ -59,8 +58,8 @@ async function safeWriteJson(
 			path.dirname(absoluteFilePath),
 			`.${path.basename(absoluteFilePath)}.new_${Date.now()}_${Math.random().toString(36).substring(2)}.tmp`,
 		)
-		const jsonData = JSON.stringify(data, replacer, space)
-		await fs.writeFile(actualTempNewFilePath, jsonData, "utf8")
+
+		await _streamDataToFile(actualTempNewFilePath, data)
 
 		// Step 2: Check if the target file exists. If so, rename it to a backup path.
 		try {
@@ -159,4 +158,58 @@ async function safeWriteJson(
 	}
 }
 
+/**
+ * Helper function to stream JSON data to a file.
+ * @param targetPath The path to write the stream to.
+ * @param data The data to stream.
+ * @returns Promise<void>
+ */
+async function _streamDataToFile(targetPath: string, data: any): Promise<void> {
+	// Stream data to avoid high memory usage for large JSON objects.
+	const fileWriteStream = fsSync.createWriteStream(targetPath, { encoding: "utf8" })
+	const disassembler = Disassembler.disassembler()
+	// Output will be compact JSON as standard Stringer is used.
+	const stringer = Stringer.stringer()
+
+	return new Promise<void>((resolve, reject) => {
+		let errorOccurred = false
+		const handleError = (_streamName: string) => (err: Error) => {
+			if (!errorOccurred) {
+				errorOccurred = true
+				if (!fileWriteStream.destroyed) {
+					fileWriteStream.destroy(err)
+				}
+				reject(err)
+			}
+		}
+
+		disassembler.on("error", handleError("Disassembler"))
+		stringer.on("error", handleError("Stringer"))
+		fileWriteStream.on("error", (err: Error) => {
+			if (!errorOccurred) {
+				errorOccurred = true
+				reject(err)
+			}
+		})
+
+		fileWriteStream.on("finish", () => {
+			if (!errorOccurred) {
+				resolve()
+			}
+		})
+
+		disassembler.pipe(stringer).pipe(fileWriteStream)
+
+		// stream-json's Disassembler might error if `data` is undefined.
+		// JSON.stringify(undefined) would produce the string "undefined" if it's the root value.
+		// Writing 'null' is a safer JSON representation for a root undefined value.
+		if (data === undefined) {
+			disassembler.write(null)
+		} else {
+			disassembler.write(data)
+		}
+		disassembler.end()
+	})
+}
+
 export { safeWriteJson }