Use the new parser

Author: asgerf

extensions/ql-vscode/src/common/jsonl-reader.ts

@@ -1,6 +1,7 @@
 import { stat } from "fs/promises";
 import { createReadStream } from "fs-extra";
-import { createInterface } from "readline";
+
+const doubleLineBreakRegexp = /\n\r?\n/;
 
 /**
  * Read a file consisting of multiple JSON objects. Each object is separated from the previous one
@@ -13,64 +14,6 @@ export async function readJsonlFile<T>(
   path: string,
   handler: (value: T) => Promise<void>,
   logger?: { log: (message: string) => void },
-): Promise<void> {
-  function parseJsonFromCurrentLines() {
-    try {
-      return JSON.parse(currentLineSequence.join("\n")) as T;
-    } catch (e) {
-      void logger?.log(
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        `Error: Failed to parse at line ${lineCount} of ${path} as JSON: ${(e as any)?.message ?? "UNKNOWN REASON"}. Problematic line below:\n${JSON.stringify(currentLineSequence, null, 2)}`,
-      );
-      throw e;
-    }
-  }
-
-  function logProgress() {
-    void logger?.log(
-      `Processed ${lineCount} lines with ${parseCounts} parses...`,
-    );
-  }
-
-  void logger?.log(
-    `Parsing ${path} (${(await stat(path)).size / 1024 / 1024} MB)...`,
-  );
-  const fileStream = createReadStream(path, "utf8");
-  const rl = createInterface({
-    input: fileStream,
-    crlfDelay: Infinity,
-  });
-
-  let lineCount = 0;
-  let parseCounts = 0;
-  let currentLineSequence: string[] = [];
-  for await (const line of rl) {
-    if (line === "") {
-      // as mentioned above: a double newline sequence indicates the end of the current JSON object, so we parse it and pass it to the handler
-      await handler(parseJsonFromCurrentLines());
-      parseCounts++;
-      currentLineSequence = [];
-    } else {
-      currentLineSequence.push(line);
-    }
-    lineCount++;
-    if (lineCount % 1000000 === 0) {
-      logProgress();
-    }
-  }
-  // in case the file is not newline-terminated, we need to handle the last JSON object
-  if (currentLineSequence.length > 0) {
-    await handler(parseJsonFromCurrentLines());
-  }
-  logProgress();
-}
-
-const doubleLineBreakRegexp = /\n\r?\n/;
-
-export async function readJsonlFile2<T>(
-  path: string,
-  handler: (value: T) => Promise<void>,
-  logger?: { log: (message: string) => void },
 ): Promise<void> {
   void logger?.log(
     `Parsing ${path} (${(await stat(path)).size / 1024 / 1024} MB)...`,

extensions/ql-vscode/test/benchmarks/jsonl-reader.bench.ts

@@ -1,9 +1,7 @@
 import { readFile } from "fs-extra";
-import { readJsonlFile, readJsonlFile2 } from "../../src/common/jsonl-reader";
+import { readJsonlFile } from "../../src/common/jsonl-reader";
 import { performance } from "perf_hooks";
 import { join } from "path";
-import { createReadStream } from "fs";
-import { createInterface } from "readline";
 
 /** An "obviously correct" implementation to test against. */
 async function readJsonlReferenceImpl<T>(
@@ -21,18 +19,6 @@ async function readJsonlReferenceImpl<T>(
   }
 }
 
-async function justReadline(
-  path: string,
-  handler: (value: unknown) => Promise<void>,
-) {
-  const stream = createReadStream(path, "utf8");
-  const rl = createInterface(stream);
-
-  for await (const line of rl) {
-    await handler(line);
-  }
-}
-
 type ParserFn = (
   text: string,
   callback: (v: unknown) => Promise<void>,
@@ -41,8 +27,6 @@ type ParserFn = (
 const parsers: Record<string, ParserFn> = {
   readJsonlReferenceImpl,
   readJsonlFile,
-  readJsonlFile2,
-  justReadline,
 };
 
 async function main() {