Merge pull request #3820 from github/esbena/parallel-log-parsing

Hackathon: Parallel evaluator summary log parsing

Author: esbena

extensions/ql-vscode/src/compare-performance/compare-performance-view.ts

@@ -1,3 +1,4 @@
+import { statSync } from "fs";
 import path from "path";
 import { ViewColumn } from "vscode";
 import type { CodeQLCliServer } from "../codeql-cli/cli";
@@ -12,6 +13,7 @@ import { showAndLogExceptionWithTelemetry } from "../common/logging";
 import { extLogger } from "../common/logging/vscode";
 import type { WebviewPanelConfig } from "../common/vscode/abstract-webview";
 import { AbstractWebview } from "../common/vscode/abstract-webview";
+import { withProgress } from "../common/vscode/progress";
 import { telemetryListener } from "../common/vscode/telemetry";
 import type { ResultsView } from "../local-queries";
 import { scanLog } from "../log-insights/log-scanner";
@@ -54,12 +56,22 @@ export class ComparePerformanceView extends AbstractWebview<
 
     await this.waitForPanelLoaded();
 
-    // TODO: try processing in (async) parallel once readJsonl is streaming
-    const fromPerf = await scanLog(
-      fromJsonLog,
-      new PerformanceOverviewScanner(),
-    );
-    const toPerf = await scanLog(toJsonLog, new PerformanceOverviewScanner());
+    function scanLogWithProgress(log: string, logDescription: string) {
+      const bytes = statSync(log).size;
+      return withProgress(
+        async (progress) =>
+          scanLog(log, new PerformanceOverviewScanner(), progress),
+
+        {
+          title: `Scanning evaluator log ${logDescription} (${(bytes / 1024 / 1024).toFixed(1)} MB)`,
+        },
+      );
+    }
+
+    const [fromPerf, toPerf] = await Promise.all([
+      scanLogWithProgress(fromJsonLog, "1/2"),
+      scanLogWithProgress(toJsonLog, "2/2"),
+    ]);
 
     // TODO: filter out irrelevant common predicates before transfer?
 

extensions/ql-vscode/src/log-insights/log-scanner.ts

@@ -1,6 +1,7 @@
-import type { SummaryEvent } from "./log-summary";
-import { readJsonlFile } from "../common/jsonl-reader";
 import type { Disposable } from "../common/disposable-object";
+import { readJsonlFile } from "../common/jsonl-reader";
+import type { ProgressCallback } from "../common/vscode/progress";
+import type { SummaryEvent } from "./log-summary";
 
 /**
  * Callback interface used to report diagnostics from a log scanner.
@@ -114,15 +115,22 @@ export class EvaluationLogScannerSet {
 }
 
 /**
- * Scan the evaluator summary log using the given scanner. For conveience, returns the scanner.
+ * Scan the evaluator summary log using the given scanner. For convenience, returns the scanner.
  *
  * @param jsonSummaryLocation The file path of the JSON summary log.
  * @param scanner The scanner to process events from the log
  */
 export async function scanLog<T extends EvaluationLogScanner>(
   jsonSummaryLocation: string,
   scanner: T,
+  progress?: ProgressCallback,
 ): Promise<T> {
+  progress?.({
+    // XXX all scans have step 1 - the backing progress tracker allows increments instead of steps - but for now we are happy with a tiny UI that says what is happening
+    message: `Scanning ...`,
+    step: 1,
+    maxStep: 2,
+  });
   await readJsonlFile<SummaryEvent>(jsonSummaryLocation, async (obj) => {
     scanner.onEvent(obj);
   });