feat: parallel log scanning

Author: esbena

extensions/ql-vscode/src/compare-performance/compare-performance-view.ts

@@ -12,6 +12,7 @@ import { showAndLogExceptionWithTelemetry } from "../common/logging";
 import { extLogger } from "../common/logging/vscode";
 import type { WebviewPanelConfig } from "../common/vscode/abstract-webview";
 import { AbstractWebview } from "../common/vscode/abstract-webview";
+import { withProgress } from "../common/vscode/progress";
 import { telemetryListener } from "../common/vscode/telemetry";
 import type { ResultsView } from "../local-queries";
 import { scanLog } from "../log-insights/log-scanner";
@@ -58,12 +59,24 @@ export class ComparePerformanceView extends AbstractWebview<
 
     await this.waitForPanelLoaded();
 
-    // TODO: try processing in (async) parallel once readJsonl is streaming
-    const fromPerf = await scanLog(
-      fromJsonLog,
-      new PerformanceOverviewScanner(),
+    const { fromPerf, toPerf } = await withProgress(
+      async (progress) => {
+        const fromPerf = scanLog(
+          fromJsonLog,
+          new PerformanceOverviewScanner(),
+          progress,
+        );
+        const toPerf = scanLog(
+          toJsonLog,
+          new PerformanceOverviewScanner(),
+          progress,
+        );
+        return { fromPerf: await fromPerf, toPerf: await toPerf };
+      },
+      {
+        title: "Parsing performance logs",
+      },
     );
-    const toPerf = await scanLog(toJsonLog, new PerformanceOverviewScanner());
 
     // TODO: filter out irrelevant common predicates before transfer?
 

extensions/ql-vscode/src/log-insights/log-scanner.ts

@@ -1,6 +1,7 @@
-import type { SummaryEvent } from "./log-summary";
-import { readJsonlFile } from "../common/jsonl-reader";
 import type { Disposable } from "../common/disposable-object";
+import { readJsonlFile } from "../common/jsonl-reader";
+import type { ProgressCallback } from "../common/vscode/progress";
+import type { SummaryEvent } from "./log-summary";
 
 /**
  * Callback interface used to report diagnostics from a log scanner.
@@ -122,7 +123,14 @@ export class EvaluationLogScannerSet {
 export async function scanLog<T extends EvaluationLogScanner>(
   jsonSummaryLocation: string,
   scanner: T,
+  progress: ProgressCallback,
 ): Promise<T> {
+  progress({
+    message: `Scanning log ${jsonSummaryLocation}`,
+    // XXX all scans have step 1 - the backing progress tracker allows increments instead of steps - but for now we are happy with a tiny UI that says what is happening
+    step: 1,
+    maxStep: 2,
+  });
   await readJsonlFile<SummaryEvent>(jsonSummaryLocation, async (obj) => {
     scanner.onEvent(obj);
   });