From c981b89d66edf9f3ea3b67e9559edb7031da75d0 Mon Sep 17 00:00:00 2001
From: Paul Gschwendtner <paulgschwendtner@gmail.com>
Date: Tue, 16 Sep 2025 12:21:56 +0000
Subject: [PATCH] refactor: ensure errors with dynamic logging are not
 swallowed

Ensures errors printed with dynamic logging are not swallowed. This is
easily confusing as the report summary is currently just printed with
e.g. zero completed app generations.
---
 package.json                               |  1 +
 runner/eval-cli.ts                         |  7 +++++-
 runner/orchestration/generate.ts           |  4 ++--
 runner/progress/dynamic-progress-logger.ts | 26 +++++++++++++++++++++-
 4 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/package.json b/package.json
index d6ef31a..f91eab4 100644
--- a/package.json
+++ b/package.json
@@ -5,6 +5,7 @@
     "build-runner": "tsc",
     "release-build": "tsx release-build.ts",
     "web-codegen-scorer": "tsx ./runner/bin/cli.ts",
+    "wcs": "pnpm -s web-codegen-scorer",
     "eval": "pnpm web-codegen-scorer eval",
     "init": "pnpm web-codegen-scorer init",
     "report": "cd report-app && CODEGEN_REPORTS_DIR=../.web-codegen-scorer/reports pnpm start",
diff --git a/runner/eval-cli.ts b/runner/eval-cli.ts
index 6e8ae00..700b93b 100644
--- a/runner/eval-cli.ts
+++ b/runner/eval-cli.ts
@@ -119,7 +119,12 @@ function builder(argv: Argv): Argv<Options> {
     })
     .option('logging', {
       type: 'string',
-      default: 'dynamic' as const,
+      default:
+        process.env['CI'] === '1'
+          ? ('text-only' as const)
+          : ('dynamic' as const),
+      defaultDescription: '`dynamic` (or `text-only` when `CI=1`)',
+      requiresArg: true,
       choices: ['text-only', 'dynamic'] as const,
       description: 'Type of logging to use during the evaluation process',
     })
diff --git a/runner/orchestration/generate.ts b/runner/orchestration/generate.ts
index 86a81bb..9f6a802 100644
--- a/runner/orchestration/generate.ts
+++ b/runner/orchestration/generate.ts
@@ -175,9 +175,9 @@ export async function generateCodeAndAssess(options: {
               stack: e instanceof Error ? e.stack : undefined,
             });
 
-            let details = `  Error: ${e}`;
+            let details = `Error: ${e}`;
             if (e instanceof Error && e.stack) {
-              details += e.stack;
+              details += `\nStack: ${e.stack}`;
             }
 
             progress.log(
diff --git a/runner/progress/dynamic-progress-logger.ts b/runner/progress/dynamic-progress-logger.ts
index e3a97b5..da950e5 100644
--- a/runner/progress/dynamic-progress-logger.ts
+++ b/runner/progress/dynamic-progress-logger.ts
@@ -6,6 +6,7 @@ import {
   ProgressType,
   progressTypeToIcon,
 } from './progress-logger.js';
+import { redX } from '../reporting/format.js';
 
 const PREFIX_WIDTH = 20;
 
@@ -17,6 +18,11 @@ export class DynamicProgressLogger implements ProgressLogger {
   private spinnerFrames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
   private currentSpinnerFrame = 0;
   private spinnerInterval: ReturnType<typeof setInterval> | undefined;
+  private errors: {
+    prompt: RootPromptDefinition;
+    message: string;
+    details?: string;
+  }[] = [];
 
   initialize(total: number): void {
     this.finalize();
@@ -72,9 +78,22 @@ export class DynamicProgressLogger implements ProgressLogger {
     this.wrapper?.stop();
     this.pendingBars.clear();
     this.wrapper = this.totalBar = this.spinnerInterval = undefined;
+
+    for (const error of this.errors) {
+      let message = `${redX()} [${error.prompt.name}] ${error.message}`;
+      if (error.details) {
+        message += `\n  ${error.details}`;
+      }
+      console.error(message);
+    }
   }
 
-  log(prompt: RootPromptDefinition, type: ProgressType, message: string): void {
+  log(
+    prompt: RootPromptDefinition,
+    type: ProgressType,
+    message: string,
+    details?: string
+  ): void {
     if (!this.wrapper || !this.totalBar) {
       return;
     }
@@ -92,6 +111,11 @@ export class DynamicProgressLogger implements ProgressLogger {
       return;
     }
 
+    // Capture errors for static printing once the dynamic progress is hidden.
+    if (type === 'error') {
+      this.errors.push({ prompt, message, details });
+    }
+
     // Pad/trim the name so they're all the same length.
     const name = this.trimString(
       prompt.name.padEnd(PREFIX_WIDTH, ' '),