diff --git a/package.json b/package.json index d6ef31a..f91eab4 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "build-runner": "tsc", "release-build": "tsx release-build.ts", "web-codegen-scorer": "tsx ./runner/bin/cli.ts", + "wcs": "pnpm -s web-codegen-scorer", "eval": "pnpm web-codegen-scorer eval", "init": "pnpm web-codegen-scorer init", "report": "cd report-app && CODEGEN_REPORTS_DIR=../.web-codegen-scorer/reports pnpm start", diff --git a/runner/eval-cli.ts b/runner/eval-cli.ts index 6e8ae00..700b93b 100644 --- a/runner/eval-cli.ts +++ b/runner/eval-cli.ts @@ -119,7 +119,12 @@ function builder(argv: Argv): Argv { }) .option('logging', { type: 'string', - default: 'dynamic' as const, + default: + process.env['CI'] === '1' + ? ('text-only' as const) + : ('dynamic' as const), + defaultDescription: '`dynamic` (or `text-only` when `CI=1`)', + requiresArg: true, choices: ['text-only', 'dynamic'] as const, description: 'Type of logging to use during the evaluation process', }) diff --git a/runner/orchestration/generate.ts b/runner/orchestration/generate.ts index 86a81bb..9f6a802 100644 --- a/runner/orchestration/generate.ts +++ b/runner/orchestration/generate.ts @@ -175,9 +175,9 @@ export async function generateCodeAndAssess(options: { stack: e instanceof Error ? e.stack : undefined, }); - let details = ` Error: ${e}`; + let details = `Error: ${e}`; if (e instanceof Error && e.stack) { - details += e.stack; + details += `\nStack: ${e.stack}`; } progress.log( diff --git a/runner/progress/dynamic-progress-logger.ts b/runner/progress/dynamic-progress-logger.ts index e3a97b5..da950e5 100644 --- a/runner/progress/dynamic-progress-logger.ts +++ b/runner/progress/dynamic-progress-logger.ts @@ -6,6 +6,7 @@ import { ProgressType, progressTypeToIcon, } from './progress-logger.js'; +import { redX } from '../reporting/format.js'; const PREFIX_WIDTH = 20; @@ -17,6 +18,11 @@ export class DynamicProgressLogger implements ProgressLogger { private spinnerFrames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; private currentSpinnerFrame = 0; private spinnerInterval: ReturnType | undefined; + private errors: { + prompt: RootPromptDefinition; + message: string; + details?: string; + }[] = []; initialize(total: number): void { this.finalize(); @@ -72,9 +78,22 @@ export class DynamicProgressLogger implements ProgressLogger { this.wrapper?.stop(); this.pendingBars.clear(); this.wrapper = this.totalBar = this.spinnerInterval = undefined; + + for (const error of this.errors) { + let message = `${redX()} [${error.prompt.name}] ${error.message}`; + if (error.details) { + message += `\n ${error.details}`; + } + console.error(message); + } } - log(prompt: RootPromptDefinition, type: ProgressType, message: string): void { + log( + prompt: RootPromptDefinition, + type: ProgressType, + message: string, + details?: string + ): void { if (!this.wrapper || !this.totalBar) { return; } @@ -92,6 +111,11 @@ export class DynamicProgressLogger implements ProgressLogger { return; } + // Capture errors for static printing once the dynamic progress is hidden. + if (type === 'error') { + this.errors.push({ prompt, message, details }); + } + // Pad/trim the name so they're all the same length. const name = this.trimString( prompt.name.padEnd(PREFIX_WIDTH, ' '),