Add schema validation for .thinktank result files (#126)

parseAndValidateResult() checks required fields before use. All commands
that load result files now validate and skip/warn on invalid data instead
of crashing. 24 new tests for valid/invalid schemas.

Generated by thinktank Opus (5 agents, 4 pass, Copeland: #1 at +3).

Closes #69

Co-authored-by: unknown <that-github-user@github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: 3 people

src/commands/apply.ts

@@ -5,6 +5,7 @@ import { promisify } from "node:util";
 import pc from "picocolors";
 import type { EnsembleResult } from "../types.js";
 import { cleanupBranches, getRepoRoot, removeWorktree } from "../utils/git.js";
+import { parseAndValidateResult } from "../utils/schema.js";
 
 const exec = promisify(execFile);
 
@@ -71,9 +72,15 @@ export async function apply(opts: ApplyOptions): Promise<void> {
   let result: EnsembleResult;
   try {
     const raw = await readFile(join(".thinktank", "latest.json"), "utf-8");
-    result = JSON.parse(raw);
-  } catch {
-    console.error("  No results found. Run `thinktank run` first.");
+    result = parseAndValidateResult(raw, "latest.json");
+  } catch (err) {
+    const msg = (err as Error).message;
+    if (msg.includes("Invalid result file")) {
+      console.error(`  ${msg}`);
+      console.error("  The result file may be corrupted. Try running `thinktank run` again.");
+    } else {
+      console.error("  No results found. Run `thinktank run` first.");
+    }
     process.exit(1);
   }
 

src/commands/evaluate.ts

@@ -3,6 +3,7 @@ import { join } from "node:path";
 import pc from "picocolors";
 import { analyzeConvergence, copelandRecommend, recommend } from "../scoring/convergence.js";
 import type { EnsembleResult } from "../types.js";
+import { parseAndValidateResult } from "../utils/schema.js";
 
 interface RunEvaluation {
   file: string;
@@ -83,9 +84,9 @@ export async function evaluate(): Promise<void> {
   for (const file of files) {
     try {
       const raw = await readFile(join(".thinktank", file), "utf-8");
-      runs.push(JSON.parse(raw) as EnsembleResult);
-    } catch {
-      // skip malformed
+      runs.push(parseAndValidateResult(raw, file));
+    } catch (err) {
+      console.warn(`  Skipping ${file}: ${(err as Error).message}`);
     }
   }
 

src/commands/list.ts

@@ -3,6 +3,7 @@ import { join } from "node:path";
 import pc from "picocolors";
 import type { EnsembleResult } from "../types.js";
 import { displayResults, padRight } from "../utils/display.js";
+import { parseAndValidateResult } from "../utils/schema.js";
 
 export interface RunSummary {
   runNumber: number;
@@ -48,9 +49,9 @@ export async function loadAllRuns(): Promise<{ filename: string; result: Ensembl
   for (const file of files) {
     try {
       const raw = await readFile(join(".thinktank", file), "utf-8");
-      runs.push({ filename: file, result: JSON.parse(raw) as EnsembleResult });
-    } catch {
-      // skip malformed files
+      runs.push({ filename: file, result: parseAndValidateResult(raw, file) });
+    } catch (err) {
+      console.warn(`  Skipping ${file}: ${(err as Error).message}`);
     }
   }
   return runs;

src/commands/stats.ts

@@ -2,6 +2,7 @@ import { readdir, readFile } from "node:fs/promises";
 import { join } from "node:path";
 import pc from "picocolors";
 import type { EnsembleResult } from "../types.js";
+import { parseAndValidateResult } from "../utils/schema.js";
 
 export interface StatsOptions {
   model?: string;
@@ -62,9 +63,9 @@ export async function stats(opts: StatsOptions = {}): Promise<void> {
   for (const file of files) {
     try {
       const raw = await readFile(join(".thinktank", file), "utf-8");
-      allResults.push(JSON.parse(raw) as EnsembleResult);
-    } catch {
-      // skip malformed files
+      allResults.push(parseAndValidateResult(raw, file));
+    } catch (err) {
+      console.warn(`  Skipping ${file}: ${(err as Error).message}`);
     }
   }
 

src/utils/schema.test.ts

@@ -0,0 +1,171 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { parseAndValidateResult, validateResult } from "./schema.js";
+
+function makeValidResult(): Record<string, unknown> {
+  return {
+    prompt: "fix the bug",
+    model: "sonnet",
+    timestamp: "2025-01-01T00:00:00Z",
+    scoring: "weighted",
+    agents: [
+      {
+        id: 1,
+        worktree: "/tmp/wt-1",
+        status: "success",
+        exitCode: 0,
+        duration: 30,
+        output: "done",
+        diff: "--- a/file\n+++ b/file",
+        filesChanged: ["file.ts"],
+        linesAdded: 1,
+        linesRemoved: 0,
+      },
+    ],
+    tests: [{ agentId: 1, passed: true, output: "ok", exitCode: 0 }],
+    convergence: [{ agents: [1], similarity: 1, filesChanged: ["file.ts"], description: "group" }],
+    recommended: 1,
+    scores: [{ agentId: 1, testPoints: 10, convergencePoints: 5, diffSizePoints: 3, total: 18 }],
+  };
+}
+
+describe("validateResult", () => {
+  it("returns null for a valid result", () => {
+    assert.equal(validateResult(makeValidResult()), null);
+  });
+
+  it("returns null when recommended is null", () => {
+    const result = makeValidResult();
+    result.recommended = null;
+    assert.equal(validateResult(result), null);
+  });
+
+  it("returns null with optional copelandScores", () => {
+    const result = makeValidResult();
+    result.copelandScores = [];
+    assert.equal(validateResult(result), null);
+  });
+
+  it("rejects null", () => {
+    assert.match(validateResult(null)!, /non-null object/);
+  });
+
+  it("rejects non-object", () => {
+    assert.match(validateResult("string")!, /non-null object/);
+  });
+
+  it("rejects missing prompt", () => {
+    const result = makeValidResult();
+    delete result.prompt;
+    assert.match(validateResult(result)!, /prompt/);
+  });
+
+  it("rejects non-string prompt", () => {
+    const result = makeValidResult();
+    result.prompt = 42;
+    assert.match(validateResult(result)!, /prompt/);
+  });
+
+  it("rejects missing model", () => {
+    const result = makeValidResult();
+    delete result.model;
+    assert.match(validateResult(result)!, /model/);
+  });
+
+  it("rejects missing timestamp", () => {
+    const result = makeValidResult();
+    delete result.timestamp;
+    assert.match(validateResult(result)!, /timestamp/);
+  });
+
+  it("rejects invalid scoring value", () => {
+    const result = makeValidResult();
+    result.scoring = "invalid";
+    assert.match(validateResult(result)!, /scoring/);
+  });
+
+  it("rejects missing scoring", () => {
+    const result = makeValidResult();
+    delete result.scoring;
+    assert.match(validateResult(result)!, /scoring/);
+  });
+
+  it("rejects non-array agents", () => {
+    const result = makeValidResult();
+    result.agents = "not-array";
+    assert.match(validateResult(result)!, /agents/);
+  });
+
+  it("rejects missing agents", () => {
+    const result = makeValidResult();
+    delete result.agents;
+    assert.match(validateResult(result)!, /agents/);
+  });
+
+  it("rejects non-array tests", () => {
+    const result = makeValidResult();
+    result.tests = {};
+    assert.match(validateResult(result)!, /tests/);
+  });
+
+  it("rejects non-array convergence", () => {
+    const result = makeValidResult();
+    result.convergence = "nope";
+    assert.match(validateResult(result)!, /convergence/);
+  });
+
+  it("rejects non-number non-null recommended", () => {
+    const result = makeValidResult();
+    result.recommended = "bad";
+    assert.match(validateResult(result)!, /recommended/);
+  });
+
+  it("rejects missing recommended", () => {
+    const result = makeValidResult();
+    delete result.recommended;
+    assert.match(validateResult(result)!, /recommended/);
+  });
+
+  it("rejects missing scores", () => {
+    const result = makeValidResult();
+    delete result.scores;
+    assert.match(validateResult(result)!, /scores/);
+  });
+
+  it("rejects non-array scores", () => {
+    const result = makeValidResult();
+    result.scores = "bad";
+    assert.match(validateResult(result)!, /scores/);
+  });
+
+  it("accepts empty arrays for agents, tests, convergence, scores", () => {
+    const result = makeValidResult();
+    result.agents = [];
+    result.tests = [];
+    result.convergence = [];
+    result.scores = [];
+    result.recommended = null;
+    assert.equal(validateResult(result), null);
+  });
+});
+
+describe("parseAndValidateResult", () => {
+  it("parses and returns a valid result", () => {
+    const json = JSON.stringify(makeValidResult());
+    const result = parseAndValidateResult(json, "test.json");
+    assert.equal(result.prompt, "fix the bug");
+    assert.equal(result.model, "sonnet");
+  });
+
+  it("throws on invalid JSON", () => {
+    assert.throws(() => parseAndValidateResult("{bad", "test.json"), /JSON/i);
+  });
+
+  it("throws on valid JSON but invalid schema", () => {
+    assert.throws(() => parseAndValidateResult('{"foo": 1}', "test.json"), /Invalid result file/);
+  });
+
+  it("includes filename in error message", () => {
+    assert.throws(() => parseAndValidateResult("{}", "run-2025.json"), /run-2025\.json/);
+  });
+});

src/utils/schema.ts

@@ -0,0 +1,55 @@
+import type { EnsembleResult } from "../types.js";
+
+/**
+ * Validate that a parsed JSON object has the required shape of an EnsembleResult.
+ * Returns null on success, or a descriptive error string on failure.
+ */
+export function validateResult(data: unknown): string | null {
+  if (data === null || typeof data !== "object") {
+    return "result must be a non-null object";
+  }
+
+  const obj = data as Record<string, unknown>;
+
+  if (typeof obj.prompt !== "string") {
+    return "missing or invalid field: prompt (expected string)";
+  }
+  if (typeof obj.model !== "string") {
+    return "missing or invalid field: model (expected string)";
+  }
+  if (typeof obj.timestamp !== "string") {
+    return "missing or invalid field: timestamp (expected string)";
+  }
+  if (obj.scoring !== "weighted" && obj.scoring !== "copeland") {
+    return 'missing or invalid field: scoring (expected "weighted" or "copeland")';
+  }
+  if (!Array.isArray(obj.agents)) {
+    return "missing or invalid field: agents (expected array)";
+  }
+  if (!Array.isArray(obj.tests)) {
+    return "missing or invalid field: tests (expected array)";
+  }
+  if (!Array.isArray(obj.convergence)) {
+    return "missing or invalid field: convergence (expected array)";
+  }
+  if (obj.recommended !== null && typeof obj.recommended !== "number") {
+    return "missing or invalid field: recommended (expected number or null)";
+  }
+  if (!Array.isArray(obj.scores)) {
+    return "missing or invalid field: scores (expected array)";
+  }
+
+  return null;
+}
+
+/**
+ * Parse JSON and validate as EnsembleResult. Returns the result or throws with a descriptive message.
+ */
+export function parseAndValidateResult(json: string, filename: string): EnsembleResult {
+  const data = JSON.parse(json);
+  const error = validateResult(data);
+  if (error) {
+    throw new Error(`Invalid result file ${filename}: ${error}`);
+  }
+  return data as EnsembleResult;
+}