scratch-f32-solo-rescore.ts

/**
 * F32 solo re-scoring of the capability (instrument-validation) runs.
 *
 * F32 established (probe v1/v2, 2026-08-30) that the hypothesis classifier
 * is batch-context-sensitive: items flip across the ext-gen boundary
 * depending on what else is in the batch, stably (3/3 in batch vs 5/5 solo
 * on the sonnet famprobe final modal). The experimental-corpus numbers
 * survived the audit (the only screened experimental item, p31b, stays
 * in-world solo), but the capability-table numbers for the licensed probes
 * were produced by batched classification and are therefore suspect in
 * BOTH directions.
 *
 * This script re-scores those runs with every hypothesis classified SOLO
 * under eval-v4 (majority of 3; non-unanimous items reported), recomputes
 * both ladders, and writes a `.solo-v4.json` sidecar next to each artifact
 * (a distinct suffix — the pipeline's `.judged*.json` files are untouched).
 * These runs are detector measurements (instrument-validation corpus role),
 * so re-scoring them under a disclosed, cleaner procedure is measurement of
 * the measurement — the corpus provenance gate never lets them near an
 * experimental statistic either way.
 *
 * ~450 judge calls (≈140 unique hypotheses × 3). Usage:
 *
 *   npx tsx scratch-f32-solo-rescore.ts
 */
import { createJudgeClient } from "./src/evaluator/judgeClient.js";
import { classifyHypothesesLLM, EVAL_V4_VERSION } from "./src/evaluator/llmClassifier.js";
import {
  computeLevels,
  INTERVENTION_ONLY_CLASSES,
  stripEvents,
  type PrivilegedEvent,
} from "./src/evaluator/study3.js";
import { readFileSync, writeFileSync, existsSync } from "node:fs";

// Same env loading as the study3 CLI: pick up ANTHROPIC_API_KEY from .env.
try {
  process.loadEnvFile();
} catch {
  // no .env file — rely on shell environment
}

const FILES = [
  "runs/s3-f30-postfix/w0-seed9195.json",
  "runs/s3-famprobe-sonnet/w0-seed9196.json",
  "runs/s3-famprobe-sonar/w0-seed9197.json",
  "runs/s3-famprobe-cerebras/w0-seed9198.json",
  "runs/s3-r38-poscontrol-v4/wd_exact-seed9192.json",
  "runs/s3-r38-poscontrol-v4/w0-seed9192.json",
  "runs/s3-famprobe-gemini/w0-seed9199.json",
].filter((f) => existsSync(f));

interface Hyp {
  label: string;
  rationale: string;
  probability: number;
  evidenceFor: number[];
  evidenceAgainst: number[];
}
interface RawArtifact {
  config: { name: string; seed: number };
  study3?: { opaqueIds?: boolean; opaqueIdHalfBits?: number | null; instrumentValidation?: boolean } | null;
  startedAt?: string;
  agents: {
    agentId: string;
    beliefTimeline: { day: number; state: { hypotheses: Hyp[]; residual: number } }[];
  }[];
  events: PrivilegedEvent[];
}

const apiKey = process.env["ANTHROPIC_API_KEY"];
if (!apiKey) throw new Error("ANTHROPIC_API_KEY is not set");
const judge = createJudgeClient({ apiKey });

for (const file of FILES) {
  const out = file.replace(/\.json$/, ".solo-v4.json");
  if (existsSync(out)) {
    console.log(`SKIP ${file} — ${out} already written (delete it to redo)`);
    continue;
  }
  const artifact = JSON.parse(readFileSync(file, "utf8")) as RawArtifact;
  if (!artifact.study3?.instrumentValidation) {
    // Guard: this script exists to re-score DETECTOR MEASUREMENTS. Refusing
    // experimental artifacts keeps solo-v4 numbers from ever being produced
    // for the corpus outside a pre-specified procedure change.
    console.log(`SKIP ${file} — not an instrument-validation artifact`);
    continue;
  }

  // Unique hypotheses in rescore's encounter order (same dedupe key).
  const seen = new Map<string, { label: string; rationale: string }>();
  for (const ag of artifact.agents) {
    for (const snap of ag.beliefTimeline) {
      for (const h of snap.state.hypotheses) {
        const key = `${h.label}${h.rationale}`;
        if (!seen.has(key)) seen.set(key, { label: h.label, rationale: h.rationale });
      }
    }
  }
  const items = [...seen.entries()];
  console.log(`\n=== ${file} — ${items.length} unique hypotheses, solo ×3 each`);

  const classMap = new Map<string, string>();
  const splits: string[] = [];
  for (const [key, item] of items) {
    const votes: string[] = [];
    for (let i = 0; i < 3; i++) {
      const [c] = await classifyHypothesesLLM([item], judge.complete, EVAL_V4_VERSION);
      votes.push(c ?? "other");
    }
    const tally = new Map<string, number>();
    for (const v of votes) tally.set(v, (tally.get(v) ?? 0) + 1);
    const majority = [...tally.entries()].sort((a, b) => b[1] - a[1])[0]![0];
    classMap.set(key, majority);
    if (tally.size > 1) {
      splits.push(`${votes.join("/")} · ${item.label.slice(0, 80)}`);
    }
  }
  for (const s of splits) console.log(`  NON-UNANIMOUS: ${s}`);

  const classify = (label: string, rationale: string) =>
    classMap.get(`${label}${rationale}`) ?? "other";
  const blind = {
    config: artifact.config,
    study3: artifact.study3,
    ...(artifact.startedAt !== undefined ? { startedAt: artifact.startedAt } : {}),
    agents: artifact.agents,
    events: stripEvents(artifact.events),
  };
  const levels = computeLevels(blind, classify);
  const levelsInterventionOnly = computeLevels(blind, classify, INTERVENTION_ONLY_CLASSES);

  // Older sidecars (pre-§16.2) carry no levelsInterventionOnly — the 9192
  // acceptance runs were judged before the secondary ladder existed. Compare
  // against whatever the stored sidecar actually has.
  const stored = JSON.parse(readFileSync(file.replace(/\.json$/, ".judged-eval-v4.json"), "utf8")) as {
    levels: typeof levels;
    levelsInterventionOnly?: typeof levelsInterventionOnly;
  };
  const fmt = (t: { tauSuspicion: number | null; tauCommitment: number | null; tauGrounded: number | null; finalLevel: number } | undefined) =>
    t ? `τ[${t.tauSuspicion},${t.tauCommitment},${t.tauGrounded}] L${t.finalLevel}` : "(not in stored sidecar — pre-§16.2)";
  for (let i = 0; i < levels.length; i++) {
    const n = levels[i]!;
    const ni = levelsInterventionOnly[i]!;
    console.log(
      `  ${n.agentId} pooled: stored ${fmt(stored.levels[i])}` +
        ` → solo τ[${n.tauSuspicion},${n.tauCommitment},${n.tauGrounded}] L${n.finalLevel} modal=${n.finalModalExtGenClass}`,
    );
    console.log(
      `  ${n.agentId} ivn-only: stored ${fmt(stored.levelsInterventionOnly?.[i])}` +
        ` → solo τ[${ni.tauSuspicion},${ni.tauCommitment},${ni.tauGrounded}] L${ni.finalLevel}`,
    );
  }

  writeFileSync(
    out,
    JSON.stringify(
      {
        source: file,
        procedure: "F32 solo re-scoring: every unique hypothesis classified alone, eval-v4, majority of 3",
        evalVersion: EVAL_V4_VERSION,
        judgeModel: judge.model,
        corpusRole: "instrument-validation",
        nonUnanimous: splits,
        classifications: items.map(([key, it]) => ({ label: it.label, class: classMap.get(key) })),
        levels,
        levelsInterventionOnly,
      },
      null,
      2,
    ),
  );
  console.log(`  → ${out}`);
}
console.log(
  "\nDone. Solo sidecars written beside artifacts (.solo-v4.json); pipeline sidecars untouched.",
);