feat: add composable target runtime config contract (#1625)

* feat(config): add composable AgentV config graph

* feat(results): add target execution envelopes

* feat(providers): isolate coding agent sdk targets

* feat(providers): add explicit codex runtime targets

* feat(runtime): add sandbox target runner

* feat(providers): add pi rpc runtime boundary

* feat(providers): align coding agent target contracts

* docs: document target runtime config graph

* test: align runtime provider fixtures with config contract

* test: use explicit codex sdk provider in CLI fixture

Author: christso

README.md

@@ -17,9 +17,9 @@ Test AI targets on real repo tasks and measure what actually works.
 - **Eval suite / imports / tests** are the task corpus: the prompts, cases, datasets, and imported benchmarks you want to evaluate.
 - **Category** is derived from where the eval lives, such as folder path and file name. Use paths to organize the corpus instead of repeating category labels in every eval.
 - **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts.
-- **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target`, either by label from `targets.yaml` or with an eval-local target object.
+- **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target` by configured target `id` or with an eval-local target object.
 - **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and target/model names out of that tag.
-- **Evaluate options** configure runner-level behavior such as repeat policy, optional timeouts, and `max_concurrency` under `evaluate_options`.
+- **Execution** configures runner-level behavior such as general suite concurrency with `execution.max_concurrency`; `evaluate_options` holds eval behaviors such as repeat policy and budgets.
 - **Default test** configures inherited per-test defaults such as score `threshold`.
 - **Run** is one concrete execution of a tagged eval against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend.
 
@@ -31,16 +31,31 @@ npm install -g agentv
 agentv init
 ```
 
-**2. Configure targets** in `.agentv/targets.yaml` — point to the system under test, such as an agent, provider, gateway, replay source, or CLI wrapper. Provider-specific budgets belong here:
+**2. Configure targets and graders** in `.agentv/config.yaml` — point to the system under test and the reusable grader. Provider settings live under `config`, and target `id` is the selection name used by evals and CLI flags:
 
 ```yaml
 targets:
-  - label: local-openai
+  - id: local-openai
     provider: openai
-    api_format: chat
-    base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
-    api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
-    model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
+    runtime: host
+    config:
+      api_format: chat
+      base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
+      api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
+      model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
+
+graders:
+  - id: local-openai-grader
+    provider: openai
+    config:
+      api_format: chat
+      base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
+      api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
+      model: ${{ LOCAL_OPENAI_PROXY_MODEL }}
+
+defaults:
+  target: local-openai
+  grader: local-openai-grader
 ```
 
 **3. Create shared test defaults** in `evals/default-test.yaml`. This is a promptfoo-style partial test config that AgentV applies to each test:
@@ -68,7 +83,7 @@ description: Code generation quality
 tags:
   experiment: with-skills
 target: local-openai
-evaluate_options:
+execution:
   max_concurrency: 1
 
 default_test: file://./default-test.yaml
@@ -104,8 +119,14 @@ description: Code generation quality with eval-local target settings
 tags:
   experiment: with-skills
 target:
-  extends: local-openai
-  model: gpt-5.4-mini
+  id: local-mini
+  provider: openai
+  runtime: host
+  config:
+    api_format: chat
+    base_url: ${{ LOCAL_OPENAI_PROXY_BASE_URL }}
+    api_key: ${{ LOCAL_OPENAI_PROXY_API_KEY }}
+    model: gpt-5.4-mini
 evaluate_options:
   repeat:
     count: 2
@@ -119,7 +140,7 @@ tests:
     input: Write FizzBuzz in Python
 ```
 
-`target: local-openai` resolves the target label from `.agentv/targets.yaml` or `targets.yaml` and uses its default provider, model, hooks, and provider settings. The object form above starts from `local-openai`, then applies the eval-local fields for this eval. If `extends` is omitted, the object defines the full target inline and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved target metadata.
+`target: local-openai` resolves the configured target id from `.agentv/config.yaml` and uses its provider, model, hooks, and provider settings. The object form above defines a full eval-local target and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved target metadata.
 
 Use `default_test.threshold` for the inherited per-test pass cutoff. `default_test` can also point at a shared file, matching promptfoo's external defaults pattern:
 
@@ -150,7 +171,7 @@ agentv results compare .agentv/results/<baseline-run-id>/index.jsonl .agentv/res
 
 ## Results
 
-Each run writes a portable bundle directly under `.agentv/results/<run_id>/`. In this example, `tags.experiment: with-skills` names the condition being measured and `target: local-openai` selects the system under test from `targets.yaml`; both are recorded as metadata, not path segments. The root `index.jsonl` manifest is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and target configuration used for the run.
+Each run writes a portable bundle directly under `.agentv/results/<run_id>/`. In this example, `tags.experiment: with-skills` names the condition being measured and `target: local-openai` selects the system under test from `.agentv/config.yaml`; both are recorded as metadata, not path segments. The root `index.jsonl` manifest is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and target configuration used for the run.
 
 ```bash
 agentv eval evals/my-eval.eval.yaml

apps/cli/package.json

@@ -28,12 +28,9 @@
     "test:watch": "bun test --watch"
   },
   "dependencies": {
-    "@anthropic-ai/claude-agent-sdk": "^0.2.49",
-    "@github/copilot-sdk": "^1.0.3",
     "@hono/node-server": "^1.19.11",
     "@inquirer/prompts": "^8.2.1",
     "@earendil-works/pi-ai": "^0.74.0",
-    "@openai/codex-sdk": "^0.136.0",
     "cmd-ts": "^0.14.3",
     "dotenv": "^16.4.5",
     "fast-glob": "^3.3.3",
@@ -44,9 +41,21 @@
     "yaml": "^2.8.3"
   },
   "peerDependencies": {
+    "@anthropic-ai/claude-agent-sdk": "^0.2.88",
+    "@github/copilot-sdk": "^1.0.3",
+    "@openai/codex-sdk": "^0.136.0",
     "@earendil-works/pi-coding-agent": "^0.74.0"
   },
   "peerDependenciesMeta": {
+    "@anthropic-ai/claude-agent-sdk": {
+      "optional": true
+    },
+    "@github/copilot-sdk": {
+      "optional": true
+    },
+    "@openai/codex-sdk": {
+      "optional": true
+    },
     "@earendil-works/pi-coding-agent": {
       "optional": true
     }

apps/cli/src/commands/eval/run-eval.ts

@@ -636,7 +636,7 @@ function normalizeOptions(
   yamlExecution?: ExecutionDefaults,
 ): NormalizedOptions {
   const cliWorkers = normalizeOptionalNumber(rawOptions.workers);
-  const configWorkers = config?.execution?.workers ?? yamlExecution?.workers;
+  const configWorkers = config?.execution?.maxConcurrency ?? yamlExecution?.max_concurrency;
   const workers = cliWorkers ?? configWorkers ?? 0;
 
   const cliOutputDir = normalizeString(rawOptions.output);

apps/cli/src/commands/eval/statistics.ts

@@ -22,6 +22,7 @@ export interface EvaluationSummary {
   readonly passedCount: number;
   readonly byFailureStage: Readonly<Record<string, number>>;
   readonly byFailureReason: Readonly<Record<string, number>>;
+  readonly byTargetErrorKind: Readonly<Record<string, number>>;
 }
 
 const HISTOGRAM_BREAKPOINTS = [0, 0.2, 0.4, 0.6, 0.8, 1];
@@ -112,6 +113,7 @@ export function calculateEvaluationSummary(
       passedCount: 0,
       byFailureStage: {},
       byFailureReason: {},
+      byTargetErrorKind: {},
     };
   }
 
@@ -150,6 +152,7 @@ export function calculateEvaluationSummary(
   // Aggregate by failure stage and reason (execution errors only)
   const byFailureStage: Record<string, number> = {};
   const byFailureReason: Record<string, number> = {};
+  const byTargetErrorKind: Record<string, number> = {};
   for (const result of executionErrors) {
     if (result.failureStage) {
       byFailureStage[result.failureStage] = (byFailureStage[result.failureStage] ?? 0) + 1;
@@ -158,6 +161,10 @@ export function calculateEvaluationSummary(
       byFailureReason[result.failureReasonCode] =
         (byFailureReason[result.failureReasonCode] ?? 0) + 1;
     }
+    const targetErrorKind = result.targetExecution?.errorKind;
+    if (targetErrorKind) {
+      byTargetErrorKind[targetErrorKind] = (byTargetErrorKind[targetErrorKind] ?? 0) + 1;
+    }
   }
 
   return {
@@ -177,6 +184,7 @@ export function calculateEvaluationSummary(
     passedCount,
     byFailureStage,
     byFailureReason,
+    byTargetErrorKind,
   };
 }
 
@@ -297,6 +305,14 @@ export function formatEvaluationSummary(
     }
   }
 
+  const targetErrorEntries = Object.entries(summary.byTargetErrorKind);
+  if (targetErrorEntries.length > 0) {
+    lines.push('\nTarget runtime errors by kind:');
+    for (const [kind, count] of targetErrorEntries) {
+      lines.push(`  ${kind}: ${count}`);
+    }
+  }
+
   return lines.join('\n');
 }
 

apps/cli/test/eval.integration.test.ts

@@ -48,7 +48,7 @@ targets:
   - label: cli-target
     provider: mock
   - label: codex-target
-    provider: codex
+    provider: codex-sdk
     model: gpt-5-default
 `;
   await writeFile(targetsPath, targetsContent, 'utf8');

apps/dashboard/src/components/ResultTable.tsx

@@ -615,6 +615,26 @@ function primaryTrialArtifactPath(trial: EvalCaseTrial): string | null {
   );
 }
 
+function formatTargetError(
+  error: string | undefined,
+  kind: string | undefined,
+): string | undefined {
+  if (!error) return kind ? `Target error: ${kind}` : undefined;
+  return kind ? `[target:${kind}] ${error}` : error;
+}
+
+function targetErrorKind(value: {
+  targetExecution?: { error_kind?: string; errorKind?: string };
+  target_execution?: { error_kind?: string; errorKind?: string };
+}): string | undefined {
+  return (
+    value.target_execution?.error_kind ??
+    value.target_execution?.errorKind ??
+    value.targetExecution?.error_kind ??
+    value.targetExecution?.errorKind
+  );
+}
+
 function TrialResultCell({
   column,
   row,
@@ -656,7 +676,12 @@ function TrialResultCell({
     case 'review':
       return <span className="text-gray-700">-</span>;
     case 'error':
-      return <TruncatedMuted value={trial.error} tone="text-red-300" />;
+      return (
+        <TruncatedMuted
+          value={formatTargetError(trial.error, targetErrorKind(trial))}
+          tone="text-red-300"
+        />
+      );
     default:
       return <span className="text-gray-700">-</span>;
   }
@@ -840,7 +865,12 @@ function ResultCell({
     case 'cost_tokens':
       return <CostTokenCell row={row} repeatGroup={repeatGroup} />;
     case 'error':
-      return <TruncatedMuted value={row.result.error} tone="text-red-300" />;
+      return (
+        <TruncatedMuted
+          value={formatTargetError(row.result.error, targetErrorKind(row.result))}
+          tone="text-red-300"
+        />
+      );
     default:
       return <span className="text-gray-600">-</span>;
   }

apps/dashboard/src/lib/types.ts

@@ -70,6 +70,41 @@ export interface TokenUsage {
   cached?: number;
 }
 
+export interface TargetExecutionEnvelope {
+  schema_version?: 'agentv.target_execution.v1';
+  status?: 'success' | 'error' | string;
+  target_id?: string;
+  provider_id?: string;
+  provider_kind?: string;
+  runtime_mode?: string;
+  runtimeMode?: string;
+  error_kind?: string;
+  errorKind?: string;
+  message?: string;
+  exit_code?: number | null;
+  exitCode?: number | null;
+  signal?: string | null;
+  duration_ms?: number;
+  durationMs?: number;
+  command?: {
+    argv?: string[];
+    command_line?: string;
+    cwd?: string;
+  };
+  artifacts?: {
+    target_execution_path?: string;
+    stdout_path?: string;
+    stderr_path?: string;
+    transcript_path?: string;
+    transcript_raw_path?: string;
+    summary_path?: string;
+    metrics_path?: string;
+    file_changes_path?: string;
+    output_path?: string;
+    answer_path?: string;
+  };
+}
+
 export interface ScoreEntry {
   name?: string;
   type?: string;
@@ -100,6 +135,11 @@ export interface EvalCaseTrial {
   scores?: ScoreEntry[];
   assertions?: AssertionEntry[];
   error?: string;
+  targetExecution?: TargetExecutionEnvelope;
+  target_execution?: TargetExecutionEnvelope;
+  target_execution_path?: string;
+  stdout_path?: string;
+  stderr_path?: string;
   execution_status?: string;
   cost_usd?: number;
   total_tokens?: number;
@@ -228,6 +268,11 @@ export interface EvalResult {
   score: number;
   executionStatus?: string;
   error?: string;
+  targetExecution?: TargetExecutionEnvelope;
+  target_execution?: TargetExecutionEnvelope;
+  target_execution_path?: string;
+  stdout_path?: string;
+  stderr_path?: string;
   costUsd?: number;
   durationMs?: number;
   tokenUsage?: TokenUsage;