feat(cli): migrate provider authoring surface (#1714)

* feat(cli): migrate provider authoring surface

* fix(cli): preserve rerun bundle target artifacts

Author: christso

README.md

@@ -1,6 +1,6 @@
 # AgentV
 
-Test AI targets on real repo tasks and measure what actually works.
+Test AI providers on real repo tasks and measure what actually works.
 
 ## Why?
 
@@ -10,18 +10,18 @@ Test AI targets on real repo tasks and measure what actually works.
 - **Version-controlled** — evals, judges, and results all live in Git
 - **Hybrid graders** — deterministic code checks + LLM-based subjective scoring
 - **CI/CD native** — exit codes, JSONL output, threshold flags for pipeline gating
-- **Any target** — run against agents, model providers, gateways, replay targets, CLI wrappers, transcript providers, and future app or service wrappers
+- **Any provider** — run against agents, model providers, gateways, replay providers, CLI wrappers, transcript providers, and future app or service wrappers
 
 ## Core Concepts
 
 - **Eval suite / tests** are the task corpus: the prompts, cases, datasets, and reusable field-local files you want to evaluate.
 - **Category** is derived from where the eval lives, such as folder path and file name. Use paths to organize the corpus instead of repeating category labels in every eval.
 - **Environment / fixtures / graders** are task-owned context: host or Docker setup, repos, setup scripts, files, fixtures, deterministic checks, and LLM grading prompts.
-- **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target` by configured target `id` or with an eval-local target object.
-- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and target/model names out of that tag.
+- **Provider** is the configured system under test: an agent, model provider, gateway, replay provider, CLI wrapper, transcript provider, or future app/service wrapper. Each provider entry uses `id` for the backend/spec and optional `label` for the stable AgentV selection and result identity.
+- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and provider/model names out of that tag.
 - **Evaluate options** configure eval run behavior such as `max_concurrency`, repeat policy, and budgets.
 - **Default test** configures inherited per-test defaults such as score `threshold`.
-- **Run** is one concrete execution of a tagged eval against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend.
+- **Run** is one concrete execution of a tagged eval against a resolved provider that writes portable artifacts for readers such as Dashboard, compare, and trend.
 
 ## Quick start
 
@@ -31,21 +31,21 @@ npm install -g agentv
 agentv init
 ```
 
-**2. Configure targets and graders** in `.agentv/config.yaml` — point to the system under test and the reusable grader. Provider settings live under `config`, and target `id` is the selection name used by evals and CLI flags:
+**2. Configure providers and graders** in `.agentv/providers.yaml` — point to the system under test and the reusable grader. Provider `id` names the backend/spec; `label` is the stable selection name used by evals and CLI flags:
 
 ```yaml
-targets:
-  - id: local-openai
-    provider: openai
+providers:
+  - id: openai
+    label: local-openai
     runtime: host
     config:
       api_format: chat
       base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
       api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
       model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}"
 
-  - id: local-openai-grader
-    provider: openai
+  - id: openai
+    label: local-openai-grader
     runtime: host
     config:
       api_format: chat
@@ -54,7 +54,7 @@ targets:
       model: "{{ env.LOCAL_OPENAI_PROXY_MODEL }}"
 
 defaults:
-  target: local-openai
+  provider: local-openai
   grader: local-openai-grader
 ```
 
@@ -82,7 +82,8 @@ options:
 description: Code generation quality
 tags:
   experiment: with-skills
-target: local-openai
+providers:
+  - local-openai
 evaluate_options:
   max_concurrency: 2
 
@@ -112,25 +113,25 @@ tests:
 Plain assertion strings are short-form rubric criteria: AgentV groups them into
 `llm-rubric` and writes grader detail to `grading.json.component_results` for
 the Dashboard. Use explicit `type: llm-rubric` when you need weights, required
-flags, `score_ranges`, a custom grader prompt, a grader target, or output
+flags, `score_ranges`, a custom grader prompt, a grader provider, or output
 transforms; use string `value` for free-form rubric checks. Executable graders
 use `type: script`.
 
-The target can be an eval-local object when this eval needs target settings of its own:
+The provider can be an eval-local object when this eval needs provider settings of its own:
 
 ```yaml
-description: Code generation quality with eval-local target settings
+description: Code generation quality with eval-local provider settings
 tags:
   experiment: with-skills
-target:
-  id: local-mini
-  provider: openai
-  runtime: host
-  config:
-    api_format: chat
-    base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
-    api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
-    model: gpt-5.4-mini
+providers:
+  - id: openai
+    label: local-mini
+    runtime: host
+    config:
+      api_format: chat
+      base_url: "{{ env.LOCAL_OPENAI_PROXY_BASE_URL }}"
+      api_key: "{{ env.LOCAL_OPENAI_PROXY_API_KEY }}"
+      model: gpt-5.4-mini
 evaluate_options:
   repeat:
     count: 2
@@ -148,7 +149,7 @@ tests:
       input: Write FizzBuzz in Python
 ```
 
-`target: local-openai` resolves the configured target id from `.agentv/config.yaml` and uses its provider, model, hooks, and provider settings. The object form above defines a full eval-local target and must include enough provider configuration to run. AgentV records the resolved target information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved target metadata.
+`providers: [local-openai]` resolves the configured provider label from `.agentv/providers.yaml` and uses its backend, model, hooks, and provider settings. The object form above defines a full eval-local provider and must include enough provider configuration to run. AgentV records the resolved provider information in run artifacts so results can be audited and replayed. The `tags.experiment` label stays `with-skills` because the condition is unchanged; the model/provider variation belongs to the resolved provider metadata.
 
 Use `default_test.threshold` for the inherited per-test pass cutoff. `default_test` can also point at a shared file:
 
@@ -179,7 +180,7 @@ agentv results compare .agentv/results/<baseline-run-id>/.internal/index.jsonl .
 
 ## Results
 
-Each run writes a portable bundle directly under `.agentv/results/<run_id>/`. In this example, `tags.experiment: with-skills` names the condition being measured and `target: local-openai` selects the system under test from `.agentv/config.yaml`; both are recorded as metadata, not path segments. The `.internal/index.jsonl` file is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and target configuration used for the run.
+Each run writes a portable bundle directly under `.agentv/results/<run_id>/`. In this example, `tags.experiment: with-skills` names the condition being measured and `providers: [local-openai]` selects the system under test from `.agentv/providers.yaml`; both are recorded as metadata, not path segments. The `.internal/index.jsonl` file is the portable row index used by scripts, CI, and `agentv results compare`; per-case sidecars include the resolved eval and provider configuration used for the run.
 
 ```bash
 agentv eval evals/my-eval.eval.yaml
@@ -192,7 +193,7 @@ Run bundle layout:
 .agentv/results/
 ├── 2026-06-30T08-30-00-000Z/     # <run_id> — one committed run bundle
 │   ├── summary.json              # run rollup: metadata, pass rate, counts, cost
-│   ├── fizzbuzz--a1b2c3d4/       # <result_dir> for one test/target row
+│   ├── fizzbuzz--a1b2c3d4/       # <result_dir> for one test/provider row
 │   │   ├── summary.json          # optional per-case rollup across samples
 │   │   ├── test/                 # generated test bundle: frozen inputs for reproducibility
 │   │   │   ├── EVAL.yaml         #   resolved eval spec

apps/cli/src/commands/create/commands.ts

@@ -42,7 +42,8 @@ export default defineAssertion(({ output }) => {
 
 const EVAL_TEMPLATES: Record<string, (name: string) => string> = {
   default: (name: string) => `description: ${name} evaluation suite
-target: default
+providers:
+  - default
 
 tests:
   - id: sample-test
@@ -54,7 +55,8 @@ tests:
         value: "well"
 `,
   rubric: (name: string) => `description: ${name} evaluation suite
-target: default
+providers:
+  - default
 
 tests:
   - id: sample-test
@@ -78,11 +80,11 @@ const PROVIDER_TEMPLATE = `#!/usr/bin/env bun
 /**
  * Custom provider scaffold.
  *
- * AgentV providers are configured via .agentv/targets.yaml using the CLI provider:
+ * AgentV providers are configured via .agentv/providers.yaml using the CLI provider:
  *
- *   targets:
- *     - name: my-target
- *       provider: cli
+ *   providers:
+ *     - id: cli
+ *       label: my-provider
  *       command: "bun run .agentv/providers/<name>.ts {PROMPT}"
  *
  * This script receives the prompt as a CLI argument and prints the response to stdout.
@@ -168,7 +170,7 @@ export const createProviderCommand = command({
     await writeFile(filePath, PROVIDER_TEMPLATE);
     console.log(`Created ${path.relative(process.cwd(), filePath)} (template: ${templateName})`);
     console.log(
-      `\nConfigure in .agentv/targets.yaml:\n  targets:\n    - name: ${name}\n      provider: cli\n      command: "bun run .agentv/providers/${name}.ts {PROMPT}"`,
+      `\nConfigure in .agentv/providers.yaml:\n  providers:\n    - id: cli\n      label: ${name}\n      command: "bun run .agentv/providers/${name}.ts {PROMPT}"`,
     );
   },
 });

apps/cli/src/commands/eval/commands/bundle.ts

@@ -67,9 +67,9 @@ function ensureTargetGraph(
         .map((entry) => entry.name)
         .sort()
         .join(', ');
-      const owner = requestedBy ? ` referenced by target '${requestedBy}'` : '';
+      const owner = requestedBy ? ` referenced by provider '${requestedBy}'` : '';
       throw new Error(
-        `Target '${name}'${owner} not found in ${targetsFilePath}. Available targets: ${available}`,
+        `Provider '${name}'${owner} not found in ${targetsFilePath}. Available providers: ${available}`,
       );
     }
     seen.add(name);
@@ -117,7 +117,7 @@ function definitionsWithEvalTargetSpec(
   if (!base) {
     const available = definitions.map((definition) => definition.name).join(', ');
     throw new Error(
-      `Target '${targetSpec.extends}' not found for eval-local target '${targetSpec.name}'. Available targets: ${available}`,
+      `Provider '${targetSpec.extends}' not found for eval-local provider '${targetSpec.name}'. Available providers: ${available}`,
     );
   }
   const effective = {
@@ -161,15 +161,37 @@ export const evalBundleCommand = command({
     target: multioption({
       type: array(string),
       long: 'target',
-      description: 'Target name to bundle (repeatable). Defaults to eval target(s) or default.',
+      description: '[Removed: use --provider <label>] Former provider selector',
     }),
     targets: option({
       type: optional(string),
       long: 'targets',
+      description: '[Removed: use --providers <path>] Former providers.yaml path',
+    }),
+    provider: multioption({
+      type: array(string),
+      long: 'provider',
+      description:
+        'Provider label to bundle (repeatable). Defaults to eval provider(s) or default.',
+    }),
+    providers: option({
+      type: optional(string),
+      long: 'providers',
       description: 'Path to providers.yaml (overrides discovery)',
     }),
   },
   handler: async (args) => {
+    if (args.target.length > 0) {
+      throw new Error(
+        `--target was removed from agentv eval bundle. Use --provider ${args.target[0]} instead.`,
+      );
+    }
+    if (args.targets !== undefined) {
+      throw new Error(
+        `--targets was removed from agentv eval bundle. Use --providers ${args.targets} instead.`,
+      );
+    }
+
     const cwd = process.cwd();
     const repoRoot = await findRepoRoot(cwd);
     const resolvedPaths = await resolveEvalPaths([args.evalPath], cwd);
@@ -192,10 +214,10 @@ export const evalBundleCommand = command({
     let targetNames: readonly string[];
     if (suite.inlineTarget) {
       definitions = [suite.inlineTarget];
-      targetNames = unique(args.target.length > 0 ? args.target : [suite.inlineTarget.name]);
+      targetNames = unique(args.provider.length > 0 ? args.provider : [suite.inlineTarget.name]);
     } else {
       const targetsFilePath = await discoverTargetsFile({
-        explicitPath: args.targets,
+        explicitPath: args.providers,
         testFilePath: evalFilePath,
         repoRoot,
         cwd,
@@ -209,8 +231,8 @@ export const evalBundleCommand = command({
       );
       const suiteTarget = await readTestSuiteTarget(evalFilePath);
       targetNames = unique(
-        args.target.length > 0
-          ? args.target
+        args.provider.length > 0
+          ? args.provider
           : (suite.targets ?? [suite.targetSpec?.name ?? suiteTarget ?? 'default']),
       );
       for (const targetName of targetNames) {

apps/cli/src/commands/eval/interactive.ts

@@ -3,7 +3,7 @@ import path from 'node:path';
 import { listTargetNames, readTargetDefinitions } from '@agentv/core';
 import { checkbox, confirm, number, search, select } from '@inquirer/prompts';
 
-import { TARGET_FILE_CANDIDATES, fileExists } from '../../utils/targets.js';
+import { PROVIDER_FILE_CANDIDATES, fileExists } from '../../utils/targets.js';
 import {
   type DiscoveredEvalFile,
   discoverEvalFiles,
@@ -98,7 +98,7 @@ async function promptMainMenu(
       choices.push({
         name: '⏯  Resume last run',
         value: 'resume',
-        description: `${dirLabel} (target: ${lastConfig.target})`,
+        description: `${dirLabel} (provider: ${lastConfig.target})`,
       });
     }
   }
@@ -108,7 +108,7 @@ async function promptMainMenu(
     choices.push({
       name: '🔄 Rerun last config',
       value: 'rerun',
-      description: `${evalCount} eval file(s), target: ${lastConfig.target}`,
+      description: `${evalCount} eval file(s), provider: ${lastConfig.target}`,
     });
   }
 
@@ -143,7 +143,7 @@ async function promptNewEvaluation(cwd: string): Promise<InteractiveConfig | und
     return undefined;
   }
 
-  // Step 3: Select target
+  // Step 3: Select provider
   const target = await promptTargetSelection(cwd, selectedFiles[0].path);
 
   // Step 4: Advanced options
@@ -217,12 +217,12 @@ async function promptTargetSelection(cwd: string, firstEvalPath: string): Promis
   }
 
   if (targetNames.length === 1) {
-    console.log(`${ANSI_DIM}Using target: ${targetNames[0]}${ANSI_RESET}`);
+    console.log(`${ANSI_DIM}Using provider: ${targetNames[0]}${ANSI_RESET}`);
     return targetNames[0];
   }
 
   return search<string>({
-    message: 'Select a target (type to search)',
+    message: 'Select a provider (type to search)',
     source: async (term) => {
       const filtered = term
         ? targetNames.filter((t) => t.toLowerCase().includes(term.toLowerCase()))
@@ -232,7 +232,7 @@ async function promptTargetSelection(cwd: string, firstEvalPath: string): Promis
         return {
           name: t,
           value: t,
-          description: def ? `provider: ${def.provider}` : undefined,
+          description: def ? `backend: ${def.provider}` : undefined,
         };
       });
     },
@@ -263,7 +263,7 @@ async function findTargetsFile(
   }
 
   for (const dir of dirsToSearch) {
-    for (const candidate of TARGET_FILE_CANDIDATES) {
+    for (const candidate of PROVIDER_FILE_CANDIDATES) {
       const fullPath = `${dir}/${candidate}`;
       if (await fileExists(fullPath)) {
         return fullPath;
@@ -314,7 +314,7 @@ async function promptReviewAndConfirm(config: InteractiveConfig, cwd: string): P
   console.log(`\n${ANSI_BOLD}Review Configuration${ANSI_RESET}`);
   console.log(`${ANSI_DIM}${'─'.repeat(40)}${ANSI_RESET}`);
   console.log(`${ANSI_GREEN}Eval files:${ANSI_RESET}\n${evalDisplay}`);
-  console.log(`${ANSI_GREEN}Target:${ANSI_RESET}    ${config.target}`);
+  console.log(`${ANSI_GREEN}Provider:${ANSI_RESET}  ${config.target}`);
   console.log(`${ANSI_GREEN}Workers:${ANSI_RESET}   ${config.workers}`);
   console.log(`${ANSI_GREEN}Cache:${ANSI_RESET}     ${config.cache ? 'yes' : 'no'}`);
   console.log(`${ANSI_DIM}${'─'.repeat(40)}${ANSI_RESET}`);