fix(grader): select grader providers from shared pool (#1718)

* fix(grader): select grader providers from shared pool

* fix(grader): validate dependencies before grader preflight

Author: christso

apps/cli/src/commands/eval/run-eval.ts

@@ -303,7 +303,7 @@ interface NormalizedOptions {
   /** Removed: the run directory always uses index.jsonl */
   readonly outputFormat?: string;
   readonly graderTarget?: string;
-  /** Config-level fallback grader target name, from `.agentv/config.yaml`'s `defaults.grader`. */
+  /** Config-level fallback grader provider name, from `.agentv/config.yaml`'s `defaults.grader`. */
   readonly defaultGraderTarget?: string;
   readonly model?: string;
   readonly outputMessages: number | 'all';
@@ -1405,7 +1405,12 @@ async function prepareFileMetadata(params: {
       : suite.tests;
   const testIds = testCases.map((value) => value.id);
   const suiteTargetSpec = suite.targetSpec;
+  const suiteDefaults = suite.defaults;
   const suiteTargets = suiteTargetSpec ? [suiteTargetSpec.name] : suite.targets;
+  const fileOptions =
+    suiteDefaults?.grader && !effectiveOptions.graderTarget
+      ? { ...effectiveOptions, defaultGraderTarget: suiteDefaults.grader }
+      : effectiveOptions;
   const defaultBudgetUsd =
     effectiveOptions.cliBudgetUsd === undefined
       ? (effectiveOptions.budgetUsd ?? suite.budgetUsd)
@@ -1414,11 +1419,11 @@ async function prepareFileMetadata(params: {
 
   if (testCases.length === 0) {
     return {
-      options: effectiveOptions,
+      options: fileOptions,
       testIds,
       testCases,
       selections: [],
-      trialsConfig: effectiveOptions.experimentTrialsConfig,
+      trialsConfig: fileOptions.experimentTrialsConfig,
       suiteTargets,
       yamlCache: suite.cacheConfig?.enabled,
       yamlCachePath: suite.cacheConfig?.cachePath,
@@ -1432,7 +1437,7 @@ async function prepareFileMetadata(params: {
 
   let selections: { selection: TargetSelection; inlineTargetLabel: string }[];
 
-  if (effectiveOptions.transcript) {
+  if (fileOptions.transcript) {
     // --transcript mode: bypass target resolution entirely.
     // Create a synthetic TargetSelection for the transcript provider.
     const transcriptSelection: TargetSelection = {
@@ -1444,15 +1449,15 @@ async function prepareFileMetadata(params: {
       },
       targetName: 'transcript',
       targetSource: 'cli',
-      targetsFilePath: effectiveOptions.transcript,
+      targetsFilePath: fileOptions.transcript,
     };
     selections = [
       {
         selection: transcriptSelection,
-        inlineTargetLabel: `transcript (${path.basename(effectiveOptions.transcript)})`,
+        inlineTargetLabel: `transcript (${path.basename(fileOptions.transcript)})`,
       },
     ];
-  } else if (suite.inlineTarget && effectiveOptions.cliTargets.length === 0) {
+  } else if (suite.inlineTarget && fileOptions.cliTargets.length === 0) {
     const targetDefinition = suite.inlineTarget;
     const resolvedTarget = resolveProviderDefinition(targetDefinition, process.env, testFilePath, {
       emitDeprecationWarnings: false,
@@ -1469,7 +1474,7 @@ async function prepareFileMetadata(params: {
         inlineTargetLabel: resolveTargetLabel(targetDefinition.name, resolvedTarget.name),
       },
     ];
-  } else if (suite.providerFactory && effectiveOptions.cliTargets.length === 0) {
+  } else if (suite.providerFactory && fileOptions.cliTargets.length === 0) {
     const taskTarget: ResolvedProviderBackend = {
       kind: 'mock',
       name: 'custom-task',
@@ -1490,12 +1495,12 @@ async function prepareFileMetadata(params: {
     ];
   } else {
     // Determine provider labels: CLI --provider flags override YAML
-    const cliTargets = effectiveOptions.cliTargets;
-    const experimentTargets = effectiveOptions.experimentTargets ?? [];
+    const cliTargets = fileOptions.cliTargets;
+    const experimentTargets = fileOptions.experimentTargets ?? [];
     const suiteTargetSpec = suite.targetSpec;
     const suiteTargets = suiteTargetSpec ? [suiteTargetSpec.name] : suite.targets;
     const suiteTargetRefs = suite.targetRefs;
-    const experimentTargetRefs = effectiveOptions.experimentTargetRefs;
+    const experimentTargetRefs = fileOptions.experimentTargetRefs;
 
     // Resolve which target names to use (precedence: CLI/experiment > suite YAML targets > default)
     let targetNames: readonly string[];
@@ -1511,6 +1516,9 @@ async function prepareFileMetadata(params: {
     } else if (suiteTargets && suiteTargets.length > 0) {
       targetNames = suiteTargets;
       targetRefs = suiteTargetRefs;
+    } else if (suiteDefaults?.provider) {
+      targetNames = [suiteDefaults.provider];
+      targetRefs = undefined;
     } else {
       targetNames = [];
       targetRefs = undefined;
@@ -1526,12 +1534,12 @@ async function prepareFileMetadata(params: {
         providerDefinitions,
         providerDefinitionsSource,
         requireExplicitProviderCatalog: true,
-        allowLegacyTargetFiles: effectiveOptions.allowLegacyTargetFiles,
+        allowLegacyTargetFiles: fileOptions.allowLegacyTargetFiles,
         env: process.env,
         targetNames,
         targetRefs,
         targetSource,
-        modelOverride: effectiveOptions.targetModelOverride,
+        modelOverride: fileOptions.targetModelOverride,
       });
 
       selections = multiSelections.map((sel) => ({
@@ -1549,18 +1557,18 @@ async function prepareFileMetadata(params: {
         providerDefinitions,
         providerDefinitionsSource,
         requireExplicitProviderCatalog: true,
-        allowLegacyTargetFiles: effectiveOptions.allowLegacyTargetFiles,
+        allowLegacyTargetFiles: fileOptions.allowLegacyTargetFiles,
         cliTargetName:
           targetSource === 'cli'
             ? targetNames.length === 1
               ? targetNames[0]
-              : effectiveOptions.target
-            : effectiveOptions.target,
+              : fileOptions.target
+            : fileOptions.target,
         fileTargetName:
           targetSource === 'test-file' && targetNames.length === 1 ? targetNames[0] : undefined,
         fileTargetSpec:
           targetSource === 'test-file' && targetNames.length === 1 ? suiteTargetSpec : undefined,
-        modelOverride: effectiveOptions.targetModelOverride,
+        modelOverride: fileOptions.targetModelOverride,
         env: process.env,
       });
 
@@ -1587,11 +1595,11 @@ async function prepareFileMetadata(params: {
   }
 
   return {
-    options: effectiveOptions,
+    options: fileOptions,
     testIds,
     testCases,
     selections,
-    trialsConfig: effectiveOptions.experimentTrialsConfig,
+    trialsConfig: fileOptions.experimentTrialsConfig,
     suiteTargets,
     yamlCache: suite.cacheConfig?.enabled,
     yamlCachePath: suite.cacheConfig?.cachePath,
@@ -1951,9 +1959,9 @@ export async function runEvalCommand(
     process.env.AGENTV_EXPERIMENT = normalizedExperiment;
   }
 
-  // Validate --grader-target / --model combinations
+  // Validate --grader-provider / --model combinations
   if (options.graderTarget === 'agentv' && !options.model) {
-    throw new Error('--grader-target agentv requires --model (e.g., --model openai:gpt-5-mini)');
+    throw new Error('--grader-provider agentv requires --model (e.g., --model openai:gpt-5-mini)');
   }
 
   if (options.removedOut) {

apps/cli/src/commands/eval/task-bundle.ts

@@ -636,12 +636,81 @@ function bundledEvalFileName(evalFilePath: string): string {
 
 function uniqueTargetDefinitions(
   selections: readonly TaskBundleTargetSelection[],
+  tests: readonly EvalTest[] = [],
 ): readonly ProviderDefinition[] {
   const selected: ProviderDefinition[] = [];
   const seen = new Set<string>();
 
+  function addDefinitions(
+    names: readonly string[],
+    definitions: readonly ProviderDefinition[],
+  ): void {
+    for (const name of names) {
+      for (const definition of selectTargetDefinitions(name, definitions)) {
+        if (seen.has(definition.name)) {
+          continue;
+        }
+        seen.add(definition.name);
+        selected.push(definition);
+      }
+    }
+  }
+
+  const graderTargetNames = collectGraderTargetNames(tests);
   for (const selection of selections) {
-    for (const definition of selectTargetDefinitions(selection.targetName, selection.definitions)) {
+    addDefinitions([selection.targetName, ...graderTargetNames], selection.definitions);
+  }
+
+  return selected;
+}
+
+function collectGraderTargetNames(tests: readonly EvalTest[]): readonly string[] {
+  const names: string[] = [];
+  const seen = new Set<string>();
+
+  function collect(value: unknown, key?: string): void {
+    if (key === 'target' && typeof value === 'string') {
+      const target = value.trim();
+      if (target.length > 0 && !target.includes('${{') && !seen.has(target)) {
+        seen.add(target);
+        names.push(target);
+      }
+      return;
+    }
+
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        collect(item);
+      }
+      return;
+    }
+
+    if (isRecord(value)) {
+      for (const [childKey, childValue] of Object.entries(value)) {
+        collect(childValue, childKey);
+      }
+    }
+  }
+
+  for (const test of tests) {
+    for (const grader of test.source?.graderDefinitions ?? []) {
+      collect(grader.definition);
+    }
+  }
+
+  return names;
+}
+
+function selectTaskBundleTargetDefinitions(
+  targetName: string,
+  definitions: readonly ProviderDefinition[],
+  tests: readonly EvalTest[],
+): readonly ProviderDefinition[] {
+  const selected: ProviderDefinition[] = [];
+  const seen = new Set<string>();
+
+  for (const name of [targetName, ...collectGraderTargetNames(tests)]) {
+    for (const definition of selectTargetDefinitions(name, definitions)) {
       if (seen.has(definition.name)) {
         continue;
       }
@@ -1133,7 +1202,11 @@ export async function materializeTaskBundle(
     return undefined;
   }
 
-  const targetDefinitions = selectTargetDefinitions(options.targetName, options.targetDefinitions);
+  const targetDefinitions = selectTaskBundleTargetDefinitions(
+    options.targetName,
+    options.targetDefinitions,
+    [options.test],
+  );
   if (targetDefinitions.length === 0) {
     return undefined;
   }
@@ -1228,7 +1301,7 @@ export async function materializeEvalBundle(
   });
   await writeYamlFile(
     providersPath,
-    serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections)),
+    serializeTargetDefinitions(uniqueTargetDefinitions(options.targetSelections, options.tests)),
   );
   await mkdir(path.dirname(configPath), { recursive: true });
   await writeYamlFile(configPath, { providers: `file://../${BUNDLE_PROVIDERS_FILENAME}` });

apps/cli/src/commands/grade/index.ts

@@ -657,7 +657,7 @@ export const gradeCommand = command({
     model: option({
       type: optional(string),
       long: 'model',
-      description: 'Override model for the grader target (e.g., "openai:gpt-5-mini")',
+      description: 'Override model for the grader provider (e.g., "openai:gpt-5-mini")',
     }),
     threshold: option({
       type: optional(number),

apps/cli/test/commands/eval/task-bundle.test.ts

@@ -55,6 +55,7 @@ describe('materializeTaskBundle', () => {
               name: 'quality',
               type: 'llm-grader',
               prompt: 'file://graders/prompt.md',
+              target: 'judge',
               command: ['bun', scriptPath, '--token', 'literal-secret'],
             },
           },
@@ -97,6 +98,11 @@ describe('materializeTaskBundle', () => {
           provider: 'mock',
           api_key: 'literal-secret',
         },
+        {
+          name: 'judge',
+          provider: 'mock',
+          api_key: '${{ JUDGE_API_KEY }}',
+        },
       ],
       outputDir: path.join(tempDir, 'out'),
       cwd: tempDir,
@@ -132,8 +138,11 @@ describe('materializeTaskBundle', () => {
       'file://files/fixtures/input.txt',
     );
     expect(assertion.prompt).toBe('file://graders/graders/prompt.md');
+    expect(assertion.target).toBe('judge');
     expect(assertion.command).toEqual(['bun', 'graders/graders/check.ts', '--token', '[redacted]']);
     expect(taskProviders).toContain('api_key: ${{ MOCK_API_KEY }}');
+    expect(taskProviders).toContain('label: judge');
+    expect(taskProviders).toContain('api_key: ${{ JUDGE_API_KEY }}');
     expect(taskProviders).toContain('api_key: "[redacted]"');
     expect(taskEval).not.toContain('literal-secret');
     expect(taskProviders).not.toContain('literal-secret');

apps/web/src/content/docs/docs/next/graders/llm-graders.mdx

@@ -162,10 +162,27 @@ tests:
 
 ## Per-Grader Provider
 
-By default, an `llm-rubric` uses `defaults.grader` from the resolved config graph.
-Override it per assertion when you need multiple grader models in one run:
+By default, an `llm-rubric` uses `tests[].options.provider`, then
+`default_test.options.provider`, then `defaults.grader` from the resolved config
+graph. Each value selects from the same `providers` pool used for candidate
+providers; AgentV does not infer a grader from the first candidate provider.
+Override the provider per assertion when you need multiple grader models in one
+run:
 
 ```yaml
+default_test:
+  options:
+    provider: grader_gpt_5_mini
+
+tests:
+  - id: strict-case
+    options:
+      provider: grader_claude_haiku
+    assert:
+      - name: semantic_quality
+        type: llm-rubric
+        value: The answer is correct and concise.
+
 assert:
   - name: grader-gpt
     type: llm-rubric

apps/web/src/content/docs/docs/next/targets/configuration.mdx

@@ -63,8 +63,11 @@ settings belong under `config`. Process-backed coding-agent providers use
 
 A grader is not a separate kind of entity — it is a provider selected for a
 grading role, either through `defaults.grader` (shown above) or an
-assertion-level `provider` override. There is no separate `graders:` list;
-authoring one is a hard error telling you to move each entry into `providers`.
+`options.provider` / assertion-level `provider` override. `default_test.options.provider`
+sets a shared grader fallback for tests, `tests[].options.provider` overrides it
+for one test, and an assertion-level `provider` wins for that assertion. There
+is no separate `graders:` list; authoring one is a hard error telling you to
+move each entry into `providers`.
 
 ## Runtime Modes
 
@@ -306,8 +309,9 @@ providers:
     reasoning_effort: high
 ```
 
-Use `defaults.grader` for the project default grader. A specific evaluator can
-still choose its own grader provider when the evaluator supports that override.
+Use `defaults.grader` for the project default grader. `default_test.options.provider`
+and `tests[].options.provider` can choose a grader provider for LLM-backed
+assertions before an assertion-level `provider` override takes final precedence.
 
 ### Environment And Lifecycle Extensions