fix(eval): migrate remaining assert fixtures

Author: christso

README.md

@@ -18,33 +18,10 @@ Test AI targets on real repo tasks and measure what actually works.
 - **Category** is derived from where the eval lives, such as folder path and file name. Use paths to organize the corpus instead of repeating category labels in every eval.
 - **Workspace / fixtures / graders** are task-owned context: repos, setup scripts, files, fixtures, isolation, deterministic checks, and LLM grading prompts.
 - **Target** is the system under test: an agent, provider, gateway, replay target, CLI wrapper, transcript provider, or future app/service wrapper. Each eval selects one `target`, either by label from `targets.yaml` or with an eval-local target object.
-- **Experiment** is the run/result grouping label being measured over that corpus, such as `with-skills` or `without-skills`. Keep suite/category and target/model names out of this label.
+- **Tags** are run/result grouping labels. `tags.experiment` is the default experiment namespace, such as `with-skills` or `without-skills`; keep suite/category and target/model names out of that tag.
 - **Evaluate options** configure runner-level behavior such as repeat policy, optional timeouts, and `max_concurrency` under `evaluate_options`.
 - **Default test** configures inherited per-test defaults such as score `threshold`.
-- **Run** is one concrete execution of an experiment against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend.
-
-```mermaid
-flowchart LR
-  corpus["Eval suite / imports / tests<br/>task corpus"]
-  category["Category<br/>path-derived grouping"]
-  context["Workspace / fixtures / graders<br/>task-owned context"]
-  experiment["Experiment<br/>named run condition"]
-  target["Target<br/>system under test"]
-  controls["Run controls<br/>execution + gates"]
-  run["Run<br/>concrete execution"]
-  artifacts["Run artifacts<br/>summary.json + index.jsonl + sidecars"]
-  readers["Dashboard / compare / trend<br/>derived readers"]
-
-  corpus --> category
-  corpus --> run
-  context --> run
-  category --> run
-  experiment --> run
-  target --> run
-  controls --> run
-  run --> artifacts
-  artifacts --> readers
-```
+- **Run** is one concrete execution of a tagged eval against a resolved target that writes portable artifacts for readers such as Dashboard, compare, and trend.
 
 ## Quick start
 

apps/cli/src/commands/eval/task-bundle.ts

@@ -494,7 +494,23 @@ function parseSourceTestCase(test: EvalTest): Record<string, unknown> {
 }
 
 function withoutLegacyAssertionKeys(testCase: Record<string, unknown>): Record<string, unknown> {
-  return Object.fromEntries(Object.entries(testCase).filter(([key]) => key !== 'assert'));
+  return Object.fromEntries(
+    Object.entries(testCase).filter(([key]) => key !== 'assert' && key !== 'assertions'),
+  );
+}
+
+function serializeGraderDefinition(
+  definition: Record<string, unknown>,
+  rewrites: ReadonlyMap<string, string>,
+): unknown {
+  const serialized = rewritePathsDeep(toSnakeCaseDeep(definition), rewrites);
+  if (!isRecord(serialized)) {
+    return serialized;
+  }
+  const { name, metric, ...rest } = serialized;
+  return typeof name === 'string' && metric === undefined
+    ? { metric: name, ...rest }
+    : serialized;
 }
 
 function buildEvalCase(
@@ -506,8 +522,8 @@ function buildEvalCase(
   if (graderDefinitions.length > 0) {
     return {
       ...withoutLegacyAssertionKeys(testCase),
-      assertions: graderDefinitions.map((grader) =>
-        rewritePathsDeep(toSnakeCaseDeep(grader.definition), rewrites),
+      assert: graderDefinitions.map((grader) =>
+        serializeGraderDefinition(grader.definition, rewrites),
       ),
     };
   }

apps/cli/test/commands/eval/artifact-writer.test.ts

@@ -2404,7 +2404,7 @@ describe('writeArtifactsFromResults', () => {
 
     const parsedEval = parseYamlValue(taskEval) as Record<string, unknown>;
     const [testCase] = parsedEval.tests as Record<string, unknown>[];
-    const [assertion] = testCase.assertions as Record<string, unknown>[];
+    const [assertion] = testCase.assert as Record<string, unknown>[];
     expect(parsedEval.target).toBe('gpt-4o');
     expect(testCase.input).toBe('file://files/src/input.txt');
     expect(assertion.prompt).toBe('file://graders/src/grader.md');

apps/cli/test/commands/eval/bundle.test.ts

@@ -75,7 +75,7 @@ describe('agentv eval bundle', () => {
           value: ../data/input.txt
         - type: text
           value: Answer using the fixture.
-  assertions:
+  assert:
     - type: contains
       value: Mock
 `,
@@ -184,7 +184,7 @@ tests: ../data/cases.yaml
 tests:
   - id: inline-case
     input: hello
-    assertions:
+    assert:
       - type: contains
         value: inline
 `,
@@ -226,7 +226,7 @@ tests:
 tests:
   - id: missing-template
     input: hello
-    assertions:
+    assert:
       - type: contains
         value: Mock
 `,

apps/cli/test/commands/eval/pipeline/fixtures/builtin-test.eval.yaml

@@ -3,12 +3,12 @@ tests:
   - id: test-01
     input: hello world
     criteria: Response echoes the input
-    assertions:
-      - name: has_hello
+    assert:
+      - metric: has_hello
         type: contains
         value: hello
-      - name: matches_pattern
+      - metric: matches_pattern
         type: regex
         value: "h[aeiou]llo"
-      - name: is_valid_json
+      - metric: is_valid_json
         type: is-json

apps/cli/test/commands/eval/pipeline/fixtures/input-test.eval.yaml

@@ -3,12 +3,12 @@ tests:
   - id: test-01
     input: hello world
     criteria: Response echoes the input
-    assertions:
-      - name: contains_hello
+    assert:
+      - metric: contains_hello
         type: script
         command: node grader-score-1.js
         weight: 1.0
-      - name: relevance
+      - metric: relevance
         type: llm-grader
         prompt: Did the response echo the input?
         weight: 2.0

apps/cli/test/commands/eval/pipeline/fixtures/no-name.eval.yaml

@@ -2,7 +2,7 @@ tests:
   - id: test-01
     input: hello world
     criteria: Response echoes the input
-    assertions:
-      - name: contains_hello
+    assert:
+      - metric: contains_hello
         type: contains
         value: hello

apps/cli/test/commands/eval/task-bundle.test.ts

@@ -121,7 +121,7 @@ describe('materializeTaskBundle', () => {
     const taskTargets = await readFile(paths?.targetsPath ?? '', 'utf8');
     const parsedEval = parseYamlValue(taskEval) as Record<string, unknown>;
     const [testCase] = parsedEval.tests as Record<string, unknown>[];
-    const [assertion] = testCase.assertions as Record<string, unknown>[];
+    const [assertion] = testCase.assert as Record<string, unknown>[];
 
     expect(parsedEval.target).toBe('selected');
     expect(parsedEval.execution).toBeUndefined();

apps/cli/test/commands/grade/grade-prepared.test.ts

@@ -94,7 +94,7 @@ targets:
     `
 workspace:
   template: ../template
-assertions:
+assert:
 ${assertionYaml
   .trim()
   .split('\n')
@@ -126,7 +126,7 @@ describe('agentv grade prepared attempts', () => {
     const { evalPath, targetMarker, graderPayloadPath } = await writeFixtureProject(
       tempDir,
       `
-- name: workspace-check
+- metric: workspace-check
   type: script
   command: ["bun", "../scripts/workspace-grader.ts"]
 `,
@@ -247,7 +247,7 @@ describe('agentv grade prepared attempts', () => {
     const { evalPath, targetMarker } = await writeFixtureProject(
       tempDir,
       `
-- name: expected-tool-sequence
+- metric: expected-tool-sequence
   type: tool-trajectory
   mode: exact
   expected:
@@ -292,7 +292,7 @@ describe('agentv grade prepared attempts', () => {
     const { evalPath, targetMarker } = await writeFixtureProject(
       tempDir,
       `
-- name: expected-tool-sequence
+- metric: expected-tool-sequence
   type: tool-trajectory
   mode: in_order
   expected:
@@ -400,7 +400,7 @@ describe('agentv grade prepared attempts', () => {
     const { evalPath } = await writeFixtureProject(
       tempDir,
       `
-- name: expected-tool-sequence
+- metric: expected-tool-sequence
   type: tool-trajectory
   mode: in_order
   expected:
@@ -443,7 +443,7 @@ describe('agentv grade prepared attempts', () => {
     const { evalPath } = await writeFixtureProject(
       tempDir,
       `
-- name: expected-tool-sequence
+- metric: expected-tool-sequence
   type: tool-trajectory
   mode: in_order
   expected:

examples/features/agent-skills-evals/README.md

@@ -62,12 +62,12 @@ tests:
             value: evals/files/sales.csv
           - type: text
             value: "Find the top 3 months by revenue."
-    assertions:
+    assert:
       - type: skill-trigger
         skill: csv-analyzer
         should_trigger: true
-      - type: rubrics
-        criteria: "Output identifies November as the highest revenue month"
+      - type: llm-rubric
+        value: "Output identifies November as the highest revenue month"
       - type: contains
         value: "$22,500"
 ```