fix(evals): infer task input types from eval data

Author: c-ehrlich

packages/ai/src/evals/builder.ts

@@ -95,7 +95,7 @@ class EvalBuilderImpl<
     }
 
     // Call existing Eval function - this handles all Vitest registration
-    Eval<TInput, TExpected, TOutput>(finalName, finalParams);
+    Eval(finalName, finalParams);
   }
 }
 

packages/ai/src/evals/eval.ts

@@ -50,6 +50,30 @@ declare module 'vitest' {
 
 const createVersionId = customAlphabet('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ', 10);
 
+type EvalDataSourceBase = { input: unknown; expected: unknown };
+type EvalDataSource =
+  | readonly EvalDataSourceBase[]
+  | Promise<readonly EvalDataSourceBase[]>
+  | (() => readonly EvalDataSourceBase[] | Promise<readonly EvalDataSourceBase[]>);
+
+type ResolveEvalData<TData extends EvalDataSource> = Awaited<
+  TData extends (...args: any[]) => infer TResult ? TResult : TData
+>;
+
+type InferEvalDataInput<TData extends EvalDataSource> =
+  ResolveEvalData<TData> extends readonly (infer TRecord)[]
+    ? TRecord extends { input: infer TInput }
+      ? TInput
+      : never
+    : never;
+
+type InferEvalDataExpected<TData extends EvalDataSource> =
+  ResolveEvalData<TData> extends readonly (infer TRecord)[]
+    ? TRecord extends { expected: infer TExpected }
+      ? TExpected
+      : never
+    : never;
+
 type RunTaskFailureDetails = {
   duration: number;
   outOfScopeFlags: OutOfScopeFlagAccess[];
@@ -111,17 +135,24 @@ function getRunTaskFailureDetails(error: unknown): RunTaskFailureDetails | undef
  * ```
  */
 export function Eval<
-  TInput,
-  TExpected,
-  TOutput,
+  TData extends EvalDataSource,
+  TInput extends string | Record<string, any> = InferEvalDataInput<TData>,
+  TExpected extends string | Record<string, any> = InferEvalDataExpected<TData>,
+  TOutput extends string | Record<string, any> = string | Record<string, any>,
   Name extends string = string,
   Capability extends string = string,
   Step extends string = string,
 >(
   name: ValidateName<Name>,
-  params: Omit<EvalParams<TInput, TExpected, TOutput>, 'capability' | 'step'> & {
+  params: Omit<
+    EvalParams<TInput, TExpected, TOutput>,
+    'capability' | 'step' | 'data' | 'task' | 'scorers'
+  > & {
     capability: ValidateName<Capability>;
     step?: ValidateName<Step> | undefined;
+    data: TData;
+    task: EvalTask<TInput, TExpected, TOutput>;
+    scorers: ReadonlyArray<ScorerLike<TInput, TExpected, TOutput>>;
   },
 ): void {
   // Record eval name for validation

packages/ai/test/evals/eval.types.test.ts

@@ -0,0 +1,54 @@
+import { describe, it } from 'vitest';
+import { Eval } from '../../src/evals';
+import { Scorer } from '../../src/scorers/scorers';
+
+describe('Eval type inference', () => {
+  it('infers task input and expected from data when scorer omits input', () => {
+    const answerSimilarityScorer = Scorer(
+      'answer-similarity',
+      ({ output, expected }: { output: string; expected: string }) => {
+        output;
+        expected;
+        return 1;
+      },
+    );
+
+    const compileOnly = () =>
+      Eval('name-apl-query', {
+        capability: 'name_query',
+        data: () => [
+          {
+            input: "['nginx-access-logs'] | where status >= 500",
+            expected: 'Nginx 5xx Errors',
+          },
+        ],
+        task: async ({ input }: { input: string }) => input,
+        scorers: [answerSimilarityScorer],
+      });
+
+    compileOnly;
+  });
+
+  it('rejects task input that conflicts with the data source', () => {
+    const OutputOnlyScorer = Scorer(
+      'output-only',
+      ({ output }: { output: string }) => output.length > 0,
+    );
+
+    const invalid = () =>
+      Eval('mismatched-task-input', {
+        capability: 'name_query',
+        data: () => [
+          {
+            input: 'foo',
+            expected: 'bar',
+          },
+        ],
+        // @ts-expect-error task input must match the data input type
+        task: async ({ input }: { input: number }) => String(input),
+        scorers: [OutputOnlyScorer],
+      });
+
+    invalid;
+  });
+});