1- import { createScorer } from ' @mastra/core/evals' ;
2- import { z } from ' zod' ;
1+ import { createScorer } from " @mastra/core/evals" ;
2+ import { z } from " zod" ;
33
4- import { DEFAULT_MODEL , NAME_PROPERTY } from ' ../constants' ;
4+ import { DEFAULT_MODEL , NAME_PROPERTY } from " ../constants" ;
55
66/** Schema for extracted person entity */
77const zPersonEntity = z
@@ -41,8 +41,8 @@ const zAnalysisResult = z.object({
4141 * (e.g., "Bill Gates" matches "William Gates").
4242 */
4343export const nerPeopleScorer = createScorer ( {
44- id : ' ner-people' ,
45- description : ' Evaluates NER people extraction against expected persons list' ,
44+ id : " ner-people" ,
45+ description : " Evaluates NER people extraction against expected persons list" ,
4646 judge : {
4747 model : DEFAULT_MODEL ,
4848 instructions : `You are an expert at comparing person names for semantic equivalence.
@@ -59,31 +59,38 @@ Be precise: only match names that clearly refer to the same real-world person.`,
5959
6060 if ( ! Array . isArray ( output ) ) {
6161 // eslint-disable-next-line no-console
62- console . warn ( '[nerPeopleScorer] Expected run.output to be an array, got:' , typeof output ) ;
62+ console . warn (
63+ "[nerPeopleScorer] Expected run.output to be an array, got:" ,
64+ typeof output ,
65+ ) ;
6366 } else {
6467 for ( const person of output ) {
6568 const personObj = person as Record < string , unknown > ;
6669 const name = personObj [ NAME_PROPERTY ] ;
67- if ( typeof name === ' string' ) {
70+ if ( typeof name === " string" ) {
6871 extractedNames . push ( name ) ;
6972 }
7073 }
7174 }
7275
7376 // Parse ground truth
7477 const groundTruth = zGroundTruth . parse ( run . groundTruth ) ;
75- const expectedNames = groundTruth . expectedPersons . map ( ( person ) => person [ NAME_PROPERTY ] ) ;
78+ const expectedNames = groundTruth . expectedPersons . map (
79+ ( person ) => person [ NAME_PROPERTY ] ,
80+ ) ;
7681
7782 return { extractedNames, expectedNames } ;
7883 } )
7984 . analyze ( {
80- description : 'Match extracted person names against expected persons using fuzzy matching' ,
85+ description :
86+ "Match extracted person names against expected persons using fuzzy matching" ,
8187 outputSchema : zAnalysisResult ,
8288 createPrompt : ( { results } ) => {
83- const { extractedNames, expectedNames } = results . preprocessStepResult as {
84- extractedNames : string [ ] ;
85- expectedNames : string [ ] ;
86- } ;
89+ const { extractedNames, expectedNames } =
90+ results . preprocessStepResult as {
91+ extractedNames : string [ ] ;
92+ expectedNames : string [ ] ;
93+ } ;
8794 return `Compare extracted person names against expected names.
8895
8996EXTRACTED PERSONS (from NER step):
@@ -113,18 +120,24 @@ Return JSON with:
113120 // - Recall (finding expected persons): 70% weight
114121 // - Precision (not having false positives): 30% weight
115122 const recall = matchedPersons . length / totalExpected ;
116- const precision = totalExtracted > 0 ? matchedPersons . length / totalExtracted : 1 ;
123+ const precision =
124+ totalExtracted > 0 ? matchedPersons . length / totalExtracted : 1 ;
117125
118126 return 0.7 * recall + 0.3 * precision ;
119127 } )
120128 . generateReason ( ( { results, score } ) => {
121- const { matchedPersons, missingPersons, extraPersons, totalExtracted, totalExpected } =
122- results . analyzeStepResult ;
129+ const {
130+ matchedPersons,
131+ missingPersons,
132+ extraPersons,
133+ totalExtracted,
134+ totalExpected,
135+ } = results . analyzeStepResult ;
123136
124137 return (
125138 `Score: ${ score . toFixed ( 2 ) } . Found ${ matchedPersons . length } /${ totalExpected } expected persons. ` +
126- `${ missingPersons . length > 0 ? `Missing: ${ missingPersons . join ( ', ' ) } . ` : '' } ` +
127- `${ extraPersons . length > 0 ? `Extra: ${ extraPersons . join ( ', ' ) } . ` : '' } ` +
139+ `${ missingPersons . length > 0 ? `Missing: ${ missingPersons . join ( ", " ) } . ` : "" } ` +
140+ `${ extraPersons . length > 0 ? `Extra: ${ extraPersons . join ( ", " ) } . ` : "" } ` +
128141 `Total extracted: ${ totalExtracted } .`
129142 ) ;
130143 } ) ;
0 commit comments