Fixed bug in output where it double reported

Author: steviec

src/testing/evals/runner.ts

@@ -169,6 +169,7 @@ export class EvalTestRunner {
       // Validate tool call success for required tools
       if (test.expected_tool_calls?.required && conversationResult.success) {
         const toolSuccessErrors = this.validateToolCallSuccess(
+          conversationResult.toolCalls,
           conversationResult.toolResults,
           test.expected_tool_calls.required
         );
@@ -268,12 +269,19 @@ export class EvalTestRunner {
   }
 
   private validateToolCallSuccess(
+    toolCalls: ToolCall<string, Record<string, unknown>>[],
     toolResults: ToolResult<string, Record<string, unknown>, unknown>[],
     requiredTools: string[]
   ): string[] {
     const errors: string[] = [];
+    const calledToolNames = toolCalls.map(call => call.toolName);
 
     for (const requiredTool of requiredTools) {
+      // Only validate success if the tool was actually called
+      if (!calledToolNames.includes(requiredTool)) {
+        continue; // Skip - this case is already handled by validateToolCalls
+      }
+
       // Find results for this required tool
       const resultsForTool = toolResults.filter(tr => tr.toolName === requiredTool);
 

test/unit/tool-validation.test.ts

@@ -119,7 +119,15 @@ describe('Tool Call Validation', () => {
     // Test the validateToolCallSuccess method
     const validateToolCallSuccess = (runner as any).validateToolCallSuccess.bind(runner);
 
-    // Mock tool results with error
+    // Mock tool calls and results with error
+    const toolCalls = [
+      {
+        toolCallId: '1',
+        toolName: 'run_flow_files',
+        args: {},
+      },
+    ];
+
     const toolResults = [
       {
         toolCallId: '1',
@@ -134,7 +142,7 @@ describe('Tool Call Validation', () => {
 
     const requiredTools = ['run_flow_files'];
 
-    const errors = validateToolCallSuccess(toolResults, requiredTools);
+    const errors = validateToolCallSuccess(toolCalls, toolResults, requiredTools);
 
     // Should produce an error because the required tool failed
     expect(errors.length).toBeGreaterThan(0);
@@ -159,7 +167,15 @@ describe('Tool Call Validation', () => {
     // Test the validateToolCallSuccess method
     const validateToolCallSuccess = (runner as any).validateToolCallSuccess.bind(runner);
 
-    // Mock tool results with success
+    // Mock tool calls and results with success
+    const toolCalls = [
+      {
+        toolCallId: '1',
+        toolName: 'run_flow_files',
+        args: {},
+      },
+    ];
+
     const toolResults = [
       {
         toolCallId: '1',
@@ -174,7 +190,7 @@ describe('Tool Call Validation', () => {
 
     const requiredTools = ['run_flow_files'];
 
-    const errors = validateToolCallSuccess(toolResults, requiredTools);
+    const errors = validateToolCallSuccess(toolCalls, toolResults, requiredTools);
 
     // Should not produce any errors
     expect(errors).toEqual([]);