Merge pull request #49 from runbasehq/dev

feat: add scoring system for tool call evaluation

Author: fveiraswww

README.md

@@ -1,4 +1,5 @@
 # mcp-check
+⚠️ It's not usable yet, but docs and a stable release are coming in the next few weeks. Stay tuned.
 
 <p align="center">
   <a href="https://x.com/fveiras_">
@@ -35,8 +36,7 @@ const mcpServer = new McpServer({
 
 // Execute a prompt with multiple AI models
 const result = await client(mcpServer, ["claude-3-haiku-20240307", "gpt-4"])
-  .prompt("What tools are available and how do they work?")
-  .execute();
+  .prompt("What tools are available and how do they work?");
 
 // Get comprehensive results
 const executionResult = result.getExecutionResult();
@@ -65,7 +65,7 @@ const mcpServer = new McpServer({
 });
 ```
 
-### client(mcpServer, models, config?)
+### client(mcpServer, models, scorers?, config?)
 
 Create a client instance to execute prompts:
 
@@ -80,13 +80,27 @@ const result = await client(mcpServer, ["claude-3-haiku-20240307", "gpt-4"], {
     onError: (data) => console.error("Error:", data.error)
   }
 })
-  .prompt("Your prompt here")
-  .execute();
+  .scorers([
+    {
+      name: "contains id",
+      tool: "list_branches",
+      scorer: ({ output }) => {
+        try {
+          const branches = JSON.parse(output[0]?.text);
+          return branches.some(branch => branch.id) ? 1 : 0;
+        } catch {
+          return 0;
+        }
+      }
+    }
+  ])
+  .prompt("Your prompt here");
 ```
 
 **Parameters:**
 - `mcpServer`: Configured MCP server instance
 - `models`: Array of AI model names to use
+- `scorers?`: Optional array of Scorer instances for tool evaluation
 - `config?`: Optional configuration including API keys, silent mode, and chunk handlers
 
 **Supported Models:**
@@ -96,17 +110,28 @@ const result = await client(mcpServer, ["claude-3-haiku-20240307", "gpt-4"], {
 ### Agent Methods
 
 #### `.prompt(text: string)`
-Set the prompt to execute against the MCP server.
+Execute the prompt against the MCP server and return results. This method automatically executes the prompt.
+
+#### `.scorers(scorers: Array<{name: string; tool: string; scorer: Function}>)`
+Configure scorers to evaluate tool call results:
+```typescript
+.scorers([
+  {
+    name: "contains_data",
+    tool: "fetch_data", 
+    scorer: ({ output, input }) => {
+      return output?.data ? 1 : 0;
+    }
+  }
+])
+```
 
 #### `.allowTools(tools: string[])`
 Restrict which tools can be used by the models.
 
-#### `.execute()`
-Execute the prompt and return comprehensive results with tool usage tracking.
-
 ### Result Methods
 
-The `execute()` method returns an `AgentsResult` object with these methods:
+The `prompt()` method returns an `AgentsResult` object with these methods:
 
 #### `getResponse(model)`
 Get the response for a specific model:
@@ -155,6 +180,55 @@ Get list of models that executed successfully.
 #### `getFailedAgents()`
 Get list of models that failed to execute.
 
+#### `getScores(model)`
+Get evaluation scores for a specific model's tool calls:
+```typescript
+const scores = result.getScores("claude-3-haiku-20240307");
+scores.forEach(score => {
+  console.log(`${score.name}: ${score.score} for tool ${score.tool}`);
+});
+```
+
+## Scorer System
+
+The scorer system allows you to evaluate and validate tool call results automatically:
+
+```typescript
+const result = await client(mcpServer, ["claude-3-haiku-20240307"])
+  .scorers([
+    {
+      name: "valid_branches",
+      tool: "list_branches",
+      scorer: ({ output, input }) => {
+        try {
+          const branches = JSON.parse(output[0]?.text);
+          return branches.every(b => b.id && b.name) ? 1 : 0;
+        } catch {
+          return 0;
+        }
+      }
+    },
+    {
+      name: "has_results",
+      tool: "search_content", 
+      scorer: ({ output }) => {
+        return output?.results?.length > 0 ? 1 : 0;
+      }
+    }
+  ])
+  .prompt("List all branches and search for content");
+
+// Get scores for evaluation
+const scores = result.getScores("claude-3-haiku-20240307");
+console.log("Evaluation results:", scores);
+```
+
+Scorer functions receive:
+- `output`: The tool's result/response
+- `input`: The tool's input arguments
+
+Return a number (typically 0-1) representing the evaluation score.
+
 ## Testing Example
 
 ```typescript
@@ -170,8 +244,14 @@ const mcpServer = new McpServer({
 describe("MCP Server Tests", () => {
   test("should use expected tools across multiple models", async () => {
     const result = await client(mcpServer, ["claude-3-haiku-20240307", "gpt-4"])
-      .prompt("Update the content using the available tools.")
-      .execute();
+      .scorers([
+        {
+          name: "tool_success",
+          tool: "update_blocks",
+          scorer: ({ output }) => output?.success ? 1 : 0
+        }
+      ])
+      .prompt("Update the content using the available tools.");
 
     // Verify execution summary
     const execution = result.getExecutionResult();

packages/mcp-check/CHANGELOG.md

@@ -1,5 +1,14 @@
 # mcp-testing-library
 
+## 0.4.0
+
+### Minor Changes
+
+- - Add scoring system for tool call evaluation (getScores() and .scorers() methods)
+  - Simplified API: Removed .execute() calls since prompt() now auto-executes
+  - Scorer System: Added comprehensive section explaining how to use scorers
+  - Updated Examples
+
 ## 0.3.3
 
 ### Patch Changes

packages/mcp-check/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mcp-check",
   "module": "dist/src/index.js",
-  "version": "0.3.3",
+  "version": "0.4.0",
   "type": "module",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",

packages/mcp-check/src/chunks/types.ts

@@ -7,10 +7,10 @@ export interface ToolCall {
 
 /**
  * Represents the result of a streaming operation.
- * 
+ *
  * Contains the final content generated and information about tools used
  * during the streaming process.
- * 
+ *
  * @example
  * ```typescript
  * const result: StreamResult = {
@@ -33,10 +33,10 @@ export interface StreamResult {
 
 /**
  * Represents a complete agent response with metadata.
- * 
+ *
  * This interface provides a comprehensive view of an AI agent's response,
  * including the model used, content generated, tools utilized, and metadata.
- * 
+ *
  * @example
  * ```typescript
  * const response: AgentResponse = {
@@ -76,10 +76,10 @@ export interface AgentResponse {
 
 /**
  * Represents the result of executing multiple agents.
- * 
+ *
  * This interface aggregates responses from multiple AI agents and provides
  * summary statistics about the execution.
- * 
+ *
  * @example
  * ```typescript
  * const executionResult: AgentsExecutionResult = {
@@ -117,25 +117,25 @@ export interface AgentsExecutionResult {
 
 /**
  * Typed tool call with generic arguments and result types.
- * 
+ *
  * This interface provides type safety for tool calls by allowing specification
  * of argument and result types through generics.
- * 
+ *
  * @template TArgs - Type of the tool arguments
  * @template TResult - Type of the tool result
- * 
+ *
  * @example
  * ```typescript
  * interface WeatherArgs {
  *   location: string;
  *   units: "celsius" | "fahrenheit";
  * }
- * 
+ *
  * interface WeatherResult {
  *   temperature: number;
  *   condition: string;
  * }
- * 
+ *
  * const typedCall: TypedToolCall<WeatherArgs, WeatherResult> = {
  *   args: { location: "NYC", units: "fahrenheit" },
  *   result: { temperature: 72, condition: "sunny" },
@@ -157,10 +157,10 @@ export interface TypedToolCall<TArgs = Record<string, any>, TResult = any> {
 
 /**
  * Statistics for tool call usage.
- * 
+ *
  * This interface tracks usage patterns and performance metrics for tools
  * used by AI agents.
- * 
+ *
  * @example
  * ```typescript
  * const stats: ToolCallStats = {
@@ -187,10 +187,10 @@ export interface ToolCallStats {
 
 /**
  * Union type of all possible normalized chunk types.
- * 
+ *
  * Defines the different types of streaming chunks that can be processed
  * from AI providers.
- * 
+ *
  * - `text_delta`: Incremental text content updates
  * - `tool_call_start`: Beginning of a tool call
  * - `tool_call_done`: Completion of a tool call
@@ -199,7 +199,7 @@ export interface ToolCallStats {
  * - `message_done`: Completion of a message
  * - `thinking_delta`: Incremental thinking/reasoning updates
  * - `error`: Error information
- * 
+ *
  * @example
  * ```typescript
  * const chunkType: NormalizedChunkType = "text_delta";
@@ -217,11 +217,11 @@ export type NormalizedChunkType =
 
 /**
  * Represents a normalized chunk of streaming data from AI providers.
- * 
+ *
  * This interface provides a unified format for handling chunks from different
  * AI providers (Anthropic, OpenAI) by normalizing their specific formats
  * into a common structure.
- * 
+ *
  * @example
  * ```typescript
  * const chunk: NormalizedChunk = {
@@ -261,13 +261,13 @@ export interface NormalizedChunk {
 
 /**
  * Generic callback function for handling any normalized chunk.
- * 
+ *
  * This type defines a callback that can process any type of normalized chunk.
  * The callback can be synchronous or asynchronous.
- * 
+ *
  * @param chunk - The normalized chunk to process
  * @returns Promise<void> | void
- * 
+ *
  * @example
  * ```typescript
  * const callback: ChunkCallback = async (chunk) => {
@@ -280,13 +280,13 @@ export type ChunkCallback = (chunk: NormalizedChunk) => void | Promise<void>;
 
 /**
  * Callback function for handling specific chunk types.
- * 
+ *
  * This type defines a callback that processes the data payload from a specific
  * chunk type. The callback can be synchronous or asynchronous.
- * 
+ *
  * @param data - The data payload from the chunk
  * @returns Promise<void> | void
- * 
+ *
  * @example
  * ```typescript
  * const textCallback: ChunkTypeCallback = async (data) => {
@@ -296,15 +296,17 @@ export type ChunkCallback = (chunk: NormalizedChunk) => void | Promise<void>;
  * };
  * ```
  */
-export type ChunkTypeCallback = (data: NormalizedChunk["data"]) => void | Promise<void>;
+export type ChunkTypeCallback = (
+  data: NormalizedChunk["data"],
+) => void | Promise<void>;
 
 /**
  * Configuration object for chunk handlers.
- * 
+ *
  * This interface defines all the callback functions that can be registered
  * to handle different types of chunks during processing. All callbacks are
  * optional and can be async.
- * 
+ *
  * @example
  * ```typescript
  * const handlers: ChunkHandlerConfig = {