Merge pull request #57 from runbasehq/dev

Release 0.4.2

Author: PineAppleGrits

packages/mcp-check/CHANGELOG.md

@@ -1,5 +1,11 @@
 # mcp-testing-library
 
+## 0.4.2
+
+### Patch Changes
+
+- improve typings
+
 ## 0.4.1
 
 ### Patch Changes

packages/mcp-check/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mcp-check",
   "module": "dist/src/index.js",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "type": "module",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",

packages/mcp-check/src/agents/client.ts

@@ -76,8 +76,7 @@ export class AgentsClient<T extends ModelName = ModelName> {
    * @example
    * ```typescript
    * const result = await client
-   *   .prompt("What tools are available?")
-   *   .execute();
+   *   .prompt("What tools are available?");
    *
    * console.log("Execution time:", result.getExecutionResult().summary.executionTime);
    * console.log("Successful models:", result.getSuccessfulAgents());

packages/mcp-check/src/chunks/types.ts

@@ -1,3 +1,6 @@
+import type { BetaRawContentBlockDeltaEvent, BetaRawContentBlockStartEvent, BetaRawContentBlockStopEvent, BetaRawMessageStreamEvent } from "@anthropic-ai/sdk/resources/beta.js";
+import type OpenAI from "openai";
+
 export interface ToolCall {
   /** Arguments passed to the tool function */
   args: Record<string, any>;
@@ -233,11 +236,9 @@ export type NormalizedChunkType =
  * };
  * ```
  */
-export interface NormalizedChunk {
+export interface BaseNormalizedChunk {
   /** The type of chunk being processed */
   type: NormalizedChunkType;
-  /** The AI provider that generated this chunk */
-  provider: "anthropic" | "openai";
   /** Timestamp when the chunk was received */
   timestamp: number;
   /** The normalized data payload for this chunk */
@@ -255,8 +256,25 @@ export interface NormalizedChunk {
     /** Additional data properties */
     [key: string]: any;
   };
-  /** The original provider-specific chunk data */
-  originalChunk?: any;
+}
+
+/**
+ * Union type of all possible normalized chunk types.
+ *
+ * Defines the different types of streaming chunks that can be processed
+ * from AI providers.
+ *
+ *
+ **/
+export type NormalizedChunk = NormalizedChunkAnthropic | NormalizedChunkOpenAI;
+export interface NormalizedChunkAnthropic extends BaseNormalizedChunk {
+  provider: "anthropic";
+  originalChunk: BetaRawMessageStreamEvent;
+}
+
+export interface NormalizedChunkOpenAI extends BaseNormalizedChunk {
+  provider: "openai";
+  originalChunk: OpenAI.Responses.ResponseStreamEvent;
 }
 
 /**
@@ -296,9 +314,7 @@ export type ChunkCallback = (chunk: NormalizedChunk) => void | Promise<void>;
  * };
  * ```
  */
-export type ChunkTypeCallback = (
-  data: NormalizedChunk["data"],
-) => void | Promise<void>;
+export type ChunkTypeCallback = (data: NormalizedChunk["data"]) => void | Promise<void>;
 
 /**
  * Configuration object for chunk handlers.
@@ -347,17 +363,17 @@ export interface ChunkHandlerConfig {
   /** Provider-specific handlers for Anthropic */
   anthropic?: {
     /** Handler for Anthropic content block delta chunks */
-    onContentBlockDelta?: (chunk: any) => void | Promise<void>;
+    onContentBlockDelta?: (chunk: BetaRawContentBlockDeltaEvent) => void | Promise<void>;
     /** Handler for Anthropic content block start chunks */
-    onContentBlockStart?: (chunk: any) => void | Promise<void>;
+    onContentBlockStart?: (chunk: BetaRawContentBlockStartEvent) => void | Promise<void>;
     /** Handler for Anthropic content block stop chunks */
-    onContentBlockStop?: (chunk: any) => void | Promise<void>;
+    onContentBlockStop?: (chunk: BetaRawContentBlockStopEvent) => void | Promise<void>;
   };
   /** Provider-specific handlers for OpenAI */
   openai?: {
     /** Handler for OpenAI response output item added chunks */
-    onResponseOutputItemAdded?: (chunk: any) => void | Promise<void>;
+    onResponseOutputItemAdded?: (chunk: OpenAI.Responses.ResponseOutputItemAddedEvent) => void | Promise<void>;
     /** Handler for OpenAI response output item done chunks */
-    onResponseOutputItemDone?: (chunk: any) => void | Promise<void>;
+    onResponseOutputItemDone?: (chunk: OpenAI.Responses.ResponseOutputItemDoneEvent) => void | Promise<void>;
   };
 }

packages/mcp-check/src/index.ts

@@ -1,12 +1,7 @@
 import { config } from "dotenv";
 import type { ModelName } from "./providers";
 import type { ProviderConfig } from "./providers/types";
-import type {
-  AgentResponse,
-  AgentsExecutionResult,
-  ToolCall,
-  ToolCallStats,
-} from "./chunks/types";
+import type { AgentResponse, AgentsExecutionResult, ToolCall, ToolCallStats } from "./chunks/types";
 import { createProvider } from "./providers";
 
 config();
@@ -31,7 +26,6 @@ export {
  * @template T - The type of model names to use
  * @param mcpServer - The MCP server configuration
  * @param models - Array of model names to execute
- * @param scorers - Array of scorers to score the output of the tools
  * @param config - Optional provider configuration (API keys, silent mode, etc.)
  * @returns A configured AgentsClient instance
  *
@@ -53,13 +47,8 @@ export {
  * ```
  */
 
-export function client<T extends ModelName>(
-  mcpServer: McpServer,
-  models: T[],
-  scorers?: Scorer[],
-  config: ProviderConfig = {},
-): AgentsClient<T> {
-  return new AgentsClient(mcpServer, models, scorers, config);
+export function client<T extends ModelName>(mcpServer: McpServer, models: T[], config: ProviderConfig = {}): AgentsClient<T> {
+  return new AgentsClient(mcpServer, models, config);
 }
 
 /**
@@ -86,15 +75,7 @@ export class Scorer {
   public scorer: (toolResponse: any) => number;
   public tool: string;
 
-  constructor({
-    name,
-    scorer,
-    tool,
-  }: {
-    name: string;
-    scorer: (toolResponse: any) => number;
-    tool: string;
-  }) {
+  constructor({ name, scorer, tool }: { name: string; scorer: (toolResponse: any) => number; tool: string }) {
     this.name = name;
     this.scorer = scorer;
     this.tool = tool;
@@ -107,17 +88,7 @@ export class McpServer {
   public name: string;
   public type: string;
 
-  constructor({
-    url,
-    authorizationToken,
-    name,
-    type,
-  }: {
-    url: string;
-    authorizationToken: string;
-    name: string;
-    type: string;
-  }) {
+  constructor({ url, authorizationToken, name, type }: { url: string; authorizationToken: string; name: string; type: string }) {
     this.url = url;
     this.authorizationToken = authorizationToken;
     this.name = name;
@@ -139,7 +110,7 @@ export class AgentsResult<T extends ModelName = ModelName> {
     endTime: number,
     models: T[],
     agentsInstance?: AgentsClient<T>,
-    scorers?: Scorer[],
+    scorers?: Scorer[]
   ) {
     this.responses = responses;
     this.executionStartTime = startTime;
@@ -189,15 +160,13 @@ export class AgentsResult<T extends ModelName = ModelName> {
     const successfulExecutions = Object.keys(this.responses).length;
     const failedExecutions = this.models.length - successfulExecutions;
 
-    const allTools = (Object.values(this.responses) as AgentResponse[]).flatMap(
-      (r) => r.usedTools,
-    );
+    const allTools = (Object.values(this.responses) as AgentResponse[]).flatMap((r) => r.usedTools);
     const toolCounts = allTools.reduce(
       (acc, tool) => {
         acc[tool] = (acc[tool] || 0) + 1;
         return acc;
       },
-      {} as Record<string, number>,
+      {} as Record<string, number>
     );
 
     const commonTools = Object.entries(toolCounts)
@@ -231,18 +200,13 @@ export class AgentsResult<T extends ModelName = ModelName> {
         }
 
         stats[toolName].callCount += (calls as any[]).length;
-        stats[toolName].lastCalled = Math.max(
-          stats[toolName].lastCalled || 0,
-          response.metadata?.timestamp || 0,
-        );
+        stats[toolName].lastCalled = Math.max(stats[toolName].lastCalled || 0, response.metadata?.timestamp || 0);
       });
     });
 
     return Object.values(stats).map((stat) => ({
       ...stat,
-      averageDuration: stat.totalDuration
-        ? stat.totalDuration / stat.callCount
-        : undefined,
+      averageDuration: stat.totalDuration ? stat.totalDuration / stat.callCount : undefined,
     }));
   }
 
@@ -286,15 +250,10 @@ export class AgentsClient<T extends ModelName = ModelName> {
   private config: ProviderConfig;
   private _scorers: Scorer[] = [];
 
-  constructor(
-    mcpServer: McpServer,
-    models: T[],
-    scorers?: Scorer[],
-    config: ProviderConfig = {},
-  ) {
+  constructor(mcpServer: McpServer, models: T[], config: ProviderConfig = {}) {
     this.models = models;
     this.mcpServer = mcpServer;
-    this._scorers = scorers || [];
+    this._scorers = config.scorers || [];
     this.config = { silent: true, ...config };
   }
 
@@ -313,26 +272,21 @@ export class AgentsClient<T extends ModelName = ModelName> {
       name: string;
       tool: string;
       scorer: (toolResponse: any) => number;
-    }>,
+    }>
   ): this {
     this._scorers = scorers.map((s) => new Scorer(s));
     return this;
   }
 
-  async execute(): Promise<AgentsResult<T>> {
+  private async execute(): Promise<AgentsResult<T>> {
     if (!this.mcpServer) {
       throw new Error("MCP server not set");
     }
 
     const executionStartTime = Date.now();
 
     this.executionPromises = this.models.map(async (model) => {
-      const provider = createProvider(
-        model,
-        this.mcpServer!,
-        this.promptText,
-        this.config,
-      );
+      const provider = createProvider(model, this.mcpServer!, this.promptText, this.config);
       const result = await provider.stream(model);
 
       this.usedTools[model] = result.usedTools;
@@ -358,16 +312,9 @@ export class AgentsClient<T extends ModelName = ModelName> {
         acc[response.model as T] = response;
         return acc;
       },
-      {} as Record<T, AgentResponse>,
+      {} as Record<T, AgentResponse>
     );
 
-    return new AgentsResult(
-      responsesMap,
-      executionStartTime,
-      executionEndTime,
-      this.models,
-      this,
-      this._scorers,
-    );
+    return new AgentsResult(responsesMap, executionStartTime, executionEndTime, this.models, this, this._scorers);
   }
 }

packages/mcp-check/src/providers/types.ts

@@ -1,3 +1,4 @@
+import type { Scorer } from "src/index.js";
 import type { ChunkHandlerConfig } from "../chunks/types.js";
 
 /**
@@ -28,6 +29,6 @@ export interface ProviderConfig {
   chunkHandlers?: ChunkHandlerConfig;
   /** Whether to suppress console output during execution */
   silent?: boolean;
-  /** Additional configuration properties */
-  [key: string]: any;
+  /** Scorers to score the output of the tools */
+  scorers?: Scorer[];
 }

packages/mcp-check/tests/index.test.ts

@@ -9,10 +9,7 @@ const mcpServer = new McpServer({
 
 describe("get branches tool", function () {
   test("should retrieve available branches from Basehub", async function () {
-    const result = await client(mcpServer, [
-      "claude-3-haiku-20240307",
-      "claude-3-5-haiku-20241022",
-    ])
+    const result = await client(mcpServer, ["claude-3-haiku-20240307", "claude-3-5-haiku-20241022"])
       .scorers([
         {
           name: "contains id",
@@ -21,9 +18,7 @@ describe("get branches tool", function () {
             try {
               const resultText = output[0]?.text;
               const branches = JSON.parse(resultText);
-              return branches.some((branch: { id: string }) => branch.id)
-                ? 1
-                : 0;
+              return branches.some((branch: { id: string }) => branch.id) ? 1 : 0;
             } catch (e) {
               console.log("Parse error:", e);
               return 0;
@@ -33,16 +28,10 @@ describe("get branches tool", function () {
       ])
       .prompt("What branches are available in this Basehub repo?");
 
-    expect(
-      result.hasUsedTool("claude-3-5-haiku-20241022", "list_branches"),
-    ).toBe(true);
-    expect(result.getUsedTools("claude-3-5-haiku-20241022")).toEqual(
-      expect.arrayContaining(["list_branches"]),
-    );
+    expect(result.hasUsedTool("claude-3-5-haiku-20241022", "list_branches")).toBe(true);
+    expect(result.getUsedTools("claude-3-5-haiku-20241022")).toEqual(expect.arrayContaining(["list_branches"]));
 
-    expect(
-      result.getToolCallCount("claude-3-5-haiku-20241022", "list_branches"),
-    ).toBeGreaterThan(0);
+    expect(result.getToolCallCount("claude-3-5-haiku-20241022", "list_branches")).toBeGreaterThan(0);
 
     const response = result.getResponse("claude-3-5-haiku-20241022");
 
@@ -56,9 +45,7 @@ describe("get branches tool", function () {
     expect(executionResult.summary.executionTime).toBeGreaterThan(0);
 
     expect(executionResult.responses["claude-3-haiku-20240307"]).toBeDefined();
-    expect(
-      executionResult.responses["claude-3-5-haiku-20241022"],
-    ).toBeDefined();
+    expect(executionResult.responses["claude-3-5-haiku-20241022"]).toBeDefined();
 
     expect(result.getResponse("claude-3-5-haiku-20241022")).toBeDefined();
     expect(result.getAgentNames()).toContain("claude-3-haiku-20240307");