From 2b36f959ba33af25e3ac0c3b6b4e5b097fdf0c29 Mon Sep 17 00:00:00 2001 From: Alon Mishne Date: Thu, 2 Oct 2025 10:41:40 -0700 Subject: [PATCH 1/3] feat: include tools and resources in MCP server info --- .../pages/report-viewer/report-viewer.html | 24 +++++++++++++++++-- runner/codegen/genkit/genkit-runner.ts | 9 ++++++- runner/codegen/llm-runner.ts | 9 ++++++- runner/orchestration/generate.ts | 22 ++++++++++------- runner/shared-interfaces.ts | 2 +- 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/report-app/src/app/pages/report-viewer/report-viewer.html b/report-app/src/app/pages/report-viewer/report-viewer.html index 274842c..deb7732 100644 --- a/report-app/src/app/pages/report-viewer/report-viewer.html +++ b/report-app/src/app/pages/report-viewer/report-viewer.html @@ -219,7 +219,7 @@

Servers

diff --git a/runner/codegen/genkit/genkit-runner.ts b/runner/codegen/genkit/genkit-runner.ts index e4b6455..a54b4b7 100644 --- a/runner/codegen/genkit/genkit-runner.ts +++ b/runner/codegen/genkit/genkit-runner.ts @@ -11,6 +11,7 @@ import { LlmGenerateTextResponse, LlmGenerateTextRequestOptions, LlmGenerateFilesRequestOptions, + McpServerDetails, } from '../llm-runner.js'; import {setTimeout} from 'node:timers/promises'; import {callWithTimeout} from '../../utils/timeout.js'; @@ -193,7 +194,7 @@ export class GenkitRunner implements LlmRunner { } } - startMcpServerHost(hostName: string, servers: McpServerOptions[]): void { + async startMcpServerHost(hostName: string, servers: McpServerOptions[]): Promise { if (this.mcpHost !== null) { throw new Error('MCP host is already started'); } @@ -210,6 +211,12 @@ export class GenkitRunner implements LlmRunner { globalLogger.startCapturingLogs(); this.mcpHost = createMcpHost({name: hostName, mcpServers}); + const tools = await this.mcpHost.getActiveTools(this.genkitInstance); + const resources = await this.mcpHost.getActiveResources(this.genkitInstance); + return { + tools: tools.map((t) => t.__action.name), + resources: resources.map((r) => r.__action.name), + }; } flushMcpServerLogs(): string[] { diff --git a/runner/codegen/llm-runner.ts b/runner/codegen/llm-runner.ts index e12dcdc..565ee29 100644 --- a/runner/codegen/llm-runner.ts +++ b/runner/codegen/llm-runner.ts @@ -46,8 +46,9 @@ export interface LlmRunner { * Optional since not all runners may support MCP. * @param hostName Name for the MCP host. * @param servers Configured servers that should be started. + * @returns Details about the created server. */ - startMcpServerHost?(hostName: string, servers: McpServerOptions[]): void; + startMcpServerHost?(hostName: string, servers: McpServerOptions[]): Promise; /** Stops tracking MCP server logs and returns the current ones. */ flushMcpServerLogs?(): string[]; @@ -179,6 +180,12 @@ export const mcpServerOptionsSchema = z.object({ /** Options used to start an MCP server. */ export type McpServerOptions = z.infer; +/** Details about an MCP server. */ +export interface McpServerDetails { + tools: string[]; + resources: string[]; +} + /** * Type for a prompt message may be passed to LLM runner in the eval tool. * diff --git a/runner/orchestration/generate.ts b/runner/orchestration/generate.ts index 95b12c4..18a2333 100644 --- a/runner/orchestration/generate.ts +++ b/runner/orchestration/generate.ts @@ -125,14 +125,16 @@ export async function generateCodeAndAssess(options: { // We need Chrome to collect runtime information. await installChrome(); - if ( - env instanceof LocalEnvironment && - options.startMcp && - env.mcpServerOptions.length && - env.llm.startMcpServerHost - ) { - env.llm.startMcpServerHost(`mcp-${env.clientSideFramework.id}`, env.mcpServerOptions); - } + const mcpServerDetails = + env instanceof LocalEnvironment && + options.startMcp && + env.mcpServerOptions.length && + env.llm.startMcpServerHost + ? await env.llm.startMcpServerHost( + `mcp-${env.clientSideFramework.id}`, + env.mcpServerOptions + ) + : undefined; progress.initialize(promptsToProcess.length); @@ -228,7 +230,9 @@ export async function generateCodeAndAssess(options: { name: m.name, command: m.command, args: m.args, - })), + tools: mcpServerDetails?.tools || [], + resources: mcpServerDetails?.resources || [], + })), logs: env.llm.flushMcpServerLogs().join('\n'), } : undefined; diff --git a/runner/shared-interfaces.ts b/runner/shared-interfaces.ts index a43cb9a..63d9040 100644 --- a/runner/shared-interfaces.ts +++ b/runner/shared-interfaces.ts @@ -391,7 +391,7 @@ export interface RunDetails { /** Information about configured MCP servers, if any. */ mcp?: { /** MCP servers that were configured. */ - servers: {name: string; command: string; args: string[]}[]; + servers: {name: string; command: string; args: string[], tools: string[], resources: string[]}[]; /** Logs produced by all of the servers. */ logs: string; From 5c0c309518f78fc0cdaf19920a1fcb5af227e55e Mon Sep 17 00:00:00 2001 From: Alon Mishne Date: Thu, 2 Oct 2025 11:58:36 -0700 Subject: [PATCH 2/3] refactor: prettier on mcp changes --- runner/codegen/genkit/genkit-runner.ts | 9 ++++++--- runner/orchestration/generate.ts | 22 +++++++++++----------- runner/shared-interfaces.ts | 8 +++++++- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/runner/codegen/genkit/genkit-runner.ts b/runner/codegen/genkit/genkit-runner.ts index a54b4b7..18eea7a 100644 --- a/runner/codegen/genkit/genkit-runner.ts +++ b/runner/codegen/genkit/genkit-runner.ts @@ -194,7 +194,10 @@ export class GenkitRunner implements LlmRunner { } } - async startMcpServerHost(hostName: string, servers: McpServerOptions[]): Promise { + async startMcpServerHost( + hostName: string, + servers: McpServerOptions[], + ): Promise { if (this.mcpHost !== null) { throw new Error('MCP host is already started'); } @@ -214,8 +217,8 @@ export class GenkitRunner implements LlmRunner { const tools = await this.mcpHost.getActiveTools(this.genkitInstance); const resources = await this.mcpHost.getActiveResources(this.genkitInstance); return { - tools: tools.map((t) => t.__action.name), - resources: resources.map((r) => r.__action.name), + tools: tools.map(t => t.__action.name), + resources: resources.map(r => r.__action.name), }; } diff --git a/runner/orchestration/generate.ts b/runner/orchestration/generate.ts index 18a2333..b6eec0e 100644 --- a/runner/orchestration/generate.ts +++ b/runner/orchestration/generate.ts @@ -125,16 +125,16 @@ export async function generateCodeAndAssess(options: { // We need Chrome to collect runtime information. await installChrome(); - const mcpServerDetails = - env instanceof LocalEnvironment && - options.startMcp && - env.mcpServerOptions.length && - env.llm.startMcpServerHost - ? await env.llm.startMcpServerHost( - `mcp-${env.clientSideFramework.id}`, - env.mcpServerOptions - ) - : undefined; + const mcpServerDetails = + env instanceof LocalEnvironment && + options.startMcp && + env.mcpServerOptions.length && + env.llm.startMcpServerHost + ? await env.llm.startMcpServerHost( + `mcp-${env.clientSideFramework.id}`, + env.mcpServerOptions, + ) + : undefined; progress.initialize(promptsToProcess.length); @@ -232,7 +232,7 @@ export async function generateCodeAndAssess(options: { args: m.args, tools: mcpServerDetails?.tools || [], resources: mcpServerDetails?.resources || [], - })), + })), logs: env.llm.flushMcpServerLogs().join('\n'), } : undefined; diff --git a/runner/shared-interfaces.ts b/runner/shared-interfaces.ts index 63d9040..7907b0c 100644 --- a/runner/shared-interfaces.ts +++ b/runner/shared-interfaces.ts @@ -391,7 +391,13 @@ export interface RunDetails { /** Information about configured MCP servers, if any. */ mcp?: { /** MCP servers that were configured. */ - servers: {name: string; command: string; args: string[], tools: string[], resources: string[]}[]; + servers: { + name: string; + command: string; + args: string[]; + tools: string[]; + resources: string[]; + }[]; /** Logs produced by all of the servers. */ logs: string; From a3f97656560206890b993e0bfe37251331a01b9c Mon Sep 17 00:00:00 2001 From: Alon Mishne Date: Fri, 3 Oct 2025 15:02:22 -0700 Subject: [PATCH 3/3] refactor: used `toToolDefinition` instead to get action name --- runner/codegen/genkit/genkit-runner.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/runner/codegen/genkit/genkit-runner.ts b/runner/codegen/genkit/genkit-runner.ts index 18eea7a..e27e509 100644 --- a/runner/codegen/genkit/genkit-runner.ts +++ b/runner/codegen/genkit/genkit-runner.ts @@ -1,4 +1,11 @@ -import {DynamicResourceAction, GenerateResponse, genkit, ModelReference, ToolAction} from 'genkit'; +import { + Action, + DynamicResourceAction, + GenerateResponse, + genkit, + ModelReference, + ToolAction, +} from 'genkit'; import {GenkitMcpHost, McpServerConfig, createMcpHost} from '@genkit-ai/mcp'; import {GenkitPlugin, GenkitPluginV2} from 'genkit/plugin'; import {z} from 'zod'; @@ -21,10 +28,18 @@ import {MODEL_PROVIDERS} from './models.js'; import {UserFacingError} from '../../utils/errors.js'; import {GenkitModelProvider, PromptDataForCounting} from './model-provider.js'; import {ToolLogEntry} from '../../shared-interfaces.js'; +import {toToolDefinition} from 'genkit/tool'; const globalLogger = new GenkitLogger(); logger.init(globalLogger); +/** + * Gets the name of a Genkit action. + */ +function getActionName(action: Action): string { + return toToolDefinition(action).name; +} + /** Runner that uses the Genkit API under the hood. */ export class GenkitRunner implements LlmRunner { readonly id = 'genkit'; @@ -217,8 +232,8 @@ export class GenkitRunner implements LlmRunner { const tools = await this.mcpHost.getActiveTools(this.genkitInstance); const resources = await this.mcpHost.getActiveResources(this.genkitInstance); return { - tools: tools.map(t => t.__action.name), - resources: resources.map(r => r.__action.name), + tools: tools.map(getActionName), + resources: resources.map(getActionName), }; }