diff --git a/report-app/src/app/pages/report-viewer/report-viewer.html b/report-app/src/app/pages/report-viewer/report-viewer.html index 274842c..deb7732 100644 --- a/report-app/src/app/pages/report-viewer/report-viewer.html +++ b/report-app/src/app/pages/report-viewer/report-viewer.html @@ -219,7 +219,7 @@

Servers

diff --git a/runner/codegen/genkit/genkit-runner.ts b/runner/codegen/genkit/genkit-runner.ts index e4b6455..e27e509 100644 --- a/runner/codegen/genkit/genkit-runner.ts +++ b/runner/codegen/genkit/genkit-runner.ts @@ -1,4 +1,11 @@ -import {DynamicResourceAction, GenerateResponse, genkit, ModelReference, ToolAction} from 'genkit'; +import { + Action, + DynamicResourceAction, + GenerateResponse, + genkit, + ModelReference, + ToolAction, +} from 'genkit'; import {GenkitMcpHost, McpServerConfig, createMcpHost} from '@genkit-ai/mcp'; import {GenkitPlugin, GenkitPluginV2} from 'genkit/plugin'; import {z} from 'zod'; @@ -11,6 +18,7 @@ import { LlmGenerateTextResponse, LlmGenerateTextRequestOptions, LlmGenerateFilesRequestOptions, + McpServerDetails, } from '../llm-runner.js'; import {setTimeout} from 'node:timers/promises'; import {callWithTimeout} from '../../utils/timeout.js'; @@ -20,10 +28,18 @@ import {MODEL_PROVIDERS} from './models.js'; import {UserFacingError} from '../../utils/errors.js'; import {GenkitModelProvider, PromptDataForCounting} from './model-provider.js'; import {ToolLogEntry} from '../../shared-interfaces.js'; +import {toToolDefinition} from 'genkit/tool'; const globalLogger = new GenkitLogger(); logger.init(globalLogger); +/** + * Gets the name of a Genkit action. + */ +function getActionName(action: Action): string { + return toToolDefinition(action).name; +} + /** Runner that uses the Genkit API under the hood. */ export class GenkitRunner implements LlmRunner { readonly id = 'genkit'; @@ -193,7 +209,10 @@ export class GenkitRunner implements LlmRunner { } } - startMcpServerHost(hostName: string, servers: McpServerOptions[]): void { + async startMcpServerHost( + hostName: string, + servers: McpServerOptions[], + ): Promise { if (this.mcpHost !== null) { throw new Error('MCP host is already started'); } @@ -210,6 +229,12 @@ export class GenkitRunner implements LlmRunner { globalLogger.startCapturingLogs(); this.mcpHost = createMcpHost({name: hostName, mcpServers}); + const tools = await this.mcpHost.getActiveTools(this.genkitInstance); + const resources = await this.mcpHost.getActiveResources(this.genkitInstance); + return { + tools: tools.map(getActionName), + resources: resources.map(getActionName), + }; } flushMcpServerLogs(): string[] { diff --git a/runner/codegen/llm-runner.ts b/runner/codegen/llm-runner.ts index e12dcdc..565ee29 100644 --- a/runner/codegen/llm-runner.ts +++ b/runner/codegen/llm-runner.ts @@ -46,8 +46,9 @@ export interface LlmRunner { * Optional since not all runners may support MCP. * @param hostName Name for the MCP host. * @param servers Configured servers that should be started. + * @returns Details about the created server. */ - startMcpServerHost?(hostName: string, servers: McpServerOptions[]): void; + startMcpServerHost?(hostName: string, servers: McpServerOptions[]): Promise; /** Stops tracking MCP server logs and returns the current ones. */ flushMcpServerLogs?(): string[]; @@ -179,6 +180,12 @@ export const mcpServerOptionsSchema = z.object({ /** Options used to start an MCP server. */ export type McpServerOptions = z.infer; +/** Details about an MCP server. */ +export interface McpServerDetails { + tools: string[]; + resources: string[]; +} + /** * Type for a prompt message may be passed to LLM runner in the eval tool. * diff --git a/runner/orchestration/generate.ts b/runner/orchestration/generate.ts index 95b12c4..b6eec0e 100644 --- a/runner/orchestration/generate.ts +++ b/runner/orchestration/generate.ts @@ -125,14 +125,16 @@ export async function generateCodeAndAssess(options: { // We need Chrome to collect runtime information. await installChrome(); - if ( + const mcpServerDetails = env instanceof LocalEnvironment && options.startMcp && env.mcpServerOptions.length && env.llm.startMcpServerHost - ) { - env.llm.startMcpServerHost(`mcp-${env.clientSideFramework.id}`, env.mcpServerOptions); - } + ? await env.llm.startMcpServerHost( + `mcp-${env.clientSideFramework.id}`, + env.mcpServerOptions, + ) + : undefined; progress.initialize(promptsToProcess.length); @@ -228,6 +230,8 @@ export async function generateCodeAndAssess(options: { name: m.name, command: m.command, args: m.args, + tools: mcpServerDetails?.tools || [], + resources: mcpServerDetails?.resources || [], })), logs: env.llm.flushMcpServerLogs().join('\n'), } diff --git a/runner/shared-interfaces.ts b/runner/shared-interfaces.ts index a43cb9a..7907b0c 100644 --- a/runner/shared-interfaces.ts +++ b/runner/shared-interfaces.ts @@ -391,7 +391,13 @@ export interface RunDetails { /** Information about configured MCP servers, if any. */ mcp?: { /** MCP servers that were configured. */ - servers: {name: string; command: string; args: string[]}[]; + servers: { + name: string; + command: string; + args: string[]; + tools: string[]; + resources: string[]; + }[]; /** Logs produced by all of the servers. */ logs: string;