Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 22 additions & 2 deletions report-app/src/app/pages/report-viewer/report-viewer.html
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ <h4>Servers</h4>
<ul>
@for (server of details.mcp.servers; track server) {
<li>
<span>
<div>
<strong>{{ server.name }}</strong
>:
<code
Expand All @@ -228,7 +228,27 @@ <h4>Servers</h4>
{{ arg }}
}
</code>
</span>
</div>
@if (server.tools && server.tools.length > 0) {
<div>
Tools
<ul>
@for (tool of server.tools; track tool) {
<li>{{ tool }}</li>
}
</ul>
</div>
}
@if (server.resources && server.resources.length > 0) {
<div>
Resources
<ul>
@for (tool of server.resources; track tool) {
<li>{{ tool }}</li>
}
</ul>
</div>
}
</li>
}
</ul>
Expand Down
29 changes: 27 additions & 2 deletions runner/codegen/genkit/genkit-runner.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
import {DynamicResourceAction, GenerateResponse, genkit, ModelReference, ToolAction} from 'genkit';
import {
Action,
DynamicResourceAction,
GenerateResponse,
genkit,
ModelReference,
ToolAction,
} from 'genkit';
import {GenkitMcpHost, McpServerConfig, createMcpHost} from '@genkit-ai/mcp';
import {GenkitPlugin, GenkitPluginV2} from 'genkit/plugin';
import {z} from 'zod';
Expand All @@ -11,6 +18,7 @@ import {
LlmGenerateTextResponse,
LlmGenerateTextRequestOptions,
LlmGenerateFilesRequestOptions,
McpServerDetails,
} from '../llm-runner.js';
import {setTimeout} from 'node:timers/promises';
import {callWithTimeout} from '../../utils/timeout.js';
Expand All @@ -20,10 +28,18 @@ import {MODEL_PROVIDERS} from './models.js';
import {UserFacingError} from '../../utils/errors.js';
import {GenkitModelProvider, PromptDataForCounting} from './model-provider.js';
import {ToolLogEntry} from '../../shared-interfaces.js';
import {toToolDefinition} from 'genkit/tool';

const globalLogger = new GenkitLogger();
logger.init(globalLogger);

/**
* Gets the name of a Genkit action.
*/
function getActionName(action: Action<any, any>): string {
return toToolDefinition(action).name;
}

/** Runner that uses the Genkit API under the hood. */
export class GenkitRunner implements LlmRunner {
readonly id = 'genkit';
Expand Down Expand Up @@ -193,7 +209,10 @@ export class GenkitRunner implements LlmRunner {
}
}

startMcpServerHost(hostName: string, servers: McpServerOptions[]): void {
async startMcpServerHost(
hostName: string,
servers: McpServerOptions[],
): Promise<McpServerDetails> {
if (this.mcpHost !== null) {
throw new Error('MCP host is already started');
}
Expand All @@ -210,6 +229,12 @@ export class GenkitRunner implements LlmRunner {

globalLogger.startCapturingLogs();
this.mcpHost = createMcpHost({name: hostName, mcpServers});
const tools = await this.mcpHost.getActiveTools(this.genkitInstance);
const resources = await this.mcpHost.getActiveResources(this.genkitInstance);
return {
tools: tools.map(getActionName),
resources: resources.map(getActionName),
};
}

flushMcpServerLogs(): string[] {
Expand Down
9 changes: 8 additions & 1 deletion runner/codegen/llm-runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,9 @@ export interface LlmRunner {
* Optional since not all runners may support MCP.
* @param hostName Name for the MCP host.
* @param servers Configured servers that should be started.
* @returns Details about the created server.
*/
startMcpServerHost?(hostName: string, servers: McpServerOptions[]): void;
startMcpServerHost?(hostName: string, servers: McpServerOptions[]): Promise<McpServerDetails>;

/** Stops tracking MCP server logs and returns the current ones. */
flushMcpServerLogs?(): string[];
Expand Down Expand Up @@ -179,6 +180,12 @@ export const mcpServerOptionsSchema = z.object({
/** Options used to start an MCP server. */
export type McpServerOptions = z.infer<typeof mcpServerOptionsSchema>;

/** Details about an MCP server. */
export interface McpServerDetails {
tools: string[];
resources: string[];
}

/**
* Type for a prompt message may be passed to LLM runner in the eval tool.
*
Expand Down
12 changes: 8 additions & 4 deletions runner/orchestration/generate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,16 @@ export async function generateCodeAndAssess(options: {
// We need Chrome to collect runtime information.
await installChrome();

if (
const mcpServerDetails =
env instanceof LocalEnvironment &&
options.startMcp &&
env.mcpServerOptions.length &&
env.llm.startMcpServerHost
) {
env.llm.startMcpServerHost(`mcp-${env.clientSideFramework.id}`, env.mcpServerOptions);
}
? await env.llm.startMcpServerHost(
`mcp-${env.clientSideFramework.id}`,
env.mcpServerOptions,
)
: undefined;

progress.initialize(promptsToProcess.length);

Expand Down Expand Up @@ -228,6 +230,8 @@ export async function generateCodeAndAssess(options: {
name: m.name,
command: m.command,
args: m.args,
tools: mcpServerDetails?.tools || [],
resources: mcpServerDetails?.resources || [],
})),
logs: env.llm.flushMcpServerLogs().join('\n'),
}
Expand Down
8 changes: 7 additions & 1 deletion runner/shared-interfaces.ts
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,13 @@ export interface RunDetails {
/** Information about configured MCP servers, if any. */
mcp?: {
/** MCP servers that were configured. */
servers: {name: string; command: string; args: string[]}[];
servers: {
name: string;
command: string;
args: string[];
tools: string[];
resources: string[];
}[];

/** Logs produced by all of the servers. */
logs: string;
Expand Down
Loading