diff --git a/src/checks.test.ts b/src/checks/checks.test.ts similarity index 97% rename from src/checks.test.ts rename to src/checks/checks.test.ts index b7c19eb..d8e2f88 100644 --- a/src/checks.test.ts +++ b/src/checks/checks.test.ts @@ -1,4 +1,4 @@ -import { createClientInitializationCheck } from './checks'; +import { createClientInitializationCheck } from './client'; describe('createClientInitializationCheck', () => { it('should return SUCCESS for a valid initialize request', () => { diff --git a/src/checks.ts b/src/checks/client.ts similarity index 97% rename from src/checks.ts rename to src/checks/client.ts index 11a1fee..0cc0ca6 100644 --- a/src/checks.ts +++ b/src/checks/client.ts @@ -1,4 +1,19 @@ -import { ConformanceCheck, CheckStatus } from './types.js'; +import { ConformanceCheck, CheckStatus } from '../types.js'; + +export function createServerInfoCheck(serverInfo: { name: string; version: string }): ConformanceCheck { + return { + id: 'server-info', + name: 'ServerInfo', + description: 'Test server info returned to client', + status: 'INFO', + timestamp: new Date().toISOString(), + specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], + details: { + serverName: serverInfo.name, + serverVersion: serverInfo.version + } + }; +} export function createClientInitializationCheck(initializeRequest: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck { const protocolVersionSent = initializeRequest?.protocolVersion; @@ -29,19 +44,4 @@ export function createClientInitializationCheck(initializeRequest: any, expected errorMessage: errors.length > 0 ? errors.join('; ') : undefined, logs: errors.length > 0 ? errors : undefined }; -} - -export function createServerInfoCheck(serverInfo: { name: string; version: string }): ConformanceCheck { - return { - id: 'server-info', - name: 'ServerInfo', - description: 'Test server info returned to client', - status: 'INFO', - timestamp: new Date().toISOString(), - specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], - details: { - serverName: serverInfo.name, - serverVersion: serverInfo.version - } - }; -} +} \ No newline at end of file diff --git a/src/checks/index.ts b/src/checks/index.ts new file mode 100644 index 0000000..0ce84aa --- /dev/null +++ b/src/checks/index.ts @@ -0,0 +1,7 @@ +// Namespaced exports for client and server checks +import * as client from './client.js'; +import * as server from './server.js'; + + +export const clientChecks = client; +export const serverChecks = server; diff --git a/src/checks/server.ts b/src/checks/server.ts new file mode 100644 index 0000000..cf4f8da --- /dev/null +++ b/src/checks/server.ts @@ -0,0 +1,36 @@ +import { ConformanceCheck, CheckStatus } from '../types.js'; + +export function createServerInitializationCheck(initializeResponse: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck { + const result = initializeResponse?.result; + const protocolVersion = result?.protocolVersion; + const serverInfo = result?.serverInfo; + const capabilities = result?.capabilities; + + const errors: string[] = []; + if (!initializeResponse?.jsonrpc) errors.push('Missing jsonrpc field'); + if (!initializeResponse?.id) errors.push('Missing id field'); + if (!result) errors.push('Missing result field'); + if (!protocolVersion) errors.push('Missing protocolVersion in result'); + if (protocolVersion !== expectedSpecVersion) errors.push(`Protocol version mismatch: expected ${expectedSpecVersion}, got ${protocolVersion}`); + if (!serverInfo) errors.push('Missing serverInfo in result'); + if (!serverInfo?.name) errors.push('Missing server name in serverInfo'); + if (!serverInfo?.version) errors.push('Missing server version in serverInfo'); + if (capabilities === undefined) errors.push('Missing capabilities in result'); + + const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE'; + + return { + id: 'mcp-server-initialization', + name: 'MCPServerInitialization', + description: 'Validates that MCP server properly responds to initialize request', + status, + timestamp: new Date().toISOString(), + specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], + details: { + expectedSpecVersion, + response: initializeResponse + }, + errorMessage: errors.length > 0 ? errors.join('; ') : undefined, + logs: errors.length > 0 ? errors : undefined + }; +} \ No newline at end of file diff --git a/src/scenarios/client/initialize.ts b/src/scenarios/client/initialize.ts index ff046f5..80e9d07 100644 --- a/src/scenarios/client/initialize.ts +++ b/src/scenarios/client/initialize.ts @@ -1,6 +1,6 @@ import http from 'http'; import { Scenario, ScenarioUrls, ConformanceCheck } from '../../types.js'; -import { createClientInitializationCheck, createServerInfoCheck } from '../../checks.js'; +import { clientChecks } from '../../checks/index.js'; export class InitializeScenario implements Scenario { name = 'initialize'; @@ -96,7 +96,7 @@ export class InitializeScenario implements Scenario { private handleInitialize(request: any, res: http.ServerResponse): void { const initializeRequest = request.params; - const check = createClientInitializationCheck(initializeRequest); + const check = clientChecks.createClientInitializationCheck(initializeRequest); this.checks.push(check); const serverInfo = { @@ -104,8 +104,7 @@ export class InitializeScenario implements Scenario { version: '1.0.0' }; - const serverInfoCheck = createServerInfoCheck(serverInfo); - this.checks.push(serverInfoCheck); + this.checks.push(clientChecks.createServerInfoCheck(serverInfo)); const response = { jsonrpc: '2.0', diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 2682a6e..0744c24 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -1,12 +1,17 @@ -import { Scenario } from '../types.js'; +import { Scenario, ClientScenario } from '../types.js'; import { InitializeScenario } from './client/initialize.js'; import { ToolsCallScenario } from './client/tools_call.js'; +import { ServerInitializeClientScenario } from './server/server_initialize.js'; export const scenarios = new Map([ ['initialize', new InitializeScenario()], ['tools-call', new ToolsCallScenario()] ]); +export const clientScenarios = new Map([ + ['initialize', new ServerInitializeClientScenario()] +]); + export function registerScenario(name: string, scenario: Scenario): void { scenarios.set(name, scenario); } @@ -15,6 +20,14 @@ export function getScenario(name: string): Scenario | undefined { return scenarios.get(name); } +export function getClientScenario(name: string): ClientScenario | undefined { + return clientScenarios.get(name); +} + export function listScenarios(): string[] { return Array.from(scenarios.keys()); } + +export function listClientScenarios(): string[] { + return Array.from(clientScenarios.keys()); +} diff --git a/src/scenarios/server/server_initialize.ts b/src/scenarios/server/server_initialize.ts new file mode 100644 index 0000000..5374997 --- /dev/null +++ b/src/scenarios/server/server_initialize.ts @@ -0,0 +1,82 @@ +import { ClientScenario, ConformanceCheck } from '../../types.js'; +import { serverChecks } from '../../checks/index.js'; + +export class ServerInitializeClientScenario implements ClientScenario { + name = 'server-initialize'; + description = 'Acts as MCP client to test external server initialization'; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json, text/event-stream', + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-06-18', + capabilities: {}, + clientInfo: { + name: 'conformance-test-client', + version: '1.0.0' + } + } + }) + }); + + if (!response.ok) { + const responseBody = await response.text(); + throw new Error(`HTTP ${response.status}: ${response.statusText}. Response body: ${responseBody}`); + } + + const responseText = await response.text(); + + // Handle SSE format + let result; + if (responseText.startsWith('event:') || responseText.includes('\ndata:')) { + // Parse SSE format - extract JSON from data: lines + const lines = responseText.split('\n'); + const dataLines = lines.filter(line => line.startsWith('data: ')); + if (dataLines.length > 0) { + const jsonData = dataLines[0].substring(6); // Remove 'data: ' prefix + result = JSON.parse(jsonData); + } else { + throw new Error(`SSE response without data line: ${responseText}`); + } + } else { + // Regular JSON response + result = JSON.parse(responseText); + } + + const check = serverChecks.createServerInitializationCheck(result); + checks.push(check); + } catch (error) { + checks.push({ + id: 'server-initialize-request', + name: 'ServerInitializeRequest', + description: 'Tests server response to initialize request', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed to send initialize request: ${error instanceof Error ? error.message : String(error)}`, + details: { + error: error instanceof Error ? error.message : String(error), + serverUrl + }, + specReferences: [ + { + id: 'MCP-Initialize', + url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle#initialization' + } + ] + }); + } + + return checks; + } +} \ No newline at end of file diff --git a/src/server-runner.ts b/src/server-runner.ts new file mode 100644 index 0000000..8228c18 --- /dev/null +++ b/src/server-runner.ts @@ -0,0 +1,101 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import { ConformanceCheck } from './types.js'; +import { getClientScenario } from './scenarios/index.js'; + +async function ensureResultsDir(): Promise { + const resultsDir = path.join(process.cwd(), 'results'); + await fs.mkdir(resultsDir, { recursive: true }); + return resultsDir; +} + +function createResultDir(scenario: string): string { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + return path.join('results', `server-${scenario}-${timestamp}`); +} + +export async function runServerConformanceTest( + serverUrl: string, + scenarioName: string +): Promise<{ + checks: ConformanceCheck[]; + resultDir: string; +}> { + await ensureResultsDir(); + const resultDir = createResultDir(scenarioName); + await fs.mkdir(resultDir, { recursive: true }); + + const scenario = getClientScenario(scenarioName); + if (!scenario) { + throw new Error(`Unknown client scenario: ${scenarioName}`); + } + + console.log(`Running client scenario '${scenarioName}' against server: ${serverUrl}`); + + const checks = await scenario.run(serverUrl); + + await fs.writeFile(path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2)); + + console.log(`Results saved to ${resultDir}`); + + return { + checks, + resultDir + }; +} + +async function main(): Promise { + const args = process.argv.slice(2); + let serverUrl: string | null = null; + let scenario: string | null = null; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '--server-url' && i + 1 < args.length) { + serverUrl = args[i + 1]; + i++; + } else if (args[i] === '--scenario' && i + 1 < args.length) { + scenario = args[i + 1]; + i++; + } + } + + if (!serverUrl || !scenario) { + console.error('Usage: server-runner --server-url --scenario '); + console.error('Example: server-runner --server-url http://localhost:3000 --scenario initialize'); + process.exit(1); + } + + try { + const result = await runServerConformanceTest(serverUrl, scenario); + + const denominator = result.checks.filter(c => c.status === 'SUCCESS' || c.status == 'FAILURE').length; + const passed = result.checks.filter(c => c.status === 'SUCCESS').length; + const failed = result.checks.filter(c => c.status === 'FAILURE').length; + + console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`); + + console.log(`\nTest Results:`); + console.log(`Passed: ${passed}/${denominator}, ${failed} failed`); + + if (failed > 0) { + console.log('\nFailed Checks:'); + result.checks + .filter(c => c.status === 'FAILURE') + .forEach(c => { + console.log(` - ${c.name}: ${c.description}`); + if (c.errorMessage) { + console.log(` Error: ${c.errorMessage}`); + } + }); + } + + process.exit(failed > 0 ? 1 : 0); + } catch (error) { + console.error('Server test runner error:', error); + process.exit(1); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} \ No newline at end of file diff --git a/src/types.ts b/src/types.ts index 1bd7733..e49d08f 100644 --- a/src/types.ts +++ b/src/types.ts @@ -30,3 +30,9 @@ export interface Scenario { stop(): Promise; getChecks(): ConformanceCheck[]; } + +export interface ClientScenario { + name: string; + description: string; + run(serverUrl: string): Promise; +}