Skip to content

Commit 3b9da98

Browse files
pcarletonpcarleton
authored
initial runner for checking server conformance (#3)
* create some checks * basic initialize working * rename
1 parent 90ab89f commit 3b9da98

File tree

9 files changed

+267
-23
lines changed

9 files changed

+267
-23
lines changed

src/checks.test.ts renamed to src/checks/checks.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { createClientInitializationCheck } from './checks';
1+
import { createClientInitializationCheck } from './client';
22

33
describe('createClientInitializationCheck', () => {
44
it('should return SUCCESS for a valid initialize request', () => {

src/checks.ts renamed to src/checks/client.ts

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,19 @@
1-
import { ConformanceCheck, CheckStatus } from './types.js';
1+
import { ConformanceCheck, CheckStatus } from '../types.js';
2+
3+
export function createServerInfoCheck(serverInfo: { name: string; version: string }): ConformanceCheck {
4+
return {
5+
id: 'server-info',
6+
name: 'ServerInfo',
7+
description: 'Test server info returned to client',
8+
status: 'INFO',
9+
timestamp: new Date().toISOString(),
10+
specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
11+
details: {
12+
serverName: serverInfo.name,
13+
serverVersion: serverInfo.version
14+
}
15+
};
16+
}
217

318
export function createClientInitializationCheck(initializeRequest: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck {
419
const protocolVersionSent = initializeRequest?.protocolVersion;
@@ -29,19 +44,4 @@ export function createClientInitializationCheck(initializeRequest: any, expected
2944
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
3045
logs: errors.length > 0 ? errors : undefined
3146
};
32-
}
33-
34-
export function createServerInfoCheck(serverInfo: { name: string; version: string }): ConformanceCheck {
35-
return {
36-
id: 'server-info',
37-
name: 'ServerInfo',
38-
description: 'Test server info returned to client',
39-
status: 'INFO',
40-
timestamp: new Date().toISOString(),
41-
specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
42-
details: {
43-
serverName: serverInfo.name,
44-
serverVersion: serverInfo.version
45-
}
46-
};
47-
}
47+
}

‎src/checks/index.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// Namespaced exports for client and server checks
2+
import * as client from './client.js';
3+
import * as server from './server.js';
4+
5+
6+
export const clientChecks = client;
7+
export const serverChecks = server;

‎src/checks/server.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import { ConformanceCheck, CheckStatus } from '../types.js';
2+
3+
export function createServerInitializationCheck(initializeResponse: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck {
4+
const result = initializeResponse?.result;
5+
const protocolVersion = result?.protocolVersion;
6+
const serverInfo = result?.serverInfo;
7+
const capabilities = result?.capabilities;
8+
9+
const errors: string[] = [];
10+
if (!initializeResponse?.jsonrpc) errors.push('Missing jsonrpc field');
11+
if (!initializeResponse?.id) errors.push('Missing id field');
12+
if (!result) errors.push('Missing result field');
13+
if (!protocolVersion) errors.push('Missing protocolVersion in result');
14+
if (protocolVersion !== expectedSpecVersion) errors.push(`Protocol version mismatch: expected ${expectedSpecVersion}, got ${protocolVersion}`);
15+
if (!serverInfo) errors.push('Missing serverInfo in result');
16+
if (!serverInfo?.name) errors.push('Missing server name in serverInfo');
17+
if (!serverInfo?.version) errors.push('Missing server version in serverInfo');
18+
if (capabilities === undefined) errors.push('Missing capabilities in result');
19+
20+
const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE';
21+
22+
return {
23+
id: 'mcp-server-initialization',
24+
name: 'MCPServerInitialization',
25+
description: 'Validates that MCP server properly responds to initialize request',
26+
status,
27+
timestamp: new Date().toISOString(),
28+
specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }],
29+
details: {
30+
expectedSpecVersion,
31+
response: initializeResponse
32+
},
33+
errorMessage: errors.length > 0 ? errors.join('; ') : undefined,
34+
logs: errors.length > 0 ? errors : undefined
35+
};
36+
}

‎src/scenarios/client/initialize.ts‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import http from 'http';
22
import { Scenario, ScenarioUrls, ConformanceCheck } from '../../types.js';
3-
import { createClientInitializationCheck, createServerInfoCheck } from '../../checks.js';
3+
import { clientChecks } from '../../checks/index.js';
44

55
export class InitializeScenario implements Scenario {
66
name = 'initialize';
@@ -96,16 +96,15 @@ export class InitializeScenario implements Scenario {
9696
private handleInitialize(request: any, res: http.ServerResponse): void {
9797
const initializeRequest = request.params;
9898

99-
const check = createClientInitializationCheck(initializeRequest);
99+
const check = clientChecks.createClientInitializationCheck(initializeRequest);
100100
this.checks.push(check);
101101

102102
const serverInfo = {
103103
name: 'test-server',
104104
version: '1.0.0'
105105
};
106106

107-
const serverInfoCheck = createServerInfoCheck(serverInfo);
108-
this.checks.push(serverInfoCheck);
107+
this.checks.push(clientChecks.createServerInfoCheck(serverInfo));
109108

110109
const response = {
111110
jsonrpc: '2.0',

‎src/scenarios/index.ts‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
1-
import { Scenario } from '../types.js';
1+
import { Scenario, ClientScenario } from '../types.js';
22
import { InitializeScenario } from './client/initialize.js';
33
import { ToolsCallScenario } from './client/tools_call.js';
4+
import { ServerInitializeClientScenario } from './server/server_initialize.js';
45

56
export const scenarios = new Map<string, Scenario>([
67
['initialize', new InitializeScenario()],
78
['tools-call', new ToolsCallScenario()]
89
]);
910

11+
export const clientScenarios = new Map<string, ClientScenario>([
12+
['initialize', new ServerInitializeClientScenario()]
13+
]);
14+
1015
export function registerScenario(name: string, scenario: Scenario): void {
1116
scenarios.set(name, scenario);
1217
}
@@ -15,6 +20,14 @@ export function getScenario(name: string): Scenario | undefined {
1520
return scenarios.get(name);
1621
}
1722

23+
export function getClientScenario(name: string): ClientScenario | undefined {
24+
return clientScenarios.get(name);
25+
}
26+
1827
export function listScenarios(): string[] {
1928
return Array.from(scenarios.keys());
2029
}
30+
31+
export function listClientScenarios(): string[] {
32+
return Array.from(clientScenarios.keys());
33+
}

‎src/scenarios/server/server_initialize.ts‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import { ClientScenario, ConformanceCheck } from '../../types.js';
2+
import { serverChecks } from '../../checks/index.js';
3+
4+
export class ServerInitializeClientScenario implements ClientScenario {
5+
name = 'server-initialize';
6+
description = 'Acts as MCP client to test external server initialization';
7+
8+
async run(serverUrl: string): Promise<ConformanceCheck[]> {
9+
const checks: ConformanceCheck[] = [];
10+
11+
try {
12+
const response = await fetch(serverUrl, {
13+
method: 'POST',
14+
headers: {
15+
'Content-Type': 'application/json',
16+
'Accept': 'application/json, text/event-stream',
17+
},
18+
body: JSON.stringify({
19+
jsonrpc: '2.0',
20+
id: 1,
21+
method: 'initialize',
22+
params: {
23+
protocolVersion: '2025-06-18',
24+
capabilities: {},
25+
clientInfo: {
26+
name: 'conformance-test-client',
27+
version: '1.0.0'
28+
}
29+
}
30+
})
31+
});
32+
33+
if (!response.ok) {
34+
const responseBody = await response.text();
35+
throw new Error(`HTTP ${response.status}: ${response.statusText}. Response body: ${responseBody}`);
36+
}
37+
38+
const responseText = await response.text();
39+
40+
// Handle SSE format
41+
let result;
42+
if (responseText.startsWith('event:') || responseText.includes('\ndata:')) {
43+
// Parse SSE format - extract JSON from data: lines
44+
const lines = responseText.split('\n');
45+
const dataLines = lines.filter(line => line.startsWith('data: '));
46+
if (dataLines.length > 0) {
47+
const jsonData = dataLines[0].substring(6); // Remove 'data: ' prefix
48+
result = JSON.parse(jsonData);
49+
} else {
50+
throw new Error(`SSE response without data line: ${responseText}`);
51+
}
52+
} else {
53+
// Regular JSON response
54+
result = JSON.parse(responseText);
55+
}
56+
57+
const check = serverChecks.createServerInitializationCheck(result);
58+
checks.push(check);
59+
} catch (error) {
60+
checks.push({
61+
id: 'server-initialize-request',
62+
name: 'ServerInitializeRequest',
63+
description: 'Tests server response to initialize request',
64+
status: 'FAILURE',
65+
timestamp: new Date().toISOString(),
66+
errorMessage: `Failed to send initialize request: ${error instanceof Error ? error.message : String(error)}`,
67+
details: {
68+
error: error instanceof Error ? error.message : String(error),
69+
serverUrl
70+
},
71+
specReferences: [
72+
{
73+
id: 'MCP-Initialize',
74+
url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle#initialization'
75+
}
76+
]
77+
});
78+
}
79+
80+
return checks;
81+
}
82+
}

‎src/server-runner.ts‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import { promises as fs } from 'fs';
2+
import path from 'path';
3+
import { ConformanceCheck } from './types.js';
4+
import { getClientScenario } from './scenarios/index.js';
5+
6+
async function ensureResultsDir(): Promise<string> {
7+
const resultsDir = path.join(process.cwd(), 'results');
8+
await fs.mkdir(resultsDir, { recursive: true });
9+
return resultsDir;
10+
}
11+
12+
function createResultDir(scenario: string): string {
13+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
14+
return path.join('results', `server-${scenario}-${timestamp}`);
15+
}
16+
17+
export async function runServerConformanceTest(
18+
serverUrl: string,
19+
scenarioName: string
20+
): Promise<{
21+
checks: ConformanceCheck[];
22+
resultDir: string;
23+
}> {
24+
await ensureResultsDir();
25+
const resultDir = createResultDir(scenarioName);
26+
await fs.mkdir(resultDir, { recursive: true });
27+
28+
const scenario = getClientScenario(scenarioName);
29+
if (!scenario) {
30+
throw new Error(`Unknown client scenario: ${scenarioName}`);
31+
}
32+
33+
console.log(`Running client scenario '${scenarioName}' against server: ${serverUrl}`);
34+
35+
const checks = await scenario.run(serverUrl);
36+
37+
await fs.writeFile(path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2));
38+
39+
console.log(`Results saved to ${resultDir}`);
40+
41+
return {
42+
checks,
43+
resultDir
44+
};
45+
}
46+
47+
async function main(): Promise<void> {
48+
const args = process.argv.slice(2);
49+
let serverUrl: string | null = null;
50+
let scenario: string | null = null;
51+
52+
for (let i = 0; i < args.length; i++) {
53+
if (args[i] === '--server-url' && i + 1 < args.length) {
54+
serverUrl = args[i + 1];
55+
i++;
56+
} else if (args[i] === '--scenario' && i + 1 < args.length) {
57+
scenario = args[i + 1];
58+
i++;
59+
}
60+
}
61+
62+
if (!serverUrl || !scenario) {
63+
console.error('Usage: server-runner --server-url <url> --scenario <scenario>');
64+
console.error('Example: server-runner --server-url http://localhost:3000 --scenario initialize');
65+
process.exit(1);
66+
}
67+
68+
try {
69+
const result = await runServerConformanceTest(serverUrl, scenario);
70+
71+
const denominator = result.checks.filter(c => c.status === 'SUCCESS' || c.status == 'FAILURE').length;
72+
const passed = result.checks.filter(c => c.status === 'SUCCESS').length;
73+
const failed = result.checks.filter(c => c.status === 'FAILURE').length;
74+
75+
console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`);
76+
77+
console.log(`\nTest Results:`);
78+
console.log(`Passed: ${passed}/${denominator}, ${failed} failed`);
79+
80+
if (failed > 0) {
81+
console.log('\nFailed Checks:');
82+
result.checks
83+
.filter(c => c.status === 'FAILURE')
84+
.forEach(c => {
85+
console.log(` - ${c.name}: ${c.description}`);
86+
if (c.errorMessage) {
87+
console.log(` Error: ${c.errorMessage}`);
88+
}
89+
});
90+
}
91+
92+
process.exit(failed > 0 ? 1 : 0);
93+
} catch (error) {
94+
console.error('Server test runner error:', error);
95+
process.exit(1);
96+
}
97+
}
98+
99+
if (import.meta.url === `file://${process.argv[1]}`) {
100+
main();
101+
}

‎src/types.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,9 @@ export interface Scenario {
3030
stop(): Promise<void>;
3131
getChecks(): ConformanceCheck[];
3232
}
33+
34+
export interface ClientScenario {
35+
name: string;
36+
description: string;
37+
run(serverUrl: string): Promise<ConformanceCheck[]>;
38+
}

0 commit comments

Comments
 (0)