readme

Author: pcarleton

README.md

@@ -0,0 +1,68 @@
+# MCP Conformance Test Framework
+
+A framework for testing MCP (Model Context Protocol) client implementations against the specification.
+
+## Quick Start
+
+```bash
+npm install
+npm run start -- --command "tsx examples/clients/typescript/test1.ts" --scenario initialize
+```
+
+## Overview
+
+The conformance test framework validates MCP client implementations by:
+1. Starting a test server for the specified scenario
+2. Running the client implementation with the test server URL
+3. Capturing MCP protocol interactions
+4. Running conformance checks against the specification
+5. Generating detailed test results
+
+## Usage
+
+```bash
+npm run start -- --command "<client-command>" --scenario <scenario-name>
+```
+
+- `--command` - The command to run your MCP client (can include flags)
+- `--scenario` - The test scenario to run (e.g., "initialize")
+
+The framework appends the server URL as the final argument to your command.
+
+## Test Results
+
+Results are saved to `results/<scenario>-<timestamp>/`:
+- `checks.json` - Array of conformance check results with pass/fail status
+- `stdout.txt` - Client stdout output
+- `stderr.txt` - Client stderr output
+
+## Example Clients
+
+- `examples/clients/typescript/test1.ts` - Valid MCP client (passes all checks)
+- `examples/clients/typescript/test-broken.ts` - Invalid client missing required fields (fails checks)
+
+## Available Scenarios
+
+- **initialize** - Tests MCP client initialization handshake
+  - Validates protocol version
+  - Validates clientInfo (name and version)
+  - Validates server response handling
+
+## Architecture
+
+See `src/runner/DESIGN.md` for detailed architecture documentation.
+
+### Key Components
+
+- **Runner** (`src/runner/`) - Orchestrates test execution and result generation
+- **Scenarios** (`src/scenarios/`) - Test scenarios with custom server implementations
+- **Checks** (`src/checks.ts`) - Conformance validation functions
+- **Types** (`src/types.ts`) - Shared type definitions
+
+## Adding New Scenarios
+
+1. Create a new directory in `src/scenarios/<scenario-name>/`
+2. Implement the `Scenario` interface with `start()`, `stop()`, and `getChecks()`
+3. Register the scenario in `src/scenarios/index.ts`
+
+See `src/scenarios/initialize/` for a reference implementation.

examples/clients/typescript/test-broken.ts

@@ -0,0 +1,59 @@
+#!/usr/bin/env node
+
+import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
+
+async function main(): Promise<void> {
+  const serverUrl = process.argv[2];
+
+  if (!serverUrl) {
+    console.error('Usage: test-broken-client <server-url>');
+    process.exit(1);
+  }
+
+  console.log(`Connecting to MCP server at: ${serverUrl}`);
+
+  try {
+    const transport = new StreamableHTTPClientTransport(
+      new URL(serverUrl)
+    );
+
+    await transport.start();
+
+    const initRequest = {
+      jsonrpc: '2.0',
+      id: 1,
+      method: 'initialize',
+      params: {
+        protocolVersion: '2025-06-18',
+        clientInfo: {
+          version: '1.0.0'
+        },
+        capabilities: {}
+      }
+    };
+
+    const response = await fetch(serverUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify(initRequest)
+    });
+
+    const result = await response.json();
+    console.log('✅ Received response:', JSON.stringify(result, null, 2));
+
+    await transport.close();
+    console.log('✅ Connection closed');
+
+    process.exit(0);
+  } catch (error) {
+    console.error('❌ Error:', error);
+    process.exit(1);
+  }
+}
+
+main().catch((error) => {
+  console.error('Unhandled error:', error);
+  process.exit(1);
+});

src/scenarios/index.ts

@@ -1,8 +1,8 @@
 import { Scenario } from '../types.js';
-import { initializeScenario } from './initialize/index.js';
+import { InitializeScenario } from './initialize.js';
 
 export const scenarios = new Map<string, Scenario>([
-  ['initialize', initializeScenario],
+  ['initialize', new InitializeScenario()],
 ]);
 
 export function registerScenario(name: string, scenario: Scenario): void {

src/scenarios/initialize/server.ts src/scenarios/initialize.tssrc/scenarios/initialize/server.ts renamed to src/scenarios/initialize.ts

@@ -1,13 +1,16 @@
 import http from 'http';
-import { ConformanceCheck } from '../../types.js';
-import { createClientInitializationCheck, createServerInfoCheck } from '../../checks.js';
+import { Scenario, ScenarioUrls, ConformanceCheck } from '../types.js';
+import { createClientInitializationCheck, createServerInfoCheck } from '../checks.js';
+
+export class InitializeScenario implements Scenario {
+  name = 'initialize';
+  description = 'Tests MCP client initialization handshake';
 
-export class InitializeTestServer {
   private server: http.Server | null = null;
   private checks: ConformanceCheck[] = [];
   private port: number = 0;
 
-  async start(): Promise<number> {
+  async start(): Promise<ScenarioUrls> {
     return new Promise((resolve, reject) => {
       this.server = http.createServer((req, res) => {
         this.handleRequest(req, res);
@@ -19,7 +22,9 @@ export class InitializeTestServer {
         const address = this.server!.address();
         if (address && typeof address === 'object') {
           this.port = address.port;
-          resolve(this.port);
+          resolve({
+            serverUrl: `http://localhost:${this.port}`
+          });
         } else {
           reject(new Error('Failed to get server address'));
         }