diff --git a/examples/clients/typescript/auth-test.ts b/examples/clients/typescript/auth-test.ts new file mode 100644 index 0000000..dee052b --- /dev/null +++ b/examples/clients/typescript/auth-test.ts @@ -0,0 +1,79 @@ +#!/usr/bin/env node + +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { ConformanceOAuthProvider } from './helpers/ConformanceOAuthProvider.js'; +import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js'; + +async function main(): Promise { + const serverUrl = process.argv[2]; + + if (!serverUrl) { + console.error('Usage: auth-test '); + process.exit(1); + } + + console.log(`Connecting to MCP server at: ${serverUrl}`); + + const client = new Client( + { + name: 'test-auth-client', + version: '1.0.0' + }, + { + capabilities: {} + } + ); + + const authProvider = new ConformanceOAuthProvider( + 'http://localhost:3000/callback', + { + client_name: 'test-auth-client', + redirect_uris: ['http://localhost:3000/callback'] + } + ); + + let transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + authProvider + }); + + // Try to connect - handle OAuth if needed + try { + await client.connect(transport); + console.log('✅ Successfully connected to MCP server'); + } catch (error) { + if (error instanceof UnauthorizedError) { + console.log('🔐 OAuth required - handling authorization...'); + + // The provider will automatically fetch the auth code + const authCode = await authProvider.getAuthCode(); + + // Complete the auth flow + await transport.finishAuth(authCode); + + // Close the old transport + await transport.close(); + + // Create a new transport with the authenticated provider + transport = new StreamableHTTPClientTransport(new URL(serverUrl), { + authProvider: authProvider + }); + + // Connect with the new transport + await client.connect(transport); + console.log('✅ Successfully connected with authentication'); + } else { + throw error; + } + } + + await client.listTools(); + console.log('✅ Successfully listed tools'); + + await transport.close(); + console.log('✅ Connection closed successfully'); + + process.exit(0); +} + +main(); diff --git a/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts b/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts new file mode 100644 index 0000000..46fdd21 --- /dev/null +++ b/examples/clients/typescript/helpers/ConformanceOAuthProvider.ts @@ -0,0 +1,95 @@ +import { OAuthClientProvider } from '@modelcontextprotocol/sdk/client/auth.js'; +import { + OAuthClientInformation, + OAuthClientInformationFull, + OAuthClientMetadata, + OAuthTokens +} from '@modelcontextprotocol/sdk/shared/auth.js'; + +export class ConformanceOAuthProvider implements OAuthClientProvider { + private _clientInformation?: OAuthClientInformationFull; + private _tokens?: OAuthTokens; + private _codeVerifier?: string; + private _authCode?: string; + private _authCodePromise?: Promise; + + constructor( + private readonly _redirectUrl: string | URL, + private readonly _clientMetadata: OAuthClientMetadata, + private readonly _clientMetadataUrl?: string | URL + ) {} + + get redirectUrl(): string | URL { + return this._redirectUrl; + } + + get clientMetadata(): OAuthClientMetadata { + return this._clientMetadata; + } + + clientInformation(): OAuthClientInformation | undefined { + if (this._clientMetadataUrl) { + console.log('Using client ID metadata URL'); + return { + client_id: this._clientMetadataUrl.toString() + }; + } + return this._clientInformation; + } + + saveClientInformation(clientInformation: OAuthClientInformationFull): void { + this._clientInformation = clientInformation; + } + + tokens(): OAuthTokens | undefined { + return this._tokens; + } + + saveTokens(tokens: OAuthTokens): void { + this._tokens = tokens; + } + + async redirectToAuthorization(authorizationUrl: URL): Promise { + try { + const response = await fetch(authorizationUrl.toString(), { + redirect: 'manual' // Don't follow redirects automatically + }); + + // Get the Location header which contains the redirect with auth code + const location = response.headers.get('location'); + if (location) { + const redirectUrl = new URL(location); + const code = redirectUrl.searchParams.get('code'); + if (code) { + this._authCode = code; + return; + } else { + throw new Error('No auth code in redirect URL'); + } + } else { + throw new Error('No redirect location received'); + } + } catch (error) { + console.error('Failed to fetch authorization URL:', error); + throw error; + } + } + + async getAuthCode(): Promise { + if (this._authCode) { + return this._authCode; + } + throw new Error('No authorization code'); + } + + saveCodeVerifier(codeVerifier: string): void { + this._codeVerifier = codeVerifier; + } + + codeVerifier(): string { + if (!this._codeVerifier) { + throw new Error('No code verifier saved'); + } + return this._codeVerifier; + } +} diff --git a/package-lock.json b/package-lock.json index 597e3a0..c29a11b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "conformance", - "version": "1.0.0", + "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "conformance", - "version": "1.0.0", + "version": "0.1.0", "dependencies": { "@modelcontextprotocol/sdk": "^1.20.1", "express": "^5.1.0", diff --git a/src/runner/index.ts b/src/runner/index.ts index e08c8cb..b042aeb 100644 --- a/src/runner/index.ts +++ b/src/runner/index.ts @@ -4,6 +4,53 @@ import path from 'path'; import { ConformanceCheck } from '../types.js'; import { getScenario } from '../scenarios/index.js'; +// ANSI color codes +const COLORS = { + RESET: '\x1b[0m', + GRAY: '\x1b[90m', + GREEN: '\x1b[32m', + YELLOW: '\x1b[33m', + RED: '\x1b[31m', + BLUE: '\x1b[36m' +}; + +function getStatusColor(status: string): string { + switch (status) { + case 'SUCCESS': + return COLORS.GREEN; + case 'FAILURE': + return COLORS.RED; + case 'INFO': + return COLORS.BLUE; + default: + return COLORS.RESET; + } +} + +function formatPrettyChecks(checks: ConformanceCheck[]): string { + // Find the longest id and status for column alignment + const maxIdLength = Math.max(...checks.map((c) => c.id.length)); + const maxStatusLength = Math.max(...checks.map((c) => c.status.length)); + + return checks + .map((check) => { + const timestamp = `${COLORS.GRAY}${check.timestamp}${COLORS.RESET}`; + const id = check.id.padEnd(maxIdLength); + const statusColor = getStatusColor(check.status); + const status = `${statusColor}${check.status.padEnd(maxStatusLength)}${COLORS.RESET}`; + const description = check.description; + const line = `${timestamp} [${id}] ${status} ${description}`; + // Add newline after outgoing responses for better visual separation + return ( + line + + (check.id.includes('outgoing') && check.id.includes('response') + ? '\n' + : '') + ); + }) + .join('\n'); +} + export interface ClientExecutionResult { exitCode: number; stdout: string; @@ -110,6 +157,21 @@ export async function runConformanceTest( timeout ); + // Print stdout/stderr if client exited with nonzero code + if (clientOutput.exitCode !== 0) { + console.error(`\nClient exited with code ${clientOutput.exitCode}`); + if (clientOutput.stdout) { + console.error(`\nStdout:\n${clientOutput.stdout}`); + } + if (clientOutput.stderr) { + console.error(`\nStderr:\n${clientOutput.stderr}`); + } + } + + if (clientOutput.timedOut) { + console.error(`\nClient timed out after ${timeout}ms`); + } + const checks = scenario.getChecks(); await fs.writeFile( @@ -175,6 +237,7 @@ async function main(): Promise { const args = process.argv.slice(2); let command: string | null = null; let scenario: string | null = null; + let verbose = false; for (let i = 0; i < args.length; i++) { if (args[i] === '--command' && i + 1 < args.length) { @@ -183,12 +246,14 @@ async function main(): Promise { } else if (args[i] === '--scenario' && i + 1 < args.length) { scenario = args[i + 1]; i++; + } else if (args[i] === '--verbose') { + verbose = true; } } if (!scenario) { console.error( - 'Usage: runner --scenario [--command ""]' + 'Usage: runner --scenario [--command ""] [--verbose]' ); console.error( 'Example: runner --scenario initialize --command "tsx examples/clients/typescript/test1.ts"' @@ -216,7 +281,11 @@ async function main(): Promise { const passed = result.checks.filter((c) => c.status === 'SUCCESS').length; const failed = result.checks.filter((c) => c.status === 'FAILURE').length; - console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`); + if (verbose) { + console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`); + } else { + console.log(`Checks:\n${formatPrettyChecks(result.checks)}`); + } console.log(`\nTest Results:`); console.log(`Passed: ${passed}/${denominator}, ${failed} failed`); diff --git a/src/scenarios/client/auth/basic-dcr.test.ts b/src/scenarios/client/auth/basic-dcr.test.ts new file mode 100644 index 0000000..05fba3f --- /dev/null +++ b/src/scenarios/client/auth/basic-dcr.test.ts @@ -0,0 +1,13 @@ +import { describe, test } from '@jest/globals'; +import { runClientAgainstScenario } from './helpers/testClient.js'; +import path from 'path'; + +describe('PRM Path-Based Discovery', () => { + test('client discovers PRM at path-based location before root', async () => { + const clientPath = path.join( + process.cwd(), + 'examples/clients/typescript/auth-test.ts' + ); + await runClientAgainstScenario(clientPath, 'auth/basic-dcr'); + }); +}); diff --git a/src/scenarios/client/auth/basic-dcr.ts b/src/scenarios/client/auth/basic-dcr.ts new file mode 100644 index 0000000..579df5a --- /dev/null +++ b/src/scenarios/client/auth/basic-dcr.ts @@ -0,0 +1,63 @@ +import type { Scenario, ConformanceCheck } from '../../../types.js'; +import { ScenarioUrls } from '../../../types.js'; +import { createAuthServer } from './helpers/createAuthServer.js'; +import { createServer } from './helpers/createServer.js'; +import { ServerLifecycle } from './helpers/serverLifecycle.js'; + +export class AuthBasicDCRScenario implements Scenario { + name = 'auth-basic-dcr'; + description = + 'Tests Basic OAuth flow with DCR, PRM at path-based location, OAuth metadata at root location, and no scopes required'; + private authServer = new ServerLifecycle(() => this.authBaseUrl); + private server = new ServerLifecycle(() => this.baseUrl); + private checks: ConformanceCheck[] = []; + private baseUrl: string = ''; + private authBaseUrl: string = ''; + + async start(): Promise { + this.checks = []; + + const authApp = createAuthServer(this.checks, () => this.authBaseUrl); + this.authBaseUrl = await this.authServer.start(authApp); + + const app = createServer( + this.checks, + () => this.baseUrl, + () => this.authBaseUrl + ); + this.baseUrl = await this.server.start(app); + + return { serverUrl: `${this.baseUrl}/mcp` }; + } + + async stop() { + await this.authServer.stop(); + await this.server.stop(); + } + + getChecks(): ConformanceCheck[] { + const expectedSlugs = [ + 'prm-pathbased-requested', + 'authorization-server-metadata', + 'client-registration', + 'authorization-request', + 'token-request' + ]; + + for (const slug of expectedSlugs) { + if (!this.checks.find((c) => c.id === slug)) { + this.checks.push({ + id: slug, + // TODO: these are redundant... + name: `Expected Check Missing: ${slug}`, + description: `Expected Check Missing: ${slug}`, + status: 'FAILURE', + timestamp: new Date().toISOString() + // TODO: ideally we'd add the spec references + }); + } + } + + return this.checks; + } +} diff --git a/src/scenarios/client/auth/basic-metadata-var1.test.ts b/src/scenarios/client/auth/basic-metadata-var1.test.ts new file mode 100644 index 0000000..99e3b0a --- /dev/null +++ b/src/scenarios/client/auth/basic-metadata-var1.test.ts @@ -0,0 +1,13 @@ +import { describe, test } from '@jest/globals'; +import { runClientAgainstScenario } from './helpers/testClient.js'; +import path from 'path'; + +describe('OAuth Metadata at OpenID Configuration Path', () => { + test('client discovers OAuth metadata at OpenID configuration path', async () => { + const clientPath = path.join( + process.cwd(), + 'examples/clients/typescript/auth-test.ts' + ); + await runClientAgainstScenario(clientPath, 'auth/basic-metadata-var1'); + }); +}); diff --git a/src/scenarios/client/auth/basic-metadata-var1.ts b/src/scenarios/client/auth/basic-metadata-var1.ts new file mode 100644 index 0000000..3e8043f --- /dev/null +++ b/src/scenarios/client/auth/basic-metadata-var1.ts @@ -0,0 +1,73 @@ +import type { Scenario, ConformanceCheck } from '../../../types.js'; +import { ScenarioUrls } from '../../../types.js'; +import { createAuthServer } from './helpers/createAuthServer.js'; +import { createServer } from './helpers/createServer.js'; +import { ServerLifecycle } from './helpers/serverLifecycle.js'; + +export class AuthBasicMetadataVar1Scenario implements Scenario { + // TODO: name should match what we put in the scenario map + name = 'auth-basic-metadata-var1'; + description = + 'Tests Basic OAuth flow with DCR, PRM at root location, OAuth metadata at OpenID discovery path, and no scopes required'; + private authServer = new ServerLifecycle(() => this.authBaseUrl); + private server = new ServerLifecycle(() => this.baseUrl); + private checks: ConformanceCheck[] = []; + private baseUrl: string = ''; + private authBaseUrl: string = ''; + + async start(): Promise { + this.checks = []; + + const authApp = createAuthServer(this.checks, () => this.authBaseUrl, { + metadataPath: '/.well-known/openid-configuration', + isOpenIdConfiguration: true + }); + this.authBaseUrl = await this.authServer.start(authApp); + + const app = createServer( + this.checks, + () => this.baseUrl, + () => this.authBaseUrl, + { + // TODO: this will put this path in the WWW-Authenticate header + // but RFC 9728 states that in that case, the resource in the PRM + // must match the URL used to make the request to the resource server. + // We'll need to establish an opinion on whether that means the + // URL for the metadata fetch, or the URL for the MCP endpoint, + // or more generally what are the valid scenarios / combos. + prmPath: '/.well-known/oauth-protected-resource' + } + ); + this.baseUrl = await this.server.start(app); + + return { serverUrl: `${this.baseUrl}/mcp` }; + } + + async stop() { + await this.authServer.stop(); + await this.server.stop(); + } + + getChecks(): ConformanceCheck[] { + const expectedSlugs = [ + 'authorization-server-metadata', + 'client-registration', + 'authorization-request', + 'token-request' + ]; + + for (const slug of expectedSlugs) { + if (!this.checks.find((c) => c.id === slug)) { + this.checks.push({ + id: slug, + name: `Expected Check Missing: ${slug}`, + description: `Expected Check Missing: ${slug}`, + status: 'FAILURE', + timestamp: new Date().toISOString() + }); + } + } + + return this.checks; + } +} diff --git a/src/scenarios/client/auth/helpers/createAuthServer.ts b/src/scenarios/client/auth/helpers/createAuthServer.ts new file mode 100644 index 0000000..4b0128d --- /dev/null +++ b/src/scenarios/client/auth/helpers/createAuthServer.ts @@ -0,0 +1,158 @@ +import express, { Request, Response } from 'express'; +import type { ConformanceCheck } from '../../../../types.js'; +import { createRequestLogger } from '../../../request-logger.js'; + +export interface AuthServerOptions { + metadataPath?: string; + isOpenIdConfiguration?: boolean; +} + +export function createAuthServer( + checks: ConformanceCheck[], + getAuthBaseUrl: () => string, + options: AuthServerOptions = {} +): express.Application { + const { + metadataPath = '/.well-known/oauth-authorization-server', + isOpenIdConfiguration = false + } = options; + const app = express(); + app.use(express.json()); + app.use(express.urlencoded({ extended: true })); + + app.use( + createRequestLogger(checks, { + incomingId: 'incoming-auth-request', + outgoingId: 'outgoing-auth-response' + }) + ); + + app.get(metadataPath, (req: Request, res: Response) => { + checks.push({ + id: 'authorization-server-metadata', + name: 'AuthorizationServerMetadata', + description: 'Client requested authorization server metadata', + status: 'SUCCESS', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'RFC-8414', + url: 'https://tools.ietf.org/html/rfc8414' + } + ], + details: { + url: req.url, + path: req.path + } + }); + + const metadata: any = { + issuer: getAuthBaseUrl(), + authorization_endpoint: `${getAuthBaseUrl()}/authorize`, + token_endpoint: `${getAuthBaseUrl()}/token`, + registration_endpoint: `${getAuthBaseUrl()}/register`, + response_types_supported: ['code'], + grant_types_supported: ['authorization_code', 'refresh_token'], + code_challenge_methods_supported: ['S256'], + token_endpoint_auth_methods_supported: ['none'] + }; + + // Add OpenID Configuration specific fields + if (isOpenIdConfiguration) { + metadata.jwks_uri = `${getAuthBaseUrl()}/.well-known/jwks.json`; + metadata.subject_types_supported = ['public']; + metadata.id_token_signing_alg_values_supported = ['RS256']; + } + + res.json(metadata); + }); + + app.get('/authorize', (req: Request, res: Response) => { + checks.push({ + id: 'authorization-request', + name: 'AuthorizationRequest', + description: 'Client made authorization request', + status: 'SUCCESS', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'RFC-6749-4.1.1', + url: 'https://tools.ietf.org/html/rfc6749#section-4.1.1' + } + ], + details: { + response_type: req.query.response_type, + client_id: req.query.client_id, + redirect_uri: req.query.redirect_uri, + state: req.query.state, + code_challenge: req.query.code_challenge ? 'present' : 'missing', + code_challenge_method: req.query.code_challenge_method + } + }); + + const redirectUri = req.query.redirect_uri as string; + const state = req.query.state as string; + const redirectUrl = new URL(redirectUri); + redirectUrl.searchParams.set('code', 'test-auth-code'); + if (state) { + redirectUrl.searchParams.set('state', state); + } + + res.redirect(redirectUrl.toString()); + }); + + app.post('/token', (req: Request, res: Response) => { + checks.push({ + id: 'token-request', + name: 'TokenRequest', + description: 'Client requested access token', + status: 'SUCCESS', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'RFC-6749-4.1.3', + url: 'https://tools.ietf.org/html/rfc6749#section-4.1.3' + } + ], + details: { + endpoint: '/token', + grantType: req.body.grant_type + } + }); + + res.json({ + access_token: 'test-token', + token_type: 'Bearer', + expires_in: 3600 + }); + }); + + app.post('/register', (req: Request, res: Response) => { + checks.push({ + id: 'client-registration', + name: 'ClientRegistration', + description: 'Client registered with authorization server', + status: 'SUCCESS', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'RFC-7591-2', + url: 'https://tools.ietf.org/html/rfc7591#section-2' + } + ], + details: { + endpoint: '/register', + clientName: req.body.client_name + } + }); + + res.status(201).json({ + client_id: 'test-client-id', + client_secret: 'test-client-secret', + client_name: req.body.client_name || 'test-client', + redirect_uris: req.body.redirect_uris || [] + }); + }); + + return app; +} diff --git a/src/scenarios/client/auth/helpers/createServer.ts b/src/scenarios/client/auth/helpers/createServer.ts new file mode 100644 index 0000000..a73e38d --- /dev/null +++ b/src/scenarios/client/auth/helpers/createServer.ts @@ -0,0 +1,116 @@ +import { Server } from '@modelcontextprotocol/sdk/server/index.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; +import { requireBearerAuth } from '@modelcontextprotocol/sdk/server/auth/middleware/bearerAuth.js'; +import express, { Request, Response, NextFunction } from 'express'; +import type { ConformanceCheck } from '../../../../types.js'; +import { createRequestLogger } from '../../../request-logger.js'; +import { MockTokenVerifier } from './mockTokenVerifier.js'; + +export interface ServerOptions { + prmPath?: string; +} + +export function createServer( + checks: ConformanceCheck[], + getBaseUrl: () => string, + getAuthServerUrl: () => string, + options: ServerOptions = {} +): express.Application { + const { prmPath = '/.well-known/oauth-protected-resource/mcp' } = options; + const server = new Server( + { + name: 'auth-prm-pathbased-server', + version: '1.0.0' + }, + { + capabilities: { + tools: {} + } + } + ); + + server.setRequestHandler(ListToolsRequestSchema, async () => { + return { + tools: [] + }; + }); + + const app = express(); + app.use(express.json()); + + app.use( + createRequestLogger(checks, { + incomingId: 'incoming-request', + outgoingId: 'outgoing-response', + mcpRoute: '/mcp' + }) + ); + + app.get(prmPath, (req: Request, res: Response) => { + checks.push({ + id: 'prm-pathbased-requested', + name: 'PRMPathBasedRequested', + description: 'Client requested PRM metadata at path-based location', + status: 'SUCCESS', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'RFC-9728-3', + url: 'https://tools.ietf.org/html/rfc9728#section-3' + } + ], + details: { + url: req.url, + path: req.path + } + }); + + res.json({ + resource: getBaseUrl(), + authorization_servers: [getAuthServerUrl()] + }); + }); + + app.post('/mcp', async (req: Request, res: Response, next: NextFunction) => { + // Apply bearer token auth per-request in order to delay setting PRM URL + // until after the server has started + // TODO: Find a way to do this w/ pre-applying middleware. + const authMiddleware = requireBearerAuth({ + verifier: new MockTokenVerifier(checks), + requiredScopes: [], + resourceMetadataUrl: `${getBaseUrl()}${prmPath}` + }); + + authMiddleware(req, res, async (err?: any) => { + if (err) return next(err); + const transport = new StreamableHTTPServerTransport({ + sessionIdGenerator: undefined + }); + + try { + await server.connect(transport); + + await transport.handleRequest(req, res, req.body); + res.on('close', () => { + transport.close(); + server.close(); + }); + } catch (error) { + console.error('Error handling MCP request:', error); + if (!res.headersSent) { + res.status(500).json({ + jsonrpc: '2.0', + error: { + code: -32603, + message: 'Internal server error' + }, + id: null + }); + } + } + }); + }); + + return app; +} diff --git a/src/scenarios/client/auth/helpers/mockTokenVerifier.ts b/src/scenarios/client/auth/helpers/mockTokenVerifier.ts new file mode 100644 index 0000000..251d620 --- /dev/null +++ b/src/scenarios/client/auth/helpers/mockTokenVerifier.ts @@ -0,0 +1,53 @@ +import { OAuthTokenVerifier } from '@modelcontextprotocol/sdk/server/auth/provider.js'; +import { AuthInfo } from '@modelcontextprotocol/sdk/server/auth/types.js'; +import type { ConformanceCheck } from '../../../../types.js'; + +export class MockTokenVerifier implements OAuthTokenVerifier { + constructor(private checks: ConformanceCheck[]) {} + + async verifyAccessToken(token: string): Promise { + if (token === 'test-token') { + this.checks.push({ + id: 'valid-bearer-token', + name: 'ValidBearerToken', + description: 'Client provided valid bearer token', + status: 'SUCCESS', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'MCP-Authorization', + url: 'https://spec.modelcontextprotocol.io/specification/architecture/#authorization' + } + ], + details: { + token: token.substring(0, 10) + '...' + } + }); + return { + token, + clientId: 'test-client', + scopes: [], + expiresAt: Math.floor(Date.now() / 1000) + 3600 + }; + } + + this.checks.push({ + id: 'invalid-bearer-token', + name: 'InvalidBearerToken', + description: 'Client provided invalid bearer token', + status: 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: [ + { + id: 'MCP-Authorization', + url: 'https://spec.modelcontextprotocol.io/specification/architecture/#authorization' + } + ], + details: { + message: 'Token verification failed', + token: token ? token.substring(0, 10) + '...' : 'missing' + } + }); + throw new Error('Invalid token'); + } +} diff --git a/src/scenarios/client/auth/helpers/serverLifecycle.ts b/src/scenarios/client/auth/helpers/serverLifecycle.ts new file mode 100644 index 0000000..74ca986 --- /dev/null +++ b/src/scenarios/client/auth/helpers/serverLifecycle.ts @@ -0,0 +1,32 @@ +import express from 'express'; + +export class ServerLifecycle { + private app: express.Application | null = null; + private httpServer: any = null; + private baseUrl: string = ''; + + constructor(private getBaseUrl: () => string) {} + + async start(app: express.Application): Promise { + this.app = app; + this.httpServer = this.app.listen(0); + const port = this.httpServer.address().port; + this.baseUrl = `http://localhost:${port}`; + return this.baseUrl; + } + + async stop(): Promise { + if (this.httpServer) { + await new Promise((resolve) => { + this.httpServer.closeAllConnections?.(); + this.httpServer.close(() => resolve()); + }); + this.httpServer = null; + } + this.app = null; + } + + getUrl(): string { + return this.baseUrl; + } +} diff --git a/src/scenarios/client/auth/helpers/testClient.ts b/src/scenarios/client/auth/helpers/testClient.ts new file mode 100644 index 0000000..6fa7f50 --- /dev/null +++ b/src/scenarios/client/auth/helpers/testClient.ts @@ -0,0 +1,123 @@ +import { getScenario } from '../../../../scenarios/index.js'; +import { spawn } from 'child_process'; + +const CLIENT_TIMEOUT = 10000; // 10 seconds for client to complete + +export async function runClientAgainstScenario( + clientPath: string, + scenarioName: string, + expectedFailureSlugs: string[] = [] +): Promise { + const scenario = getScenario(scenarioName); + if (!scenario) { + throw new Error(`Scenario ${scenarioName} not found`); + } + + // Start the scenario server + const urls = await scenario.start(); + const serverUrl = urls.serverUrl; + + try { + // Run the client + await new Promise((resolve, reject) => { + const clientProcess = spawn('npx', ['tsx', clientPath, serverUrl], { + stdio: ['ignore', 'pipe', 'pipe'] + }); + + let stdout = ''; + let stderr = ''; + + clientProcess.stdout?.on('data', (data) => { + stdout += data.toString(); + }); + + clientProcess.stderr?.on('data', (data) => { + stderr += data.toString(); + }); + + const timeout = setTimeout(() => { + clientProcess.kill('SIGTERM'); + reject( + new Error( + `Client failed to complete within ${CLIENT_TIMEOUT}ms\nStdout: ${stdout}\nStderr: ${stderr}` + ) + ); + }, CLIENT_TIMEOUT); + + clientProcess.on('exit', (code) => { + clearTimeout(timeout); + if (code === 0) { + resolve(); + } else { + reject( + new Error( + `Client exited with code ${code}\nStdout: ${stdout}\nStderr: ${stderr}` + ) + ); + } + }); + + clientProcess.on('error', (error) => { + clearTimeout(timeout); + reject( + new Error( + `Failed to start client: ${error.message}\nStdout: ${stdout}\nStderr: ${stderr}` + ) + ); + }); + }); + + // Get checks from the scenario + const checks = scenario.getChecks(); + + // Verify checks were returned + if (checks.length === 0) { + throw new Error('No checks returned from scenario'); + } + + // Filter out INFO checks + const nonInfoChecks = checks.filter((c) => c.status !== 'INFO'); + + // Check for expected failures + if (expectedFailureSlugs.length > 0) { + // Verify that the expected failures are present + for (const slug of expectedFailureSlugs) { + const check = checks.find((c) => c.id === slug); + if (!check) { + throw new Error(`Expected failure check ${slug} not found`); + } + } + + // Verify that only the expected checks failed + const failures = nonInfoChecks.filter((c) => c.status === 'FAILURE'); + const failureSlugs = failures.map((c) => c.id); + if ( + failureSlugs.sort().join(',') !== expectedFailureSlugs.sort().join(',') + ) { + throw new Error( + `Expected failures ${expectedFailureSlugs.sort().join(', ')} but got ${failureSlugs.sort().join(', ')}` + ); + } + } else { + // Default: expect all checks to pass + const failures = nonInfoChecks.filter((c) => c.status === 'FAILURE'); + if (failures.length > 0) { + const failureMessages = failures + .map((c) => `${c.name}: ${c.errorMessage || c.description}`) + .join('\n '); + throw new Error(`Scenario failed with checks:\n ${failureMessages}`); + } + + // All non-INFO checks should be SUCCESS + const successes = nonInfoChecks.filter((c) => c.status === 'SUCCESS'); + if (successes.length !== nonInfoChecks.length) { + throw new Error( + `Expected all checks to pass but got ${successes.length}/${nonInfoChecks.length}` + ); + } + } + } finally { + // Stop the scenario server + await scenario.stop(); + } +} diff --git a/src/scenarios/client/tools_call.ts b/src/scenarios/client/tools_call.ts index 3ca1f60..7e3a298 100644 --- a/src/scenarios/client/tools_call.ts +++ b/src/scenarios/client/tools_call.ts @@ -5,8 +5,9 @@ import { ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js'; import type { Scenario, ConformanceCheck } from '../../types.js'; -import express, { Request, Response, NextFunction } from 'express'; +import express, { Request, Response } from 'express'; import { ScenarioUrls } from '../../types.js'; +import { createRequestLogger } from '../request-logger.js'; function createServer(checks: ConformanceCheck[]): express.Application { const server = new Server( @@ -86,33 +87,13 @@ function createServer(checks: ConformanceCheck[]): express.Application { const app = express(); app.use(express.json()); - app.use((req: Request, res: Response, next: NextFunction) => { - // Log incoming requests for debugging - // console.log(`Incoming request: ${req.method} ${req.url}`); - checks.push({ - id: 'incoming-request', - name: 'IncomingRequest', - description: `Received ${req.method} request for ${req.url}`, - status: 'INFO', - timestamp: new Date().toISOString(), - details: { - body: JSON.stringify(req.body) - } - }); - next(); - checks.push({ - id: 'outgoing-response', - name: 'OutgoingResponse', - // TODO: include MCP method? - description: `Sent ${res.statusCode} response`, - status: 'INFO', - timestamp: new Date().toISOString(), - details: { - // TODO: Response body not available in express middleware - statusCode: res.statusCode - } - }); - }); + app.use( + createRequestLogger(checks, { + incomingId: 'incoming-request', + outgoingId: 'outgoing-response', + mcpRoute: '/mcp' + }) + ); app.post('/mcp', async (req: Request, res: Response) => { const transport = new StreamableHTTPServerTransport({ diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index a546d86..409c18d 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -1,6 +1,8 @@ import { Scenario, ClientScenario } from '../types.js'; import { InitializeScenario } from './client/initialize.js'; import { ToolsCallScenario } from './client/tools_call.js'; +import { AuthBasicDCRScenario } from './client/auth/basic-dcr.js'; +import { AuthBasicMetadataVar1Scenario } from './client/auth/basic-metadata-var1.js'; // Import all new server test scenarios import { ServerInitializeScenario } from './server/lifecycle.js'; @@ -43,7 +45,9 @@ import { export const scenarios = new Map([ ['initialize', new InitializeScenario()], - ['tools-call', new ToolsCallScenario()] + ['tools-call', new ToolsCallScenario()], + ['auth/basic-dcr', new AuthBasicDCRScenario()], + ['auth/basic-metadata-var1', new AuthBasicMetadataVar1Scenario()] ]); export const clientScenarios = new Map([ diff --git a/src/scenarios/request-logger.ts b/src/scenarios/request-logger.ts new file mode 100644 index 0000000..742db4e --- /dev/null +++ b/src/scenarios/request-logger.ts @@ -0,0 +1,115 @@ +import { Request, Response, NextFunction } from 'express'; +import { ConformanceCheck } from '../types.js'; + +export interface LoggerOptions { + incomingId: string; + outgoingId: string; + mcpRoute?: string; +} + +export function createRequestLogger( + checks: ConformanceCheck[], + options: LoggerOptions +) { + return (req: Request, res: Response, next: NextFunction) => { + // Log incoming request + let requestDescription = `Received ${req.method} request for ${req.path}`; + const requestDetails: any = { + method: req.method, + path: req.path + }; + + // Add query parameters to details if they exist + if (Object.keys(req.query).length > 0) { + requestDetails.query = req.query; + } + + // Extract MCP method if this is the MCP route + if ( + options.mcpRoute && + req.path === options.mcpRoute && + req.get('content-type')?.includes('application/json') && + req.body && + typeof req.body === 'object' && + req.body.method + ) { + const mcpMethod = req.body.method; + requestDescription += ` (method: ${mcpMethod})`; + requestDetails.mcpMethod = mcpMethod; + } + + checks.push({ + id: options.incomingId, + name: + options.incomingId.charAt(0).toUpperCase() + + options.incomingId.slice(1), + description: requestDescription, + status: 'INFO', + timestamp: new Date().toISOString(), + details: requestDetails + }); + + // Capture response body + const oldWrite = res.write.bind(res); + const oldEnd = res.end.bind(res); + const chunks: (Buffer | string)[] = []; + + (res.write as any) = function (chunk: any, ...args: any[]) { + chunks.push(chunk); + return oldWrite(chunk, ...args); + }; + + (res.end as any) = function (chunk?: any, ...args: any[]) { + if (chunk) { + chunks.push(chunk); + } + + const buffers = chunks.map((c) => + typeof c === 'string' ? Buffer.from(c) : c + ); + const body = Buffer.concat(buffers).toString('utf8'); + let responseDescription = `Sent ${res.statusCode} response for ${req.method} ${req.path}`; + const responseDetails: any = { + method: req.method, + path: req.path, + statusCode: res.statusCode + }; + + // Include MCP method in response log if present + if (requestDetails.mcpMethod) { + responseDescription += ` (method: ${requestDetails.mcpMethod})`; + responseDetails.mcpMethod = requestDetails.mcpMethod; + } + + // Add response headers + const headers = res.getHeaders(); + if (Object.keys(headers).length > 0) { + responseDetails.headers = headers; + } + + // Add response body if available + if (body) { + try { + responseDetails.body = JSON.parse(body); + } catch { + responseDetails.body = body; + } + } + + checks.push({ + id: options.outgoingId, + name: + options.outgoingId.charAt(0).toUpperCase() + + options.outgoingId.slice(1), + description: responseDescription, + status: 'INFO', + timestamp: new Date().toISOString(), + details: responseDetails + }); + + return oldEnd(chunk, ...args); + }; + + next(); + }; +}