Add server conformance testing (#1375)

Author: pcarleton

.github/workflows/conformance.yml

@@ -6,13 +6,17 @@ on:
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: conformance-${{ github.ref }}
+  cancel-in-progress: true
+
 permissions:
   contents: read
 
 jobs:
   client-conformance:
     runs-on: ubuntu-latest
-    continue-on-error: true  # Non-blocking initially
+    continue-on-error: true
     steps:
       - uses: actions/checkout@v4
       - name: Install pnpm
@@ -27,3 +31,21 @@ jobs:
       - run: pnpm install
       - run: pnpm run build:all
       - run: pnpm run test:conformance:client:all
+
+  server-conformance:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          run_install: false
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: pnpm
+          cache-dependency-path: pnpm-lock.yaml
+      - run: pnpm install
+      - run: pnpm run build:all
+      - run: pnpm run test:conformance:server

.gitignore

@@ -133,3 +133,6 @@ dist/
 
 # IDE
 .idea/
+
+# Conformance test results
+results/

package.json

@@ -32,7 +32,11 @@
         "test:all": "pnpm -r test",
         "test:conformance:client": "conformance client --command 'npx tsx src/conformance/everything-client.ts'",
         "test:conformance:client:all": "conformance client --command 'npx tsx src/conformance/everything-client.ts' --suite all",
-        "test:conformance:client:run": "npx tsx src/conformance/everything-client.ts"
+        "test:conformance:client:run": "npx tsx src/conformance/everything-client.ts",
+        "test:conformance:server": "scripts/run-server-conformance.sh",
+        "test:conformance:server:all": "scripts/run-server-conformance.sh --suite all",
+        "test:conformance:server:run": "npx tsx src/conformance/everything-server.ts",
+        "test:conformance:all": "pnpm run test:conformance:client:all && pnpm run test:conformance:server:all"
     },
     "devDependencies": {
         "@cfworker/json-schema": "catalog:runtimeShared",
@@ -41,6 +45,7 @@
         "@eslint/js": "catalog:devTools",
         "@modelcontextprotocol/client": "workspace:^",
         "@modelcontextprotocol/conformance": "0.1.9",
+        "@modelcontextprotocol/server": "workspace:^",
         "@types/content-type": "catalog:devTools",
         "@types/cors": "catalog:devTools",
         "@types/cross-spawn": "catalog:devTools",
@@ -51,9 +56,11 @@
         "@types/supertest": "catalog:devTools",
         "@types/ws": "catalog:devTools",
         "@typescript/native-preview": "catalog:devTools",
+        "cors": "catalog:runtimeServerOnly",
         "eslint": "catalog:devTools",
         "eslint-config-prettier": "catalog:devTools",
         "eslint-plugin-n": "catalog:devTools",
+        "express": "catalog:runtimeServerOnly",
         "prettier": "catalog:devTools",
         "supertest": "catalog:devTools",
         "tsdown": "catalog:devTools",

pnpm-lock.yaml

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Script to run server conformance tests
+# Starts the conformance server, runs conformance tests, then stops the server
+
+set -e
+
+PORT="${PORT:-3000}"
+SERVER_URL="http://localhost:${PORT}/mcp"
+
+# Navigate to the repo root
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR/.."
+
+# Start the server in the background
+echo "Starting conformance test server on port ${PORT}..."
+npx tsx src/conformance/everything-server.ts &
+SERVER_PID=$!
+
+# Function to cleanup on exit
+cleanup() {
+    echo "Stopping server (PID: ${SERVER_PID})..."
+    kill $SERVER_PID 2>/dev/null || true
+    wait $SERVER_PID 2>/dev/null || true
+}
+trap cleanup EXIT
+
+# Wait for server to be ready
+echo "Waiting for server to be ready..."
+MAX_RETRIES=30
+RETRY_COUNT=0
+while ! curl -s "${SERVER_URL}" > /dev/null 2>&1; do
+    RETRY_COUNT=$((RETRY_COUNT + 1))
+    if [ $RETRY_COUNT -ge $MAX_RETRIES ]; then
+        echo "Server failed to start after ${MAX_RETRIES} attempts"
+        exit 1
+    fi
+    sleep 0.5
+done
+
+echo "Server is ready. Running conformance tests..."
+
+# Run conformance tests - pass through all arguments
+npx @modelcontextprotocol/conformance server --url "${SERVER_URL}" "$@"
+
+echo "Conformance tests completed."

scripts/run-server-conformance.sh

@@ -0,0 +1,64 @@
+# Conformance Tests
+
+This directory contains conformance test implementations for the TypeScript MCP SDK.
+
+## Client Conformance Tests
+
+Tests the SDK's client implementation against a conformance test server.
+
+```bash
+# Run all client tests
+pnpm run test:conformance:client:all
+
+# Run specific suite
+pnpm run test:conformance:client -- --suite auth
+
+# Run single scenario
+pnpm run test:conformance:client -- --scenario auth/basic-cimd
+```
+
+## Server Conformance Tests
+
+Tests the SDK's server implementation by running a conformance server.
+
+```bash
+# Run all active server tests
+pnpm run test:conformance:server
+
+# Run all server tests (including pending)
+pnpm run test:conformance:server:all
+```
+
+## Local Development
+
+### Running Tests Against Local Conformance Repo
+
+Link the local conformance package:
+```bash
+cd ~/code/mcp/typescript-sdk
+pnpm link ~/code/mcp/conformance
+```
+
+Then run tests as above.
+
+### Debugging Server Tests
+
+Start the server manually:
+```bash
+pnpm run test:conformance:server:run
+```
+
+In another terminal, run specific tests:
+```bash
+npx @modelcontextprotocol/conformance server \
+  --url http://localhost:3000/mcp \
+  --scenario server-initialize
+```
+
+## Files
+
+- `everything-client.ts` - Client that handles all client conformance scenarios
+- `everything-server.ts` - Server that implements all server conformance features
+- `helpers/` - Shared utilities for conformance tests
+
+Scripts are in `scripts/` at the repo root.