|
| 1 | +import { spawn } from 'child_process'; |
| 2 | +import { promises as fs } from 'fs'; |
| 3 | +import path from 'path'; |
| 4 | +import { ConformanceCheck } from '../types.js'; |
| 5 | +import { getScenario } from '../scenarios/index.js'; |
| 6 | + |
| 7 | +export interface ClientExecutionResult { |
| 8 | + exitCode: number; |
| 9 | + stdout: string; |
| 10 | + stderr: string; |
| 11 | + timedOut: boolean; |
| 12 | +} |
| 13 | + |
| 14 | +async function ensureResultsDir(): Promise<string> { |
| 15 | + const resultsDir = path.join(process.cwd(), 'results'); |
| 16 | + await fs.mkdir(resultsDir, { recursive: true }); |
| 17 | + return resultsDir; |
| 18 | +} |
| 19 | + |
| 20 | +function createResultDir(scenario: string): string { |
| 21 | + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); |
| 22 | + return path.join('results', `${scenario}-${timestamp}`); |
| 23 | +} |
| 24 | + |
| 25 | +async function executeClient( |
| 26 | + command: string, |
| 27 | + serverUrl: string, |
| 28 | + timeout: number = 30000 |
| 29 | +): Promise<ClientExecutionResult> { |
| 30 | + const commandParts = command.split(' '); |
| 31 | + const executable = commandParts[0]; |
| 32 | + const args = [...commandParts.slice(1), serverUrl]; |
| 33 | + |
| 34 | + let stdout = ''; |
| 35 | + let stderr = ''; |
| 36 | + let timedOut = false; |
| 37 | + |
| 38 | + return new Promise((resolve) => { |
| 39 | + const process = spawn(executable, args, { |
| 40 | + shell: true, |
| 41 | + stdio: 'pipe' |
| 42 | + }); |
| 43 | + |
| 44 | + const timeoutHandle = setTimeout(() => { |
| 45 | + timedOut = true; |
| 46 | + process.kill(); |
| 47 | + }, timeout); |
| 48 | + |
| 49 | + if (process.stdout) { |
| 50 | + process.stdout.on('data', (data) => { |
| 51 | + stdout += data.toString(); |
| 52 | + }); |
| 53 | + } |
| 54 | + |
| 55 | + if (process.stderr) { |
| 56 | + process.stderr.on('data', (data) => { |
| 57 | + stderr += data.toString(); |
| 58 | + }); |
| 59 | + } |
| 60 | + |
| 61 | + process.on('close', (code) => { |
| 62 | + clearTimeout(timeoutHandle); |
| 63 | + resolve({ |
| 64 | + exitCode: code || 0, |
| 65 | + stdout, |
| 66 | + stderr, |
| 67 | + timedOut |
| 68 | + }); |
| 69 | + }); |
| 70 | + |
| 71 | + process.on('error', (error) => { |
| 72 | + clearTimeout(timeoutHandle); |
| 73 | + resolve({ |
| 74 | + exitCode: -1, |
| 75 | + stdout, |
| 76 | + stderr: stderr + `\nProcess error: ${error.message}`, |
| 77 | + timedOut |
| 78 | + }); |
| 79 | + }); |
| 80 | + }); |
| 81 | +} |
| 82 | + |
| 83 | +export async function runConformanceTest( |
| 84 | + clientCommand: string, |
| 85 | + scenarioName: string, |
| 86 | + timeout: number = 30000 |
| 87 | +): Promise<{ |
| 88 | + checks: ConformanceCheck[]; |
| 89 | + clientOutput: ClientExecutionResult; |
| 90 | + resultDir: string; |
| 91 | +}> { |
| 92 | + await ensureResultsDir(); |
| 93 | + const resultDir = createResultDir(scenarioName); |
| 94 | + await fs.mkdir(resultDir, { recursive: true }); |
| 95 | + |
| 96 | + const scenario = getScenario(scenarioName); |
| 97 | + if (!scenario) { |
| 98 | + throw new Error(`Unknown scenario: ${scenarioName}`); |
| 99 | + } |
| 100 | + |
| 101 | + console.log(`Starting scenario: ${scenarioName}`); |
| 102 | + const urls = await scenario.start(); |
| 103 | + |
| 104 | + console.log(`Executing client: ${clientCommand} ${urls.serverUrl}`); |
| 105 | + |
| 106 | + try { |
| 107 | + const clientOutput = await executeClient(clientCommand, urls.serverUrl, timeout); |
| 108 | + |
| 109 | + const checks = scenario.getChecks(); |
| 110 | + |
| 111 | + await fs.writeFile( |
| 112 | + path.join(resultDir, 'checks.json'), |
| 113 | + JSON.stringify(checks, null, 2) |
| 114 | + ); |
| 115 | + |
| 116 | + await fs.writeFile( |
| 117 | + path.join(resultDir, 'stdout.txt'), |
| 118 | + clientOutput.stdout |
| 119 | + ); |
| 120 | + |
| 121 | + await fs.writeFile( |
| 122 | + path.join(resultDir, 'stderr.txt'), |
| 123 | + clientOutput.stderr |
| 124 | + ); |
| 125 | + |
| 126 | + console.log(`Results saved to ${resultDir}`); |
| 127 | + |
| 128 | + return { |
| 129 | + checks, |
| 130 | + clientOutput, |
| 131 | + resultDir |
| 132 | + }; |
| 133 | + } finally { |
| 134 | + await scenario.stop(); |
| 135 | + } |
| 136 | +} |
| 137 | + |
| 138 | +async function main(): Promise<void> { |
| 139 | + const args = process.argv.slice(2); |
| 140 | + let command: string | null = null; |
| 141 | + let scenario: string | null = null; |
| 142 | + |
| 143 | + for (let i = 0; i < args.length; i++) { |
| 144 | + if (args[i] === '--command' && i + 1 < args.length) { |
| 145 | + command = args[i + 1]; |
| 146 | + i++; |
| 147 | + } else if (args[i] === '--scenario' && i + 1 < args.length) { |
| 148 | + scenario = args[i + 1]; |
| 149 | + i++; |
| 150 | + } |
| 151 | + } |
| 152 | + |
| 153 | + if (!command || !scenario) { |
| 154 | + console.error('Usage: runner --command "<command>" --scenario <scenario>'); |
| 155 | + console.error('Example: runner --command "tsx examples/clients/typescript/test1.ts" --scenario initialize'); |
| 156 | + process.exit(1); |
| 157 | + } |
| 158 | + |
| 159 | + try { |
| 160 | + const result = await runConformanceTest(command, scenario); |
| 161 | + |
| 162 | + const passed = result.checks.filter(c => c.status === 'SUCCESS').length; |
| 163 | + const failed = result.checks.filter(c => c.status === 'FAILURE').length; |
| 164 | + |
| 165 | + console.log(`\nTest Results:`); |
| 166 | + console.log(`Passed: ${passed}/${result.checks.length}`); |
| 167 | + console.log(`Failed: ${failed}/${result.checks.length}`); |
| 168 | + |
| 169 | + if (failed > 0) { |
| 170 | + console.log('\nFailed Checks:'); |
| 171 | + result.checks |
| 172 | + .filter(c => c.status === 'FAILURE') |
| 173 | + .forEach(c => { |
| 174 | + console.log(` - ${c.name}: ${c.description}`); |
| 175 | + if (c.errorMessage) { |
| 176 | + console.log(` Error: ${c.errorMessage}`); |
| 177 | + } |
| 178 | + }); |
| 179 | + } |
| 180 | + |
| 181 | + process.exit(failed > 0 ? 1 : 0); |
| 182 | + } catch (error) { |
| 183 | + console.error('Test runner error:', error); |
| 184 | + process.exit(1); |
| 185 | + } |
| 186 | +} |
| 187 | + |
| 188 | +main(); |
0 commit comments