diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.ts new file mode 100644 index 000000000..6a9fcec4b --- /dev/null +++ b/packages/cli/src/commands/mcp.ts @@ -0,0 +1,168 @@ +import { AuthCommand } from './authCommand' +import { Flags } from '@oclif/core' +import * as fs from 'node:fs/promises' +import * as path from 'node:path' +import * as os from 'node:os' +import { spawn } from 'node:child_process' +import { EXAMPLE_SPEC_TS, CHECKLY_CONFIG_TS, TSCONFIG_JSON, PACKAGE_JSON } from '../templates/packed-files' + +// Default playwright.config.ts since it's not in packed-files +const PLAYWRIGHT_CONFIG_TS = `import { defineConfig, devices } from '@playwright/test'; + +export default defineConfig({ + testDir: './', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, + reporter: 'html', + use: { + trace: 'on-first-retry', + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +});` + +export default class McpCommand extends AuthCommand { + static coreCommand = true + static hidden = false + static description = 'Run MCP server tests with Playwright on Checkly.' + static state = 'beta' + + static flags = { + verbose: Flags.boolean({ + char: 'v', + description: 'Show all logs, errors, and stdout from processes', + default: false, + }), + } + + async run(): Promise { + const { flags } = await this.parse(McpCommand) + const { verbose } = flags + + let tempDir: string | null = null + let pinggyUrl: string | null = null + // Output initial "in progress" message + this.log('starting mcp server in progress (this can take a while)..') + this.log('it will run for 20 minutes and then exit automatically') + try { + // Create temporary directory + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-mcp-')) + + // Write all files to temp directory + await fs.writeFile(path.join(tempDir, 'example.spec.ts'), EXAMPLE_SPEC_TS) + await fs.writeFile(path.join(tempDir, 'checkly.config.ts'), CHECKLY_CONFIG_TS) + await fs.writeFile(path.join(tempDir, 'tsconfig.json'), TSCONFIG_JSON) + await fs.writeFile(path.join(tempDir, 'package.json'), PACKAGE_JSON) + await fs.writeFile(path.join(tempDir, 'playwright.config.ts'), PLAYWRIGHT_CONFIG_TS) + + // Run npm install in the temp directory + const { execa } = await import('execa') + if (verbose) { + this.log('Installing dependencies...') + } + await execa('npm', ['install'], { + cwd: tempDir, + stdio: verbose ? 'inherit' : 'pipe' // Show output in verbose mode + }) + + + + // Save current directory + const originalCwd = process.cwd() + + // Change to temp directory + process.chdir(tempDir) + + try { + // Run pw-test command with stream-logs enabled using spawn to capture output + // Use the same node and checkly command that was used to run this command + const nodeExecutable = process.argv[0] + const checklyExecutable = process.argv[1] + const args = [checklyExecutable, 'pw-test', '--stream-logs'] + + const pwTestProcess = spawn(nodeExecutable, args, { + cwd: tempDir, + env: { ...process.env }, + stdio: ['inherit', 'pipe', 'pipe'] + }) + + // Capture stdout + pwTestProcess.stdout?.on('data', (data) => { + const output = data.toString() + + // In verbose mode, show all output + if (verbose) { + process.stdout.write(output) + } + + // Parse for Pinggy URL + const urlMatch = output.match(/🌐 PINGGY PUBLIC URL:\s*(https:\/\/[a-z0-9-]+\.a\.free\.pinggy\.link)/) + if (urlMatch && !pinggyUrl) { + pinggyUrl = urlMatch[1] + // Output JSON immediately when URL is found + this.log(JSON.stringify({ serverUrl: pinggyUrl })) + } + }) + + // Capture stderr as well + pwTestProcess.stderr?.on('data', (data) => { + const output = data.toString() + + // In verbose mode, show all error output + if (verbose) { + process.stderr.write(output) + } + + // Also check stderr for URLs (pinggy might output there) + const urlMatch = output.match(/🌐 PINGGY PUBLIC URL:\s*(https:\/\/[a-z0-9-]+\.a\.free\.pinggy\.link)/) + if (urlMatch && !pinggyUrl) { + pinggyUrl = urlMatch[1] + // Output JSON immediately when URL is found + this.log(JSON.stringify({ serverUrl: pinggyUrl })) + } + }) + + // Wait for process to complete + await new Promise((resolve, reject) => { + pwTestProcess.on('close', (code) => { + if (code === 0) { + resolve() + } else { + reject(new Error(`Process exited with code ${code}`)) + } + }) + + pwTestProcess.on('error', (error) => { + reject(error) + }) + }) + + } finally { + // Always restore original directory + process.chdir(originalCwd) + } + + } catch (error) { + // Show errors in verbose mode + if (verbose && error instanceof Error) { + this.error(error.message) + } + process.exit(1) + } finally { + // Clean up temp directory + if (tempDir) { + try { + await fs.rm(tempDir, { recursive: true, force: true }) + } catch (cleanupError) { + // Suppress cleanup errors + } + } + } + } +} \ No newline at end of file diff --git a/packages/cli/src/commands/pw-test.ts b/packages/cli/src/commands/pw-test.ts index 090efbb62..56bfcbb94 100644 --- a/packages/cli/src/commands/pw-test.ts +++ b/packages/cli/src/commands/pw-test.ts @@ -83,6 +83,10 @@ export default class PwTestCommand extends AuthCommand { 'create-check': Flags.boolean({ description: 'Create a Checkly check from the Playwright test.', default: false, + }), + 'stream-logs': Flags.boolean({ + description: 'Stream logs from the test run to the console.', + default: false, }) } @@ -104,6 +108,7 @@ export default class PwTestCommand extends AuthCommand { record, 'test-session-name': testSessionName, 'create-check': createCheck, + 'stream-logs': streamLogs, } = flags const { configDirectory, configFilenames } = splitConfigFilePath(configFilename) const { @@ -206,6 +211,10 @@ export default class PwTestCommand extends AuthCommand { const checkBundles = Object.values(projectBundle.data.check) + if (this.fancy) { + ux.action.stop() + } + if (!checkBundles.length) { this.log(`Unable to find checks to run`) return @@ -233,6 +242,7 @@ export default class PwTestCommand extends AuthCommand { configDirectory, // TODO: ADD PROPER RETRY STRATEGY HANDLING null, // testRetryStrategy + streamLogs, ) runner.on(Events.RUN_STARTED, @@ -284,6 +294,9 @@ export default class PwTestCommand extends AuthCommand { reporters.forEach(r => r.onError(err)) process.exitCode = 1 }) + runner.on(Events.STREAM_LOGS, (check: any, sequenceId: SequenceId, logs) => { + reporters.forEach(r => r.onStreamLogs(check, sequenceId, logs)) + }) await runner.run() } @@ -294,6 +307,7 @@ export default class PwTestCommand extends AuthCommand { } return arg }) + const input = parseArgs.join(' ') || '' const inputLogicalId = cased(input, 'kebab-case').substring(0, 50) const testCommand = await PwTestCommand.getTestCommand(dir, input) diff --git a/packages/cli/src/reporters/abstract-list.ts b/packages/cli/src/reporters/abstract-list.ts index c9edae8b1..ed6f77699 100644 --- a/packages/cli/src/reporters/abstract-list.ts +++ b/packages/cli/src/reporters/abstract-list.ts @@ -20,6 +20,7 @@ export type checkFilesMap = Map> export default abstract class AbstractListReporter implements Reporter { @@ -96,6 +97,17 @@ export default abstract class AbstractListReporter implements Reporter { printLn(chalk.red('Unable to run checks: ') + err.message) } + onStreamLogs (check: any, sequenceId: SequenceId, logs: string[] | undefined) { + const checkFile = this.checkFilesMap!.get(check.getSourceFile?.())!.get(sequenceId)! + const logList = logs || [] + if (!checkFile.logs) { + checkFile.logs = [] + } + // checkFile.logs.push(...logs.logs) + logList.forEach((log: string) => printLn(log, 2)) + return + } + // Clear the summary which was printed by _printStatus from stdout // TODO: Rather than clearing the whole status bar, we could overwrite the exact lines that changed. // This might look a bit smoother and reduce the flickering effects. diff --git a/packages/cli/src/reporters/reporter.ts b/packages/cli/src/reporters/reporter.ts index c3b32a65e..7a0bb218b 100644 --- a/packages/cli/src/reporters/reporter.ts +++ b/packages/cli/src/reporters/reporter.ts @@ -14,6 +14,7 @@ export interface Reporter { onEnd(): void; onError(err: Error): void, onSchedulingDelayExceeded(): void + onStreamLogs(check: any, sequenceId: SequenceId, logs: string[]): void } export type ReporterType = 'list' | 'dot' | 'ci' | 'github' | 'json' diff --git a/packages/cli/src/rest/test-sessions.ts b/packages/cli/src/rest/test-sessions.ts index 479f09dd4..94b94e00c 100644 --- a/packages/cli/src/rest/test-sessions.ts +++ b/packages/cli/src/rest/test-sessions.ts @@ -12,6 +12,7 @@ type RunTestSessionRequest = { repoInfo?: GitInformation | null, environment?: string | null, shouldRecord: boolean, + streamLogs?: boolean, } type TriggerTestSessionRequest = { diff --git a/packages/cli/src/services/abstract-check-runner.ts b/packages/cli/src/services/abstract-check-runner.ts index 73ee504cd..1f84b7eb4 100644 --- a/packages/cli/src/services/abstract-check-runner.ts +++ b/packages/cli/src/services/abstract-check-runner.ts @@ -18,7 +18,8 @@ export enum Events { RUN_STARTED = 'RUN_STARTED', RUN_FINISHED = 'RUN_FINISHED', ERROR = 'ERROR', - MAX_SCHEDULING_DELAY_EXCEEDED = 'MAX_SCHEDULING_DELAY_EXCEEDED' + MAX_SCHEDULING_DELAY_EXCEEDED = 'MAX_SCHEDULING_DELAY_EXCEEDED', + STREAM_LOGS = 'STREAM_LOGS', } export type PrivateRunLocation = { @@ -160,6 +161,10 @@ export default abstract class AbstractCheckRunner extends EventEmitter { this.disableTimeout(sequenceId) this.emit(Events.CHECK_FAILED, sequenceId, check, message) this.emit(Events.CHECK_FINISHED, check) + } else if (subtopic === 'stream-logs') { + const { logs } = message + this.emit(Events.STREAM_LOGS, check, sequenceId, logs) + } } diff --git a/packages/cli/src/services/test-runner.ts b/packages/cli/src/services/test-runner.ts index c12ed1836..a6140f6ca 100644 --- a/packages/cli/src/services/test-runner.ts +++ b/packages/cli/src/services/test-runner.ts @@ -21,6 +21,7 @@ export default class TestRunner extends AbstractCheckRunner { updateSnapshots: boolean baseDirectory: string testRetryStrategy: RetryStrategy | null + streamLogs?: boolean constructor ( accountId: string, @@ -36,6 +37,7 @@ export default class TestRunner extends AbstractCheckRunner { updateSnapshots: boolean, baseDirectory: string, testRetryStrategy: RetryStrategy | null, + streamLogs?: boolean, ) { super(accountId, timeout, verbose) this.projectBundle = projectBundle @@ -48,6 +50,7 @@ export default class TestRunner extends AbstractCheckRunner { this.updateSnapshots = updateSnapshots this.baseDirectory = baseDirectory this.testRetryStrategy = testRetryStrategy + this.streamLogs = streamLogs ?? false } async scheduleChecks ( @@ -89,6 +92,7 @@ export default class TestRunner extends AbstractCheckRunner { repoInfo: this.repoInfo, environment: this.environment, shouldRecord: this.shouldRecord, + streamLogs: this.streamLogs, }) const { testSessionId, sequenceIds } = data const checks = this.checkBundles.map(({ construct: check }) => ({ check, sequenceId: sequenceIds?.[check.logicalId] })) diff --git a/packages/cli/src/templates/packed-files.ts b/packages/cli/src/templates/packed-files.ts new file mode 100644 index 000000000..a0487fdf9 --- /dev/null +++ b/packages/cli/src/templates/packed-files.ts @@ -0,0 +1,256 @@ +// Packed files generated automatically + +export const EXAMPLE_SPEC_TS = `import { spawn } from 'child_process'; +import { test } from '@playwright/test'; +import * as net from 'net'; + +test('has title', async ({ page }) => { + const browserPath = '/checkly/browsers/chromium-1181/chrome-linux/chrome'; + const MCP_PORT = 8931; + const TEN_MINUTES_MS = 20 * 60 * 1000; // 20 minutes in milliseconds + + // Set timeout for the test + test.setTimeout(TEN_MINUTES_MS); + + // Function to check if a port is in use + function isPortInUse(port: number): Promise { + return new Promise((resolve) => { + const server = net.createServer(); + + server.listen(port, () => { + server.once('close', () => { + resolve(false); // Port is available + }); + server.close(); + }); + + server.on('error', () => { + resolve(true); // Port is in use + }); + }); + } + + // Function to run pinggy to expose MCP port + async function runPinggy(port: number) { + console.log(\`Starting pinggy to expose port \${port}...\`); + + const pinggyProcess = spawn('ssh', [ + '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'LogLevel=ERROR', + '-p', '443', + '-R', \`0:localhost:\${port}\`, + 'a.pinggy.io' + ], { + stdio: 'pipe', + shell: false + }); + + // Handle pinggy output + pinggyProcess.stdout?.on('data', (data) => { + const output = data.toString(); + console.log(\`Pinggy stdout: \${output}\`); + + // Look for HTTPS URL in the output + const httpsMatch = output.match(/https:\\/\\/[a-z0-9-]+\\.a\\.free\\.pinggy\\.link/); + if (httpsMatch) { + const publicUrl = httpsMatch[0]; + console.log(\`🌐 PINGGY PUBLIC URL: \${publicUrl}\`); + } + }); + + pinggyProcess.stderr?.on('data', (data) => { + const output = data.toString(); + console.log(\`Pinggy stderr: \${output}\`); + + // Also check stderr for URLs (pinggy might output there) + const httpsMatch = output.match(/https:\\/\\/[a-z0-9-]+\\.a\\.free\\.pinggy\\.link/); + if (httpsMatch) { + const publicUrl = httpsMatch[0]; + console.log(\`🌐 PINGGY PUBLIC URL: \${publicUrl}\`); + } + }); + + pinggyProcess.on('close', (code) => { + console.log(\`Pinggy process exited with code \${code}\`); + }); + + pinggyProcess.on('error', (error) => { + console.error(\`Pinggy process error: \${error}\`); + }); + + return pinggyProcess; + } + + // Function to run MCP server + async function runMCPServer() { + // Check if port is already in use + const portInUse = await isPortInUse(MCP_PORT); + if (portInUse) { + console.log(\`Port \${MCP_PORT} is already in use. MCP server may already be running.\`); + return null; + } + + console.log('Starting MCP server...'); + + const mcpProcess = spawn('npx', ['-y', '@playwright/mcp@latest', '--port', \`\${MCP_PORT}\`, '--executable-path', browserPath, '--isolated'], { + stdio: 'pipe', + shell: true + }); + + // Handle process output + mcpProcess.stdout?.on('data', (data) => { + console.log(\`MCP stdout: \${data.toString().trim()}\`); + }); + + mcpProcess.stderr?.on('data', (data) => { + console.log(\`MCP stderr: \${data.toString().trim()}\`); + }); + + mcpProcess.on('close', (code) => { + console.log(\`MCP process exited with code \${code}\`); + }); + + mcpProcess.on('error', (error) => { + console.error(\`MCP process error: \${error}\`); + }); + + return mcpProcess; + } + + // Main async function to run all setup + async function setupAndRun() { + console.log("Running MCP server with a timeout of 30 seconds..."); + const mcpServerProcess = await runMCPServer(); + + if (mcpServerProcess) { + console.log("MCP server started. You can now run your Playwright tests."); + + // Start pinggy to expose the MCP server + console.log("Starting pinggy to expose MCP server..."); + const pinggyProcess = await runPinggy(MCP_PORT); + } else { + console.log("MCP server not started (port already in use)."); + } + } + + // Run the setup + await setupAndRun(); + + console.log("logging in a test"); + + // Wait for the specified timeout + await new Promise(resolve => setTimeout(resolve, TEN_MINUTES_MS)); +});`; + +export const CHECKLY_CONFIG_TS = `import { defineConfig } from 'checkly' +import { Frequency } from 'checkly/constructs' + +export default defineConfig({ + projectName: 'MCP Server Instance', + logicalId: 'cool-website-monitoring', + repoUrl: 'https://github.com/acme/website', + checks: { + playwrightConfigPath: './playwright.config.ts', + playwrightChecks: [ + { + /** + * Create a multi-browser check that runs + * every 10 mins in two locations. + */ + logicalId: 'multi-browser', + name: 'Playwright MCP Server', + // Use one project (or multiple projects) defined in your Playwright config + pwProjects: ['chromium'], + // Use one tag (or multiple tags) defined in your spec files + //pwTags: '@smoke-tests', + frequency: Frequency.EVERY_10M, + locations: ['us-east-1', 'eu-west-1'], + }, + // { + // /** + // * Create a check that runs the \`@critical\` tagged tests + // * every 5 mins in three locations. + // */ + // logicalId: 'critical-tagged', + // name: 'Critical Tagged tests', + // // Use one project (or multiple projects) defined in your Playwright config + // pwProjects: ['chromium'], + // // Use one tag (or multiple tags) defined in your spec files + // pwTags: '@critical', + // frequency: Frequency.EVERY_5M, + // locations: ['us-east-1', 'eu-central-1', 'ap-southeast-2'], + // }, + ], + }, + cli: { + runLocation: 'us-east-1', + retries: 0, + }, +}) +`; + +export const TSCONFIG_JSON = `{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020"], + "outDir": "./dist", + "rootDir": "./", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true + }, + "include": [ + "*.ts" + ], + "exclude": [ + "node_modules", + "dist" + ], + "ts-node": { + "esm": false, + "compilerOptions": { + "module": "commonjs" + } + } +} +`; + +export const PACKAGE_JSON = `{ + "name": "hackathon_pwmcp", + "version": "1.0.0", + "description": "MCP Manager API for Playwright server instances with ngrok", + "main": "mcp-manager-api.js", + "scripts": { + "build": "tsc", + "start": "node dist/mcp-manager-api.js", + "dev": "ts-node mcp-manager-api.ts", + "test": "npx playwright test" + }, + "keywords": ["mcp", "playwright", "ngrok", "api", "typescript"], + "author": "", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.54.1", + "@types/express": "^5.0.3", + "@types/node": "^24.0.15", + "@types/uuid": "^10.0.0", + "checkly": "^0.0.0-pr.1111.e932b3e", + "jiti": "^2.4.2", + "ts-node": "^10.9.2", + "typescript": "^5.8.3" + }, + "dependencies": { + "express": "^4.18.2", + "ngrok": "^5.0.0-beta.2", + "uuid": "^10.0.0" + } +} +`; +