Danielpaulus/sc 0/add mcp cmd

packages/cli/src/commands/mcp.ts

@@ -0,0 +1,168 @@
+import { AuthCommand } from './authCommand'
+import { Flags } from '@oclif/core'
+import * as fs from 'node:fs/promises'
+import * as path from 'node:path'
+import * as os from 'node:os'
+import { spawn } from 'node:child_process'
+import { EXAMPLE_SPEC_TS, CHECKLY_CONFIG_TS, TSCONFIG_JSON, PACKAGE_JSON } from '../templates/packed-files'
+
+// Default playwright.config.ts since it's not in packed-files
+const PLAYWRIGHT_CONFIG_TS = `import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: process.env.CI ? 1 : undefined,
+  reporter: 'html',
+  use: {
+    trace: 'on-first-retry',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+});`
+
+export default class McpCommand extends AuthCommand {
+  static coreCommand = true
+  static hidden = false
+  static description = 'Run MCP server tests with Playwright on Checkly.'
+  static state = 'beta'
+
+  static flags = {
+    verbose: Flags.boolean({
+      char: 'v',
+      description: 'Show all logs, errors, and stdout from processes',
+      default: false,
+    }),
+  }
+
+  async run(): Promise<void> {
+    const { flags } = await this.parse(McpCommand)
+    const { verbose } = flags
+
+    let tempDir: string | null = null
+    let pinggyUrl: string | null = null
+      // Output initial "in progress" message
+    this.log('starting mcp server in progress (this can take a while)..')
+    this.log('it will run for 20 minutes and then exit automatically')
+    try {
+      // Create temporary directory
+      tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'checkly-mcp-'))
+
+      // Write all files to temp directory
+      await fs.writeFile(path.join(tempDir, 'example.spec.ts'), EXAMPLE_SPEC_TS)
+      await fs.writeFile(path.join(tempDir, 'checkly.config.ts'), CHECKLY_CONFIG_TS)
+      await fs.writeFile(path.join(tempDir, 'tsconfig.json'), TSCONFIG_JSON)
+      await fs.writeFile(path.join(tempDir, 'package.json'), PACKAGE_JSON)
+      await fs.writeFile(path.join(tempDir, 'playwright.config.ts'), PLAYWRIGHT_CONFIG_TS)
+
+      // Run npm install in the temp directory
+      const { execa } = await import('execa')
+      if (verbose) {
+        this.log('Installing dependencies...')
+      }
+      await execa('npm', ['install'], {
+        cwd: tempDir,
+        stdio: verbose ? 'inherit' : 'pipe' // Show output in verbose mode
+      })
+
+
+
+      // Save current directory
+      const originalCwd = process.cwd()
+
+      // Change to temp directory
+      process.chdir(tempDir)
+
+      try {
+        // Run pw-test command with stream-logs enabled using spawn to capture output
+        // Use the same node and checkly command that was used to run this command
+        const nodeExecutable = process.argv[0]
+        const checklyExecutable = process.argv[1]
+        const args = [checklyExecutable, 'pw-test', '--stream-logs']
+
+        const pwTestProcess = spawn(nodeExecutable, args, {
+          cwd: tempDir,
+          env: { ...process.env },
+          stdio: ['inherit', 'pipe', 'pipe']
+        })
+
+        // Capture stdout
+        pwTestProcess.stdout?.on('data', (data) => {
+          const output = data.toString()
+
+          // In verbose mode, show all output
+          if (verbose) {
+            process.stdout.write(output)
+          }
+
+          // Parse for Pinggy URL
+          const urlMatch = output.match(/🌐 PINGGY PUBLIC URL:\s*(https:\/\/[a-z0-9-]+\.a\.free\.pinggy\.link)/)
+          if (urlMatch && !pinggyUrl) {
+            pinggyUrl = urlMatch[1]
+            // Output JSON immediately when URL is found
+            this.log(JSON.stringify({ serverUrl: pinggyUrl }))
+          }
+        })
+
+        // Capture stderr as well
+        pwTestProcess.stderr?.on('data', (data) => {
+          const output = data.toString()
+
+          // In verbose mode, show all error output
+          if (verbose) {
+            process.stderr.write(output)
+          }
+
+          // Also check stderr for URLs (pinggy might output there)
+          const urlMatch = output.match(/🌐 PINGGY PUBLIC URL:\s*(https:\/\/[a-z0-9-]+\.a\.free\.pinggy\.link)/)
+          if (urlMatch && !pinggyUrl) {
+            pinggyUrl = urlMatch[1]
+            // Output JSON immediately when URL is found
+            this.log(JSON.stringify({ serverUrl: pinggyUrl }))
+          }
+        })
+
+        // Wait for process to complete
+        await new Promise<void>((resolve, reject) => {
+          pwTestProcess.on('close', (code) => {
+            if (code === 0) {
+              resolve()
+            } else {
+              reject(new Error(`Process exited with code ${code}`))
+            }
+          })
+
+          pwTestProcess.on('error', (error) => {
+            reject(error)
+          })
+        })
+
+      } finally {
+        // Always restore original directory
+        process.chdir(originalCwd)
+      }
+
+    } catch (error) {
+      // Show errors in verbose mode
+      if (verbose && error instanceof Error) {
+        this.error(error.message)
+      }
+      process.exit(1)
+    } finally {
+      // Clean up temp directory
+      if (tempDir) {
+        try {
+          await fs.rm(tempDir, { recursive: true, force: true })
+        } catch (cleanupError) {
+          // Suppress cleanup errors
+        }
+      }
+    }
+  }
+}

packages/cli/src/commands/pw-test.ts

@@ -83,6 +83,10 @@ export default class PwTestCommand extends AuthCommand {
     'create-check': Flags.boolean({
       description: 'Create a Checkly check from the Playwright test.',
       default: false,
+    }),
+    'stream-logs': Flags.boolean({
+      description: 'Stream logs from the test run to the console.',
+      default: false,
     })
   }
 
@@ -104,6 +108,7 @@ export default class PwTestCommand extends AuthCommand {
       record,
       'test-session-name': testSessionName,
       'create-check': createCheck,
+      'stream-logs': streamLogs,
     } = flags
     const { configDirectory, configFilenames } = splitConfigFilePath(configFilename)
     const {
@@ -206,6 +211,10 @@ export default class PwTestCommand extends AuthCommand {
 
     const checkBundles = Object.values(projectBundle.data.check)
 
+    if (this.fancy) {
+      ux.action.stop()
+    }
+
     if (!checkBundles.length) {
       this.log(`Unable to find checks to run`)
       return
@@ -233,6 +242,7 @@ export default class PwTestCommand extends AuthCommand {
       configDirectory,
       // TODO: ADD PROPER RETRY STRATEGY HANDLING
       null, // testRetryStrategy
+      streamLogs,
     )
 
     runner.on(Events.RUN_STARTED,
@@ -284,6 +294,9 @@ export default class PwTestCommand extends AuthCommand {
       reporters.forEach(r => r.onError(err))
       process.exitCode = 1
     })
+    runner.on(Events.STREAM_LOGS, (check: any, sequenceId: SequenceId, logs) => {
+      reporters.forEach(r => r.onStreamLogs(check, sequenceId, logs))
+    })
     await runner.run()
     }
 
@@ -294,6 +307,7 @@ export default class PwTestCommand extends AuthCommand {
         }
         return arg
       })
+
       const input = parseArgs.join(' ') || ''
       const inputLogicalId = cased(input, 'kebab-case').substring(0, 50)
       const testCommand = await PwTestCommand.getTestCommand(dir, input)

packages/cli/src/reporters/abstract-list.ts

@@ -20,6 +20,7 @@ export type checkFilesMap = Map<string|undefined, Map<SequenceId, {
   testResultId?: string,
   links?: TestResultsShortLinks,
   numRetries: number,
+  logs?: string[],
 }>>
 
 export default abstract class AbstractListReporter implements Reporter {
@@ -96,6 +97,17 @@ export default abstract class AbstractListReporter implements Reporter {
     printLn(chalk.red('Unable to run checks: ') + err.message)
   }
 
+  onStreamLogs (check: any, sequenceId: SequenceId, logs: string[] | undefined) {
+    const checkFile = this.checkFilesMap!.get(check.getSourceFile?.())!.get(sequenceId)!
+    const logList = logs || []
+    if (!checkFile.logs) {
+      checkFile.logs = []
+    }
+    // checkFile.logs.push(...logs.logs)
+    logList.forEach((log: string) => printLn(log, 2))
+    return
+  }
+
   // Clear the summary which was printed by _printStatus from stdout
   // TODO: Rather than clearing the whole status bar, we could overwrite the exact lines that changed.
   // This might look a bit smoother and reduce the flickering effects.

packages/cli/src/reporters/reporter.ts

@@ -14,6 +14,7 @@ export interface Reporter {
   onEnd(): void;
   onError(err: Error): void,
   onSchedulingDelayExceeded(): void
+  onStreamLogs(check: any, sequenceId: SequenceId, logs: string[]): void
 }
 
 export type ReporterType = 'list' | 'dot' | 'ci' | 'github' | 'json'

packages/cli/src/rest/test-sessions.ts

@@ -12,6 +12,7 @@ type RunTestSessionRequest = {
   repoInfo?: GitInformation | null,
   environment?: string | null,
   shouldRecord: boolean,
+  streamLogs?: boolean,
 }
 
 type TriggerTestSessionRequest = {

packages/cli/src/services/abstract-check-runner.ts

@@ -18,7 +18,8 @@ export enum Events {
   RUN_STARTED = 'RUN_STARTED',
   RUN_FINISHED = 'RUN_FINISHED',
   ERROR = 'ERROR',
-  MAX_SCHEDULING_DELAY_EXCEEDED = 'MAX_SCHEDULING_DELAY_EXCEEDED'
+  MAX_SCHEDULING_DELAY_EXCEEDED = 'MAX_SCHEDULING_DELAY_EXCEEDED',
+  STREAM_LOGS = 'STREAM_LOGS',
 }
 
 export type PrivateRunLocation = {
@@ -160,6 +161,10 @@ export default abstract class AbstractCheckRunner extends EventEmitter {
       this.disableTimeout(sequenceId)
       this.emit(Events.CHECK_FAILED, sequenceId, check, message)
       this.emit(Events.CHECK_FINISHED, check)
+    } else if (subtopic === 'stream-logs') {
+      const { logs } = message
+      this.emit(Events.STREAM_LOGS, check, sequenceId, logs)
+
     }
   }
 

packages/cli/src/services/test-runner.ts

@@ -21,6 +21,7 @@ export default class TestRunner extends AbstractCheckRunner {
   updateSnapshots: boolean
   baseDirectory: string
   testRetryStrategy: RetryStrategy | null
+  streamLogs?: boolean
 
   constructor (
     accountId: string,
@@ -36,6 +37,7 @@ export default class TestRunner extends AbstractCheckRunner {
     updateSnapshots: boolean,
     baseDirectory: string,
     testRetryStrategy: RetryStrategy | null,
+    streamLogs?: boolean,
   ) {
     super(accountId, timeout, verbose)
     this.projectBundle = projectBundle
@@ -48,6 +50,7 @@ export default class TestRunner extends AbstractCheckRunner {
     this.updateSnapshots = updateSnapshots
     this.baseDirectory = baseDirectory
     this.testRetryStrategy = testRetryStrategy
+    this.streamLogs = streamLogs ?? false
   }
 
   async scheduleChecks (
@@ -89,6 +92,7 @@ export default class TestRunner extends AbstractCheckRunner {
         repoInfo: this.repoInfo,
         environment: this.environment,
         shouldRecord: this.shouldRecord,
+        streamLogs: this.streamLogs,
       })
       const { testSessionId, sequenceIds } = data
       const checks = this.checkBundles.map(({ construct: check }) => ({ check, sequenceId: sequenceIds?.[check.logicalId] }))