feat: add webserver guardrail for playwright checks [tra-1789] (#1196)

* feat: add webserver guardrail [tra-1789]

* fix: add warning

* fix: catch [tra-1789]

* fix: merge [tra-1789]

* fix: test [tra-1789]

Author: pilimartinez

packages/cli/src/commands/pw-test.ts

@@ -18,7 +18,6 @@ import { Flags } from '@oclif/core'
 import { createReporters, ReporterType } from '../reporters/reporter'
 import TestRunner from '../services/test-runner'
 import {
-  DEFAULT_CHECK_RUN_TIMEOUT_SECONDS,
   DEFAULT_PLAYWRIGHT_CHECK_RUN_TIMEOUT_SECONDS,
   Events,
   SequenceId,
@@ -95,6 +94,12 @@ export default class PwTestCommand extends AuthCommand {
       default: true,
       allowNo: true,
     }),
+    'include': Flags.string({
+      char: 'i',
+      description: 'File patterns to include when bundling the test project (e.g., "utils/**/*").',
+      multiple: true,
+      default: [],
+    }),
   }
 
   async run (): Promise<void> {
@@ -116,6 +121,7 @@ export default class PwTestCommand extends AuthCommand {
       'test-session-name': testSessionName,
       'create-check': createCheck,
       'stream-logs': streamLogs,
+      'include': includeFlag,
     } = flags
     const { configDirectory, configFilenames } = splitConfigFilePath(configFilename)
     const pwPathFlag = this.getConfigPath(playwrightFlags)
@@ -174,8 +180,10 @@ export default class PwTestCommand extends AuthCommand {
       verifyRuntimeDependencies: false,
       checklyConfigConstructs,
       playwrightConfigPath,
-      include: checklyConfig.checks?.include,
+      include: includeFlag.length ? includeFlag : checklyConfig.checks?.include,
+      includeFlagProvided: includeFlag.length > 0,
       playwrightChecks: [playwrightCheck],
+      currentCommand: 'pw-test',
       checkFilter: check => {
         // Skip non Playwright checks
         if (!(check instanceof PlaywrightCheck)) {

packages/cli/src/constructs/__tests__/fixtures/playwright-check-webserver/package.json

@@ -0,0 +1,6 @@
+{
+ "name": "playwright-check-webserver-fixture",
+ "devDependencies": {
+  "@playwright/test": "^1.55.1"
+ }
+}

packages/cli/src/constructs/__tests__/fixtures/playwright-check-webserver/playwright.config.ts

@@ -0,0 +1,9 @@
+import { defineConfig } from '@playwright/test'
+
+export default defineConfig({
+  testDir: './tests',
+  webServer: {
+    command: 'npm run start',
+    url: 'http://localhost:3000',
+  },
+})

packages/cli/src/constructs/__tests__/playwright-check.spec.ts

@@ -1,6 +1,6 @@
 import path from 'node:path'
 
-import { describe, it, expect, beforeEach, vi } from 'vitest'
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
 import { AxiosHeaders } from 'axios'
 
 import { CheckGroupV2, Diagnostics, PlaywrightCheck, RetryStrategyBuilder } from '../index'
@@ -19,6 +19,13 @@ describe('PlaywrightCheck', () => {
         headers: new AxiosHeaders(),
       },
     })
+    Session.currentCommand = undefined
+    Session.includeFlagProvided = undefined
+  })
+
+  afterEach(() => {
+    Session.currentCommand = undefined
+    Session.includeFlagProvided = undefined
   })
 
   it('should synthesize groupName', async () => {
@@ -366,6 +373,78 @@ describe('PlaywrightCheck', () => {
         }),
       ]))
     })
+
+    it('should warn if webServer is configured in playwright config when running pw-test', async () => {
+      Session.basePath = path.resolve(__dirname, './fixtures/playwright-check-webserver')
+      Session.project = new Project('project-id', {
+        name: 'Test Project',
+        repoUrl: 'https://github.com/checkly/checkly-cli',
+      })
+      Session.currentCommand = 'pw-test'
+
+      const check = new PlaywrightCheck('foo', {
+        name: 'Test Check',
+        playwrightConfigPath: path.resolve(__dirname, './fixtures/playwright-check-webserver/playwright.config.ts'),
+      })
+
+      const diags = new Diagnostics()
+      await check.validate(diags)
+
+      expect(diags.isFatal()).toEqual(false)
+      expect(diags.observations).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          title: expect.stringContaining('webServer configuration detected'),
+          message: expect.stringContaining('webServer configuration requires additional files'),
+        }),
+      ]))
+    })
+
+    it('should not warn about webServer when not running pw-test command', async () => {
+      Session.basePath = path.resolve(__dirname, './fixtures/playwright-check-webserver')
+      Session.project = new Project('project-id', {
+        name: 'Test Project',
+        repoUrl: 'https://github.com/checkly/checkly-cli',
+      })
+      Session.currentCommand = 'test'
+
+      const check = new PlaywrightCheck('foo', {
+        name: 'Test Check',
+        playwrightConfigPath: path.resolve(__dirname, './fixtures/playwright-check-webserver/playwright.config.ts'),
+      })
+
+      const diags = new Diagnostics()
+      await check.validate(diags)
+
+      expect(diags.observations).not.toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          title: expect.stringContaining('webServer configuration detected'),
+        }),
+      ]))
+    })
+
+    it('should not warn about webServer when --include flag is provided', async () => {
+      Session.basePath = path.resolve(__dirname, './fixtures/playwright-check-webserver')
+      Session.project = new Project('project-id', {
+        name: 'Test Project',
+        repoUrl: 'https://github.com/checkly/checkly-cli',
+      })
+      Session.currentCommand = 'pw-test'
+      Session.includeFlagProvided = true
+
+      const check = new PlaywrightCheck('foo', {
+        name: 'Test Check',
+        playwrightConfigPath: path.resolve(__dirname, './fixtures/playwright-check-webserver/playwright.config.ts'),
+      })
+
+      const diags = new Diagnostics()
+      await check.validate(diags)
+
+      expect(diags.observations).not.toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          title: expect.stringContaining('webServer configuration detected'),
+        }),
+      ]))
+    })
   })
 
   describe('defaults', () => {

packages/cli/src/constructs/playwright-check.ts

@@ -14,6 +14,7 @@ import {
   InvalidPropertyValueDiagnostic,
   UnsupportedPropertyDiagnostic,
 } from './construct-diagnostics'
+import { WarningDiagnostic } from './diagnostics'
 import { Diagnostics } from './diagnostics'
 import { PlaywrightCheckBundle } from './playwright-check-bundle'
 import { Session } from './project'
@@ -283,12 +284,42 @@ export class PlaywrightCheck extends RuntimeCheck {
     }
   }
 
+  protected async validateWebServerConfig (diagnostics: Diagnostics): Promise<void> {
+    // Only show webServer warning for pw-test command when --include is not provided
+    if (Session.currentCommand !== 'pw-test' || Session.includeFlagProvided) {
+      return
+    }
+
+    try {
+      const playwrightConfig = await Session.loadFile(this.playwrightConfigPath)
+
+      if (playwrightConfig && typeof playwrightConfig === 'object' && 'webServer' in playwrightConfig) {
+        diagnostics.add(new WarningDiagnostic({
+          title: 'webServer configuration detected',
+          message:
+            `webServer configuration requires additional files. `
+            + `Include all files needed to start the server (e.g., server scripts, config files) `
+            + `by passing them via the --include flag.`,
+        }))
+      }
+    } catch (err: any) {
+      diagnostics.add(new InvalidPropertyValueDiagnostic(
+        'playwrightConfigPath',
+        new Error(
+          `Unable to parse Playwright config "${this.playwrightConfigPath}": ${err.message}`,
+          { cause: err },
+        ),
+      ))
+    }
+  }
+
   async validate (diagnostics: Diagnostics): Promise<void> {
     await super.validate(diagnostics)
     await this.validateRetryStrategy(diagnostics)
 
     try {
       await fs.access(this.playwrightConfigPath, fs.constants.R_OK)
+      await this.validateWebServerConfig(diagnostics)
     } catch (err: any) {
       diagnostics.add(new InvalidPropertyValueDiagnostic(
         'playwrightConfigPath',

packages/cli/src/constructs/project.ts

@@ -244,6 +244,8 @@ export class Session {
   static parsers = new Map<string, Parser>()
   static constructExports: ConstructExport[] = []
   static ignoreDirectoriesMatch: string[] = []
+  static currentCommand?: 'pw-test' | 'test' | 'deploy'
+  static includeFlagProvided?: boolean
 
   static async loadFile<T = unknown> (filePath: string): Promise<T> {
     const loader = this.loader

packages/cli/src/services/project-parser.ts

@@ -33,7 +33,9 @@ type ProjectParseOpts = {
   checklyConfigConstructs?: Construct[]
   playwrightConfigPath?: string
   include?: string | string[]
+  includeFlagProvided?: boolean
   playwrightChecks?: PlaywrightSlimmedProp[]
+  currentCommand?: 'pw-test' | 'test' | 'deploy'
 }
 
 const BASE_CHECK_DEFAULTS = {
@@ -59,7 +61,9 @@ export async function parseProject (opts: ProjectParseOpts): Promise<Project> {
     checklyConfigConstructs,
     playwrightConfigPath,
     include,
+    includeFlagProvided,
     playwrightChecks,
+    currentCommand,
   } = opts
   const project = new Project(projectLogicalId, {
     name: projectName,
@@ -82,6 +86,8 @@ export async function parseProject (opts: ProjectParseOpts): Promise<Project> {
   Session.defaultRuntimeId = defaultRuntimeId
   Session.verifyRuntimeDependencies = verifyRuntimeDependencies ?? true
   Session.ignoreDirectoriesMatch = ignoreDirectoriesMatch
+  Session.currentCommand = currentCommand
+  Session.includeFlagProvided = includeFlagProvided
 
   // TODO: Do we really need all of the ** globs, or could we just put node_modules?
   const ignoreDirectories = ['**/node_modules/**', '**/.git/**', ...ignoreDirectoriesMatch]