fix: fix broken and flaky e2e test (#1008)

* feat: disable fancy output for e2e tests because some are super flaky

* fix: flaky e2e test only passed due to varying progress messages

Author: sorccu

packages/cli/e2e/__tests__/deploy.spec.ts

@@ -234,7 +234,29 @@ Update and Unchanged:
     })
     expect(resultOne.status).toBe(0)
     expect(resultTwo.status).toBe(0)
-    expect(resultOne.stdout).not.toEqual(resultTwo.stdout)
+    expect(resultOne.stdout).toContain(
+`Create:
+    ApiCheck: api-check
+    ApiCheck: api-check-high-freq
+    HeartbeatCheck: heartbeat-check-1
+    BrowserCheck: homepage-browser-check
+    CheckGroup: my-group-1
+    Dashboard: dashboard-1
+    MaintenanceWindow: maintenance-window-1
+    PrivateLocation: private-location-1
+`)
+    expect(resultTwo.stdout).toContain(
+`Create:
+    ApiCheck: api-check
+    ApiCheck: api-check-high-freq
+    HeartbeatCheck: heartbeat-check-1
+    BrowserCheck: homepage-browser-check
+    BrowserCheck: snapshot-test.test.ts
+    CheckGroup: my-group-1
+    Dashboard: dashboard-1
+    MaintenanceWindow: maintenance-window-1
+    PrivateLocation: private-location-1
+`)
   })
 
   it('Should terminate when no resources are found', async () => {

packages/cli/e2e/__tests__/fixtures/deploy-project/snapshot-test.test.ts

@@ -0,0 +1,8 @@
+import { expect, test } from '@playwright/test'
+
+test.use({ actionTimeout: 10000 })
+
+test('Danube Snapshot Test', async ({ page }) => {
+  await page.goto('https://danube-web.shop')
+  await expect(page).toHaveScreenshot({ maxDiffPixels: 10000 })
+})

packages/cli/e2e/run-checkly.ts

@@ -37,6 +37,7 @@ export function runChecklyCli (options: {
         // Once 4.8.0 has been released, we can remove the 4.8.0 fallback here.
         CHECKLY_CLI_VERSION: cliVersion ?? '4.8.0',
         CHECKLY_E2E_PROMPTS_INJECTIONS: promptsInjection?.length ? JSON.stringify(promptsInjection) : undefined,
+        CHECKLY_E2E_DISABLE_FANCY_OUTPUT: '1',
         ...env,
       },
       cwd: directory ?? process.cwd(),

packages/cli/src/commands/baseCommand.ts

@@ -10,6 +10,7 @@ export type BaseCommandClass = typeof Command & {
 export abstract class BaseCommand extends Command {
   static coreCommand = false
   static hidden = true
+  fancy = true
 
   protected async init (): Promise<void> {
     let version = process.env.CHECKLY_CLI_VERSION ?? this.config.version
@@ -35,6 +36,10 @@ export abstract class BaseCommand extends Command {
       }
     }
 
+    if (process.env.CHECKLY_E2E_DISABLE_FANCY_OUTPUT) {
+      this.fancy = false
+    }
+
     return super.init()
   }
 

packages/cli/src/commands/deploy.ts

@@ -74,7 +74,9 @@ export default class Deploy extends AuthCommand {
   }
 
   async run (): Promise<void> {
-    ux.action.start('Parsing your project', undefined, { stdout: true })
+    if (this.fancy) {
+      ux.action.start('Parsing your project', undefined, { stdout: true })
+    }
     const { flags } = await this.parse(Deploy)
     const {
       force,
@@ -113,7 +115,10 @@ export default class Deploy extends AuthCommand {
       checklyConfigConstructs,
     })
     const repoInfo = getGitInformation(project.repoUrl)
-    ux.action.stop()
+
+    if (this.fancy) {
+      ux.action.stop()
+    }
 
     if (!preview) {
       for (const check of Object.values(project.data.check)) {

packages/cli/src/commands/sync-playwright.ts

@@ -12,7 +12,9 @@ export default class SyncPlaywright extends BaseCommand {
   static description = 'Copy Playwright config into the Checkly config file.'
 
   async run (): Promise<void> {
-    ux.action.start('Syncing Playwright config to the Checkly config file', undefined, { stdout: true })
+    if (this.fancy) {
+      ux.action.start('Syncing Playwright config to the Checkly config file', undefined, { stdout: true })
+    }
 
     const config = await loadPlaywrightConfig()
     if (!config) {
@@ -40,13 +42,17 @@ export default class SyncPlaywright extends BaseCommand {
     const dir = path.resolve(path.dirname(configFile.fileName))
     this.reWriteChecklyConfigFile(checklyConfigData, configFile.fileName, dir)
 
-    ux.action.stop('✅ ')
+    if (this.fancy) {
+      ux.action.stop('✅ ')
+    }
     this.log('Successfully updated Checkly config file')
     this.exit(0)
   }
 
   private handleError (message: string) {
-    ux.action.stop('❌')
+    if (this.fancy) {
+      ux.action.stop('❌')
+    }
     this.log(message)
     this.exit(1)
   }

packages/cli/src/commands/test.ts

@@ -124,7 +124,9 @@ export default class Test extends AuthCommand {
   static strict = false
 
   async run (): Promise<void> {
-    ux.action.start('Parsing your project', undefined, { stdout: true })
+    if (this.fancy) {
+      ux.action.start('Parsing your project', undefined, { stdout: true })
+    }
 
     const { flags, argv } = await this.parse(Test)
     const {
@@ -228,7 +230,9 @@ export default class Test extends AuthCommand {
       check.snapshots = await uploadSnapshots(check.rawSnapshots)
     }
 
-    ux.action.stop()
+    if (this.fancy) {
+      ux.action.stop()
+    }
 
     if (!checks.length) {
       this.log(`Unable to find checks to run${filePatterns[0] !== '.*' ? ' using [FILEARGS]=\'' + filePatterns + '\'' : ''}.`)