👷 do not rely on monitors for deployment gate (#3973)

Author: thomas-lebeau

.gitlab/deploy-auto.yml

@@ -17,7 +17,7 @@ stages:
     - VERSION=$(node -p -e "require('./lerna.json').version")
     - yarn
     - yarn build:bundle
-    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $UPLOAD_PATH --check-monitors
+    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $UPLOAD_PATH --check-telemetry-errors
 
 step-1_deploy-prod-minor-dcs:
   when: manual

.gitlab/deploy-manual.yml

@@ -19,7 +19,7 @@ stages:
     - VERSION=$(node -p -e "require('./lerna.json').version")
     - yarn
     - yarn build:bundle
-    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $UPLOAD_PATH --no-check-monitors
+    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $UPLOAD_PATH --no-check-telemetry-errors
 
 step-1_deploy-prod-minor-dcs:
   extends:

scripts/deploy/check-monitors.ts

@@ -0,0 +1,98 @@
+import assert from 'node:assert/strict'
+import path from 'node:path'
+import { beforeEach, before, describe, it, mock, type Mock } from 'node:test'
+import { browserSdkVersion } from '../lib/browserSdkVersion.ts'
+import type { CommandDetail } from './lib/testHelpers.ts'
+import { mockModule, mockCommandImplementation } from './lib/testHelpers.ts'
+
+const currentBrowserSdkVersionMajor = browserSdkVersion.split('.')[0]
+
+describe('deploy-prod-dc', () => {
+  const commandMock = mock.fn()
+  const checkTelemetryErrorsMock: Mock<(datacenters: string[], version: string) => Promise<void>> = mock.fn()
+
+  let commands: CommandDetail[]
+  let checkTelemetryErrorsCalls: Array<{ version: string; datacenters: string[] }>
+
+  before(async () => {
+    await mockModule(path.resolve(import.meta.dirname, '../lib/command.ts'), { command: commandMock })
+    await mockModule(path.resolve(import.meta.dirname, '../lib/executionUtils.ts'), {
+      timeout: () => Promise.resolve(),
+    })
+    await mockModule(path.resolve(import.meta.dirname, './lib/checkTelemetryErrors.ts'), {
+      checkTelemetryErrors: checkTelemetryErrorsMock,
+    })
+  })
+
+  beforeEach(() => {
+    commands = mockCommandImplementation(commandMock)
+    checkTelemetryErrorsCalls = []
+    checkTelemetryErrorsMock.mock.mockImplementation((datacenters: string[], version: string) => {
+      checkTelemetryErrorsCalls.push({ version, datacenters })
+      return Promise.resolve()
+    })
+  })
+
+  it('should deploy a given datacenter', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'us1')
+
+    // Should not call checkTelemetryErrors by default (no flag)
+    assert.strictEqual(checkTelemetryErrorsCalls.length, 0)
+
+    assert.deepEqual(commands, [
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 us1' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 us1' },
+    ])
+  })
+
+  it('should deploy a given datacenter with check telemetry errors', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'us1', '--check-telemetry-errors')
+
+    // Should call checkTelemetryErrors 31 times: 1 initial + 30 during gating
+    assert.strictEqual(checkTelemetryErrorsCalls.length, 31)
+    assert.deepEqual(checkTelemetryErrorsCalls[0], {
+      version: `${currentBrowserSdkVersionMajor}.*`,
+      datacenters: ['us1'],
+    }) // Initial check
+    assert.deepEqual(checkTelemetryErrorsCalls[30], { version: browserSdkVersion, datacenters: ['us1'] }) // Last gating check
+
+    assert.deepEqual(commands, [
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 us1' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 us1' },
+    ])
+  })
+
+  it('should only check telemetry errors before deploying if the upload path is root', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'root', '--check-telemetry-errors')
+
+    // Should only call checkTelemetryErrors once (no gating for root)
+    assert.strictEqual(checkTelemetryErrorsCalls.length, 1)
+    assert.deepEqual(checkTelemetryErrorsCalls[0], {
+      version: `${currentBrowserSdkVersionMajor}.*`,
+      datacenters: ['root'],
+    })
+
+    assert.deepEqual(commands, [
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 root' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 root' },
+    ])
+  })
+
+  it('should deploy all minor datacenters', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'minor-dcs', '--no-check-telemetry-errors')
+
+    // Should not call checkTelemetryErrors when --no-check-telemetry-errors is used
+    assert.strictEqual(checkTelemetryErrorsCalls.length, 0)
+
+    assert.deepEqual(commands, [
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 us3,us5,ap1,ap2,prtest00' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 us3,us5,ap1,ap2,prtest00' },
+    ])
+  })
+})
+
+async function runScript(scriptPath: string, ...args: string[]): Promise<void> {
+  const { main } = (await import(scriptPath)) as { main: (...args: string[]) => Promise<void> }
+
+  return main(...args)
+}

scripts/deploy/deploy-prod-dc.spec.ts

@@ -2,6 +2,8 @@ import { parseArgs } from 'node:util'
 import { printLog, runMain, timeout } from '../lib/executionUtils.ts'
 import { command } from '../lib/command.ts'
 import { siteByDatacenter } from '../lib/datacenter.ts'
+import { browserSdkVersion } from '../lib/browserSdkVersion.ts'
+import { checkTelemetryErrors } from './lib/checkTelemetryErrors.ts'
 
 /**
  * Orchestrate the deployments of the artifacts for specific DCs
@@ -21,43 +23,51 @@ function getAllMinorDcs(): string[] {
   return Object.keys(siteByDatacenter).filter((dc) => !MAJOR_DCS.includes(dc))
 }
 
-const {
-  values: { 'check-monitors': checkMonitors },
-  positionals,
-} = parseArgs({
-  allowPositionals: true,
-  allowNegative: true,
-  options: {
-    'check-monitors': {
-      type: 'boolean',
+if (!process.env.NODE_TEST_CONTEXT) {
+  runMain(() => main(...process.argv.slice(2)))
+}
+
+export async function main(...args: string[]): Promise<void> {
+  const {
+    values: { 'check-telemetry-errors': shouldCheckTelemetryErrors },
+    positionals,
+  } = parseArgs({
+    args,
+    allowPositionals: true,
+    allowNegative: true,
+    options: {
+      'check-telemetry-errors': {
+        type: 'boolean',
+        default: false,
+      },
     },
-  },
-})
+  })
 
-const version = positionals[0]
-const uploadPath = positionals[1] === 'minor-dcs' ? getAllMinorDcs().join(',') : positionals[1]
+  const version = positionals[0]
+  const uploadPath = positionals[1] === 'minor-dcs' ? getAllMinorDcs().join(',') : positionals[1]
 
-if (!uploadPath) {
-  throw new Error('UPLOAD_PATH argument is required')
-}
+  if (!uploadPath) {
+    throw new Error('UPLOAD_PATH argument is required')
+  }
 
-runMain(async () => {
-  if (checkMonitors) {
-    command`node ./scripts/deploy/check-monitors.ts ${uploadPath}`.withLogs().run()
+  if (shouldCheckTelemetryErrors) {
+    // Make sure system is in a good state before deploying
+    const currentBrowserSdkVersionMajor = browserSdkVersion.split('.')[0]
+    await checkTelemetryErrors(uploadPath.split(','), `${currentBrowserSdkVersionMajor}.*`)
   }
 
   command`node ./scripts/deploy/deploy.ts prod ${version} ${uploadPath}`.withLogs().run()
   command`node ./scripts/deploy/upload-source-maps.ts ${version} ${uploadPath}`.withLogs().run()
 
-  if (checkMonitors && uploadPath !== 'root') {
-    await gateMonitors(uploadPath)
+  if (shouldCheckTelemetryErrors && uploadPath !== 'root') {
+    await gateTelemetryErrors(uploadPath)
   }
-})
+}
 
-async function gateMonitors(uploadPath: string): Promise<void> {
-  printLog(`Check monitors for ${uploadPath} during ${GATE_DURATION / ONE_MINUTE_IN_SECOND} minutes`)
+async function gateTelemetryErrors(uploadPath: string): Promise<void> {
+  printLog(`Check telemetry errors for ${uploadPath} during ${GATE_DURATION / ONE_MINUTE_IN_SECOND} minutes`)
   for (let i = 0; i < GATE_DURATION; i += GATE_INTERVAL) {
-    command`node ./scripts/deploy/check-monitors.ts ${uploadPath}`.run()
+    await checkTelemetryErrors(uploadPath.split(','), browserSdkVersion)
     process.stdout.write('.') // progress indicator
     await timeout(GATE_INTERVAL * 1000)
   }