👷 do not rely on hardcoded list of DCs

.gitlab/deploy-auto.yml

@@ -17,7 +17,7 @@ stages:
     - VERSION=$(node -p -e "require('./lerna.json').version")
     - yarn
     - yarn build:bundle
-    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $UPLOAD_PATH --check-monitors
+    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $DATACENTER --check-monitors
 
 step-1_deploy-prod-minor-dcs:
   when: manual
@@ -26,7 +26,7 @@ step-1_deploy-prod-minor-dcs:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: minor-dcs
+    DATACENTER: minor-dcs
 
 step-2_deploy-prod-eu1:
   needs:
@@ -35,7 +35,7 @@ step-2_deploy-prod-eu1:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: eu1
+    DATACENTER: eu1
 
 step-3_deploy-prod-us1:
   needs:
@@ -44,7 +44,7 @@ step-3_deploy-prod-us1:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: us1
+    DATACENTER: us1
 
 step-4_deploy-prod-gov:
   needs:
@@ -53,7 +53,7 @@ step-4_deploy-prod-gov:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: root
+    DATACENTER: gov
 
 step-5_publish-npm:
   needs:

.gitlab/deploy-manual.yml

@@ -19,35 +19,35 @@ stages:
     - VERSION=$(node -p -e "require('./lerna.json').version")
     - yarn
     - yarn build:bundle
-    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $UPLOAD_PATH --no-check-monitors
+    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $DATACENTER --no-check-monitors
 
 step-1_deploy-prod-minor-dcs:
   extends:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: minor-dcs
+    DATACENTER: minor-dcs
 
 step-2_deploy-prod-eu1:
   extends:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: eu1
+    DATACENTER: eu1
 
 step-3_deploy-prod-us1:
   extends:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: us1
+    DATACENTER: us1
 
 step-4_deploy-prod-gov:
   extends:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: root
+    DATACENTER: root
 
 step-5_publish-npm:
   stage: deploy
@@ -80,7 +80,7 @@ step-7_create-github-release:
     - node scripts/release/create-github-release.ts
 
 # This step is used to deploy the SDK to a new datacenter.
-# the `UPLOAD_PATH` variable needs to be provided as an argument when starting the manual job
+# the `DATACENTER` variable needs to be provided as an argument when starting the manual job
 optional_step-deploy-to-new-datacenter:
   extends:
     - .base-configuration

scripts/deploy/check-monitors.ts

@@ -5,50 +5,66 @@
  */
 import { printLog, runMain, fetchHandlingError } from '../lib/executionUtils.ts'
 import { getTelemetryOrgApiKey, getTelemetryOrgApplicationKey } from '../lib/secrets.ts'
-import { monitorIdsByDatacenter, siteByDatacenter } from '../lib/datacenter.ts'
-
-interface MonitorStatus {
-  id: number
-  name: string
-  overall_state: string
-}
+import { getSite } from '../lib/datacenter.ts'
+import { browserSdkVersion } from '../lib/browserSdkVersion.ts'
 
 const datacenters = process.argv[2].split(',')
 
 runMain(async () => {
   for (const datacenter of datacenters) {
-    if (!monitorIdsByDatacenter[datacenter]) {
-      printLog(`No monitors configured for datacenter ${datacenter}`)
+    const site = getSite(datacenter)
+    const apiKey = getTelemetryOrgApiKey(site)
+    const applicationKey = getTelemetryOrgApplicationKey(site)
+
+    if (!apiKey || !applicationKey) {
+      printLog(`No API key or application key found for ${site}, skipping...`)
       continue
     }
-    const monitorIds = monitorIdsByDatacenter[datacenter]
-    const site = siteByDatacenter[datacenter]
-    const monitorStatuses = await Promise.all(monitorIds.map((monitorId) => fetchMonitorStatus(site, monitorId)))
-    for (const monitorStatus of monitorStatuses) {
-      printLog(`${monitorStatus.overall_state} - ${monitorStatus.name}`)
-      if (monitorStatus.overall_state !== 'OK') {
-        throw new Error(
-          `Monitor ${monitorStatus.name} is in state ${monitorStatus.overall_state}, see ${computeMonitorLink(site, monitorStatus.id)}`
-        )
-      }
+
+    const errorLogsCount = await queryErrorLogsCount(site, apiKey, applicationKey)
+
+    if (errorLogsCount > 0) {
+      throw new Error(`Errors found in the last 30 minutes,
+see ${computeMonitorLink(site)}`)
+    } else {
+      printLog(`No errors found in the last 30 minutes for ${datacenter}`)
     }
   }
 })
 
-async function fetchMonitorStatus(site: string, monitorId: number): Promise<MonitorStatus> {
-  const response = await fetchHandlingError(`https://api.${site}/api/v1/monitor/${monitorId}`, {
-    method: 'GET',
+async function queryErrorLogsCount(site: string, apiKey: string, applicationKey: string): Promise<number> {
+  const response = await fetchHandlingError(`https://api.${site}/api/v2/logs/events/search`, {
+    method: 'POST',
     headers: {
-      Accept: 'application/json',
-      'DD-API-KEY': getTelemetryOrgApiKey(site),
-      'DD-APPLICATION-KEY': getTelemetryOrgApplicationKey(site),
+      'Content-Type': 'application/json',
+      'DD-API-KEY': apiKey,
+      'DD-APPLICATION-KEY': applicationKey,
     },
+    body: JSON.stringify({
+      filter: {
+        from: 'now-30m',
+        to: 'now',
+        query: `source:browser status:error version:${browserSdkVersion}`,
+      },
+    }),
   })
-  return response.json() as Promise<MonitorStatus>
+
+  const data = (await response.json()) as { data: unknown[] }
+
+  return data.data.length
 }
 
-function computeMonitorLink(site: string, monitorId: number): string {
-  return `https://${computeTelemetryOrgDomain(site)}/monitors/${monitorId}`
+function computeMonitorLink(site: string): string {
+  const now = Date.now()
+  const thirtyMinutesAgo = now - 30 * 60 * 1000
+
+  const queryParams = new URLSearchParams({
+    query: `source:browser status:error version:${browserSdkVersion}`,
+    from_ts: `${thirtyMinutesAgo}`,
+    to_ts: `${now}`,
+  })
+
+  return `https://${computeTelemetryOrgDomain(site)}/logs?${queryParams.toString()}`
 }
 
 function computeTelemetryOrgDomain(site: string): string {

scripts/deploy/deploy-prod-dc.spec.ts

@@ -0,0 +1,79 @@
+import assert from 'node:assert/strict'
+import path from 'node:path'
+import { beforeEach, before, describe, it, mock, afterEach } from 'node:test'
+import type { CommandDetail } from './lib/testHelpers.ts'
+import { mockCommandImplementation, mockModule } from './lib/testHelpers.ts'
+
+describe('deploy-prod-dc', () => {
+  const commandMock = mock.fn()
+  let commands: CommandDetail[]
+
+  before(async () => {
+    await mockModule(path.resolve(import.meta.dirname, '../lib/command.ts'), { command: commandMock })
+    await mockModule(path.resolve(import.meta.dirname, '../lib/executionUtils.ts'), {
+      timeout: () => Promise.resolve(),
+    })
+  })
+
+  beforeEach(() => {
+    commands = mockCommandImplementation(commandMock)
+  })
+
+  afterEach(() => {
+    mock.restoreAll()
+  })
+
+  it('should deploy a given datacenter', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'us1')
+
+    assert.deepEqual(commands, [
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 us1' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 us1' },
+    ])
+  })
+
+  it('should deploy a given datacenter with check monitors', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'us1', '--check-monitors')
+
+    assert.deepEqual(commands, [
+      { command: 'node ./scripts/deploy/check-monitors.ts us1' },
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 us1' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 us1' },
+      // 1 monitor check per minute for 30 minutes
+      ...Array.from({ length: 30 }, () => ({ command: 'node ./scripts/deploy/check-monitors.ts us1' })),
+    ])
+  })
+
+  it('should deploy all minor datacenters', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'minor-dcs', '--no-check-monitors')
+
+    assert.deepEqual(commands, [
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 ap1,ap2,us3,us5' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 ap1,ap2,us3,us5' },
+    ])
+  })
+
+  it('should deploy all private regions', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'private-regions', '--no-check-monitors')
+
+    assert.deepEqual(commands, [
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 prtest00,prtest01' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 prtest00,prtest01' },
+    ])
+  })
+
+  it('should deploy gov datacenters to the root upload path', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'gov', '--no-check-monitors')
+
+    assert.deepEqual(commands, [
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 root' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 root' },
+    ])
+  })
+})
+
+async function runScript(scriptPath: string, ...args: string[]): Promise<void> {
+  const { main } = (await import(scriptPath)) as { main: (...args: string[]) => Promise<void> }
+
+  return main(...args)
+}

scripts/deploy/deploy-prod-dc.ts

@@ -1,7 +1,7 @@
 import { parseArgs } from 'node:util'
 import { printLog, runMain, timeout } from '../lib/executionUtils.ts'
 import { command } from '../lib/command.ts'
-import { siteByDatacenter } from '../lib/datacenter.ts'
+import { getAllMinorDcs, getAllPrivateDcs } from '../lib/datacenter.ts'
 
 /**
  * Orchestrate the deployments of the artifacts for specific DCs
@@ -12,54 +12,77 @@ const ONE_MINUTE_IN_SECOND = 60
 const GATE_DURATION = 30 * ONE_MINUTE_IN_SECOND
 const GATE_INTERVAL = ONE_MINUTE_IN_SECOND
 
-// Major DCs are the ones that are deployed last.
-// They have their own step jobs in `deploy-manual.yml` and `deploy-auto.yml`.
-const MAJOR_DCS = ['root', 'us1', 'eu1']
-
-// Minor DCs are all the DCs from `siteByDatacenter` that are not in `MAJOR_DCS`.
-function getAllMinorDcs(): string[] {
-  return Object.keys(siteByDatacenter).filter((dc) => !MAJOR_DCS.includes(dc))
+if (!process.env.NODE_TEST_CONTEXT) {
+  runMain(() => main(...process.argv.slice(2)))
 }
 
-const {
-  values: { 'check-monitors': checkMonitors },
-  positionals,
-} = parseArgs({
-  allowPositionals: true,
-  allowNegative: true,
-  options: {
-    'check-monitors': {
-      type: 'boolean',
+export async function main(...args: string[]): Promise<void> {
+  const {
+    values: { 'check-monitors': checkMonitors },
+    positionals,
+  } = parseArgs({
+    args,
+    allowPositionals: true,
+    allowNegative: true,
+    options: {
+      'check-monitors': {
+        type: 'boolean',
+        default: false,
+      },
     },
-  },
-})
+  })
 
-const version = positionals[0]
-const uploadPath = positionals[1] === 'minor-dcs' ? getAllMinorDcs().join(',') : positionals[1]
+  const version = positionals[0]
+  const datacenters = getDatacenters(positionals[1])
 
-if (!uploadPath) {
-  throw new Error('UPLOAD_PATH argument is required')
-}
+  if (!datacenters) {
+    throw new Error('DATACENTER argument is required')
+  }
 
-runMain(async () => {
   if (checkMonitors) {
-    command`node ./scripts/deploy/check-monitors.ts ${uploadPath}`.withLogs().run()
+    command`node ./scripts/deploy/check-monitors.ts ${datacenters.join(',')}`.withLogs().run()
   }
 
-  command`node ./scripts/deploy/deploy.ts prod ${version} ${uploadPath}`.withLogs().run()
-  command`node ./scripts/deploy/upload-source-maps.ts ${version} ${uploadPath}`.withLogs().run()
+  const uploadPathTypes = toDatacenterUploadPathType(datacenters).join(',')
 
-  if (checkMonitors && uploadPath !== 'root') {
-    await gateMonitors(uploadPath)
+  command`node ./scripts/deploy/deploy.ts prod ${version} ${uploadPathTypes}`.withLogs().run()
+  command`node ./scripts/deploy/upload-source-maps.ts ${version} ${uploadPathTypes}`.withLogs().run()
+
+  if (checkMonitors) {
+    await gateMonitors(datacenters)
   }
-})
+}
+
+async function gateMonitors(datacenters: string[]): Promise<void> {
+  printLog(`Check monitors for ${datacenters.join(',')} during ${GATE_DURATION / ONE_MINUTE_IN_SECOND} minutes`)
 
-async function gateMonitors(uploadPath: string): Promise<void> {
-  printLog(`Check monitors for ${uploadPath} during ${GATE_DURATION / ONE_MINUTE_IN_SECOND} minutes`)
   for (let i = 0; i < GATE_DURATION; i += GATE_INTERVAL) {
-    command`node ./scripts/deploy/check-monitors.ts ${uploadPath}`.run()
+    command`node ./scripts/deploy/check-monitors.ts ${datacenters.join(',')}`.run()
     process.stdout.write('.') // progress indicator
     await timeout(GATE_INTERVAL * 1000)
   }
+
   printLog() // new line
 }
+
+function getDatacenters(datacenterGroup: string): string[] {
+  if (datacenterGroup === 'minor-dcs') {
+    return getAllMinorDcs()
+  }
+
+  if (datacenterGroup === 'private-regions') {
+    return getAllPrivateDcs()
+  }
+
+  return datacenterGroup.split(',')
+}
+
+function toDatacenterUploadPathType(datacenters: string[]): string[] {
+  return datacenters.map((datacenter) => {
+    if (datacenter === 'gov') {
+      return 'root'
+    }
+
+    return datacenter
+  })
+}