👷 refactor deployment scripts to remove hardcoded list of DCs and sites

Author: thomas-lebeau

.gitlab/deploy-auto.yml

@@ -17,7 +17,7 @@ stages:
     - VERSION=$(node -p -e "require('./lerna.json').version")
     - yarn
     - yarn build:bundle
-    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $UPLOAD_PATH --check-monitors
+    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $DATACENTER --check-monitors
 
 step-1_deploy-prod-minor-dcs:
   when: manual
@@ -26,7 +26,7 @@ step-1_deploy-prod-minor-dcs:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: minor-dcs
+    DATACENTER: minor-dcs
 
 step-2_deploy-prod-eu1:
   needs:
@@ -35,7 +35,7 @@ step-2_deploy-prod-eu1:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: eu1
+    DATACENTER: eu1
 
 step-3_deploy-prod-us1:
   needs:
@@ -44,7 +44,7 @@ step-3_deploy-prod-us1:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: us1
+    DATACENTER: us1
 
 step-4_deploy-prod-gov:
   needs:
@@ -53,7 +53,7 @@ step-4_deploy-prod-gov:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: root
+    DATACENTER: gov
 
 step-5_publish-npm:
   needs:

.gitlab/deploy-manual.yml

@@ -19,35 +19,35 @@ stages:
     - VERSION=$(node -p -e "require('./lerna.json').version")
     - yarn
     - yarn build:bundle
-    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $UPLOAD_PATH --no-check-monitors
+    - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $DATACENTER --no-check-monitors
 
 step-1_deploy-prod-minor-dcs:
   extends:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: minor-dcs
+    DATACENTER: minor-dcs
 
 step-2_deploy-prod-eu1:
   extends:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: eu1
+    DATACENTER: eu1
 
 step-3_deploy-prod-us1:
   extends:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: us1
+    DATACENTER: us1
 
 step-4_deploy-prod-gov:
   extends:
     - .base-configuration
     - .deploy-prod
   variables:
-    UPLOAD_PATH: root
+    DATACENTER: root
 
 step-5_publish-npm:
   stage: deploy
@@ -80,7 +80,7 @@ step-7_create-github-release:
     - node scripts/release/create-github-release.ts
 
 # This step is used to deploy the SDK to a new datacenter.
-# the `UPLOAD_PATH` variable needs to be provided as an argument when starting the manual job
+# the `DATACENTER` variable needs to be provided as an argument when starting the manual job
 optional_step-deploy-to-new-datacenter:
   extends:
     - .base-configuration

scripts/deploy/check-monitors.ts

@@ -5,14 +5,14 @@
  */
 import { printLog, runMain, fetchHandlingError } from '../lib/executionUtils.ts'
 import { getTelemetryOrgApiKey, getTelemetryOrgApplicationKey } from '../lib/secrets.ts'
-import { siteByDatacenter } from '../lib/datacenter.ts'
+import { getSite } from '../lib/datacenter.ts'
 import { browserSdkVersion } from '../lib/browserSdkVersion.ts'
 
 const datacenters = process.argv[2].split(',')
 
 runMain(async () => {
   for (const datacenter of datacenters) {
-    const site = siteByDatacenter[datacenter]
+    const site = getSite(datacenter)
     const apiKey = getTelemetryOrgApiKey(site)
     const applicationKey = getTelemetryOrgApplicationKey(site)
 

scripts/deploy/deploy-prod-dc.spec.ts

@@ -1,13 +1,11 @@
 import assert from 'node:assert/strict'
 import path from 'node:path'
-import { beforeEach, before, describe, it, mock } from 'node:test'
+import { beforeEach, before, describe, it, mock, afterEach } from 'node:test'
 import type { CommandDetail } from './lib/testHelpers.ts'
-import { mockModule, mockCommandImplementation } from './lib/testHelpers.ts'
+import { mockCommandImplementation, mockModule } from './lib/testHelpers.ts'
 
-// eslint-disable-next-line
-describe.only('deploy-prod-dc', () => {
+describe('deploy-prod-dc', () => {
   const commandMock = mock.fn()
-
   let commands: CommandDetail[]
 
   before(async () => {
@@ -21,6 +19,10 @@ describe.only('deploy-prod-dc', () => {
     commands = mockCommandImplementation(commandMock)
   })
 
+  afterEach(() => {
+    mock.restoreAll()
+  })
+
   it('should deploy a given datacenter', async () => {
     await runScript('./deploy-prod-dc.ts', 'v6', 'us1')
 
@@ -42,22 +44,30 @@ describe.only('deploy-prod-dc', () => {
     ])
   })
 
-  it('should only check monitors before deploying if the upload path is root', async () => {
-    await runScript('./deploy-prod-dc.ts', 'v6', 'root', '--check-monitors')
+  it('should deploy all minor datacenters', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'minor-dcs', '--no-check-monitors')
 
     assert.deepEqual(commands, [
-      { command: 'node ./scripts/deploy/check-monitors.ts root' },
-      { command: 'node ./scripts/deploy/deploy.ts prod v6 root' },
-      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 root' },
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 ap1,ap2,us3,us5' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 ap1,ap2,us3,us5' },
     ])
   })
 
-  it('should deploy all minor datacenters', async () => {
-    await runScript('./deploy-prod-dc.ts', 'v6', 'minor-dcs', '--no-check-monitors')
+  it('should deploy all private regions', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'private-regions', '--no-check-monitors')
+
+    assert.deepEqual(commands, [
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 prtest00,prtest01' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 prtest00,prtest01' },
+    ])
+  })
+
+  it('should deploy gov datacenters to the root upload path', async () => {
+    await runScript('./deploy-prod-dc.ts', 'v6', 'gov', '--no-check-monitors')
 
     assert.deepEqual(commands, [
-      { command: 'node ./scripts/deploy/deploy.ts prod v6 us3,us5,ap1,ap2,prtest00' },
-      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 us3,us5,ap1,ap2,prtest00' },
+      { command: 'node ./scripts/deploy/deploy.ts prod v6 root' },
+      { command: 'node ./scripts/deploy/upload-source-maps.ts v6 root' },
     ])
   })
 })

scripts/deploy/deploy-prod-dc.ts

@@ -1,7 +1,7 @@
 import { parseArgs } from 'node:util'
 import { printLog, runMain, timeout } from '../lib/executionUtils.ts'
 import { command } from '../lib/command.ts'
-import { siteByDatacenter } from '../lib/datacenter.ts'
+import { getAllMinorDcs, getAllPrivateDcs } from '../lib/datacenter.ts'
 
 /**
  * Orchestrate the deployments of the artifacts for specific DCs
@@ -12,15 +12,6 @@ const ONE_MINUTE_IN_SECOND = 60
 const GATE_DURATION = 30 * ONE_MINUTE_IN_SECOND
 const GATE_INTERVAL = ONE_MINUTE_IN_SECOND
 
-// Major DCs are the ones that are deployed last.
-// They have their own step jobs in `deploy-manual.yml` and `deploy-auto.yml`.
-const MAJOR_DCS = ['root', 'us1', 'eu1']
-
-// Minor DCs are all the DCs from `siteByDatacenter` that are not in `MAJOR_DCS`.
-function getAllMinorDcs(): string[] {
-  return Object.keys(siteByDatacenter).filter((dc) => !MAJOR_DCS.includes(dc))
-}
-
 if (!process.env.NODE_TEST_CONTEXT) {
   runMain(() => main(...process.argv.slice(2)))
 }
@@ -42,30 +33,56 @@ export async function main(...args: string[]): Promise<void> {
   })
 
   const version = positionals[0]
-  const uploadPath = positionals[1] === 'minor-dcs' ? getAllMinorDcs().join(',') : positionals[1]
+  const datacenters = getDatacenters(positionals[1])
 
-  if (!uploadPath) {
-    throw new Error('UPLOAD_PATH argument is required')
+  if (!datacenters) {
+    throw new Error('DATACENTER argument is required')
   }
 
   if (checkMonitors) {
-    command`node ./scripts/deploy/check-monitors.ts ${uploadPath}`.withLogs().run()
+    command`node ./scripts/deploy/check-monitors.ts ${datacenters.join(',')}`.withLogs().run()
   }
 
-  command`node ./scripts/deploy/deploy.ts prod ${version} ${uploadPath}`.withLogs().run()
-  command`node ./scripts/deploy/upload-source-maps.ts ${version} ${uploadPath}`.withLogs().run()
+  const uploadPathTypes = toDatacenterUploadPathType(datacenters).join(',')
 
-  if (checkMonitors && uploadPath !== 'root') {
-    await gateMonitors(uploadPath)
+  command`node ./scripts/deploy/deploy.ts prod ${version} ${uploadPathTypes}`.withLogs().run()
+  command`node ./scripts/deploy/upload-source-maps.ts ${version} ${uploadPathTypes}`.withLogs().run()
+
+  if (checkMonitors) {
+    await gateMonitors(datacenters)
   }
 }
 
-async function gateMonitors(uploadPath: string): Promise<void> {
-  printLog(`Check monitors for ${uploadPath} during ${GATE_DURATION / ONE_MINUTE_IN_SECOND} minutes`)
+async function gateMonitors(datacenters: string[]): Promise<void> {
+  printLog(`Check monitors for ${datacenters.join(',')} during ${GATE_DURATION / ONE_MINUTE_IN_SECOND} minutes`)
+
   for (let i = 0; i < GATE_DURATION; i += GATE_INTERVAL) {
-    command`node ./scripts/deploy/check-monitors.ts ${uploadPath}`.run()
+    command`node ./scripts/deploy/check-monitors.ts ${datacenters.join(',')}`.run()
     process.stdout.write('.') // progress indicator
     await timeout(GATE_INTERVAL * 1000)
   }
+
   printLog() // new line
 }
+
+function getDatacenters(datacenterGroup: string): string[] {
+  if (datacenterGroup === 'minor-dcs') {
+    return getAllMinorDcs()
+  }
+
+  if (datacenterGroup === 'private-regions') {
+    return getAllPrivateDcs()
+  }
+
+  return datacenterGroup.split(',')
+}
+
+function toDatacenterUploadPathType(datacenters: string[]): string[] {
+  return datacenters.map((datacenter) => {
+    if (datacenter === 'gov') {
+      return 'root'
+    }
+
+    return datacenter
+  })
+}

scripts/deploy/lib/testHelpers.ts

@@ -62,9 +62,7 @@ export function mockCommandImplementation(mockFn: Mock<(...args: any[]) => void>
       withCurrentWorkingDirectory: () => result,
       withLogs: () => result,
       run(): string | undefined {
-        commands.push(commandDetail)
-
-        if (command.includes('aws sts assume-role')) {
+        if (command.startsWith('aws sts assume-role')) {
           return JSON.stringify({
             Credentials: {
               AccessKeyId: FAKE_AWS_ENV_CREDENTIALS.AWS_ACCESS_KEY_ID,
@@ -73,6 +71,22 @@ export function mockCommandImplementation(mockFn: Mock<(...args: any[]) => void>
             },
           })
         }
+
+        if (command.startsWith('ddtool datacenters list')) {
+          return JSON.stringify([
+            { name: 'ap1.prod.dog', site: 'ap1.datadoghq.com' },
+            { name: 'ap2.prod.dog', site: 'ap2.datadoghq.com' },
+            { name: 'eu1.prod.dog', site: 'datadoghq.eu' },
+            { name: 'us1.prod.dog', site: 'datadoghq.com' },
+            { name: 'us3.prod.dog', site: 'us3.datadoghq.com' },
+            { name: 'us5.prod.dog', site: 'us5.datadoghq.com' },
+            { name: 'prtest00.prod.dog', site: 'prtest00.datadoghq.com' },
+            { name: 'prtest01.prod.dog', site: 'prtest01.datadoghq.com' },
+          ])
+        }
+
+        // don't push command details for the above mock commands
+        commands.push(commandDetail)
       },
     }
     return result

scripts/deploy/upload-source-maps.spec.ts

@@ -1,7 +1,7 @@
 import assert from 'node:assert/strict'
 import path from 'node:path'
-import { beforeEach, before, describe, it, mock } from 'node:test'
-import { siteByDatacenter } from '../lib/datacenter.ts'
+import { beforeEach, before, describe, it, mock, afterEach } from 'node:test'
+import { getAllDatacenters, getSite } from '../lib/datacenter.ts'
 import { mockModule, mockCommandImplementation, replaceChunkHashes } from './lib/testHelpers.ts'
 
 const FAKE_API_KEY = 'FAKE_API_KEY'
@@ -49,9 +49,13 @@ describe('upload-source-maps', () => {
     commands = mockCommandImplementation(commandMock)
   })
 
+  afterEach(() => {
+    mock.restoreAll()
+  })
+
   function forEachDatacenter(callback: (site: string) => void): void {
-    for (const site of Object.values(siteByDatacenter)) {
-      callback(site)
+    for (const datacenter of getAllDatacenters()) {
+      callback(getSite(datacenter))
     }
   }
 

scripts/deploy/upload-source-maps.ts

@@ -3,7 +3,7 @@ import { printLog, runMain } from '../lib/executionUtils.ts'
 import { command } from '../lib/command.ts'
 import { getBuildEnvValue } from '../lib/buildEnv.ts'
 import { getTelemetryOrgApiKey } from '../lib/secrets.ts'
-import { siteByDatacenter } from '../lib/datacenter.ts'
+import { getSite, getAllDatacenters } from '../lib/datacenter.ts'
 import { forEachFile } from '../lib/filesUtils.ts'
 import { buildRootUploadPath, buildDatacenterUploadPath, buildBundleFolder, packages } from './lib/deploymentUtils.ts'
 
@@ -20,7 +20,8 @@ function getSitesByVersion(version: string): string[] {
     case 'canary':
       return ['datadoghq.com']
     default:
-      return Object.values(siteByDatacenter)
+      // TODO: do we upload to root for all DCs?
+      return getAllDatacenters().map(getSite)
   }
 }
 
@@ -56,7 +57,7 @@ async function uploadSourceMaps(
       uploadPath = buildRootUploadPath(packageName, version)
       await renameFilesWithVersionSuffix(bundleFolder, version)
     } else {
-      sites = [siteByDatacenter[uploadPathType]]
+      sites = [getSite(uploadPathType)]
       uploadPath = buildDatacenterUploadPath(uploadPathType, packageName, version)
     }
     const prefix = path.dirname(`/${uploadPath}`)