diff --git a/.gitlab/deploy-auto.yml b/.gitlab/deploy-auto.yml index 43b14a6cb3..c0713b3171 100644 --- a/.gitlab/deploy-auto.yml +++ b/.gitlab/deploy-auto.yml @@ -17,7 +17,7 @@ stages: - VERSION=$(node -p -e "require('./lerna.json').version") - yarn - yarn build:bundle - - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $UPLOAD_PATH --check-monitors + - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $DATACENTER --check-monitors step-1_deploy-prod-minor-dcs: when: manual @@ -26,7 +26,7 @@ step-1_deploy-prod-minor-dcs: - .base-configuration - .deploy-prod variables: - UPLOAD_PATH: minor-dcs + DATACENTER: minor-dcs step-2_deploy-prod-eu1: needs: @@ -35,7 +35,7 @@ step-2_deploy-prod-eu1: - .base-configuration - .deploy-prod variables: - UPLOAD_PATH: eu1 + DATACENTER: eu1 step-3_deploy-prod-us1: needs: @@ -44,7 +44,7 @@ step-3_deploy-prod-us1: - .base-configuration - .deploy-prod variables: - UPLOAD_PATH: us1 + DATACENTER: us1 step-4_deploy-prod-gov: needs: @@ -53,7 +53,7 @@ step-4_deploy-prod-gov: - .base-configuration - .deploy-prod variables: - UPLOAD_PATH: root + DATACENTER: gov step-5_publish-npm: needs: diff --git a/.gitlab/deploy-manual.yml b/.gitlab/deploy-manual.yml index b3f2849ccd..97b5656693 100644 --- a/.gitlab/deploy-manual.yml +++ b/.gitlab/deploy-manual.yml @@ -19,35 +19,35 @@ stages: - VERSION=$(node -p -e "require('./lerna.json').version") - yarn - yarn build:bundle - - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $UPLOAD_PATH --no-check-monitors + - node ./scripts/deploy/deploy-prod-dc.ts v${VERSION%%.*} $DATACENTER --no-check-monitors step-1_deploy-prod-minor-dcs: extends: - .base-configuration - .deploy-prod variables: - UPLOAD_PATH: minor-dcs + DATACENTER: minor-dcs step-2_deploy-prod-eu1: extends: - .base-configuration - .deploy-prod variables: - UPLOAD_PATH: eu1 + DATACENTER: eu1 step-3_deploy-prod-us1: extends: - .base-configuration - .deploy-prod variables: - UPLOAD_PATH: us1 + DATACENTER: us1 step-4_deploy-prod-gov: extends: - .base-configuration - .deploy-prod variables: - UPLOAD_PATH: root + DATACENTER: root step-5_publish-npm: stage: deploy @@ -80,7 +80,7 @@ step-7_create-github-release: - node scripts/release/create-github-release.ts # This step is used to deploy the SDK to a new datacenter. -# the `UPLOAD_PATH` variable needs to be provided as an argument when starting the manual job +# the `DATACENTER` variable needs to be provided as an argument when starting the manual job optional_step-deploy-to-new-datacenter: extends: - .base-configuration diff --git a/scripts/deploy/check-monitors.ts b/scripts/deploy/check-monitors.ts index f70e36afdf..ae361cdb0e 100644 --- a/scripts/deploy/check-monitors.ts +++ b/scripts/deploy/check-monitors.ts @@ -5,50 +5,66 @@ */ import { printLog, runMain, fetchHandlingError } from '../lib/executionUtils.ts' import { getTelemetryOrgApiKey, getTelemetryOrgApplicationKey } from '../lib/secrets.ts' -import { monitorIdsByDatacenter, siteByDatacenter } from '../lib/datacenter.ts' - -interface MonitorStatus { - id: number - name: string - overall_state: string -} +import { getSite } from '../lib/datacenter.ts' +import { browserSdkVersion } from '../lib/browserSdkVersion.ts' const datacenters = process.argv[2].split(',') runMain(async () => { for (const datacenter of datacenters) { - if (!monitorIdsByDatacenter[datacenter]) { - printLog(`No monitors configured for datacenter ${datacenter}`) + const site = getSite(datacenter) + const apiKey = getTelemetryOrgApiKey(site) + const applicationKey = getTelemetryOrgApplicationKey(site) + + if (!apiKey || !applicationKey) { + printLog(`No API key or application key found for ${site}, skipping...`) continue } - const monitorIds = monitorIdsByDatacenter[datacenter] - const site = siteByDatacenter[datacenter] - const monitorStatuses = await Promise.all(monitorIds.map((monitorId) => fetchMonitorStatus(site, monitorId))) - for (const monitorStatus of monitorStatuses) { - printLog(`${monitorStatus.overall_state} - ${monitorStatus.name}`) - if (monitorStatus.overall_state !== 'OK') { - throw new Error( - `Monitor ${monitorStatus.name} is in state ${monitorStatus.overall_state}, see ${computeMonitorLink(site, monitorStatus.id)}` - ) - } + + const errorLogsCount = await queryErrorLogsCount(site, apiKey, applicationKey) + + if (errorLogsCount > 0) { + throw new Error(`Errors found in the last 30 minutes, +see ${computeMonitorLink(site)}`) + } else { + printLog(`No errors found in the last 30 minutes for ${datacenter}`) } } }) -async function fetchMonitorStatus(site: string, monitorId: number): Promise { - const response = await fetchHandlingError(`https://api.${site}/api/v1/monitor/${monitorId}`, { - method: 'GET', +async function queryErrorLogsCount(site: string, apiKey: string, applicationKey: string): Promise { + const response = await fetchHandlingError(`https://api.${site}/api/v2/logs/events/search`, { + method: 'POST', headers: { - Accept: 'application/json', - 'DD-API-KEY': getTelemetryOrgApiKey(site), - 'DD-APPLICATION-KEY': getTelemetryOrgApplicationKey(site), + 'Content-Type': 'application/json', + 'DD-API-KEY': apiKey, + 'DD-APPLICATION-KEY': applicationKey, }, + body: JSON.stringify({ + filter: { + from: 'now-30m', + to: 'now', + query: `source:browser status:error version:${browserSdkVersion}`, + }, + }), }) - return response.json() as Promise + + const data = (await response.json()) as { data: unknown[] } + + return data.data.length } -function computeMonitorLink(site: string, monitorId: number): string { - return `https://${computeTelemetryOrgDomain(site)}/monitors/${monitorId}` +function computeMonitorLink(site: string): string { + const now = Date.now() + const thirtyMinutesAgo = now - 30 * 60 * 1000 + + const queryParams = new URLSearchParams({ + query: `source:browser status:error version:${browserSdkVersion}`, + from_ts: `${thirtyMinutesAgo}`, + to_ts: `${now}`, + }) + + return `https://${computeTelemetryOrgDomain(site)}/logs?${queryParams.toString()}` } function computeTelemetryOrgDomain(site: string): string { diff --git a/scripts/deploy/deploy-prod-dc.spec.ts b/scripts/deploy/deploy-prod-dc.spec.ts new file mode 100644 index 0000000000..b8ac24c022 --- /dev/null +++ b/scripts/deploy/deploy-prod-dc.spec.ts @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict' +import path from 'node:path' +import { beforeEach, before, describe, it, mock, afterEach } from 'node:test' +import type { CommandDetail } from './lib/testHelpers.ts' +import { mockCommandImplementation, mockModule } from './lib/testHelpers.ts' + +describe('deploy-prod-dc', () => { + const commandMock = mock.fn() + let commands: CommandDetail[] + + before(async () => { + await mockModule(path.resolve(import.meta.dirname, '../lib/command.ts'), { command: commandMock }) + await mockModule(path.resolve(import.meta.dirname, '../lib/executionUtils.ts'), { + timeout: () => Promise.resolve(), + }) + }) + + beforeEach(() => { + commands = mockCommandImplementation(commandMock) + }) + + afterEach(() => { + mock.restoreAll() + }) + + it('should deploy a given datacenter', async () => { + await runScript('./deploy-prod-dc.ts', 'v6', 'us1') + + assert.deepEqual(commands, [ + { command: 'node ./scripts/deploy/deploy.ts prod v6 us1' }, + { command: 'node ./scripts/deploy/upload-source-maps.ts v6 us1' }, + ]) + }) + + it('should deploy a given datacenter with check monitors', async () => { + await runScript('./deploy-prod-dc.ts', 'v6', 'us1', '--check-monitors') + + assert.deepEqual(commands, [ + { command: 'node ./scripts/deploy/check-monitors.ts us1' }, + { command: 'node ./scripts/deploy/deploy.ts prod v6 us1' }, + { command: 'node ./scripts/deploy/upload-source-maps.ts v6 us1' }, + // 1 monitor check per minute for 30 minutes + ...Array.from({ length: 30 }, () => ({ command: 'node ./scripts/deploy/check-monitors.ts us1' })), + ]) + }) + + it('should deploy all minor datacenters', async () => { + await runScript('./deploy-prod-dc.ts', 'v6', 'minor-dcs', '--no-check-monitors') + + assert.deepEqual(commands, [ + { command: 'node ./scripts/deploy/deploy.ts prod v6 ap1,ap2,us3,us5' }, + { command: 'node ./scripts/deploy/upload-source-maps.ts v6 ap1,ap2,us3,us5' }, + ]) + }) + + it('should deploy all private regions', async () => { + await runScript('./deploy-prod-dc.ts', 'v6', 'private-regions', '--no-check-monitors') + + assert.deepEqual(commands, [ + { command: 'node ./scripts/deploy/deploy.ts prod v6 prtest00,prtest01' }, + { command: 'node ./scripts/deploy/upload-source-maps.ts v6 prtest00,prtest01' }, + ]) + }) + + it('should deploy gov datacenters to the root upload path', async () => { + await runScript('./deploy-prod-dc.ts', 'v6', 'gov', '--no-check-monitors') + + assert.deepEqual(commands, [ + { command: 'node ./scripts/deploy/deploy.ts prod v6 root' }, + { command: 'node ./scripts/deploy/upload-source-maps.ts v6 root' }, + ]) + }) +}) + +async function runScript(scriptPath: string, ...args: string[]): Promise { + const { main } = (await import(scriptPath)) as { main: (...args: string[]) => Promise } + + return main(...args) +} diff --git a/scripts/deploy/deploy-prod-dc.ts b/scripts/deploy/deploy-prod-dc.ts index 56e83161c0..a87700dd94 100644 --- a/scripts/deploy/deploy-prod-dc.ts +++ b/scripts/deploy/deploy-prod-dc.ts @@ -1,7 +1,7 @@ import { parseArgs } from 'node:util' import { printLog, runMain, timeout } from '../lib/executionUtils.ts' import { command } from '../lib/command.ts' -import { siteByDatacenter } from '../lib/datacenter.ts' +import { getAllMinorDcs, getAllPrivateDcs } from '../lib/datacenter.ts' /** * Orchestrate the deployments of the artifacts for specific DCs @@ -12,54 +12,77 @@ const ONE_MINUTE_IN_SECOND = 60 const GATE_DURATION = 30 * ONE_MINUTE_IN_SECOND const GATE_INTERVAL = ONE_MINUTE_IN_SECOND -// Major DCs are the ones that are deployed last. -// They have their own step jobs in `deploy-manual.yml` and `deploy-auto.yml`. -const MAJOR_DCS = ['root', 'us1', 'eu1'] - -// Minor DCs are all the DCs from `siteByDatacenter` that are not in `MAJOR_DCS`. -function getAllMinorDcs(): string[] { - return Object.keys(siteByDatacenter).filter((dc) => !MAJOR_DCS.includes(dc)) +if (!process.env.NODE_TEST_CONTEXT) { + runMain(() => main(...process.argv.slice(2))) } -const { - values: { 'check-monitors': checkMonitors }, - positionals, -} = parseArgs({ - allowPositionals: true, - allowNegative: true, - options: { - 'check-monitors': { - type: 'boolean', +export async function main(...args: string[]): Promise { + const { + values: { 'check-monitors': checkMonitors }, + positionals, + } = parseArgs({ + args, + allowPositionals: true, + allowNegative: true, + options: { + 'check-monitors': { + type: 'boolean', + default: false, + }, }, - }, -}) + }) -const version = positionals[0] -const uploadPath = positionals[1] === 'minor-dcs' ? getAllMinorDcs().join(',') : positionals[1] + const version = positionals[0] + const datacenters = getDatacenters(positionals[1]) -if (!uploadPath) { - throw new Error('UPLOAD_PATH argument is required') -} + if (!datacenters) { + throw new Error('DATACENTER argument is required') + } -runMain(async () => { if (checkMonitors) { - command`node ./scripts/deploy/check-monitors.ts ${uploadPath}`.withLogs().run() + command`node ./scripts/deploy/check-monitors.ts ${datacenters.join(',')}`.withLogs().run() } - command`node ./scripts/deploy/deploy.ts prod ${version} ${uploadPath}`.withLogs().run() - command`node ./scripts/deploy/upload-source-maps.ts ${version} ${uploadPath}`.withLogs().run() + const uploadPathTypes = toDatacenterUploadPathType(datacenters).join(',') - if (checkMonitors && uploadPath !== 'root') { - await gateMonitors(uploadPath) + command`node ./scripts/deploy/deploy.ts prod ${version} ${uploadPathTypes}`.withLogs().run() + command`node ./scripts/deploy/upload-source-maps.ts ${version} ${uploadPathTypes}`.withLogs().run() + + if (checkMonitors) { + await gateMonitors(datacenters) } -}) +} + +async function gateMonitors(datacenters: string[]): Promise { + printLog(`Check monitors for ${datacenters.join(',')} during ${GATE_DURATION / ONE_MINUTE_IN_SECOND} minutes`) -async function gateMonitors(uploadPath: string): Promise { - printLog(`Check monitors for ${uploadPath} during ${GATE_DURATION / ONE_MINUTE_IN_SECOND} minutes`) for (let i = 0; i < GATE_DURATION; i += GATE_INTERVAL) { - command`node ./scripts/deploy/check-monitors.ts ${uploadPath}`.run() + command`node ./scripts/deploy/check-monitors.ts ${datacenters.join(',')}`.run() process.stdout.write('.') // progress indicator await timeout(GATE_INTERVAL * 1000) } + printLog() // new line } + +function getDatacenters(datacenterGroup: string): string[] { + if (datacenterGroup === 'minor-dcs') { + return getAllMinorDcs() + } + + if (datacenterGroup === 'private-regions') { + return getAllPrivateDcs() + } + + return datacenterGroup.split(',') +} + +function toDatacenterUploadPathType(datacenters: string[]): string[] { + return datacenters.map((datacenter) => { + if (datacenter === 'gov') { + return 'root' + } + + return datacenter + }) +} diff --git a/scripts/deploy/lib/testHelpers.ts b/scripts/deploy/lib/testHelpers.ts index e2c241ede4..32a16ef76d 100644 --- a/scripts/deploy/lib/testHelpers.ts +++ b/scripts/deploy/lib/testHelpers.ts @@ -62,9 +62,7 @@ export function mockCommandImplementation(mockFn: Mock<(...args: any[]) => void> withCurrentWorkingDirectory: () => result, withLogs: () => result, run(): string | undefined { - commands.push(commandDetail) - - if (command.includes('aws sts assume-role')) { + if (command.startsWith('aws sts assume-role')) { return JSON.stringify({ Credentials: { AccessKeyId: FAKE_AWS_ENV_CREDENTIALS.AWS_ACCESS_KEY_ID, @@ -73,6 +71,22 @@ export function mockCommandImplementation(mockFn: Mock<(...args: any[]) => void> }, }) } + + if (command.startsWith('ddtool datacenters list')) { + return JSON.stringify([ + { name: 'ap1.prod.dog', site: 'ap1.datadoghq.com' }, + { name: 'ap2.prod.dog', site: 'ap2.datadoghq.com' }, + { name: 'eu1.prod.dog', site: 'datadoghq.eu' }, + { name: 'us1.prod.dog', site: 'datadoghq.com' }, + { name: 'us3.prod.dog', site: 'us3.datadoghq.com' }, + { name: 'us5.prod.dog', site: 'us5.datadoghq.com' }, + { name: 'prtest00.prod.dog', site: 'prtest00.datadoghq.com' }, + { name: 'prtest01.prod.dog', site: 'prtest01.datadoghq.com' }, + ]) + } + + // don't push command details for the above mock commands + commands.push(commandDetail) }, } return result diff --git a/scripts/deploy/upload-source-maps.spec.ts b/scripts/deploy/upload-source-maps.spec.ts index cd67d62d2c..69759f4dfe 100644 --- a/scripts/deploy/upload-source-maps.spec.ts +++ b/scripts/deploy/upload-source-maps.spec.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict' import path from 'node:path' -import { beforeEach, before, describe, it, mock } from 'node:test' -import { siteByDatacenter } from '../lib/datacenter.ts' +import { beforeEach, before, describe, it, mock, afterEach } from 'node:test' +import { getAllDatacenters, getSite } from '../lib/datacenter.ts' import { mockModule, mockCommandImplementation, replaceChunkHashes } from './lib/testHelpers.ts' const FAKE_API_KEY = 'FAKE_API_KEY' @@ -49,9 +49,13 @@ describe('upload-source-maps', () => { commands = mockCommandImplementation(commandMock) }) + afterEach(() => { + mock.restoreAll() + }) + function forEachDatacenter(callback: (site: string) => void): void { - for (const site of Object.values(siteByDatacenter)) { - callback(site) + for (const datacenter of getAllDatacenters()) { + callback(getSite(datacenter)) } } diff --git a/scripts/deploy/upload-source-maps.ts b/scripts/deploy/upload-source-maps.ts index b8137c7e34..4ebc262e35 100644 --- a/scripts/deploy/upload-source-maps.ts +++ b/scripts/deploy/upload-source-maps.ts @@ -3,7 +3,7 @@ import { printLog, runMain } from '../lib/executionUtils.ts' import { command } from '../lib/command.ts' import { getBuildEnvValue } from '../lib/buildEnv.ts' import { getTelemetryOrgApiKey } from '../lib/secrets.ts' -import { siteByDatacenter } from '../lib/datacenter.ts' +import { getSite, getAllDatacenters } from '../lib/datacenter.ts' import { forEachFile } from '../lib/filesUtils.ts' import { buildRootUploadPath, buildDatacenterUploadPath, buildBundleFolder, packages } from './lib/deploymentUtils.ts' @@ -20,7 +20,8 @@ function getSitesByVersion(version: string): string[] { case 'canary': return ['datadoghq.com'] default: - return Object.values(siteByDatacenter) + // TODO: do we upload to root for all DCs? + return getAllDatacenters().map(getSite) } } @@ -56,7 +57,7 @@ async function uploadSourceMaps( uploadPath = buildRootUploadPath(packageName, version) await renameFilesWithVersionSuffix(bundleFolder, version) } else { - sites = [siteByDatacenter[uploadPathType]] + sites = [getSite(uploadPathType)] uploadPath = buildDatacenterUploadPath(uploadPathType, packageName, version) } const prefix = path.dirname(`/${uploadPath}`) diff --git a/scripts/lib/datacenter.ts b/scripts/lib/datacenter.ts index 73a8692889..80f9028cdd 100644 --- a/scripts/lib/datacenter.ts +++ b/scripts/lib/datacenter.ts @@ -1,24 +1,51 @@ -export const siteByDatacenter: Record = { - us1: 'datadoghq.com', - eu1: 'datadoghq.eu', - us3: 'us3.datadoghq.com', - us5: 'us5.datadoghq.com', - ap1: 'ap1.datadoghq.com', - ap2: 'ap2.datadoghq.com', - prtest00: 'prtest00.datad0g.com', +import { command } from './command.ts' + +// Major DCs are the ones that are deployed last. +// They have their own step jobs in `deploy-manual.yml` and `deploy-auto.yml`. +const MAJOR_DCS = ['gov', 'us1', 'eu1'] + +export function getSite(datacenter: string): string { + return getAllDatacentersMetadata()[datacenter].site +} + +export function getAllDatacenters(): string[] { + return Object.keys(getAllDatacentersMetadata()) +} + +export function getAllMinorDcs(): string[] { + return getAllDatacenters().filter((dc) => !MAJOR_DCS.includes(dc) && !dc.startsWith('pr')) +} + +export function getAllPrivateDcs(): string[] { + return getAllDatacenters().filter((dc) => dc.startsWith('pr')) } -/** - * Each datacenter has 3 monitor IDs: - * - Telemetry errors - * - Telemetry errors on specific org - * - Telemetry errors on specific message - */ -export const monitorIdsByDatacenter: Record = { - us1: [72055549, 68975047, 110519972], - eu1: [5855803, 5663834, 9896387], - us3: [164368, 160677, 329066], - us5: [22388, 20646, 96049], - ap1: [858, 859, 2757030], - ap2: [1234, 1235, 1236], +interface Datacenter { + name: string + site: string +} + +let cachedDatacenters: Record | undefined + +function getAllDatacentersMetadata(): Record { + if (cachedDatacenters) { + return cachedDatacenters + } + + const selector = 'datacenter.environment == "prod" && datacenter.flavor == "site"' + const rawDatacenters = command`ddtool datacenters list --selector ${selector}`.run().trim() + const jsonDatacenters = JSON.parse(rawDatacenters) as Datacenter[] + + cachedDatacenters = {} + + for (const datacenter of jsonDatacenters) { + const shortName = datacenter.name.split('.')[0] + + cachedDatacenters[shortName] = { + name: datacenter.name, + site: datacenter.site, + } + } + + return cachedDatacenters }