From 9befbe4bd0aff957e3a19aa872e4d02faa19c369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Mon, 17 Feb 2025 09:48:49 +0100 Subject: [PATCH 01/11] Turn existing CLI multi-command --- packages/compass-smoke-tests/src/cli.ts | 116 +++++++++++++----------- 1 file changed, 62 insertions(+), 54 deletions(-) diff --git a/packages/compass-smoke-tests/src/cli.ts b/packages/compass-smoke-tests/src/cli.ts index 2ff69ce5bf9..7e018c4d1bb 100755 --- a/packages/compass-smoke-tests/src/cli.ts +++ b/packages/compass-smoke-tests/src/cli.ts @@ -54,7 +54,7 @@ function getDefaultArch() { } } -const argv = yargs(hideBin(process.argv)) +yargs(hideBin(process.argv)) .scriptName('smoke-tests') .detectLocale(false) .version(false) @@ -67,51 +67,68 @@ const argv = yargs(hideBin(process.argv)) type: 'string', default: process.env.EVERGREEN_BUCKET_KEY_PREFIX, }) - .option('platform', { - choices: SUPPORTED_PLATFORMS, - demandOption: true, - default: getDefaultPlatform(), - }) - .option('arch', { - choices: SUPPORTED_ARCHS, - demandOption: true, - default: getDefaultArch(), - }) - .option('package', { - type: 'string', - choices: SUPPORTED_PACKAGES, - demandOption: true, - description: 'Which package to test', - }) - .option('forceDownload', { - type: 'boolean', - description: 'Force download all assets before starting', - }) - .option('localPackage', { - type: 'boolean', - description: 'Use the local package instead of downloading', - }) - .option('skipCleanup', { - type: 'boolean', - description: 'Do not delete the sandboxes after a run', - default: false, - }) - .option('skipUninstall', { - type: 'boolean', - description: 'Do not uninstall after a run', - default: false, - }) - .option('tests', { - type: 'array', - string: true, - choices: SUPPORTED_TESTS, - description: 'Which tests to run', - }) - .default('tests', SUPPORTED_TESTS.slice()); - -async function run() { - const context: SmokeTestsContext = argv.parseSync(); + .command( + '$0', + 'Run smoke tests', + (argv) => + argv + .option('platform', { + choices: SUPPORTED_PLATFORMS, + demandOption: true, + default: getDefaultPlatform(), + }) + .option('arch', { + choices: SUPPORTED_ARCHS, + demandOption: true, + default: getDefaultArch(), + }) + .option('package', { + type: 'string', + choices: SUPPORTED_PACKAGES, + demandOption: true, + description: 'Which package to test', + }) + .option('forceDownload', { + type: 'boolean', + description: 'Force download all assets before starting', + }) + .option('localPackage', { + type: 'boolean', + description: 'Use the local package instead of downloading', + }) + .option('skipCleanup', { + type: 'boolean', + description: 'Do not delete the sandboxes after a run', + default: false, + }) + .option('skipUninstall', { + type: 'boolean', + description: 'Do not uninstall after a run', + default: false, + }) + .option('tests', { + type: 'array', + string: true, + choices: SUPPORTED_TESTS, + description: 'Which tests to run', + }) + .default('tests', SUPPORTED_TESTS.slice()), + async (args) => { + await run(args); + } + ) + .parseAsync() + .then( + () => { + debug('done'); + }, + (err) => { + console.error(err.stack); + process.exitCode = 1; + } + ); +async function run(context: SmokeTestsContext) { function cleanupMaybe() { if (context.skipCleanup) { console.log('Skipped cleanup of sandboxes'); @@ -160,12 +177,3 @@ async function run() { } } } - -run() - .then(function () { - debug('done'); - }) - .catch(function (err) { - console.error(err.stack); - process.exitCode = 1; - }); From fb5c262613a14bcb71f4dbf372c80857a85bcc56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Mon, 17 Feb 2025 14:14:06 +0100 Subject: [PATCH 02/11] Add dispatch command --- .github/workflows/test-installers.yml | 11 +- package-lock.json | 398 ++++++++++++++++++- packages/compass-smoke-tests/package.json | 3 +- packages/compass-smoke-tests/src/cli.ts | 40 ++ packages/compass-smoke-tests/src/dispatch.ts | 127 ++++++ 5 files changed, 573 insertions(+), 6 deletions(-) create mode 100644 packages/compass-smoke-tests/src/dispatch.ts diff --git a/.github/workflows/test-installers.yml b/.github/workflows/test-installers.yml index 650af5d5d16..6de69ec00c8 100644 --- a/.github/workflows/test-installers.yml +++ b/.github/workflows/test-installers.yml @@ -6,6 +6,10 @@ permissions: on: workflow_dispatch: inputs: + version: + type: string + description: 'Version of the installer to download' + required: true bucket_name: type: string description: 'S3 bucket to download installers from' @@ -14,10 +18,11 @@ on: type: string description: 'S3 bucket key prefix to download installers from' required: true - version: + nonce: type: string - description: 'Version of the installer to download' - required: true + description: 'A random string to track the run from dispatch to watching' + +run-name: Test Installers ${{ github.event.inputs.version }} / (nonce = ${{ github.event.inputs.nonce || 'not set' }}) jobs: test: diff --git a/package-lock.json b/package-lock.json index 61fe9f9020a..3c44c40c7ce 100644 --- a/package-lock.json +++ b/package-lock.json @@ -922,6 +922,215 @@ "node": ">=10.0.0" } }, + "node_modules/@actions/github": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz", + "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@actions/http-client": "^2.2.0", + "@octokit/core": "^5.0.1", + "@octokit/plugin-paginate-rest": "^9.0.0", + "@octokit/plugin-rest-endpoint-methods": "^10.0.0" + } + }, + "node_modules/@actions/github/node_modules/@octokit/auth-token": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/core": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.0.tgz", + "integrity": "sha512-1LFfa/qnMQvEOAdzlQymH0ulepxbxnCYAKJZfMci/5XJyIHWgEYnDmgnKakbTh7CH2tFQ5O60oYDvns4i9RAIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.1.0", + "@octokit/request": "^8.3.1", + "@octokit/request-error": "^5.1.0", + "@octokit/types": "^13.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/endpoint": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz", + "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/graphql": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.0.tgz", + "integrity": "sha512-r+oZUH7aMFui1ypZnAvZmn0KSqAUgE1/tUXIWaqUCa1758ts/Jio84GZuzsvUkme98kv0WFY8//n0J1Z+vsIsQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/request": "^8.3.0", + "@octokit/types": "^13.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/openapi-types": { + "version": "23.0.1", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-23.0.1.tgz", + "integrity": "sha512-izFjMJ1sir0jn0ldEKhZ7xegCTj/ObmEDlEfpFrx4k/JyZSMRHbO3/rBwgE7f3m2DHt+RrNGIVw4wSmwnm3t/g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-paginate-rest": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz", + "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^12.6.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": "5" + } + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^20.0.0" + } + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz", + "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^12.6.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": "5" + } + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@actions/github/node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^20.0.0" + } + }, + "node_modules/@actions/github/node_modules/@octokit/request": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz", + "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/endpoint": "^9.0.6", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/request-error": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz", + "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.1.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@actions/github/node_modules/@octokit/types": { + "version": "13.8.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.8.0.tgz", + "integrity": "sha512-x7DjTIbEpEWXK99DMd01QfWy0hd5h4EN+Q7shkdKds3otGQP+oWE/y0A76i1OvH9fygo4ddvNf7ZvF0t78P98A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^23.0.1" + } + }, + "node_modules/@actions/http-client": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz", + "integrity": "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tunnel": "^0.0.6", + "undici": "^5.25.4" + } + }, + "node_modules/@actions/http-client/node_modules/undici": { + "version": "5.28.5", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.5.tgz", + "integrity": "sha512-zICwjrDrcrUE0pyyJc1I2QzBkLM8FINsgOrt6WjA+BgajVq9Nxu2PbFFXUrAggLfDXlZGZBVZYw7WNV5KiBiBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/@ampproject/remapping": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", @@ -4758,6 +4967,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/@floating-ui/core": { "version": "1.6.9", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.9.tgz", @@ -40880,7 +41099,7 @@ "version": "0.0.6", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", - "optional": true, + "devOptional": true, "engines": { "node": ">=0.6.11 <=0.7.0 || >=0.7.3" } @@ -47056,6 +47275,7 @@ "version": "1.1.2", "license": "SSPL", "devDependencies": { + "@actions/github": "^6.0.0", "@mongodb-js/eslint-config-compass": "^1.3.2", "@mongodb-js/prettier-config-compass": "^1.2.2", "@mongodb-js/tsconfig-compass": "^1.2.2", @@ -51000,6 +51220,173 @@ } }, "dependencies": { + "@actions/github": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz", + "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==", + "dev": true, + "requires": { + "@actions/http-client": "^2.2.0", + "@octokit/core": "^5.0.1", + "@octokit/plugin-paginate-rest": "^9.0.0", + "@octokit/plugin-rest-endpoint-methods": "^10.0.0" + }, + "dependencies": { + "@octokit/auth-token": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "dev": true + }, + "@octokit/core": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.0.tgz", + "integrity": "sha512-1LFfa/qnMQvEOAdzlQymH0ulepxbxnCYAKJZfMci/5XJyIHWgEYnDmgnKakbTh7CH2tFQ5O60oYDvns4i9RAIg==", + "dev": true, + "requires": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.1.0", + "@octokit/request": "^8.3.1", + "@octokit/request-error": "^5.1.0", + "@octokit/types": "^13.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/endpoint": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz", + "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==", + "dev": true, + "requires": { + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.0.tgz", + "integrity": "sha512-r+oZUH7aMFui1ypZnAvZmn0KSqAUgE1/tUXIWaqUCa1758ts/Jio84GZuzsvUkme98kv0WFY8//n0J1Z+vsIsQ==", + "dev": true, + "requires": { + "@octokit/request": "^8.3.0", + "@octokit/types": "^13.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/openapi-types": { + "version": "23.0.1", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-23.0.1.tgz", + "integrity": "sha512-izFjMJ1sir0jn0ldEKhZ7xegCTj/ObmEDlEfpFrx4k/JyZSMRHbO3/rBwgE7f3m2DHt+RrNGIVw4wSmwnm3t/g==", + "dev": true + }, + "@octokit/plugin-paginate-rest": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz", + "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==", + "dev": true, + "requires": { + "@octokit/types": "^12.6.0" + }, + "dependencies": { + "@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "dev": true + }, + "@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "dev": true, + "requires": { + "@octokit/openapi-types": "^20.0.0" + } + } + } + }, + "@octokit/plugin-rest-endpoint-methods": { + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz", + "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==", + "dev": true, + "requires": { + "@octokit/types": "^12.6.0" + }, + "dependencies": { + "@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "dev": true + }, + "@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "dev": true, + "requires": { + "@octokit/openapi-types": "^20.0.0" + } + } + } + }, + "@octokit/request": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz", + "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==", + "dev": true, + "requires": { + "@octokit/endpoint": "^9.0.6", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.1.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/request-error": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz", + "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==", + "dev": true, + "requires": { + "@octokit/types": "^13.1.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/types": { + "version": "13.8.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.8.0.tgz", + "integrity": "sha512-x7DjTIbEpEWXK99DMd01QfWy0hd5h4EN+Q7shkdKds3otGQP+oWE/y0A76i1OvH9fygo4ddvNf7ZvF0t78P98A==", + "dev": true, + "requires": { + "@octokit/openapi-types": "^23.0.1" + } + } + } + }, + "@actions/http-client": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz", + "integrity": "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==", + "dev": true, + "requires": { + "tunnel": "^0.0.6", + "undici": "^5.25.4" + }, + "dependencies": { + "undici": { + "version": "5.28.5", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.5.tgz", + "integrity": "sha512-zICwjrDrcrUE0pyyJc1I2QzBkLM8FINsgOrt6WjA+BgajVq9Nxu2PbFFXUrAggLfDXlZGZBVZYw7WNV5KiBiBA==", + "dev": true, + "requires": { + "@fastify/busboy": "^2.0.0" + } + } + } + }, "@ampproject/remapping": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", @@ -53871,6 +54258,12 @@ } } }, + "@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "dev": true + }, "@floating-ui/core": { "version": "1.6.9", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.9.tgz", @@ -58945,6 +59338,7 @@ "@mongodb-js/compass-smoke-tests": { "version": "file:packages/compass-smoke-tests", "requires": { + "@actions/github": "^6.0.0", "@mongodb-js/eslint-config-compass": "^1.3.2", "@mongodb-js/prettier-config-compass": "^1.2.2", "@mongodb-js/tsconfig-compass": "^1.2.2", @@ -88019,7 +88413,7 @@ "version": "0.0.6", "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", - "optional": true + "devOptional": true }, "tunnel-agent": { "version": "0.6.0", diff --git a/packages/compass-smoke-tests/package.json b/packages/compass-smoke-tests/package.json index c830194ef1e..3ebeb7fa1b9 100644 --- a/packages/compass-smoke-tests/package.json +++ b/packages/compass-smoke-tests/package.json @@ -30,10 +30,11 @@ "reformat": "npm run eslint . -- --fix && npm run prettier -- --write ." }, "devDependencies": { - "@types/node": "^20", + "@actions/github": "^6.0.0", "@mongodb-js/eslint-config-compass": "^1.3.2", "@mongodb-js/prettier-config-compass": "^1.2.2", "@mongodb-js/tsconfig-compass": "^1.2.2", + "@types/node": "^20", "compass-e2e-tests": "^1.29.2", "depcheck": "^1.4.1", "debug": "^4.3.4", diff --git a/packages/compass-smoke-tests/src/cli.ts b/packages/compass-smoke-tests/src/cli.ts index 7e018c4d1bb..ce600b5211d 100755 --- a/packages/compass-smoke-tests/src/cli.ts +++ b/packages/compass-smoke-tests/src/cli.ts @@ -1,8 +1,10 @@ #!/usr/bin/env npx ts-node +import assert from 'node:assert/strict'; import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import { pick } from 'lodash'; import createDebug from 'debug'; + import { SUPPORTED_TESTS } from './tests/types'; import { type SmokeTestsContext } from './context'; import { SUPPORTED_PACKAGES } from './packages'; @@ -10,6 +12,7 @@ import { testTimeToFirstQuery } from './tests/time-to-first-query'; import { testAutoUpdateFrom } from './tests/auto-update-from'; import { testAutoUpdateTo } from './tests/auto-update-to'; import { deleteSandboxesDirectory } from './directories'; +import { dispatchAndWait } from './dispatch'; const debug = createDebug('compass:smoketests'); @@ -117,6 +120,43 @@ yargs(hideBin(process.argv)) await run(args); } ) + .command( + 'dispatch', + 'Dispatch smoke tests on GitHub and watch to completion', + (argv) => + argv + .option('ref', { + type: 'string', + description: 'Git reference to dispatch the workflow from', + }) + .option('version', { + type: 'string', + description: 'Compass version to run tests for', + default: process.env.DEV_VERSION_IDENTIFIER, + }), + async ({ version, bucketName, bucketKeyPrefix, ref }) => { + const { GITHUB_TOKEN } = process.env; + assert( + typeof GITHUB_TOKEN === 'string', + 'Expected a GITHUB_TOKEN environment variable' + ); + assert( + typeof version === 'string', + 'Expected a version to dispatch tests for' + ); + assert( + typeof bucketName === 'string' && typeof bucketKeyPrefix === 'string', + 'Bucket name and key prefix are needed to download' + ); + await dispatchAndWait({ + githubToken: GITHUB_TOKEN, + ref, + version, + bucketName, + bucketKeyPrefix, + }); + } + ) .parseAsync() .then( () => { diff --git a/packages/compass-smoke-tests/src/dispatch.ts b/packages/compass-smoke-tests/src/dispatch.ts new file mode 100644 index 00000000000..04773403c64 --- /dev/null +++ b/packages/compass-smoke-tests/src/dispatch.ts @@ -0,0 +1,127 @@ +import cp from 'node:child_process'; +import crypto from 'node:crypto'; + +import * as github from '@actions/github'; +import createDebug from 'debug'; + +const GITHUB_OWNER = 'mongodb-js'; +const GITHUB_REPO = 'compass'; +const GITHUB_WORKFLOW_ID = 'test-installers.yml'; +const MAX_GET_LATEST_ATTEMPTS = 10; +const WATCH_POLL_TIMEOUT_MS = 1000 * 60 * 60; // 1 hour + +const debug = createDebug('compass:smoketests:dispatch'); + +function createNonce() { + return crypto.randomBytes(8).toString('hex'); +} + +async function getWorkflowRun( + octokit: ReturnType, + expectedRunName: string +) { + const { + data: { workflow_runs: workflowRuns }, + } = await octokit.rest.actions.listWorkflowRuns({ + owner: GITHUB_OWNER, + repo: GITHUB_REPO, + workflow_id: GITHUB_WORKFLOW_ID, + }); + return workflowRuns.find((run) => run.name === expectedRunName); +} + +async function getWorkflowRunRetrying( + octokit: ReturnType, + expectedRunName: string, + pollDelayMs = 1000 +) { + for (let attempt = 0; attempt < MAX_GET_LATEST_ATTEMPTS; attempt++) { + const run = await getWorkflowRun(octokit, expectedRunName); + debug(`Attempt %d finding run named "%s"`, attempt, expectedRunName); + if (run) { + return run; + } + await new Promise((resolve) => setTimeout(resolve, pollDelayMs)); + } + throw new Error( + `Failed to find run with name "${expectedRunName}" after ${MAX_GET_LATEST_ATTEMPTS} attempts` + ); +} + +function getDefaultRef() { + // TODO: Read this from an environment variable if possible + return cp + .spawnSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { + encoding: 'utf8', + }) + .stdout.trim(); +} + +type DispatchOptions = { + githubToken: string; + ref: string | undefined; + version: string; + bucketName: string; + bucketKeyPrefix: string; + + /** + * Delay in milliseconds to wait between requests when polling while watching the run. + */ + watchPollDelayMs?: number | undefined; +}; + +export async function dispatchAndWait({ + githubToken, + ref = getDefaultRef(), + version, + bucketName, + bucketKeyPrefix, + watchPollDelayMs = 5000, +}: DispatchOptions) { + const octokit = github.getOctokit(githubToken); + const nonce = createNonce(); + // Dispatch + await octokit.rest.actions.createWorkflowDispatch({ + owner: GITHUB_OWNER, + repo: GITHUB_REPO, + workflow_id: GITHUB_WORKFLOW_ID, + ref, + inputs: { + version, + bucket_name: bucketName, + bucket_key_prefix: bucketKeyPrefix, + nonce, + }, + }); + + // Find the next run we just dispatched + const run = await getWorkflowRunRetrying( + octokit, + `Test Installers ${version} / (nonce = ${nonce})` + ); + + console.log(`Dispatched run #${run.run_number} (${run.html_url})`); + for ( + const start = new Date(); + new Date().getTime() - start.getTime() < WATCH_POLL_TIMEOUT_MS; + + ) { + const { + data: { status, conclusion }, + } = await octokit.rest.actions.getWorkflowRun({ + owner: GITHUB_OWNER, + repo: GITHUB_REPO, + run_id: run.id, + }); + console.log( + `Status = ${status || 'null'}, conclusion = ${conclusion || 'null'}` + ); + if (status === 'completed' && conclusion === 'success') { + return; + } + await new Promise((resolve) => setTimeout(resolve, watchPollDelayMs)); + } + throw new Error( + `Run did not complete successfully within ${WATCH_POLL_TIMEOUT_MS}ms` + ); +} From 3565c872283d6fe17a3310310410c564cce89430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Mon, 17 Feb 2025 15:57:31 +0100 Subject: [PATCH 03/11] Dispatch on GHA instead of running smoke tests on Evergreen --- .evergreen/buildvariants-and-tasks.in.yml | 61 ++++------------------- .evergreen/buildvariants-and-tasks.yml | 38 +++----------- .evergreen/functions.yml | 41 ++------------- 3 files changed, 22 insertions(+), 118 deletions(-) diff --git a/.evergreen/buildvariants-and-tasks.in.yml b/.evergreen/buildvariants-and-tasks.in.yml index 812a69fef2f..d8457ed798a 100644 --- a/.evergreen/buildvariants-and-tasks.in.yml +++ b/.evergreen/buildvariants-and-tasks.in.yml @@ -60,39 +60,6 @@ const PACKAGE_BUILD_VARIANTS = [ } ]; -const SMOKETEST_BUILD_VARIANTS = [ - { - name: 'smoketest-ubuntu', - display_name: 'Smoketest Ubuntu', - run_on: 'ubuntu2004-large', - depends_on: 'package-ubuntu', - }, - { - name: 'smoketest-windows', - display_name: 'Smoketest Windows', - run_on: 'windows-vsCurrent-large', - depends_on: 'package-windows', - }, - { - name: 'smoketest-rhel', - display_name: 'Smoketest RHEL', - run_on: 'rhel80-large', - depends_on: 'package-rhel', - }, - { - name: 'smoketest-macos-x64', - display_name: 'Smoketest MacOS Intel', - run_on: 'macos-14-gui', - depends_on: 'package-macos-x64', - }, - { - name: 'smoketest-macos-arm', - display_name: 'Smoketest MacOS Arm64', - run_on: 'macos-14-arm64-gui', - depends_on: 'package-macos-arm', - } -]; - const TEST_PACKAGED_APP_BUILD_VARIANTS = [ { name: 'test-packaged-app-ubuntu', @@ -216,18 +183,16 @@ buildvariants: <% } %> <% } %> -<% for (const buildVariant of SMOKETEST_BUILD_VARIANTS) { %> -<% for (const distribution of ['compass']) { %> - - name: <%= buildVariant.name %>-<%= distribution %> - display_name: <%= buildVariant.display_name %> (<%= distribution %>) - run_on: <%= buildVariant.run_on %> + - name: smoketest-packaged-app + display_name: Smoke Test via GitHub Actions + run_on: ubuntu2004-large depends_on: - - name: package-<%= distribution %> - variant: <%= buildVariant.depends_on %> + <% for (const buildVariant of PACKAGE_BUILD_VARIANTS) { %> + - name: package-compass + variant: <%= buildVariant.name %> + <% } %> tasks: - - name: smoketest-<%= distribution %> -<% } %> -<% } %> + - name: smoketest-packaged-app - name: test-eol-servers display_name: Test EoL Servers @@ -468,21 +433,17 @@ tasks: compass_distribution: <%= distribution %> <% } %> -<% for (const distribution of ['compass']) { %> - - name: smoketest-<%= distribution %> + - name: smoketest-packaged-app tags: ['required-for-publish', 'run-on-pr'] commands: - func: prepare - func: install - func: bootstrap vars: - scope: 'compass-e2e-tests' - - func: smoketest-packaged-app + scope: '@mongodb-js/compass-smoke-tests' + - func: smoketest-on-github-actions vars: - mongodb_version: <%= LATEST_MAINTAINED_SERVER_VERSION.version %> - compass_distribution: <%= distribution %> debug: 'compass-e2e-tests*,electron*,hadron*,mongo*' -<% } %> <% for (const serverVersion of SERVER_VERSIONS) { %> <% for(const group of E2E_TEST_GROUPS) { %> diff --git a/.evergreen/buildvariants-and-tasks.yml b/.evergreen/buildvariants-and-tasks.yml index 7612fdf3bc5..4d1b2a9a0c8 100644 --- a/.evergreen/buildvariants-and-tasks.yml +++ b/.evergreen/buildvariants-and-tasks.yml @@ -76,46 +76,22 @@ buildvariants: - name: package-compass - name: package-compass-isolated - name: package-compass-readonly - - name: smoketest-ubuntu-compass - display_name: Smoketest Ubuntu (compass) + - name: smoketest-packaged-app + display_name: Smoke Test via GitHub Actions run_on: ubuntu2004-large depends_on: - name: package-compass variant: package-ubuntu - tasks: - - name: smoketest-compass - - name: smoketest-windows-compass - display_name: Smoketest Windows (compass) - run_on: windows-vsCurrent-large - depends_on: - name: package-compass variant: package-windows - tasks: - - name: smoketest-compass - - name: smoketest-rhel-compass - display_name: Smoketest RHEL (compass) - run_on: rhel80-large - depends_on: - name: package-compass variant: package-rhel - tasks: - - name: smoketest-compass - - name: smoketest-macos-x64-compass - display_name: Smoketest MacOS Intel (compass) - run_on: macos-14-gui - depends_on: - name: package-compass variant: package-macos-x64 - tasks: - - name: smoketest-compass - - name: smoketest-macos-arm-compass - display_name: Smoketest MacOS Arm64 (compass) - run_on: macos-14-arm64-gui - depends_on: - name: package-compass variant: package-macos-arm tasks: - - name: smoketest-compass + - name: smoketest-packaged-app - name: test-eol-servers display_name: Test EoL Servers run_on: ubuntu1804-large @@ -508,7 +484,7 @@ tasks: - func: save-all-artifacts vars: compass_distribution: compass-readonly - - name: smoketest-compass + - name: smoketest-packaged-app tags: - required-for-publish - run-on-pr @@ -517,11 +493,9 @@ tasks: - func: install - func: bootstrap vars: - scope: compass-e2e-tests - - func: smoketest-packaged-app + scope: '@mongodb-js/compass-smoke-tests' + - func: smoketest-on-github-actions vars: - mongodb_version: 8.0.x-enterprise - compass_distribution: compass debug: compass-e2e-tests*,electron*,hadron*,mongo* - name: test-server-40x-community-1 tags: diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index 74b0ef0b70e..79295ec0059 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -648,14 +648,12 @@ functions: npm run --unsafe-perm --workspace compass-e2e-tests test-packaged-ci - smoketest-packaged-app: + smoketest-on-github-actions: - command: github.generate_token params: - owner: 10gen - repo: compass-mongodb-com expansion_name: generated_token permissions: # optional - contents: read + actions: write - command: shell.exec # Fail the task if it's idle for 10 mins timeout_secs: 600 @@ -664,43 +662,14 @@ functions: shell: bash env: <<: *compass-env - <<: *compass-e2e-secrets DEBUG: ${debug|} - MONGODB_VERSION: ${mongodb_version|} - MONGODB_RUNNER_VERSION: ${mongodb_version|} - HADRON_DISTRIBUTION: ${compass_distribution} + GITHUB_TOKEN: ${generated_token} script: | set -e # Load environment variables eval $(.evergreen/print-compass-env.sh) - - npm i -w packages/compass-smoke-tests https://x-access-token:${generated_token}@github.com/10gen/compass-mongodb-com --engine-strict=false - - if [[ "$IS_WINDOWS" == "true" ]]; then - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_setup --tests time-to-first-query - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests auto-update-from - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests auto-update-from - fi - - if [[ "$IS_OSX" == "true" ]]; then - echo "Disabling clipboard usage in e2e tests (TODO: https://jira.mongodb.org/browse/BUILD-14780)" - export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true" - # NOTE: We're also skipping auto-update of the macOS app in CI - # because it doesn't work. Running a different test to make sure it - # can install and run successfully at least. - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip --tests=time-to-first-query - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg --tests=time-to-first-query - fi - - if [[ "$IS_UBUNTU" == "true" ]]; then - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_deb --tests=time-to-first-query - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_tar --tests=time-to-first-query - fi - - if [[ "$IS_RHEL" == "true" ]]; then - # npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_rpm --tests=time-to-first-query - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=linux_tar --tests=time-to-first-query - fi + # Start the smoke tests on GitHub Actions and wait for successful completion + npm run --workspace @mongodb-js/compass-smoke-tests start -- dispatch test-web-sandbox: - command: shell.exec From 7fd9c7078ec3b6d9219a991e883a427bab6f18a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Mon, 17 Feb 2025 16:43:56 +0100 Subject: [PATCH 04/11] Print run url on failures --- packages/compass-smoke-tests/src/dispatch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compass-smoke-tests/src/dispatch.ts b/packages/compass-smoke-tests/src/dispatch.ts index 04773403c64..ad7c7280971 100644 --- a/packages/compass-smoke-tests/src/dispatch.ts +++ b/packages/compass-smoke-tests/src/dispatch.ts @@ -122,6 +122,6 @@ export async function dispatchAndWait({ await new Promise((resolve) => setTimeout(resolve, watchPollDelayMs)); } throw new Error( - `Run did not complete successfully within ${WATCH_POLL_TIMEOUT_MS}ms` + `Run did not complete successfully within ${WATCH_POLL_TIMEOUT_MS}ms: See ${run.html_url} for details.` ); } From f5d9c23cffc9faf958a2ca54bdad69473446f962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Wed, 19 Feb 2025 21:51:07 +0100 Subject: [PATCH 05/11] Move ref default to cli.ts --- packages/compass-smoke-tests/src/cli.ts | 11 +++++++++++ packages/compass-smoke-tests/src/dispatch.ts | 14 ++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/compass-smoke-tests/src/cli.ts b/packages/compass-smoke-tests/src/cli.ts index ce600b5211d..c7572d9785d 100755 --- a/packages/compass-smoke-tests/src/cli.ts +++ b/packages/compass-smoke-tests/src/cli.ts @@ -1,4 +1,5 @@ #!/usr/bin/env npx ts-node +import cp from 'node:child_process'; import assert from 'node:assert/strict'; import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; @@ -57,6 +58,15 @@ function getDefaultArch() { } } +function getDefaultRef() { + // TODO: Read this from an environment variable if possible + return cp + .spawnSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { + encoding: 'utf8', + }) + .stdout.trim(); +} + yargs(hideBin(process.argv)) .scriptName('smoke-tests') .detectLocale(false) @@ -128,6 +138,7 @@ yargs(hideBin(process.argv)) .option('ref', { type: 'string', description: 'Git reference to dispatch the workflow from', + default: getDefaultRef(), }) .option('version', { type: 'string', diff --git a/packages/compass-smoke-tests/src/dispatch.ts b/packages/compass-smoke-tests/src/dispatch.ts index ad7c7280971..dcb0d54ae0b 100644 --- a/packages/compass-smoke-tests/src/dispatch.ts +++ b/packages/compass-smoke-tests/src/dispatch.ts @@ -1,4 +1,3 @@ -import cp from 'node:child_process'; import crypto from 'node:crypto'; import * as github from '@actions/github'; @@ -48,18 +47,9 @@ async function getWorkflowRunRetrying( ); } -function getDefaultRef() { - // TODO: Read this from an environment variable if possible - return cp - .spawnSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { - encoding: 'utf8', - }) - .stdout.trim(); -} - type DispatchOptions = { githubToken: string; - ref: string | undefined; + ref: string; version: string; bucketName: string; bucketKeyPrefix: string; @@ -72,7 +62,7 @@ type DispatchOptions = { export async function dispatchAndWait({ githubToken, - ref = getDefaultRef(), + ref, version, bucketName, bucketKeyPrefix, From 6ff413cd959f674cc386a0f3d1a53fdb4fd27abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Wed, 19 Feb 2025 21:51:24 +0100 Subject: [PATCH 06/11] Use expansion to pass --ref --- .evergreen/functions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index 79295ec0059..5cc3c96c192 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -669,7 +669,7 @@ functions: # Load environment variables eval $(.evergreen/print-compass-env.sh) # Start the smoke tests on GitHub Actions and wait for successful completion - npm run --workspace @mongodb-js/compass-smoke-tests start -- dispatch + npm run --workspace @mongodb-js/compass-smoke-tests start -- dispatch --ref ${branch_name} test-web-sandbox: - command: shell.exec From f6f26a57281a922d0e2961cb9830e6c2698ef966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Thu, 20 Feb 2025 08:33:37 +0100 Subject: [PATCH 07/11] Try using trigger_branch --- .evergreen/functions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index 5cc3c96c192..a94d2e7c752 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -669,7 +669,7 @@ functions: # Load environment variables eval $(.evergreen/print-compass-env.sh) # Start the smoke tests on GitHub Actions and wait for successful completion - npm run --workspace @mongodb-js/compass-smoke-tests start -- dispatch --ref ${branch_name} + npm run --workspace @mongodb-js/compass-smoke-tests start -- dispatch --ref ${trigger_branch} test-web-sandbox: - command: shell.exec From 8d10c1749c274a6036bd450c02c98530ba5f95ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Thu, 20 Feb 2025 11:55:01 +0100 Subject: [PATCH 08/11] Add --github-pr-number to derive ref from PR # --- .evergreen/functions.yml | 6 ++++- packages/compass-smoke-tests/src/cli.ts | 16 +++++++++++--- packages/compass-smoke-tests/src/dispatch.ts | 23 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index a94d2e7c752..e0813e27d8c 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -669,7 +669,11 @@ functions: # Load environment variables eval $(.evergreen/print-compass-env.sh) # Start the smoke tests on GitHub Actions and wait for successful completion - npm run --workspace @mongodb-js/compass-smoke-tests start -- dispatch --ref ${trigger_branch} + if [[ "${requester}" == "github_pr" ]]; then + npm run --workspace @mongodb-js/compass-smoke-tests start -- dispatch --github-pr-number ${github_pr_number} + else + npm run --workspace @mongodb-js/compass-smoke-tests start -- dispatch --ref ${branch_name} + fi test-web-sandbox: - command: shell.exec diff --git a/packages/compass-smoke-tests/src/cli.ts b/packages/compass-smoke-tests/src/cli.ts index c7572d9785d..de49e9b0323 100755 --- a/packages/compass-smoke-tests/src/cli.ts +++ b/packages/compass-smoke-tests/src/cli.ts @@ -13,7 +13,7 @@ import { testTimeToFirstQuery } from './tests/time-to-first-query'; import { testAutoUpdateFrom } from './tests/auto-update-from'; import { testAutoUpdateTo } from './tests/auto-update-to'; import { deleteSandboxesDirectory } from './directories'; -import { dispatchAndWait } from './dispatch'; +import { dispatchAndWait, getRefFromGithubPr } from './dispatch'; const debug = createDebug('compass:smoketests'); @@ -135,6 +135,10 @@ yargs(hideBin(process.argv)) 'Dispatch smoke tests on GitHub and watch to completion', (argv) => argv + .option('github-pr-number', { + type: 'number', + description: 'GitHub PR number used to determine ref', + }) .option('ref', { type: 'string', description: 'Git reference to dispatch the workflow from', @@ -145,7 +149,7 @@ yargs(hideBin(process.argv)) description: 'Compass version to run tests for', default: process.env.DEV_VERSION_IDENTIFIER, }), - async ({ version, bucketName, bucketKeyPrefix, ref }) => { + async ({ version, bucketName, bucketKeyPrefix, ref, githubPrNumber }) => { const { GITHUB_TOKEN } = process.env; assert( typeof GITHUB_TOKEN === 'string', @@ -161,7 +165,13 @@ yargs(hideBin(process.argv)) ); await dispatchAndWait({ githubToken: GITHUB_TOKEN, - ref, + ref: + typeof githubPrNumber === 'number' + ? await getRefFromGithubPr({ + githubToken: GITHUB_TOKEN, + githubPrNumber, + }) + : ref, version, bucketName, bucketKeyPrefix, diff --git a/packages/compass-smoke-tests/src/dispatch.ts b/packages/compass-smoke-tests/src/dispatch.ts index dcb0d54ae0b..edac0ce3452 100644 --- a/packages/compass-smoke-tests/src/dispatch.ts +++ b/packages/compass-smoke-tests/src/dispatch.ts @@ -47,6 +47,29 @@ async function getWorkflowRunRetrying( ); } +type RefFromGithubPrOptions = { + githubToken: string; + githubPrNumber: number; +}; + +export async function getRefFromGithubPr({ + githubToken, + githubPrNumber, +}: RefFromGithubPrOptions) { + const octokit = github.getOctokit(githubToken); + const { + data: { + head: { ref }, + }, + } = await octokit.rest.pulls.get({ + owner: GITHUB_OWNER, + repo: GITHUB_REPO, + pull_number: githubPrNumber, + }); + console.log(`Got ref "${ref}" from PR #${githubPrNumber}`); + return ref; +} + type DispatchOptions = { githubToken: string; ref: string; From bbf9c12b5a485859113ef4601349f3a8915d0f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Mon, 24 Feb 2025 16:08:29 +0100 Subject: [PATCH 09/11] Use hadron build info to determine the version --- .../compass-smoke-tests/src/build-info.ts | 49 +++++++++++++++---- packages/compass-smoke-tests/src/cli.ts | 15 ++---- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/packages/compass-smoke-tests/src/build-info.ts b/packages/compass-smoke-tests/src/build-info.ts index ffff4ba3f85..bc77ca7172c 100644 --- a/packages/compass-smoke-tests/src/build-info.ts +++ b/packages/compass-smoke-tests/src/build-info.ts @@ -2,14 +2,16 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import createDebug from 'debug'; +import { pick } from 'lodash'; import { handler as writeBuildInfo } from 'hadron-build/commands/info'; import { type PackageKind } from './packages'; import { type SmokeTestsContextWithSandbox } from './context'; -import { pick } from 'lodash'; +import { createSandbox } from './directories'; const debug = createDebug('compass:smoketests:build-info'); +const COMPASS_PATH = path.resolve(__dirname, '../../compass'); const SUPPORTED_CHANNELS = ['dev', 'beta', 'stable'] as const; @@ -229,16 +231,18 @@ export function readPackageDetails( return getPackageDetails(kind, result); } -export function writeAndReadPackageDetails( - context: SmokeTestsContextWithSandbox -): PackageDetails { - const compassDir = path.resolve(__dirname, '../../compass'); +export function writeAndReadPackageDetails({ + package: packageKind, + platform, + arch, + sandboxPath, +}: SmokeTestsContextWithSandbox): PackageDetails { const infoArgs = { format: 'json', - dir: compassDir, - platform: context.platform, - arch: context.arch, - out: path.resolve(context.sandboxPath, 'target.json'), + dir: COMPASS_PATH, + platform, + arch, + out: path.resolve(sandboxPath, 'target.json'), }; debug({ infoArgs }); @@ -258,5 +262,30 @@ export function writeAndReadPackageDetails( ]) ); writeBuildInfo(infoArgs); - return readPackageDetails(context.package, infoArgs.out); + return readPackageDetails(packageKind, infoArgs.out); +} + +export function getCompassVersionFromBuildInfo(): string { + const out = path.resolve(createSandbox(), 'target.json'); + // We're hardcoding the platform and arch here because we're only interested in the version + if (typeof process.env.HADRON_DISTRIBUTION !== 'string') { + // TODO: Ideally we would interface directly with the `Target` from the "hadron-build" package so we could pass this directly + process.env.HADRON_DISTRIBUTION = 'compass'; + } + writeBuildInfo({ + format: 'json', + dir: COMPASS_PATH, + platform: 'linux', + arch: 'x64', + out, + }); + const result = readJson(out); + assert(typeof result === 'object', 'Expected hadron to write an object'); + assert( + 'version' in result, + 'Expected hadron to write an object with a version' + ); + const { version } = result; + assert(typeof version === 'string', 'Expected version to be a string'); + return version; } diff --git a/packages/compass-smoke-tests/src/cli.ts b/packages/compass-smoke-tests/src/cli.ts index de49e9b0323..0abdd1cf292 100755 --- a/packages/compass-smoke-tests/src/cli.ts +++ b/packages/compass-smoke-tests/src/cli.ts @@ -14,6 +14,7 @@ import { testAutoUpdateFrom } from './tests/auto-update-from'; import { testAutoUpdateTo } from './tests/auto-update-to'; import { deleteSandboxesDirectory } from './directories'; import { dispatchAndWait, getRefFromGithubPr } from './dispatch'; +import { getCompassVersionFromBuildInfo } from './build-info'; const debug = createDebug('compass:smoketests'); @@ -143,26 +144,18 @@ yargs(hideBin(process.argv)) type: 'string', description: 'Git reference to dispatch the workflow from', default: getDefaultRef(), - }) - .option('version', { - type: 'string', - description: 'Compass version to run tests for', - default: process.env.DEV_VERSION_IDENTIFIER, }), - async ({ version, bucketName, bucketKeyPrefix, ref, githubPrNumber }) => { + async ({ bucketName, bucketKeyPrefix, ref, githubPrNumber }) => { const { GITHUB_TOKEN } = process.env; assert( typeof GITHUB_TOKEN === 'string', 'Expected a GITHUB_TOKEN environment variable' ); - assert( - typeof version === 'string', - 'Expected a version to dispatch tests for' - ); assert( typeof bucketName === 'string' && typeof bucketKeyPrefix === 'string', 'Bucket name and key prefix are needed to download' ); + await dispatchAndWait({ githubToken: GITHUB_TOKEN, ref: @@ -172,7 +165,7 @@ yargs(hideBin(process.argv)) githubPrNumber, }) : ref, - version, + version: getCompassVersionFromBuildInfo(), bucketName, bucketKeyPrefix, }); From 67afc54a4b08a95b2ad38970f61213acdba7597d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Mon, 24 Feb 2025 20:28:52 +0100 Subject: [PATCH 10/11] Pass version only for "dev" --- .github/workflows/test-installers.yml | 13 ++++---- .../compass-smoke-tests/src/build-info.ts | 32 ++++--------------- packages/compass-smoke-tests/src/cli.ts | 3 +- packages/compass-smoke-tests/src/dispatch.ts | 8 ++--- 4 files changed, 17 insertions(+), 39 deletions(-) diff --git a/.github/workflows/test-installers.yml b/.github/workflows/test-installers.yml index 6de69ec00c8..d2aeb5171aa 100644 --- a/.github/workflows/test-installers.yml +++ b/.github/workflows/test-installers.yml @@ -6,10 +6,6 @@ permissions: on: workflow_dispatch: inputs: - version: - type: string - description: 'Version of the installer to download' - required: true bucket_name: type: string description: 'S3 bucket to download installers from' @@ -18,11 +14,14 @@ on: type: string description: 'S3 bucket key prefix to download installers from' required: true + dev_version: + type: string + description: 'Dev version of the installer to download' nonce: type: string description: 'A random string to track the run from dispatch to watching' -run-name: Test Installers ${{ github.event.inputs.version }} / (nonce = ${{ github.event.inputs.nonce || 'not set' }}) +run-name: Test Installers ${{ github.event.inputs.dev_version || github.ref_name }} / (nonce = ${{ github.event.inputs.nonce || 'not set' }}) jobs: test: @@ -162,7 +161,7 @@ jobs: - name: Cache downloads uses: actions/cache@v4 with: - key: smoke-tests-downloads-${{ inputs.version }}-${{ runner.os }}-${{ runner.arch }}-${{ matrix.package }} + key: smoke-tests-downloads-${{ inputs.dev_version || github.ref_name }}-${{ runner.os }}-${{ runner.arch }}-${{ matrix.package }} path: packages/compass-smoke-tests/.downloads - name: Install dependencies and build packages run: npm ci @@ -195,8 +194,8 @@ jobs: env: EVERGREEN_BUCKET_NAME: ${{ inputs.bucket_name }} EVERGREEN_BUCKET_KEY_PREFIX: ${{ inputs.bucket_key_prefix }} - DEV_VERSION_IDENTIFIER: ${{ inputs.version }} HADRON_DISTRIBUTION: ${{ matrix.hadron-distribution }} + DEV_VERSION_IDENTIFIER: ${{ inputs.dev_version }} PLATFORM: ${{ matrix.hadron-platform }} ARCH: ${{ matrix.arch }} # Useful to pass a "fake" DISTRO_ID environment to inform the "mongodb-download-url" package diff --git a/packages/compass-smoke-tests/src/build-info.ts b/packages/compass-smoke-tests/src/build-info.ts index bc77ca7172c..ed922b8159f 100644 --- a/packages/compass-smoke-tests/src/build-info.ts +++ b/packages/compass-smoke-tests/src/build-info.ts @@ -8,7 +8,6 @@ import { handler as writeBuildInfo } from 'hadron-build/commands/info'; import { type PackageKind } from './packages'; import { type SmokeTestsContextWithSandbox } from './context'; -import { createSandbox } from './directories'; const debug = createDebug('compass:smoketests:build-info'); const COMPASS_PATH = path.resolve(__dirname, '../../compass'); @@ -246,6 +245,12 @@ export function writeAndReadPackageDetails({ }; debug({ infoArgs }); + // Removing the DEV_VERSION_IDENTIFIER variable if it's empty + // - to avoid any potential (future) issues from CI setting it without providing a value + if (process.env.DEV_VERSION_IDENTIFIER === '') { + delete process.env.DEV_VERSION_IDENTIFIER; + } + // These are known environment variables that will affect the way // writeBuildInfo works. Log them as a reminder and for our own sanity debug( @@ -264,28 +269,3 @@ export function writeAndReadPackageDetails({ writeBuildInfo(infoArgs); return readPackageDetails(packageKind, infoArgs.out); } - -export function getCompassVersionFromBuildInfo(): string { - const out = path.resolve(createSandbox(), 'target.json'); - // We're hardcoding the platform and arch here because we're only interested in the version - if (typeof process.env.HADRON_DISTRIBUTION !== 'string') { - // TODO: Ideally we would interface directly with the `Target` from the "hadron-build" package so we could pass this directly - process.env.HADRON_DISTRIBUTION = 'compass'; - } - writeBuildInfo({ - format: 'json', - dir: COMPASS_PATH, - platform: 'linux', - arch: 'x64', - out, - }); - const result = readJson(out); - assert(typeof result === 'object', 'Expected hadron to write an object'); - assert( - 'version' in result, - 'Expected hadron to write an object with a version' - ); - const { version } = result; - assert(typeof version === 'string', 'Expected version to be a string'); - return version; -} diff --git a/packages/compass-smoke-tests/src/cli.ts b/packages/compass-smoke-tests/src/cli.ts index 0abdd1cf292..e83aef45ec3 100755 --- a/packages/compass-smoke-tests/src/cli.ts +++ b/packages/compass-smoke-tests/src/cli.ts @@ -14,7 +14,6 @@ import { testAutoUpdateFrom } from './tests/auto-update-from'; import { testAutoUpdateTo } from './tests/auto-update-to'; import { deleteSandboxesDirectory } from './directories'; import { dispatchAndWait, getRefFromGithubPr } from './dispatch'; -import { getCompassVersionFromBuildInfo } from './build-info'; const debug = createDebug('compass:smoketests'); @@ -165,7 +164,7 @@ yargs(hideBin(process.argv)) githubPrNumber, }) : ref, - version: getCompassVersionFromBuildInfo(), + devVersion: process.env.DEV_VERSION_IDENTIFIER, bucketName, bucketKeyPrefix, }); diff --git a/packages/compass-smoke-tests/src/dispatch.ts b/packages/compass-smoke-tests/src/dispatch.ts index edac0ce3452..ce0bee999dc 100644 --- a/packages/compass-smoke-tests/src/dispatch.ts +++ b/packages/compass-smoke-tests/src/dispatch.ts @@ -73,9 +73,9 @@ export async function getRefFromGithubPr({ type DispatchOptions = { githubToken: string; ref: string; - version: string; bucketName: string; bucketKeyPrefix: string; + devVersion?: string; /** * Delay in milliseconds to wait between requests when polling while watching the run. @@ -86,7 +86,7 @@ type DispatchOptions = { export async function dispatchAndWait({ githubToken, ref, - version, + devVersion, bucketName, bucketKeyPrefix, watchPollDelayMs = 5000, @@ -100,7 +100,7 @@ export async function dispatchAndWait({ workflow_id: GITHUB_WORKFLOW_ID, ref, inputs: { - version, + dev_version: devVersion, bucket_name: bucketName, bucket_key_prefix: bucketKeyPrefix, nonce, @@ -110,7 +110,7 @@ export async function dispatchAndWait({ // Find the next run we just dispatched const run = await getWorkflowRunRetrying( octokit, - `Test Installers ${version} / (nonce = ${nonce})` + `Test Installers ${devVersion || ref} / (nonce = ${nonce})` ); console.log(`Dispatched run #${run.run_number} (${run.html_url})`); From 466b39dd3c33978537f9c7393fb2c62008771627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kr=C3=A6n=20Hansen?= Date: Mon, 24 Feb 2025 21:44:53 +0100 Subject: [PATCH 11/11] Add all build variants as deps of smoketest-packaged-app --- .evergreen/buildvariants-and-tasks.in.yml | 4 +++- .evergreen/buildvariants-and-tasks.yml | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.evergreen/buildvariants-and-tasks.in.yml b/.evergreen/buildvariants-and-tasks.in.yml index d8457ed798a..f6deee81f91 100644 --- a/.evergreen/buildvariants-and-tasks.in.yml +++ b/.evergreen/buildvariants-and-tasks.in.yml @@ -187,10 +187,12 @@ buildvariants: display_name: Smoke Test via GitHub Actions run_on: ubuntu2004-large depends_on: + <% for (const distribution of COMPASS_DISTRIBUTIONS) { %> <% for (const buildVariant of PACKAGE_BUILD_VARIANTS) { %> - - name: package-compass + - name: package-<%= distribution %> variant: <%= buildVariant.name %> <% } %> + <% } %> tasks: - name: smoketest-packaged-app diff --git a/.evergreen/buildvariants-and-tasks.yml b/.evergreen/buildvariants-and-tasks.yml index 4d1b2a9a0c8..c71791c3925 100644 --- a/.evergreen/buildvariants-and-tasks.yml +++ b/.evergreen/buildvariants-and-tasks.yml @@ -90,6 +90,26 @@ buildvariants: variant: package-macos-x64 - name: package-compass variant: package-macos-arm + - name: package-compass-isolated + variant: package-ubuntu + - name: package-compass-isolated + variant: package-windows + - name: package-compass-isolated + variant: package-rhel + - name: package-compass-isolated + variant: package-macos-x64 + - name: package-compass-isolated + variant: package-macos-arm + - name: package-compass-readonly + variant: package-ubuntu + - name: package-compass-readonly + variant: package-windows + - name: package-compass-readonly + variant: package-rhel + - name: package-compass-readonly + variant: package-macos-x64 + - name: package-compass-readonly + variant: package-macos-arm tasks: - name: smoketest-packaged-app - name: test-eol-servers