diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index 09d4f2ca403..5b075ade7e6 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -649,6 +649,13 @@ functions: npm run --unsafe-perm --workspace compass-e2e-tests test-packaged-ci smoketest-packaged-app: + - command: github.generate_token + params: + owner: 10gen + repo: compass-mongodb-com + expansion_name: generated_token + permissions: # optional + contents: read - command: shell.exec # Fail the task if it's idle for 10 mins timeout_secs: 600 @@ -667,16 +674,22 @@ functions: # Load environment variables eval $(.evergreen/print-compass-env.sh) + npm i -w packages/compass-smoke-tests https://x-access-token:${generated_token}@github.com/10gen/compass-mongodb-com --engine-strict=false + if [[ "$IS_WINDOWS" == "true" ]]; then - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi + # TODO: windows_setup + npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests=auto-update-from + npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests=auto-update-from fi if [[ "$IS_OSX" == "true" ]]; then echo "Disabling clipboard usage in e2e tests (TODO: https://jira.mongodb.org/browse/BUILD-14780)" export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true" - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg - npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip + # NOTE: We're also skipping auto-update of the macOS app in CI + # because it doesn't work. Running a different test to make sure it + # can install and run successfully at least. + npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_zip --tests=time-to-first-query + npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=osx_dmg --tests=time-to-first-query fi #if [[ "$IS_UBUNTU" == "true" ]]; then diff --git a/.evergreen/preinstall.sh b/.evergreen/preinstall.sh index 3093b9063c0..41a3511ef2a 100755 --- a/.evergreen/preinstall.sh +++ b/.evergreen/preinstall.sh @@ -13,9 +13,9 @@ echo "APPDATA: $APPDATA" echo "PATH: $PATH" # these are super useful if you want to run the smoke tests locally -echo "DEV_VERSION_IDENTIFIER: $DEV_VERSION_IDENTIFIER" -echo "EVERGREEN_BUCKET_NAME: $EVERGREEN_BUCKET_NAME" -echo "EVERGREEN_BUCKET_KEY_PREFIX: $EVERGREEN_BUCKET_KEY_PREFIX" +echo "export DEV_VERSION_IDENTIFIER=$DEV_VERSION_IDENTIFIER" +echo "export EVERGREEN_BUCKET_KEY_PREFIX=$EVERGREEN_BUCKET_KEY_PREFIX" +echo "export EVERGREEN_BUCKET_NAME=$EVERGREEN_BUCKET_NAME" echo "IS_OSX: $IS_OSX" echo "IS_LINUX: $IS_LINUX" diff --git a/packages/compass-e2e-tests/helpers/selectors.ts b/packages/compass-e2e-tests/helpers/selectors.ts index 449bf6590f3..7ed8ee95ff4 100644 --- a/packages/compass-e2e-tests/helpers/selectors.ts +++ b/packages/compass-e2e-tests/helpers/selectors.ts @@ -1380,3 +1380,12 @@ export const GlobalWrites = { SampleFindingDocuments: '[data-testid="sample-finding-documents"]', SampleInsertingDocuments: '[data-testid="sample-inserting-documents"]', }; + +// Auto-update toasts +export const AutoUpdateToast = '[data-testid="toast-compass-update"]'; +export const AutoUpdateRestartButton = + '[data-testid="auto-update-restart-button"]'; +export const AutoUpdateDownloadLink = + '[data-testid="auto-update-download-link"]'; +export const AutoUpdateReleaseNotesLink = + '[data-testid="auto-update-release-notes-link"]'; diff --git a/packages/compass-e2e-tests/tests/auto-update.test.ts b/packages/compass-e2e-tests/tests/auto-update.test.ts new file mode 100644 index 00000000000..aa399e5ed9e --- /dev/null +++ b/packages/compass-e2e-tests/tests/auto-update.test.ts @@ -0,0 +1,78 @@ +import { expect } from 'chai'; +import { + init, + cleanup, + Selectors, + screenshotPathName, +} from '../helpers/compass'; + +function wait(ms: number) { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +describe('Auto-update', function () { + it('auto-update from', async function () { + if (process.env.TEST_NAME !== 'auto-update-from') { + // we don't want this test to execute along with all the others under + // normal circumstances because it is destructive - it overwrites Compass + // itself + this.skip(); + } + + // run the app and wait for it to auto-update + console.log('starting compass the first time'); + const compass = await init('auto-update from', { firstRun: true }); + const { browser } = compass; + try { + await browser.$(Selectors.AutoUpdateToast).waitForDisplayed(); + + if (process.env.AUTO_UPDATE_UPDATABLE === 'true') { + const restartButton = browser.$(Selectors.AutoUpdateRestartButton); + await restartButton.waitForDisplayed(); + + // We could click the restart button to apply the update and restart the + // app, but restarting the app confuses webdriverio or at least our test + // helpers. So we're going to just restart the app manually. + await browser.pause(1000); + } else { + // When auto-update is not supported the toast contains a link to + // download + const linkElement = browser.$(Selectors.AutoUpdateDownloadLink); + await linkElement.waitForDisplayed(); + expect(await linkElement.getAttribute('href')).to.equal( + 'https://www.mongodb.com/try/download/compass?utm_source=compass&utm_medium=product' + ); + } + } finally { + await browser.screenshot(screenshotPathName('auto-update-from')); + await cleanup(compass); + } + + if (process.env.AUTO_UPDATE_UPDATABLE === 'true') { + console.log( + 'pause to make sure the app properly exited before starting again' + ); + await wait(10_000); + + console.log('starting compass a second time'); + // run the app again and check that the version changed + const compass = await init('auto-update from restart', { + firstRun: false, + }); + const { browser } = compass; + try { + await browser.$(Selectors.AutoUpdateToast).waitForDisplayed(); + await browser + .$(Selectors.AutoUpdateReleaseNotesLink) + .waitForDisplayed(); + } finally { + await browser.screenshot( + screenshotPathName('auto-update-from-restart') + ); + await cleanup(compass); + } + } + }); +}); diff --git a/packages/compass-smoke-tests/.depcheckrc b/packages/compass-smoke-tests/.depcheckrc index 1cd5ad0cb1c..8630a3aa6b7 100644 --- a/packages/compass-smoke-tests/.depcheckrc +++ b/packages/compass-smoke-tests/.depcheckrc @@ -1,3 +1,4 @@ ignores: - '@mongodb-js/prettier-config-compass' - '@mongodb-js/tsconfig-compass' + - compass-mongodb-com diff --git a/packages/compass-smoke-tests/src/cli.ts b/packages/compass-smoke-tests/src/cli.ts index b1678aaf40b..1f3facf907b 100755 --- a/packages/compass-smoke-tests/src/cli.ts +++ b/packages/compass-smoke-tests/src/cli.ts @@ -2,11 +2,12 @@ import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; +import { once } from 'node:events'; import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import { pick } from 'lodash'; -import { execute } from './execute'; +import { execute, executeAsync } from './execute'; import { type PackageDetails, readPackageDetails, @@ -25,6 +26,8 @@ import { installWindowsMSI } from './installers/windows-msi'; const SUPPORTED_PLATFORMS = ['win32', 'darwin', 'linux'] as const; const SUPPORTED_ARCHS = ['x64', 'arm64'] as const; +const SUPPORTED_TESTS = ['time-to-first-query', 'auto-update-from'] as const; + function isSupportedPlatform( value: unknown ): value is typeof SUPPORTED_PLATFORMS[number] { @@ -104,7 +107,14 @@ const argv = yargs(hideBin(process.argv)) type: 'boolean', description: 'Do not delete the sandbox after a run', default: false, - }); + }) + .option('tests', { + type: 'array', + string: true, + choices: SUPPORTED_TESTS, + description: 'Which tests to run', + }) + .default('tests', SUPPORTED_TESTS.slice()); type TestSubject = PackageDetails & { filepath: string; @@ -184,23 +194,47 @@ async function run() { 'platform', 'arch', 'package', + 'tests', ]) ); - const { kind, filepath, appName } = await getTestSubject(context); + const { kind, appName, filepath, autoUpdatable } = await getTestSubject( + context + ); const install = getInstaller(kind); try { - const { appPath, uninstall } = install({ - appName, - filepath, - destinationPath: context.sandboxPath, - }); + if (context.tests.length === 0) { + console.log('Warning: not performing any tests. Pass --tests.'); + } - try { - runTest({ appName, appPath }); - } finally { - await uninstall(); + for (const testName of context.tests) { + const { appPath, uninstall } = install({ + appName, + filepath, + destinationPath: context.sandboxPath, + }); + + try { + if (testName === 'time-to-first-query') { + // Auto-update does not work on mac in CI at the moment. So in that case + // we just run the E2E tests to make sure the app at least starts up. + runTimeToFirstQuery({ + appName, + appPath, + }); + } + if (testName === 'auto-update-from') { + await runUpdateTest({ + appName, + appPath, + autoUpdatable, + testName, + }); + } + } finally { + await uninstall(); + } } } finally { if (context.skipCleanup) { @@ -212,12 +246,30 @@ async function run() { } } -type RunTestOptions = { +async function importUpdateServer() { + try { + return (await import('compass-mongodb-com')).default; + } catch (err: unknown) { + console.log('Remember to npm link compass-mongodb-com'); + throw err; + } +} + +async function startAutoUpdateServer() { + console.log('Starting auto-update server'); + const { httpServer, updateChecker, start } = (await importUpdateServer())(); + start(); + await once(updateChecker, 'refreshed'); + + return httpServer; +} + +type RunE2ETestOptions = { appName: string; appPath: string; }; -function runTest({ appName, appPath }: RunTestOptions) { +function runTimeToFirstQuery({ appName, appPath }: RunE2ETestOptions) { execute( 'npm', [ @@ -241,6 +293,63 @@ function runTest({ appName, appPath }: RunTestOptions) { ); } +type RunUpdateTestOptions = { + appName: string; + appPath: string; + autoUpdatable?: boolean; + testName: string; +}; + +async function runUpdateTest({ + appName, + appPath, + autoUpdatable, + testName, +}: RunUpdateTestOptions) { + process.env.PORT = '0'; // dynamic port + process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES = 'true'; + + const server = await startAutoUpdateServer(); + + const address = server.address(); + assert(typeof address === 'object' && address !== null); + const port = address.port; + const HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE = `http://localhost:${port}`; + console.log({ HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE }); + + try { + // must be async because the update server is running in the same process + await executeAsync( + 'npm', + [ + 'run', + '--unsafe-perm', + 'test-packaged', + '--workspace', + 'compass-e2e-tests', + '--', + '--test-filter=auto-update', + ], + { + // We need to use a shell to get environment variables setup correctly + shell: true, + env: { + ...process.env, + HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE, + AUTO_UPDATE_UPDATABLE: (!!autoUpdatable).toString(), + TEST_NAME: testName, + COMPASS_APP_NAME: appName, + COMPASS_APP_PATH: appPath, + }, + } + ); + } finally { + console.log('Stopping auto-update server'); + server.close(); + delete process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES; + } +} + run() .then(function () { console.log('done'); diff --git a/packages/compass-smoke-tests/src/compass-mongodb-com.d.ts b/packages/compass-smoke-tests/src/compass-mongodb-com.d.ts new file mode 100644 index 00000000000..e5c5c295e30 --- /dev/null +++ b/packages/compass-smoke-tests/src/compass-mongodb-com.d.ts @@ -0,0 +1,9 @@ +declare module 'compass-mongodb-com' { + import type http from 'http'; + function updateServer(): { + start: () => void; + httpServer: http.Server; + updateChecker: NodeJS.EventEmitter; + }; + export = updateServer; +} diff --git a/packages/compass-smoke-tests/src/context.ts b/packages/compass-smoke-tests/src/context.ts index 483c7c50e49..268ff41b03c 100644 --- a/packages/compass-smoke-tests/src/context.ts +++ b/packages/compass-smoke-tests/src/context.ts @@ -9,5 +9,6 @@ export type SmokeTestsContext = { forceDownload?: boolean; localPackage?: boolean; sandboxPath: string; + tests: string[]; skipCleanup: boolean; }; diff --git a/packages/compass-smoke-tests/src/execute.ts b/packages/compass-smoke-tests/src/execute.ts index 3b23f1283e3..a6ce9349639 100644 --- a/packages/compass-smoke-tests/src/execute.ts +++ b/packages/compass-smoke-tests/src/execute.ts @@ -1,4 +1,4 @@ -import { spawnSync, type SpawnOptions } from 'node:child_process'; +import { spawn, spawnSync, type SpawnOptions } from 'node:child_process'; export class ExecuteFailure extends Error { constructor( @@ -27,3 +27,40 @@ export function execute( throw new ExecuteFailure(command, args, status, signal); } } + +export function executeAsync( + command: string, + args: string[], + options?: SpawnOptions +): Promise { + return new Promise((resolve, reject) => { + console.log(command, ...args); + const child = spawn(command, args, { + stdio: 'inherit', + ...options, + }); + const killChild = () => child.kill(); + const interruptChild = () => child.kill('SIGINT'); + process.once('exit', killChild); + process.once('SIGINT', interruptChild); + child.once('error', reject); + child.once('exit', (code, signal) => { + process.off('exit', killChild); + process.off('SIGINT', interruptChild); + if (code !== null) { + if (code === 0) { + resolve(); + } else { + reject(new ExecuteFailure(command, args, code, null)); + } + } else { + if (signal !== null) { + reject(new ExecuteFailure(command, args, null, signal)); + } else { + // shouldn't happen + reject(new ExecuteFailure(command, args, null, null)); + } + } + }); + }); +} diff --git a/packages/compass-smoke-tests/src/installers/mac-dmg.ts b/packages/compass-smoke-tests/src/installers/mac-dmg.ts index 4cccabd71cf..f95c0a69029 100644 --- a/packages/compass-smoke-tests/src/installers/mac-dmg.ts +++ b/packages/compass-smoke-tests/src/installers/mac-dmg.ts @@ -1,5 +1,9 @@ import path from 'node:path'; import fs from 'node:fs'; +import { + assertFileNotQuarantined, + removeApplicationSupportForApp, +} from './mac-utils'; import type { InstalledAppInfo, InstallablePackage } from './types'; import { execute } from '../execute'; @@ -24,25 +28,9 @@ export function installMacDMG({ execute('hdiutil', ['detach', volumePath]); } - // TODO: Consider instrumenting the app to use a settings directory in the sandbox - // TODO: Move this somewhere shared between mac installers - if (process.env.HOME) { - const settingsDir = path.resolve( - process.env.HOME, - 'Library', - 'Application Support', - appName - ); + removeApplicationSupportForApp(appName); - if (fs.existsSync(settingsDir)) { - console.log(`${settingsDir} already exists. Removing.`); - fs.rmSync(settingsDir, { recursive: true }); - } - } - - // see if the executable will run without being quarantined or similar - // TODO: Move this somewhere shared between mac installers - execute(path.resolve(appPath, 'Contents/MacOS', appName), ['--version']); + assertFileNotQuarantined(appPath); return { appPath: appPath, diff --git a/packages/compass-smoke-tests/src/installers/mac-utils.ts b/packages/compass-smoke-tests/src/installers/mac-utils.ts new file mode 100644 index 00000000000..335405ab0ab --- /dev/null +++ b/packages/compass-smoke-tests/src/installers/mac-utils.ts @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import { execSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; + +// TODO: Consider instrumenting the app to use a settings directory in the sandbox +export function removeApplicationSupportForApp(appName: string) { + assert(typeof process.env.HOME === 'string', 'HOME env var not set'); + + const settingsDir = path.resolve( + process.env.HOME, + 'Library', + 'Application Support', + appName + ); + + if (fs.existsSync(settingsDir)) { + console.log(`${settingsDir} already exists. Removing.`); + fs.rmSync(settingsDir, { recursive: true }); + } +} + +export function assertFileNotQuarantined(appPath: string) { + try { + execSync(`xattr -p com.apple.quarantine "${appPath}"`, { + encoding: 'utf8', + }); + assert.fail(`Expected no com.apple.quarantine attr on ${appPath}`); + } catch (err: unknown) { + assert( + typeof err === 'object' && + err !== null && + 'message' in err && + typeof err.message === 'string', + 'Expected err to be an error' + ); + assert( + /No such xattr/.test(err.message), + `Expected no com.apple.quarantine attr on ${appPath}` + ); + } +} diff --git a/packages/compass-smoke-tests/src/installers/mac-zip.ts b/packages/compass-smoke-tests/src/installers/mac-zip.ts index 083ba5869c4..26d77370425 100644 --- a/packages/compass-smoke-tests/src/installers/mac-zip.ts +++ b/packages/compass-smoke-tests/src/installers/mac-zip.ts @@ -1,6 +1,8 @@ import path from 'node:path'; -import fs from 'node:fs'; - +import { + assertFileNotQuarantined, + removeApplicationSupportForApp, +} from './mac-utils'; import type { InstalledAppInfo, InstallablePackage } from './types'; import { execute } from '../execute'; @@ -14,25 +16,9 @@ export function installMacZIP({ execute('ditto', ['-xk', filepath, destinationPath]); - // TODO: Consider instrumenting the app to use a settings directory in the sandbox - // TODO: Move this somewhere shared between mac installers - if (process.env.HOME) { - const settingsDir = path.resolve( - process.env.HOME, - 'Library', - 'Application Support', - appName - ); - - if (fs.existsSync(settingsDir)) { - console.log(`${settingsDir} already exists. Removing.`); - fs.rmSync(settingsDir, { recursive: true }); - } - } + removeApplicationSupportForApp(appName); - // see if the executable will run without being quarantined or similar - // TODO: Move this somewhere shared between mac installers - execute(path.resolve(appPath, 'Contents/MacOS', appName), ['--version']); + assertFileNotQuarantined(appPath); return { appPath: appPath, diff --git a/packages/compass-smoke-tests/tsconfig.json b/packages/compass-smoke-tests/tsconfig.json index 7d16ff38a49..5bc36e9d772 100644 --- a/packages/compass-smoke-tests/tsconfig.json +++ b/packages/compass-smoke-tests/tsconfig.json @@ -3,7 +3,11 @@ "compilerOptions": { "noEmit": true, "types": ["node"], - "lib": ["ES2021"] + "lib": ["ES2021"], + "module": "NodeNext", + "paths": { + "compass-mongodb-com": ["./src/compass-mongodb-com.d.ts"] + } }, "include": ["src/**/*"] } diff --git a/packages/compass/src/main/auto-update-manager.ts b/packages/compass/src/main/auto-update-manager.ts index 46c2a6772a2..7b82e179eef 100644 --- a/packages/compass/src/main/auto-update-manager.ts +++ b/packages/compass/src/main/auto-update-manager.ts @@ -478,7 +478,10 @@ const STATE_UPDATE: Record< mongoLogId(1001000129), 'AutoUpdateManager', 'Error Downloading Update', - { message: error.message } + { + message: error.message, + attr: { code: error.code, domain: error.domain }, + } ); }, },