Skip to content

feat(smoke-tests): test updating from the latest release to the newly packaged app COMPASS-8532 COMPASS-8535 #6669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Feb 7, 2025
Merged

feat(smoke-tests): test updating from the latest release to the newly packaged app COMPASS-8532 COMPASS-8535 #6669

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
28bbb85
test updating from the latest release to the newly packaged one
lerouxb Jan 30, 2025
4074347
also run this in CI
lerouxb Jan 30, 2025
3c7162d
better type assertions
lerouxb Jan 31, 2025
1ec2465
auto-updating TO doesn't support windows yet because we don't have .e…
lerouxb Jan 31, 2025
a897d93
the installer to use for the latest release is not necessarily the sa…
lerouxb Jan 31, 2025
1c90eba
check the text in the toasts
lerouxb Jan 31, 2025
76b2984
Merge branch 'main' into autoupdate-to
lerouxb Feb 4, 2025
1011053
Merge branch 'main' into autoupdate-to
lerouxb Feb 4, 2025
56d0879
refactor
lerouxb Feb 6, 2025
9bd0524
Update packages/compass-smoke-tests/src/releases.ts
lerouxb Feb 6, 2025
21efcb5
Merge branch 'main' into autoupdate-to
lerouxb Feb 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .evergreen/functions.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -678,8 +678,8 @@ functions:

if [[ "$IS_WINDOWS" == "true" ]]; then
# TODO: windows_setup
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests=auto-update-from
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests=auto-update-from
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests auto-update-from auto-update-to
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests auto-update-from auto-update-to
fi

if [[ "$IS_OSX" == "true" ]]; then
Expand Down
113 changes: 59 additions & 54 deletions packages/compass-e2e-tests/tests/auto-update.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,66 +13,71 @@ function wait(ms: number) {
}

describe('Auto-update', function () {
it('auto-update from', async function () {
if (process.env.TEST_NAME !== 'auto-update-from') {
// we don't want this test to execute along with all the others under
// normal circumstances because it is destructive - it overwrites Compass
// itself
this.skip();
}

// run the app and wait for it to auto-update
console.log('starting compass the first time');
const compass = await init('auto-update from', { firstRun: true });
const { browser } = compass;
try {
await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();

if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
const restartButton = browser.$(Selectors.AutoUpdateRestartButton);
await restartButton.waitForDisplayed();

// We could click the restart button to apply the update and restart the
// app, but restarting the app confuses webdriverio or at least our test
// helpers. So we're going to just restart the app manually.
await browser.pause(1000);
} else {
// When auto-update is not supported the toast contains a link to
// download
const linkElement = browser.$(Selectors.AutoUpdateDownloadLink);
await linkElement.waitForDisplayed();
expect(await linkElement.getAttribute('href')).to.equal(
'https://www.mongodb.com/try/download/compass?utm_source=compass&utm_medium=product'
);
for (const testName of ['auto-update-from', 'auto-update-to']) {
it(testName, async function () {
if (process.env.TEST_NAME !== testName) {
// we don't want this test to execute along with all the others under
// normal circumstances because it is destructive - it overwrites Compass
// itself
this.skip();
}
} finally {
await browser.screenshot(screenshotPathName('auto-update-from'));
await cleanup(compass);
}

if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
console.log(
'pause to make sure the app properly exited before starting again'
);
await wait(10_000);

console.log('starting compass a second time');
// run the app again and check that the version changed
const compass = await init('auto-update from restart', {
firstRun: false,
});
// run the app and wait for it to auto-update
console.log('starting compass the first time');
const compass = await init(testName, { firstRun: true });
const { browser } = compass;
try {
await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
await browser
.$(Selectors.AutoUpdateReleaseNotesLink)
.waitForDisplayed();

if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
const restartButton = browser.$(Selectors.AutoUpdateRestartButton);
await restartButton.waitForDisplayed();

// We could click the restart button to apply the update and restart the
// app, but restarting the app confuses webdriverio or at least our test
// helpers. So we're going to just restart the app manually.
await browser.pause(1000);
} else {
// When auto-update is not supported the toast contains a link to
// download
const linkElement = browser.$(Selectors.AutoUpdateDownloadLink);
await linkElement.waitForDisplayed();
expect(await linkElement.getAttribute('href')).to.equal(
'https://www.mongodb.com/try/download/compass?utm_source=compass&utm_medium=product'
);

// TODO: when updating to a known version we know the version, so
// check for the text
}
} finally {
await browser.screenshot(
screenshotPathName('auto-update-from-restart')
);
await browser.screenshot(screenshotPathName(testName));
await cleanup(compass);
}
}
});

if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
console.log(
'pause to make sure the app properly exited before starting again'
);
await wait(10_000);
Copy link
Contributor

@kraenhansen kraenhansen Feb 5, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we poll the ps with some pid instead? 🤔 Ideally from within the cleanup function. These timeouts always makes me nervous and often adds unneeded delays.

Copy link
Contributor Author

@lerouxb lerouxb Feb 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, but where do we get the pid? I think we can clean it up but in an effort to get somewhere let's do that as a follow-up.


console.log('starting compass a second time');
// run the app again and check that the version changed
const compass = await init(`${testName} restart`, {
firstRun: false,
});
const { browser } = compass;
try {
await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
await browser
.$(Selectors.AutoUpdateReleaseNotesLink)
.waitForDisplayed();
// TODO: when updating to a known version we know the version, so
// check for the text
} finally {
await browser.screenshot(screenshotPathName(`${testName}-restart`));
await cleanup(compass);
}
}
});
}
});
16 changes: 14 additions & 2 deletions packages/compass-smoke-tests/src/build-info.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ import { type PackageKind } from './packages';
import { type SmokeTestsContext } from './context';
import { pick } from 'lodash';

const SUPPORTED_CHANNELS = ['dev', 'beta', 'stable'] as const;

export type Channel = typeof SUPPORTED_CHANNELS[number];

function assertObjectHasKeys(
obj: unknown,
name: string,
Expand All @@ -25,13 +29,21 @@ function assertObjectHasKeys(

// subsets of the hadron-build info result

export const commonKeys = ['productName'] as const;
export type CommonBuildInfo = Record<typeof commonKeys[number], string>;
export const commonKeys = ['productName', 'version', 'channel'] as const;
export type CommonBuildInfo = Record<typeof commonKeys[number], string> & {
channel: Channel;
};

export function assertCommonBuildInfo(
buildInfo: unknown
): asserts buildInfo is CommonBuildInfo {
assertObjectHasKeys(buildInfo, 'buildInfo', commonKeys);
assert(
SUPPORTED_CHANNELS.includes((buildInfo as { channel: Channel }).channel),
`Expected ${
(buildInfo as { channel: Channel }).channel
} to be in ${SUPPORTED_CHANNELS.join(',')}`
);
}

export const windowsFilenameKeys = [
Expand Down
156 changes: 41 additions & 115 deletions packages/compass-smoke-tests/src/cli.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,10 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { once } from 'node:events';

import yargs from 'yargs';
import { hideBin } from 'yargs/helpers';
import { pick } from 'lodash';
import { execute, executeAsync } from './execute';
import {
type PackageDetails,
readPackageDetails,
Expand All @@ -16,18 +14,22 @@ import {
import { createSandbox } from './directories';
import { downloadFile } from './downloads';
import { type PackageKind, SUPPORTED_PACKAGES } from './packages';
import { getLatestRelease } from './releases';
import { SUPPORTED_TESTS } from './tests/types';
import { type SmokeTestsContext } from './context';

import { installMacDMG } from './installers/mac-dmg';
import { installMacZIP } from './installers/mac-zip';
import { installWindowsZIP } from './installers/windows-zip';
import { installWindowsMSI } from './installers/windows-msi';

import { testTimeToFirstQuery } from './tests/time-to-first-query';
import { testAutoUpdateFrom } from './tests/auto-update-from';
import { testAutoUpdateTo } from './tests/auto-update-to';

const SUPPORTED_PLATFORMS = ['win32', 'darwin', 'linux'] as const;
const SUPPORTED_ARCHS = ['x64', 'arm64'] as const;

const SUPPORTED_TESTS = ['time-to-first-query', 'auto-update-from'] as const;

function isSupportedPlatform(
value: unknown
): value is typeof SUPPORTED_PLATFORMS[number] {
Expand Down Expand Up @@ -198,9 +200,13 @@ async function run() {
])
);

const { kind, appName, filepath, autoUpdatable } = await getTestSubject(
context
);
const {
kind,
appName,
filepath,
buildInfo: { channel, version },
autoUpdatable,
} = await getTestSubject(context);
const install = getInstaller(kind);

try {
Expand All @@ -209,27 +215,51 @@ async function run() {
}

for (const testName of context.tests) {
const installerPath =
testName === 'auto-update-to'
? await getLatestRelease(
channel,
context.arch,
kind,
context.forceDownload
)
: filepath;
Copy link
Contributor

@kraenhansen kraenhansen Feb 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Won't this depend on the order if multiple tests are passed to the CLI?
I suggest we make this a bit less powerful and run only one test per invocation of the CLI (effectively changing context.tests to context.test) to make it easier to reason about what installer needs to run first.

Copy link
Contributor

@kraenhansen kraenhansen Feb 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking at this again, it does feel a bit misplaced here, instead of being a part of the actual test function, which would increase the cohesion 🤔

Copy link
Contributor Author

@lerouxb lerouxb Feb 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK what I've done now is move all this stuff into the tests. I also changed it so that auto-update-to does not unnecessarily download the newly packaged app when it is going to be installing the latest release anyway.

Let me know what you think.


const { appPath, uninstall } = install({
appName,
filepath,
filepath: installerPath,
destinationPath: context.sandboxPath,
});

try {
if (testName === 'time-to-first-query') {
// Auto-update does not work on mac in CI at the moment. So in that case
// we just run the E2E tests to make sure the app at least starts up.
runTimeToFirstQuery({
testTimeToFirstQuery({
appName,
appPath,
});
}
if (testName === 'auto-update-from') {
await runUpdateTest({
await testAutoUpdateFrom({
appName,
appPath,
autoUpdatable,
});
}
if (testName === 'auto-update-to') {
assert(
context.bucketKeyPrefix !== undefined,
'Bucket key prefix is needed to download'
);

await testAutoUpdateTo({
appName,
appPath,
autoUpdatable,
testName,
channel,
bucketKeyPrefix: context.bucketKeyPrefix,
version,
});
}
} finally {
Expand All @@ -246,110 +276,6 @@ async function run() {
}
}

async function importUpdateServer() {
try {
return (await import('compass-mongodb-com')).default;
} catch (err: unknown) {
console.log('Remember to npm link compass-mongodb-com');
throw err;
}
}

async function startAutoUpdateServer() {
console.log('Starting auto-update server');
const { httpServer, updateChecker, start } = (await importUpdateServer())();
start();
await once(updateChecker, 'refreshed');

return httpServer;
}

type RunE2ETestOptions = {
appName: string;
appPath: string;
};

function runTimeToFirstQuery({ appName, appPath }: RunE2ETestOptions) {
execute(
'npm',
[
'run',
'--unsafe-perm',
'test-packaged',
'--workspace',
'compass-e2e-tests',
'--',
'--test-filter=time-to-first-query',
],
{
// We need to use a shell to get environment variables setup correctly
shell: true,
env: {
...process.env,
COMPASS_APP_NAME: appName,
COMPASS_APP_PATH: appPath,
},
}
);
}

type RunUpdateTestOptions = {
appName: string;
appPath: string;
autoUpdatable?: boolean;
testName: string;
};

async function runUpdateTest({
appName,
appPath,
autoUpdatable,
testName,
}: RunUpdateTestOptions) {
process.env.PORT = '0'; // dynamic port
process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES = 'true';

const server = await startAutoUpdateServer();

const address = server.address();
assert(typeof address === 'object' && address !== null);
const port = address.port;
const HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE = `http://localhost:${port}`;
console.log({ HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE });

try {
// must be async because the update server is running in the same process
await executeAsync(
'npm',
[
'run',
'--unsafe-perm',
'test-packaged',
'--workspace',
'compass-e2e-tests',
'--',
'--test-filter=auto-update',
],
{
// We need to use a shell to get environment variables setup correctly
shell: true,
env: {
...process.env,
HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE,
AUTO_UPDATE_UPDATABLE: (!!autoUpdatable).toString(),
TEST_NAME: testName,
COMPASS_APP_NAME: appName,
COMPASS_APP_PATH: appPath,
},
}
);
} finally {
console.log('Stopping auto-update server');
server.close();
delete process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES;
}
}

run()
.then(function () {
console.log('done');
Expand Down
Loading
Loading