Skip to content

Commit d2983c2

Browse files
authored
tests: hook up E2E tests to CI (#3307)
## Problem E2E tests must be ran manually ## Solution Run them automatically through CodeBuild
1 parent bd69954 commit d2983c2

File tree

14 files changed

+496
-175
lines changed

14 files changed

+496
-175
lines changed

buildspec/linuxE2ETests.yml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
version: 0.2
2+
3+
env:
4+
variables:
5+
AWS_TOOLKIT_TEST_NO_COLOR: '1'
6+
NO_COVERAGE: 'true'
7+
# Suppress noisy apt-get/dpkg warnings like "debconf: unable to initialize frontend: Dialog").
8+
DEBIAN_FRONTEND: 'noninteractive'
9+
10+
phases:
11+
install:
12+
commands:
13+
- '>/dev/null add-apt-repository universe'
14+
- '>/dev/null apt-get -qq install -y apt-transport-https'
15+
- '>/dev/null apt-get -qq update'
16+
- '>/dev/null apt-get -qq install -y ca-certificates'
17+
- 'apt-get install --reinstall ca-certificates'
18+
# Dependencies for running vscode.
19+
- '>/dev/null apt-get -yqq install libatk1.0-0 libgtk-3-dev libxss1 xvfb libnss3-dev libasound2 libasound2-plugins libsecret-1-0'
20+
21+
build:
22+
commands:
23+
- npm ci --unsafe-perm
24+
# We cannot run `code` as root during tests
25+
# From: https://github.com/aws/aws-codebuild-docker-images/blob/2f796bb9c81fcfbc8585832b99a5f780ae2b2f52/ubuntu/standard/6.0/Dockerfile#L56
26+
- mkdir -p /home/codebuild-user
27+
- chown -R codebuild-user:codebuild-user /tmp /home/codebuild-user .
28+
- su codebuild-user -c "xvfb-run npm run testE2E"
29+
- VCS_COMMIT_ID="${CODEBUILD_RESOLVED_SOURCE_VERSION}"
30+
- CI_BUILD_URL=$(echo $CODEBUILD_BUILD_URL | sed 's/#/%23/g')
31+
- CI_BUILD_ID="${CODEBUILD_BUILD_ID}"
32+
finally:
33+
- rm -rf ~/.aws/sso/cache || true
34+
reports:
35+
e2e-test:
36+
files:
37+
- '*'
38+
base-directory: '.test-reports'

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3422,6 +3422,7 @@
34223422
"postinstall": "npm run generateTelemetry && npm run generateConfigurationAttributes",
34233423
"testCompile": "npm run buildScripts && tsc -p ./ && npm run instrument",
34243424
"test": "npm run testCompile && ts-node ./scripts/test/test.ts && npm run report",
3425+
"testE2E": "npm run testCompile && ts-node ./scripts/test/testE2E.ts && npm run report",
34253426
"integrationTest": "npm run testCompile && ts-node ./scripts/test/integrationTest.ts && npm run report",
34263427
"lint": "eslint -c .eslintrc.js --ext .ts .",
34273428
"lintfix": "eslint -c .eslintrc.js --fix --ext .ts .",

scripts/test/launchTestUtilities.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,12 @@ async function setupVSCode(): Promise<string> {
124124
await installVSCodeExtension(vsCodeExecutablePath, VSCODE_EXTENSION_ID.go)
125125
await installVSCodeExtension(vsCodeExecutablePath, VSCODE_EXTENSION_ID.java)
126126
await installVSCodeExtension(vsCodeExecutablePath, VSCODE_EXTENSION_ID.javadebug)
127+
128+
// On stable/insiders we can install the ssh extension during the tests but not on minver
129+
if (process.env[envvarVscodeTestVersion] === minimum) {
130+
await installVSCodeExtension(vsCodeExecutablePath, VSCODE_EXTENSION_ID.remotessh)
131+
}
132+
127133
console.log('VS Code Test instance has been set up')
128134
return vsCodeExecutablePath
129135
}
@@ -146,6 +152,7 @@ export async function runToolkitTests(suiteName: string, relativeEntryPoint: str
146152
VSCODE_EXTENSION_ID.java,
147153
VSCODE_EXTENSION_ID.javadebug,
148154
VSCODE_EXTENSION_ID.git,
155+
VSCODE_EXTENSION_ID.remotessh,
149156
],
150157
})
151158
const args = {

src/codecatalyst/tools.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import { getLogger } from '../shared/logger'
2424
import { getIdeProperties } from '../shared/extensionUtilities'
2525
import { showConfirmationMessage } from '../shared/utilities/messages'
2626
import { getSshConfigPath } from '../shared/extensions/ssh'
27+
import { VSCODE_EXTENSION_ID } from '../shared/extensions'
2728

2829
interface DependencyPaths {
2930
readonly vsc: string
@@ -39,8 +40,8 @@ interface MissingTool {
3940
export const hostNamePrefix = 'aws-devenv-'
4041

4142
export async function ensureDependencies(): Promise<Result<DependencyPaths, CancellationError | Error>> {
42-
if (!isExtensionInstalled('ms-vscode-remote.remote-ssh')) {
43-
showInstallExtensionMsg('ms-vscode-remote.remote-ssh', 'Remote SSH', 'Connecting to Dev Environment')
43+
if (!isExtensionInstalled(VSCODE_EXTENSION_ID.remotessh)) {
44+
showInstallExtensionMsg(VSCODE_EXTENSION_ID.remotessh, 'Remote SSH', 'Connecting to Dev Environment')
4445

4546
return Result.err(
4647
new ToolkitError('Remote SSH extension not installed', {

src/integrationTest/globalSetup.test.ts

Lines changed: 29 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -6,48 +6,29 @@
66
/**
77
* Before/After hooks for all integration tests.
88
*/
9-
import * as assert from 'assert'
9+
import * as vscode from 'vscode'
1010
import { VSCODE_EXTENSION_ID } from '../shared/extensions'
1111
import { getLogger } from '../shared/logger'
1212
import { WinstonToolkitLogger } from '../shared/logger/winstonToolkitLogger'
1313
import { activateExtension } from '../shared/utilities/vsCodeUtils'
14+
import { patchObject, setRunnableTimeout } from '../test/setupUtil'
15+
import { getTestWindow, resetTestWindow } from '../test/shared/vscode/window'
1416

1517
// ASSUMPTION: Tests are not run concurrently
1618

17-
const timeout: { id: NodeJS.Timeout | undefined; name: string | undefined } = { id: undefined, name: undefined }
18-
function clearTestTimeout() {
19-
if (timeout.id !== undefined) {
20-
clearTimeout(timeout.id)
21-
timeout.id = undefined
22-
timeout.name = undefined
23-
}
24-
}
25-
26-
/**
27-
* Used in integration tests to avoid hangs, because Mocha's timeout() does not
28-
* seem to work.
29-
*
30-
* TODO: See if Mocha's timeout() works after upgrading to Mocha
31-
* 8.x, then this function can be removed.
32-
*/
33-
export function setTestTimeout(testName: string | undefined, ms: number) {
34-
if (!testName) {
35-
throw Error()
36-
}
37-
if (timeout.id !== undefined) {
38-
throw Error(`timeout set by previous test was not cleared: "${timeout.name}"`)
39-
}
40-
timeout.name = testName
41-
timeout.id = setTimeout(function () {
42-
const name = timeout.name
43-
clearTestTimeout()
44-
assert.fail(`Exceeded timeout of ${(ms / 1000).toFixed(1)} seconds: "${name}"`)
45-
}, ms)
46-
}
19+
let windowPatch: vscode.Disposable
20+
const maxTestDuration = 300_000
4721

48-
// Before all tests begin, once only:
49-
before(async function () {
22+
export async function mochaGlobalSetup(this: Mocha.Runner) {
5023
console.log('globalSetup: before()')
24+
25+
// Prevent CI from hanging by forcing a timeout on both hooks and tests
26+
this.on('hook', hook => setRunnableTimeout(hook, maxTestDuration))
27+
this.on('test', test => setRunnableTimeout(test, maxTestDuration))
28+
29+
// Set up a listener for proxying login requests
30+
patchWindow()
31+
5132
// Needed for getLogger().
5233
await activateExtension(VSCODE_EXTENSION_ID.awstoolkit, false)
5334

@@ -56,26 +37,21 @@ before(async function () {
5637
if (getLogger() instanceof WinstonToolkitLogger) {
5738
;(getLogger() as WinstonToolkitLogger).logToConsole()
5839
}
59-
})
60-
// After all tests end, once only:
61-
after(async function () {
40+
}
41+
42+
export async function mochaGlobalTeardown(this: Mocha.Context) {
6243
console.log('globalSetup: after()')
63-
})
44+
windowPatch.dispose()
45+
}
6446

65-
afterEach(function () {
66-
clearTestTimeout()
67-
})
47+
export const mochaHooks = {
48+
afterEach(this: Mocha.Context) {
49+
patchWindow()
50+
},
51+
}
6852

69-
// TODO: migrate to mochaHooks (requires mocha 8.x)
70-
// https://mochajs.org/#available-root-hooks
71-
//
72-
// export mochaHooks = {
73-
// // Before all tests begin, once only:
74-
// beforeAll(async () => {
75-
// // Needed for getLogger().
76-
// await activateExtension(VSCODE_EXTENSION_ID.awstoolkit)
77-
// }),
78-
// // After all tests end, once only:
79-
// afterAll(async () => {
80-
// }),
81-
// }
53+
function patchWindow() {
54+
windowPatch?.dispose()
55+
resetTestWindow()
56+
windowPatch = patchObject(vscode, 'window', getTestWindow())
57+
}

src/integrationTest/sam.test.ts

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import { fileExists, tryRemoveFolder } from '../shared/filesystemUtilities'
2020
import { AddSamDebugConfigurationInput } from '../shared/sam/debugger/commands/addSamDebugConfiguration'
2121
import { findParentProjectFile } from '../shared/utilities/workspaceUtils'
2222
import * as testUtils from './integrationTestsUtilities'
23-
import { setTestTimeout } from './globalSetup.test'
2423
import { waitUntil } from '../shared/utilities/timeoutUtils'
2524
import { AwsSamDebuggerConfiguration } from '../shared/sam/debugger/awsSamDebugConfiguration.gen'
2625
import { AwsSamTargetType } from '../shared/sam/debugger/awsSamDebugConfiguration'
@@ -34,8 +33,6 @@ const projectFolder = testUtils.getTestWorkspaceFolder()
3433
/* Test constants go here */
3534
const codelensTimeout: number = 60000
3635
const codelensRetryInterval: number = 200
37-
// note: this refers to the _test_ timeout, not the invocation timeout
38-
const debugTimeout: number = 240000
3936
const noDebugSessionTimeout: number = 5000
4037
const noDebugSessionInterval: number = 100
4138

@@ -562,7 +559,6 @@ describe('SAM Integration Tests', async function () {
562559
this.skip()
563560
}
564561

565-
setTestTimeout(this.test?.fullTitle(), debugTimeout)
566562
await testTarget('api', {
567563
api: {
568564
path: '/hello',
@@ -577,7 +573,6 @@ describe('SAM Integration Tests', async function () {
577573
this.skip()
578574
}
579575

580-
setTestTimeout(this.test?.fullTitle(), debugTimeout)
581576
await testTarget('template')
582577
})
583578

src/shared/extensions.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export const VSCODE_EXTENSION_ID = {
2222
java: 'redhat.java',
2323
javadebug: 'vscjava.vscode-java-debug',
2424
git: 'vscode.git',
25+
remotessh: 'ms-vscode-remote.remote-ssh',
2526
}
2627

2728
/**

src/shared/sam/activation.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -392,7 +392,7 @@ async function promptInstallYamlPlugin(disposables: vscode.Disposable[]) {
392392
case installBtn:
393393
// Available options are:
394394
// extension.open: opens extension page in VS Code extension marketplace view
395-
// workspace.extension.installPlugin: autoinstalls plugin with no additional feedback
395+
// workbench.extensions.installExtension: autoinstalls plugin with no additional feedback
396396
// workspace.extension.search: preloads and executes a search in the extension sidebar with the given term
397397

398398
// not sure if these are 100% stable.

src/shared/systemUtilities.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ export class SystemUtilities {
123123
'code', // $PATH
124124
]
125125
for (const vsc of vscs) {
126-
if (vsc !== 'code' && !fs.existsSync(vsc)) {
126+
if (!vsc || (vsc !== 'code' && !fs.existsSync(vsc))) {
127127
continue
128128
}
129129
const proc = new ChildProcess(vsc, ['--version'])
@@ -132,7 +132,7 @@ export class SystemUtilities {
132132
SystemUtilities.vscPath = vsc
133133
return vsc
134134
}
135-
getLogger().warn('getVscodeCliPath: failed: %s', proc)
135+
getLogger().warn('getVscodeCliPath: failed: %s %O', proc, proc.result())
136136
}
137137

138138
return undefined

src/test/globalSetup.test.ts

Lines changed: 44 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ import { activateExtension } from '../shared/utilities/vsCodeUtils'
2222
import { FakeExtensionContext, FakeMemento } from './fakeExtensionContext'
2323
import { TestLogger } from './testLogger'
2424
import * as testUtil from './testUtil'
25-
import { printPendingUiElements, getTestWindow, resetTestWindow } from './shared/vscode/window'
25+
import { getTestWindow, resetTestWindow } from './shared/vscode/window'
26+
import { setRunnableTimeout } from './setupUtil'
2627

2728
const testReportDir = join(__dirname, '../../../.test-reports')
2829
const testLogOutput = join(testReportDir, 'testLog.log')
@@ -33,7 +34,7 @@ const maxTestDuration = 30_000
3334
let testLogger: TestLogger | undefined
3435
let openExternalStub: sinon.SinonStub<Parameters<typeof vscode['env']['openExternal']>, Thenable<boolean>>
3536

36-
before(async function () {
37+
export async function mochaGlobalSetup(this: Mocha.Context) {
3738
// Clean up and set up test logs
3839
try {
3940
await remove(testLogOutput)
@@ -48,68 +49,51 @@ before(async function () {
4849
fakeContext.globalStorageUri = (await testUtil.createTestWorkspaceFolder('globalStoragePath')).uri
4950
fakeContext.extensionPath = globals.context.extensionPath
5051
Object.assign(globals, { context: fakeContext })
51-
})
52+
}
5253

53-
after(async function () {
54+
export async function mochaGlobalTeardown(this: Mocha.Context) {
5455
testUtil.deleteTestTempDirs()
55-
})
56-
57-
beforeEach(async function () {
58-
// Set every test up so that TestLogger is the logger used by toolkit code
59-
testLogger = setupTestLogger()
60-
globals.templateRegistry = new CloudFormationTemplateRegistry()
61-
globals.codelensRootRegistry = new CodelensRootRegistry()
62-
63-
// In general, we do not want to "fake" the `vscode` API. The only exception is for things
64-
// that _require_ user input apart of a workflow. Even then, these replacements are intended
65-
// to be minimally intrusive and as close to the real thing as possible.
66-
globalSandbox.replace(vscode, 'window', getTestWindow())
67-
openExternalStub = globalSandbox.stub(vscode.env, 'openExternal')
68-
openExternalStub.rejects(
69-
new Error('No return value has been set. Use `getOpenExternalStub().resolves` to set one.')
70-
)
56+
}
7157

72-
// Wraps the test function to bubble up errors that occurred in events from `TestWindow`
73-
if (this.currentTest?.fn) {
74-
const testFn = this.currentTest.fn
75-
this.currentTest.fn = async function (done) {
76-
return Promise.race([
77-
testFn.call(this, done),
78-
new Promise<void>((_, reject) => {
79-
getTestWindow().onError(({ event, error }) => {
80-
event.dispose()
81-
reject(error)
82-
})
83-
84-
// Set a hard time limit per-test so CI doesn't hang
85-
// Mocha's `timeout` method isn't used because we want to emit a custom message
86-
setTimeout(() => {
87-
const message = `Test length exceeded max duration\n${printPendingUiElements()}`
88-
reject(new Error(message))
89-
}, maxTestDuration)
90-
}),
91-
])
58+
export const mochaHooks = {
59+
async beforeEach(this: Mocha.Context) {
60+
// Set every test up so that TestLogger is the logger used by toolkit code
61+
testLogger = setupTestLogger()
62+
globals.templateRegistry = new CloudFormationTemplateRegistry()
63+
globals.codelensRootRegistry = new CodelensRootRegistry()
64+
65+
// In general, we do not want to "fake" the `vscode` API. The only exception is for things
66+
// that _require_ user input apart of a workflow. Even then, these replacements are intended
67+
// to be minimally intrusive and as close to the real thing as possible.
68+
globalSandbox.replace(vscode, 'window', getTestWindow())
69+
openExternalStub = globalSandbox.stub(vscode.env, 'openExternal')
70+
openExternalStub.rejects(
71+
new Error('No return value has been set. Use `getOpenExternalStub().resolves` to set one.')
72+
)
73+
74+
// Wraps the test function to bubble up errors that occurred in events from `TestWindow`
75+
if (this.currentTest?.fn) {
76+
setRunnableTimeout(this.currentTest, maxTestDuration)
9277
}
93-
}
94-
95-
// Enable telemetry features for tests. The metrics won't actually be posted.
96-
globals.telemetry.telemetryEnabled = true
97-
globals.telemetry.clearRecords()
98-
globals.telemetry.logger.clear()
99-
;(globals.context as FakeExtensionContext).globalState = new FakeMemento()
100-
101-
await testUtil.closeAllEditors()
102-
})
103-
104-
afterEach(function () {
105-
// Prevent other tests from using the same TestLogger instance
106-
teardownTestLogger(this.currentTest?.fullTitle() as string)
107-
testLogger = undefined
108-
resetTestWindow()
109-
globals.templateRegistry.dispose()
110-
globals.codelensRootRegistry.dispose()
111-
globalSandbox.restore()
112-
})
78+
79+
// Enable telemetry features for tests. The metrics won't actually be posted.
80+
globals.telemetry.telemetryEnabled = true
81+
globals.telemetry.clearRecords()
82+
globals.telemetry.logger.clear()
83+
;(globals.context as FakeExtensionContext).globalState = new FakeMemento()
84+
85+
await testUtil.closeAllEditors()
86+
},
87+
afterEach(this: Mocha.Context) {
88+
// Prevent other tests from using the same TestLogger instance
89+
teardownTestLogger(this.currentTest?.fullTitle() as string)
90+
testLogger = undefined
91+
resetTestWindow()
92+
globals.templateRegistry.dispose()
93+
globals.codelensRootRegistry.dispose()
94+
globalSandbox.restore()
95+
},
96+
}
11397

11498
/**
11599
* Provides the TestLogger to tests that want to access it.

0 commit comments

Comments
 (0)