tests(codecatalyst): improve reporting when tests fail (#3391)

* Drop remote SSH minver to `0.74.0`
* Change messaging when minver is not found

Author: JadenSimon

scripts/test/launchTestUtilities.ts

@@ -125,11 +125,6 @@ async function setupVSCode(): Promise<string> {
     await installVSCodeExtension(vsCodeExecutablePath, VSCODE_EXTENSION_ID.java)
     await installVSCodeExtension(vsCodeExecutablePath, VSCODE_EXTENSION_ID.javadebug)
 
-    // On stable/insiders we can install the ssh extension during the tests but not on minver
-    if (process.env[envvarVscodeTestVersion] === minimum) {
-        await installVSCodeExtension(vsCodeExecutablePath, VSCODE_EXTENSION_ID.remotessh)
-    }
-
     console.log('VS Code Test instance has been set up')
     return vsCodeExecutablePath
 }

src/codecatalyst/tools.ts

@@ -48,12 +48,22 @@ export async function ensureDependencies(): Promise<Result<DependencyPaths, Canc
             vscodeExtensionMinVersion.remotessh
         )
 
-        return Result.err(
-            new ToolkitError('Remote SSH extension not installed', {
-                cancelled: true,
-                code: 'MissingExtension',
-            })
-        )
+        if (isExtensionInstalled(VSCODE_EXTENSION_ID.remotessh)) {
+            return Result.err(
+                new ToolkitError('Remote SSH extension version is too low', {
+                    cancelled: true,
+                    code: 'ExtensionVersionTooLow',
+                    details: { expected: vscodeExtensionMinVersion.remotessh },
+                })
+            )
+        } else {
+            return Result.err(
+                new ToolkitError('Remote SSH extension not installed', {
+                    cancelled: true,
+                    code: 'MissingExtension',
+                })
+            )
+        }
     }
 
     const tools = await ensureTools()

src/integrationTest/globalSetup.test.ts

@@ -11,7 +11,7 @@ import { VSCODE_EXTENSION_ID } from '../shared/extensions'
 import { getLogger } from '../shared/logger'
 import { WinstonToolkitLogger } from '../shared/logger/winstonToolkitLogger'
 import { activateExtension } from '../shared/utilities/vsCodeUtils'
-import { patchObject, setRunnableTimeout } from '../test/setupUtil'
+import { mapTestErrors, normalizeError, patchObject, setRunnableTimeout } from '../test/setupUtil'
 import { getTestWindow, resetTestWindow } from '../test/shared/vscode/window'
 
 // ASSUMPTION: Tests are not run concurrently
@@ -26,10 +26,8 @@ export async function mochaGlobalSetup(this: Mocha.Runner) {
     this.on('hook', hook => setRunnableTimeout(hook, maxTestDuration))
     this.on('test', test => setRunnableTimeout(test, maxTestDuration))
 
-    // Mocha won't show the full error chain
-    this.on('fail', (test, err) => {
-        getLogger().error(`test "${test.title}" failed: %s`, err)
-    })
+    // Shows the full error chain when tests fail
+    mapTestErrors(this, normalizeError)
 
     // Set up a listener for proxying login requests
     patchWindow()

src/shared/extensions.ts

@@ -26,7 +26,7 @@ export const VSCODE_EXTENSION_ID = {
 }
 
 export const vscodeExtensionMinVersion = {
-    remotessh: '0.98.0',
+    remotessh: '0.74.0',
 }
 
 /**

src/test/globalSetup.test.ts

@@ -23,7 +23,7 @@ import { FakeExtensionContext, FakeMemento } from './fakeExtensionContext'
 import { TestLogger } from './testLogger'
 import * as testUtil from './testUtil'
 import { getTestWindow, resetTestWindow } from './shared/vscode/window'
-import { setRunnableTimeout } from './setupUtil'
+import { mapTestErrors, normalizeError, setRunnableTimeout } from './setupUtil'
 
 const testReportDir = join(__dirname, '../../../.test-reports')
 const testLogOutput = join(testReportDir, 'testLog.log')
@@ -32,15 +32,18 @@ const maxTestDuration = 30_000
 
 // Expectation: Tests are not run concurrently
 let testLogger: TestLogger | undefined
-let openExternalStub: sinon.SinonStub<Parameters<typeof vscode['env']['openExternal']>, Thenable<boolean>>
+let openExternalStub: sinon.SinonStub<Parameters<(typeof vscode)['env']['openExternal']>, Thenable<boolean>>
 
-export async function mochaGlobalSetup(this: Mocha.Context) {
+export async function mochaGlobalSetup(this: Mocha.Runner) {
     // Clean up and set up test logs
     try {
         await remove(testLogOutput)
     } catch (e) {}
     mkdirpSync(testReportDir)
 
+    // Shows the full error chain when tests fail
+    mapTestErrors(this, normalizeError)
+
     // Extension activation has many side-effects such as changing globals
     // For stability in tests we will wait until the extension has activated prior to injecting mocks
     const activationLogger = (msg: string, ...meta: any[]) => console.log(format(msg, ...meta))

src/test/setupUtil.ts

@@ -9,6 +9,7 @@ import * as vscode from 'vscode'
 import { getLogger } from '../shared/logger'
 import { hasKey } from '../shared/utilities/tsUtils'
 import { getTestWindow, printPendingUiElements } from './shared/vscode/window'
+import { ToolkitError, formatError } from '../shared/errors'
 
 const runnableTimeout = Symbol('runnableTimeout')
 
@@ -54,6 +55,49 @@ export function setRunnableTimeout(test: Mocha.Runnable, maxTestDuration: number
     return test
 }
 
+export function skipTest(testOrCtx: Mocha.Context | Mocha.Test | undefined, reason?: string) {
+    const test =
+        testOrCtx?.type === 'test' ? (testOrCtx as Mocha.Test) : (testOrCtx as Mocha.Context | undefined)?.currentTest
+
+    if (test) {
+        test.title += ` (skipped${reason ? ` - ${reason}` : ''})`
+        test.skip()
+    }
+}
+
+export function skipSuite(suite: Mocha.Suite, reason?: string) {
+    suite.eachTest(test => skipTest(test, reason))
+}
+
+export function mapTestErrors(runner: Mocha.Runner, fn: (err: unknown, test: Mocha.Test) => any) {
+    return runner.prependListener('fail', (test, err) => {
+        test.err = fn(err, test) || err
+    })
+}
+
+/**
+ * Formats any known sub-classes of {@link Error} for better compatability with test reporters.
+ *
+ * Most test reporters will only output the name + message + stack trace so any relevant
+ * info must go into those fields.
+ */
+export function normalizeError(err?: unknown) {
+    if (err instanceof ToolkitError) {
+        // Error has to be mutated to show up in the report:
+        // https://github.com/michaelleeallen/mocha-junit-reporter/blob/4b17772f8da33d580fafa4d124e5c11142a70c1f/index.js#L262
+        //
+        // We'll just patch the message/stack trace even though it's arguably incorrect (and looks kind of ugly)
+        // Once `cause` is more common in the JS ecosystem we'll start to see support from test reporters
+
+        return Object.assign(err, {
+            message: formatError(err).replace(`${err.name}: `, ''),
+            stack: err.stack?.replace(err.message, err.trace.replace(`${err.name}: `, '') + '\n'),
+        })
+    }
+
+    return err
+}
+
 export function patchObject<T extends Record<string, any>, U extends keyof T>(
     obj: T,
     key: U,

src/testE2E/codecatalyst/client.test.ts

@@ -22,9 +22,9 @@ import { waitUntil } from '../../shared/utilities/timeoutUtils'
 import { AccessDeniedException } from '@aws-sdk/client-sso-oidc'
 import { GetDevEnvironmentRequest } from 'aws-sdk/clients/codecatalyst'
 import { getTestWindow } from '../../test/shared/vscode/window'
-import { patchObject, registerAuthHook, using } from '../../test/setupUtil'
+import { patchObject, registerAuthHook, skipTest, using } from '../../test/setupUtil'
 import { isExtensionInstalled } from '../../shared/utilities/vsCodeUtils'
-import { VSCODE_EXTENSION_ID, vscodeExtensionMinVersion } from '../../shared/extensions'
+import { VSCODE_EXTENSION_ID } from '../../shared/extensions'
 import { captureEventOnce } from '../../test/testUtil'
 import { toStream } from '../../shared/utilities/collectionUtils'
 import { toCollection } from '../../shared/utilities/asyncCollection'
@@ -194,8 +194,8 @@ describe('Test how this codebase uses the CodeCatalyst API', function () {
         })
 
         it('prompts to install the ssh extension if not available', async function () {
-            if (isExtensionInstalled(VSCODE_EXTENSION_ID.remotessh, vscodeExtensionMinVersion.remotessh)) {
-                this.skip()
+            if (isExtensionInstalled(VSCODE_EXTENSION_ID.remotessh)) {
+                skipTest(this, 'remote ssh already installed')
             }
 
             await assert.rejects(
@@ -226,7 +226,9 @@ describe('Test how this codebase uses the CodeCatalyst API', function () {
 
         it('connects to a running Dev Environment', async function () {
             // Get necessary objects to run the ssh command.
-            const { SessionProcess, hostname, sshPath } = await prepareDevEnvConnection(client, { ...defaultDevEnv })
+            const { SessionProcess, hostname, sshPath } = await prepareDevEnvConnection(client, {
+                ...defaultDevEnv,
+            })
 
             // Through the actual ssh connection, run 'ls' command in the dev env.
             const lsOutput = (await new SessionProcess(sshPath, [hostname, 'ls', '/projects']).run()).stdout