build(test): add automatic retry to all mocha tests. (#6005)

## Problem
There are many unreliable tests that seem to fail very rarely or even
only once. If they are truly failing so rarely, retrying them should
keep CI green. This also allows us to avoid investigating flaky tests
that aren't having a meaningful negative impact on the dev experience.

## Solution
- retry failed tests up to 3 times. 

---

<!--- REMINDER: Ensure that your PR meets the guidelines in
CONTRIBUTING.md -->

License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

Author: Hweinstock

packages/core/src/test/testRunner.ts

@@ -22,7 +22,7 @@ export async function runTests(
     testFolder: string | string[],
     extensionId: string,
     initTests: string[] = [],
-    testFiles?: string[]
+    options?: { retries?: number; testFiles?: string[] }
 ): Promise<void> {
     if (!process.env['AWS_TOOLKIT_AUTOMATION']) {
         throw new Error('Expected the "AWS_TOOLKIT_AUTOMATION" environment variable to be set for tests.')
@@ -79,6 +79,7 @@ export async function runTests(
                 mochaFile: outputFile,
             },
         },
+        retries: options?.retries ?? 0,
         timeout: 0,
     })
 
@@ -91,7 +92,7 @@ export async function runTests(
         testFilePath = testFile ? path.resolve(dist, testFile) : undefined
     }
 
-    if (testFile && testFiles) {
+    if (testFile && options?.testFiles) {
         throw new Error('Individual file and list of files given to run tests on. One must be chosen.')
     }
 
@@ -143,8 +144,8 @@ export async function runTests(
     }
 
     let files: string[] = []
-    if (testFiles) {
-        files = testFiles
+    if (options?.testFiles) {
+        files = options.testFiles
     } else {
         for (const f of Array.isArray(testFolder) ? testFolder : [testFolder]) {
             files = [...files, ...(await glob(testFilePath ?? `**/${f}/**/**.test.js`, { cwd: dist }))]

packages/toolkit/test/unit/index.ts

@@ -7,7 +7,10 @@ import { runTests } from 'aws-core-vscode/test'
 import { VSCODE_EXTENSION_ID } from 'aws-core-vscode/utils'
 
 export function run(): Promise<void> {
-    return runTests(process.env.TEST_DIR ?? ['test/unit', '../../core/dist/src/test'], VSCODE_EXTENSION_ID.awstoolkit, [
-        '../../core/dist/src/test/globalSetup.test.ts',
-    ])
+    return runTests(
+        process.env.TEST_DIR ?? ['test/unit', '../../core/dist/src/test'],
+        VSCODE_EXTENSION_ID.awstoolkit,
+        ['../../core/dist/src/test/globalSetup.test.ts'],
+        { retries: 3 }
+    )
 }