tests: enable e2e, retries, bug fixes (#4144)

* tests: disable fail fast for windows tests, enable e2e

Failures for windows insiders will cancel stable test.

For e2e tests, will we monitor for any throttling.

* tests(e2e): add retries to CoCa client, fix bugs

- Retry strategy chosen somewhat arbitrarily, but these settings don't seem to delay the test suite by much if at all. In recent runs
with these, throttling has stopped.

- Adds retries with jitter by utilizing retries in the SDK, see
https://github.com/aws/aws-sdk-js/blob/3e616251947c73d5239178c167a9d73d985ca581/lib/util.js#L884.

- Also fixes various bugs with the tests that causes errors to be thrown inadvertently, including type checking/verifty the code of some
exceptions. In one case, this threw an AccessDeniedException when checking a dev env that was already deleted previously.

* update comments

Author: hayemaxi

.github/workflows/node.js.yml

@@ -57,6 +57,7 @@ jobs:
         name: test Windows
         runs-on: windows-2019
         strategy:
+            fail-fast: false
             matrix:
                 node-version: [16.x]
                 vscode-version: [stable, insiders]

src/shared/clients/codecatalystClient.ts

@@ -30,6 +30,7 @@ import {
 import { truncateProps } from '../utilities/textUtilities'
 import { SsoConnection } from '../../auth/connection'
 import { DevSettings } from '../settings'
+import { RetryDelayOptions } from 'aws-sdk/lib/config-base'
 
 export interface CodeCatalystConfig {
     readonly region: string
@@ -119,16 +120,24 @@ function toBranch(
     }
 }
 
+interface RetryOptions {
+    retryDelayOptions?: RetryDelayOptions
+    maxRetries?: number
+}
+
 async function createCodeCatalystClient(
     connection: SsoConnection,
-    regionCode = getCodeCatalystConfig().region,
-    endpoint = getCodeCatalystConfig().endpoint
+    regionCode: string,
+    endpoint: string | AWS.Endpoint,
+    retryOptions: RetryOptions
 ): Promise<CodeCatalyst> {
     const c = await globals.sdkClientBuilder.createAwsService(CodeCatalyst, {
         region: regionCode,
         correctClockSkew: true,
         endpoint: endpoint,
         token: new TokenProvider(connection),
+        retryDelayOptions: retryOptions.retryDelayOptions,
+        maxRetries: retryOptions.maxRetries,
     } as ServiceConfigurationOptions)
 
     return c
@@ -155,9 +164,10 @@ export type CodeCatalystClientFactory = () => Promise<CodeCatalystClient>
 export async function createClient(
     connection: SsoConnection,
     regionCode = getCodeCatalystConfig().region,
-    endpoint = getCodeCatalystConfig().endpoint
+    endpoint = getCodeCatalystConfig().endpoint,
+    retryOptions: RetryOptions = {}
 ): Promise<CodeCatalystClient> {
-    const sdkClient = await createCodeCatalystClient(connection, regionCode, endpoint)
+    const sdkClient = await createCodeCatalystClient(connection, regionCode, endpoint, retryOptions)
     const c = new CodeCatalystClientInternal(connection, sdkClient)
     await c.verifySession()
 

src/test/techdebt.test.ts

@@ -44,14 +44,6 @@ describe('tech debt', function () {
         )
     })
 
-    it('stop skipping CodeCatalyst E2E Tests', function () {
-        // https://issues.amazon.com/issues/IDE-10496
-        assert(
-            new Date() < new Date(2024, 1, 15),
-            'Re-evaluate if we should still keep skipping CodeCatalyst E2E Tests'
-        )
-    })
-
     it('stop not using latest python extension version in integration CI tests', function () {
         /**
          * The explicitly set version is done in {@link installVSCodeExtension}

src/testE2E/codecatalyst/client.test.ts

@@ -29,13 +29,14 @@ import { captureEventOnce } from '../../test/testUtil'
 import { toStream } from '../../shared/utilities/collectionUtils'
 import { toCollection } from '../../shared/utilities/asyncCollection'
 import { getLogger } from '../../shared/logger'
-import { isAwsError } from '../../shared/errors'
+import { isAwsError, ToolkitError } from '../../shared/errors'
 import {
     scopesCodeCatalyst,
     createBuilderIdProfile,
     isValidCodeCatalystConnection,
     SsoConnection,
 } from '../../auth/connection'
+import { hasKey } from '../../shared/utilities/tsUtils'
 
 let spaceName: CodeCatalystOrg['name']
 let projectName: CodeCatalystProject['name']
@@ -86,7 +87,7 @@ let projectName: CodeCatalystProject['name']
  *     integ tests, but using the ssh hostname that we get from
  *     {@link prepareDevEnvConnection}.
  */
-describe.skip('Test how this codebase uses the CodeCatalyst API', function () {
+describe('Test how this codebase uses the CodeCatalyst API', function () {
     let client: CodeCatalystClient
     let commands: CodeCatalystCommands
     let webviewClient: CodeCatalystCreateWebview
@@ -193,11 +194,11 @@ describe.skip('Test how this codebase uses the CodeCatalyst API', function () {
             assert.strictEqual(actualDevEnv.persistentStorage.sizeInGiB, 32)
         })
 
-        it('creates a Dev Environment using an existing branch', async function () {
-            // TODO: The CC API does not provide a way to create a repository
-            // due to this we'll want to revisit this test.
-            // For now, we can manually create repository in the test project
-            // and then continue this test from that point.
+        it.skip('creates a Dev Environment using an existing branch', async function () {
+            // TODO: Write this test now that an API is available in the SDK:
+            // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/CodeCatalyst.html#createSourceRepository-property
+            // https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/CodeCatalyst.html#createSourceRepositoryBranch-property
+            // For now, the repository is manually created in the account.
         })
 
         it('prompts to install the ssh extension if not available', async function () {
@@ -354,16 +355,15 @@ describe.skip('Test how this codebase uses the CodeCatalyst API', function () {
             assert.ok(spaces.find(space => space.name === spaceName))
         })
 
-        it('lists all projects for the given user', async function () {
+        // TODO: Re-add this test when the CoCa SDK offers a way to delete projects.
+        it.skip('lists all projects for the given user', async function () {
             // Create lots of projects
-            // const ephemeralProjectNames = []
-            // for (let index = 0; index < 30; index++) {
-            //     ephemeralProjectNames.push(`ephemeral-project-${index}`)
-            // }
-            // await Promise.all(
-            //     ephemeralProjectNames.map(name => tryCreateTestProject(spaceName, name))
-            // )
-            // const projects = await client.listProjects({spaceName}).flatten().promise()
+            const ephemeralProjectNames = []
+            for (let index = 0; index < 30; index++) {
+                ephemeralProjectNames.push(`ephemeral-project-${index}`)
+            }
+            await Promise.all(ephemeralProjectNames.map(name => tryCreateTestProject(spaceName, name)))
+            await client.listProjects({ spaceName }).flatten().promise()
         })
 
         function buildDevEnvSettings(
@@ -425,7 +425,15 @@ describe.skip('Test how this codebase uses the CodeCatalyst API', function () {
      */
     async function createTestCodeCatalystClient(auth: Auth): Promise<CodeCatalystClient> {
         const conn = await useCodeCatalystSsoConnection(auth)
-        return await createCodeCatalystClient(conn)
+        return await createCodeCatalystClient(conn, undefined, undefined, {
+            // Add retries for tests since many may be running in parallel in github CI.
+            // AWS SDK adds jitter automatically.
+            // https://github.com/aws/aws-sdk-js/blob/3e616251947c73d5239178c167a9d73d985ca581/lib/util.js#L884
+            retryDelayOptions: {
+                base: 1200, // ms
+            },
+            maxRetries: 5,
+        })
     }
 
     /**
@@ -457,7 +465,8 @@ describe.skip('Test how this codebase uses the CodeCatalyst API', function () {
                 description: 'This project is autogenerated by the AWS Toolkit VSCode Integ Test.',
             })
         } catch (e) {
-            if (isAwsError(e) && e.code === 'ConflictException') {
+            if ((isAwsError(e) || e instanceof ToolkitError) && e.code === 'ConflictException') {
+                getLogger().debug(`Tried to create test project but it already exists: ${spaceName}/${projectName}`)
                 return client.getProject({
                     spaceName,
                     name: projectName,
@@ -578,8 +587,10 @@ describe.skip('Test how this codebase uses the CodeCatalyst API', function () {
                 let devEnvBeingDeleted: DevEnvironment
                 try {
                     devEnvBeingDeleted = await client.getDevEnvironment({ spaceName, projectName, id })
-                } catch (e) {
-                    if (e instanceof Error && e.name === AccessDeniedException.name) {
+                } catch (e: any) {
+                    // Cannot use isAwsError() because the client actually returns a regular Error
+                    // with a 'code' property for this call.
+                    if (hasKey(e, 'code') && e.code === AccessDeniedException.name) {
                         // This error is thrown because the dev env does not exist anymore
                         // on the server. The name doesn't make it obvious IMO.
                         return true