SAM: fix failing SAM tests (#4170)

* test: Bump sam version in CI to see if it passes now

- remove the old tech debt test to signal to us to check if a new version works. We do not need this anymore now.
- Always download the latest version of sam cli in CI instead of using the fixed version. This is back to what it was before we hard set a version.

Signed-off-by: nkomonen <[email protected]>

* remove node14 from sam tests

This is indicated as deprecated

Signed-off-by: nkomonen <[email protected]>

* fix: sam python tests failing due to race condition

We would run continueDebugger() to "click" the continue button
when debugging, but this failed due to there being no breakpoints
and the debugger never pausing. When trying to continue the call would
throw an error since it wasn't possible.

The spirit of the test was to click the continue button to ensure the
debugger completed. Since we do not pause anywhere, the solution this commit
does is to instead wait for the debug session to return nothing, assuming this indicates it is done.

Signed-off-by: nkomonen <[email protected]>

* install older version of python extension so CI passes

Signed-off-by: nkomonen <[email protected]>

---------

Signed-off-by: nkomonen <[email protected]>

Author: nkomonen-amazon

buildspec/linuxIntegrationTests.yml

@@ -54,8 +54,7 @@ phases:
             - bash buildspec/shared/linux-pre_build.sh
             # Where non-root "pip3 install" puts things:
             - 'export PATH="$HOME/.local/bin:$PATH"'
-            # Explicit older version since we assume the newer version is causing our CI to fail (python 3.10 sam tests specifically)
-            - '>/dev/null pip install aws-sam-cli==1.94.0'
+            - '>/dev/null pip3 install --upgrade aws-sam-cli'
             - '>/dev/null pip3 install --upgrade awscli'
             # Print info about sam (version, location, …).
             - 'pip3 show aws-sam-cli'

scripts/test/launchTestUtilities.ts

@@ -76,6 +76,13 @@ export async function invokeVSCodeCli(vsCodeExecutablePath: string, args: string
 }
 
 export async function installVSCodeExtension(vsCodeExecutablePath: string, extensionIdentifier: string): Promise<void> {
+    // HACK: `sam.test.ts` Codelens test was failing for python due to bug in newer version, so lock to last working version.
+    // Edge Case: This specific python version does not work with the "minimum" vscode version, so we do not override it as it
+    // will choose its own python extension version that works.
+    if (extensionIdentifier === VSCODE_EXTENSION_ID.python && process.env[envvarVscodeTestVersion] !== minimum) {
+        extensionIdentifier = `${VSCODE_EXTENSION_ID.python}@2023.20.0`
+    }
+
     console.log(`Installing VS Code Extension: ${extensionIdentifier}`)
     console.log(await invokeVSCodeCli(vsCodeExecutablePath, ['--install-extension', extensionIdentifier]))
 }

src/test/techdebt.test.ts

@@ -6,6 +6,7 @@
 import assert from 'assert'
 import * as semver from 'semver'
 import * as env from '../shared/vscode/env'
+import { installVSCodeExtension } from '../../scripts/test/launchTestUtilities'
 
 // Checks project config and dependencies, to remind us to remove old things
 // when possible.
@@ -43,20 +44,23 @@ describe('tech debt', function () {
         )
     })
 
-    it('remove explicit sam cli version', function () {
-        // Indicate to start using the latest aws-sam-cli version in our CI
-        // https://issues.amazon.com/issues/IDE-11386
+    it('stop skipping CodeCatalyst E2E Tests', function () {
+        // https://issues.amazon.com/issues/IDE-10496
         assert(
             new Date() < new Date(2024, 1, 15),
-            'Remove use of 1.94.0 for aws-sam-cli in linuxIntegrationTests.yml and see if integration tests are passing now'
+            'Re-evaluate if we should still keep skipping CodeCatalyst E2E Tests'
         )
     })
 
-    it('stop skipping CodeCatalyst E2E Tests', function () {
-        // https://issues.amazon.com/issues/IDE-10496
+    it('stop not using latest python extension version in integration CI tests', function () {
+        /**
+         * The explicitly set version is done in {@link installVSCodeExtension}
+         * The parent ticket for SAM test issues: IDE-12295
+         * Python Extension Bug Issue (if this is fixed, then this should be too): https://github.com/microsoft/vscode-python/issues/22659
+         */
         assert(
             new Date() < new Date(2024, 1, 15),
-            'Re-evaluate if we should still keep skipping CodeCatalyst E2E Tests'
+            'Re-evaluate if we can use the latest python extension version in CI integration tests'
         )
     })
 })

src/testInteg/sam.test.ts

@@ -24,15 +24,15 @@ import { waitUntil } from '../shared/utilities/timeoutUtils'
 import { AwsSamDebuggerConfiguration } from '../shared/sam/debugger/awsSamDebugConfiguration.gen'
 import { AwsSamTargetType } from '../shared/sam/debugger/awsSamDebugConfiguration'
 import { insertTextIntoFile } from '../shared/utilities/textUtilities'
-import { sleep } from '../shared/utilities/timeoutUtils'
 import globals from '../shared/extensionGlobals'
 import { closeAllEditors } from '../test/testUtil'
+import { ToolkitError } from '../shared/errors'
 
 const projectFolder = testUtils.getTestWorkspaceFolder()
 
 /* Test constants go here */
 const codelensTimeout: number = 60000
-const codelensRetryInterval: number = 200
+const codelensRetryInterval: number = 5000
 const noDebugSessionTimeout: number = 5000
 const noDebugSessionInterval: number = 100
 
@@ -55,15 +55,6 @@ interface TestScenario {
 // to reduce the chance of automated tests timing out.
 const scenarios: TestScenario[] = [
     // zips
-    {
-        runtime: 'nodejs14.x',
-        displayName: 'nodejs14.x (ZIP)',
-        path: 'hello-world/app.js',
-        debugSessionType: 'pwa-node',
-        language: 'javascript',
-        dependencyManager: 'npm',
-        vscodeMinimum: '1.50.0',
-    },
     {
         runtime: 'nodejs16.x',
         displayName: 'nodejs16.x (ZIP)',
@@ -150,16 +141,6 @@ const scenarios: TestScenario[] = [
     // },
 
     // images
-    {
-        runtime: 'nodejs14.x',
-        displayName: 'nodejs14.x (Image)',
-        baseImage: 'amazon/nodejs14.x-base',
-        path: 'hello-world/app.js',
-        debugSessionType: 'pwa-node',
-        language: 'javascript',
-        dependencyManager: 'npm',
-        vscodeMinimum: '1.50.0',
-    },
     {
         runtime: 'nodejs16.x',
         displayName: 'nodejs16.x (Image)',
@@ -326,25 +307,32 @@ async function startDebugger(
         )
     })
 
-    // Executes the 'F5' action
+    // Executes the 'F5' action to start debugging
     await vscode.debug.startDebugging(undefined, testConfig)
     if (!vscode.debug.activeDebugSession) {
         logSession('EXIT', `${testConfig.name} (exited immediately)`)
         return
     }
     logSession('START', vscode.debug.activeDebugSession.name)
-    await sleep(400)
-    await continueDebugger()
-    await sleep(400)
-    await continueDebugger()
-    await sleep(400)
-    await continueDebugger()
 
-    return success
-}
+    // Some tests need to hit debug continue to the debugger to complete.
+    // Testing locally seemed to take 600-800 millis to run.
+    const result = await waitUntil(
+        async () => {
+            try {
+                await vscode.commands.executeCommand('workbench.action.debug.continue')
+                return true
+            } catch {
+                return false
+            }
+        },
+        { interval: 500, timeout: 10_000 }
+    )
+    if (result === undefined) {
+        throw new ToolkitError(`Toolkit: Debug session did not stop. Something may have gotten stuck.`)
+    }
 
-async function continueDebugger(): Promise<void> {
-    await vscode.commands.executeCommand('workbench.action.debug.continue')
+    return success
 }
 
 async function stopDebugger(logMsg: string | undefined): Promise<void> {
@@ -507,13 +495,13 @@ describe('SAM Integration Tests', async function () {
                     ) {
                         this.skip()
                     }
-
                     const codeLenses = await testUtils.getAddConfigCodeLens(
                         samAppCodeUri,
                         codelensTimeout,
                         codelensRetryInterval
                     )
-                    assert.ok(codeLenses && codeLenses.length === 2)
+                    assert.ok(codeLenses, 'No CodeLenses provided')
+                    assert.strictEqual(codeLenses.length, 2, 'Incorrect amount of CodeLenses provided')
 
                     let manifestFile: RegExp
                     switch (scenario.language) {