feat(sam): improve run/debug error handling and telemetry (#2865)

## Problem
Our SAM run/debug implementation has several problems with error handling:
* Inconsistent error message UX (some are placed in the logs, some are shown directly to the user)
* Telemetry only covers a portion of the relevant code paths and does not emit reasons for failure
* Some errors are swallowed while others are bubbled up to the VSC API

## Solution
* Bubble up all errors but use a single error handler to prevent the VSC error modal from showing
    * Added a button to open the launch config to preserve current UX
* Use telemetry spans to cover almost the entire code path
* Remove sam_attachDebugger metric as it is redundant. Failures to attach counts as a failure for the whole execution.
* Add some basic error codes to better determine why a run/debug execution failed

Author: JadenSimon

src/shared/errors.ts

@@ -8,7 +8,7 @@ import { Result } from './telemetry/telemetry'
 import { CancellationError } from './utilities/timeoutUtils'
 import { isNonNullable } from './utilities/tsUtils'
 
-interface ErrorInformation {
+export interface ErrorInformation {
     /**
      * Error names are optional, but if provided they should be generic yet self-explanatory.
      *

src/shared/sam/cli/samCliLocalInvoke.ts

@@ -214,7 +214,7 @@ export class SamCliLocalInvokeInvocation {
         this.args.skipPullImage = !!this.args.skipPullImage
     }
 
-    public async execute(timeout?: Timeout): Promise<void> {
+    public async execute(timeout?: Timeout): Promise<ChildProcess> {
         await this.validate()
 
         const sam = await this.config.getOrDetectSamCli()
@@ -251,7 +251,7 @@ export class SamCliLocalInvokeInvocation {
         )
         invokeArgs.push(...(this.args.extraArgs ?? []))
 
-        await this.args.invoker.invoke({
+        return await this.args.invoker.invoke({
             options: {
                 env: {
                     ...process.env,

src/shared/sam/debugger/awsSamDebugger.ts

@@ -7,7 +7,6 @@ import * as semver from 'semver'
 import * as vscode from 'vscode'
 import * as fs from 'fs-extra'
 import * as nls from 'vscode-nls'
-import { Runtime } from 'aws-sdk/clients/lambda'
 import {
     getCodeRoot,
     getHandlerName,
@@ -17,6 +16,7 @@ import {
     GoDebugConfiguration,
     getTemplate,
     getArchitecture,
+    isImageLambdaConfig,
 } from '../../../lambda/local/debugConfiguration'
 import {
     Architecture,
@@ -53,10 +53,9 @@ import { makeInputTemplate, makeJsonFiles } from '../localLambdaRunner'
 import { SamLocalInvokeCommand } from '../cli/samCliLocalInvoke'
 import { getCredentialsFromStore } from '../../../credentials/credentialsStore'
 import { fromString } from '../../../credentials/providers/credentials'
-import { notifyUserInvalidCredentials } from '../../../credentials/credentialsUtilities'
 import { Credentials } from '@aws-sdk/types'
 import { CloudFormation } from '../../cloudformation/cloudformation'
-import { getSamCliVersion } from '../cli/samCliContext'
+import { getSamCliContext, getSamCliVersion } from '../cli/samCliContext'
 import {
     MINIMUM_SAM_CLI_VERSION_INCLUSIVE_FOR_IMAGE_SUPPORT,
     MINIMUM_SAM_CLI_VERSION_INCLUSIVE_FOR_GO_SUPPORT,
@@ -65,9 +64,53 @@ import { getIdeProperties, isCloud9 } from '../../extensionUtilities'
 import { resolve } from 'path'
 import globals from '../../extensionGlobals'
 import { LoginManager } from '../../../credentials/loginManager'
+import { Runtime, telemetry } from '../../telemetry/telemetry'
+import { ErrorInformation, isUserCancelledError, ToolkitError } from '../../errors'
+import { openLaunchJsonFile } from './commands/addSamDebugConfiguration'
+import { Logging } from '../../logger/commands'
+import { credentialHelpUrl } from '../../constants'
 
 const localize = nls.loadMessageBundle()
 
+interface NotificationButton<T = unknown> {
+    readonly label: string
+    readonly onClick: () => Promise<T> | T
+}
+
+class SamLaunchRequestError extends ToolkitError.named('SamLaunchRequestError') {
+    private readonly buttons: NotificationButton[]
+
+    public constructor(message: string, info?: ErrorInformation & { readonly extraButtons?: NotificationButton[] }) {
+        super(message, info)
+        this.buttons = info?.extraButtons ?? [
+            {
+                label: localize('AWS.generic.message.openConfig', 'Open Launch Config'),
+                onClick: openLaunchJsonFile,
+            },
+        ]
+    }
+
+    public async showNotification(): Promise<void> {
+        if (isUserCancelledError(this)) {
+            getLogger().verbose(`SAM run/debug: user cancelled`)
+            return
+        }
+
+        const logId = getLogger().error(this.trace)
+
+        const viewLogsButton = {
+            label: localize('AWS.generic.message.viewLogs', 'View Logs...'),
+            onClick: () => Logging.declared.viewLogsAtMessage.execute(logId),
+        }
+
+        const buttonsWithLogs = [viewLogsButton, ...this.buttons]
+
+        await vscode.window.showErrorMessage(this.message, ...buttonsWithLogs.map(b => b.label)).then(resp => {
+            return buttonsWithLogs.find(({ label }) => label === resp)?.onClick()
+        })
+    }
+}
+
 /**
  * SAM-specific launch attributes (which are not part of the DAP).
  *
@@ -310,12 +353,31 @@ export class SamDebugConfigProvider implements vscode.DebugConfigurationProvider
         folder: vscode.WorkspaceFolder | undefined,
         config: AwsSamDebuggerConfiguration,
         token?: vscode.CancellationToken
-    ): Promise<undefined> {
-        const resolvedConfig = await this.makeConfig(folder, config, token)
-        if (!resolvedConfig) {
-            return undefined
+    ): Promise<void> {
+        try {
+            if (config.invokeTarget.target === 'api') {
+                await telemetry.apigateway_invokeLocal.run(async span => {
+                    const resolved = await this.makeConfig(folder, config, token)
+                    span.record({ httpMethod: resolved.api?.httpMethod })
+
+                    return this.invokeConfig(resolved)
+                })
+            } else {
+                await telemetry.lambda_invokeLocal.run(async () => {
+                    const resolved = await this.makeConfig(folder, config, token)
+
+                    return this.invokeConfig(resolved)
+                })
+            }
+        } catch (err) {
+            if (err instanceof SamLaunchRequestError) {
+                err.showNotification()
+            } else if (err instanceof ToolkitError) {
+                new SamLaunchRequestError(err.message, { ...err }).showNotification()
+            } else {
+                SamLaunchRequestError.chain(err, 'Failed to run launch configuration').showNotification()
+            }
         }
-        await this.invokeConfig(resolvedConfig)
     }
 
     /**
@@ -332,22 +394,21 @@ export class SamDebugConfigProvider implements vscode.DebugConfigurationProvider
         folder: vscode.WorkspaceFolder | undefined,
         config: AwsSamDebuggerConfiguration,
         token?: vscode.CancellationToken
-    ): Promise<SamLaunchRequestArgs | undefined> {
+    ): Promise<SamLaunchRequestArgs> {
         if (token?.isCancellationRequested) {
-            return undefined
+            throw new ToolkitError('Cancellation requested', { cancelled: true })
         }
+
         folder =
             folder ?? (vscode.workspace.workspaceFolders?.length ? vscode.workspace.workspaceFolders[0] : undefined)
         if (!folder) {
-            getLogger().error(`SAM debug: no workspace folder`)
-            vscode.window.showErrorMessage(
-                localize(
-                    'AWS.sam.debugger.noWorkspace',
-                    '{0} SAM debug: choose a workspace, then try again',
-                    getIdeProperties().company
-                )
+            const message = localize(
+                'AWS.sam.debugger.noWorkspace',
+                'Choose a workspace, then try again',
+                getIdeProperties().company
             )
-            return undefined
+
+            throw new SamLaunchRequestError(message, { code: 'NoWorkspaceFolder', extraButtons: [] })
         }
 
         // If "request" field is missing this means launch.json does not exist.
@@ -356,28 +417,24 @@ export class SamDebugConfigProvider implements vscode.DebugConfigurationProvider
         const configValidator: AwsSamDebugConfigurationValidator = new DefaultAwsSamDebugConfigurationValidator(folder)
 
         if (!hasLaunchJson) {
-            vscode.window
-                .showErrorMessage(
-                    localize(
-                        'AWS.sam.debugger.noLaunchJson',
-                        '{0} SAM: To debug a Lambda locally, create a launch.json from the Run panel, then select a configuration.',
-                        getIdeProperties().company
-                    ),
-                    localize('AWS.gotoRunPanel', 'Run panel')
-                )
-                .then(async result => {
-                    if (!result) {
-                        return
-                    }
-                    await vscode.commands.executeCommand('workbench.view.debug')
-                })
-            return undefined
+            const message = localize(
+                'AWS.sam.debugger.noLaunchJson',
+                'To debug a Lambda locally, create a launch.json from the Run panel, then select a configuration.'
+            )
+
+            throw new SamLaunchRequestError(message, {
+                code: 'NoLaunchConfig',
+                extraButtons: [
+                    {
+                        label: localize('AWS.gotoRunPanel', 'Run panel'),
+                        onClick: () => vscode.commands.executeCommand('workbench.view.debug'),
+                    },
+                ],
+            })
         } else {
             const rv = configValidator.validate(config)
             if (!rv.isValid) {
-                getLogger().error(`SAM debug: invalid config: ${rv.message!}`)
-                vscode.window.showErrorMessage(rv.message!)
-                return undefined
+                throw new ToolkitError(`Invalid launch configuration: ${rv.message}`, { code: 'BadLaunchConfig' })
             } else if (rv.message) {
                 vscode.window.showInformationMessage(rv.message)
             }
@@ -433,27 +490,23 @@ export class SamDebugConfigProvider implements vscode.DebugConfigurationProvider
         if (!isZip) {
             const samCliVersion = await getSamCliVersion(this.ctx.samCliContext())
             if (semver.lt(samCliVersion, MINIMUM_SAM_CLI_VERSION_INCLUSIVE_FOR_IMAGE_SUPPORT)) {
-                getLogger().error(`SAM debug: version (${samCliVersion}) too low for Image lambdas: ${config})`)
-                vscode.window.showErrorMessage(
-                    localize(
-                        'AWS.output.sam.no.image.support',
-                        'Support for Image-based Lambdas requires a minimum SAM CLI version of 1.13.0.'
-                    )
+                const message = localize(
+                    'AWS.output.sam.no.image.support',
+                    'Support for Image-based Lambdas requires a minimum SAM CLI version of 1.13.0.'
                 )
-                return undefined
+
+                throw new SamLaunchRequestError(message, { code: 'UnsupportedSamVersion', details: { samCliVersion } })
             }
         }
 
         if (!runtime) {
-            getLogger().error(`SAM debug: failed to launch config (missing runtime): ${config})`)
-            vscode.window.showErrorMessage(
-                localize(
-                    'AWS.sam.debugger.failedLaunch.missingRuntime',
-                    'Toolkit could not infer a runtime for config: {0}. Add a "lambda.runtime" field to your launch configuration.',
-                    config.name
-                )
+            const message = localize(
+                'AWS.sam.debugger.failedLaunch.missingRuntime',
+                'Toolkit could not infer a runtime for config: {0}. Add a "lambda.runtime" field to your launch configuration.',
+                config.name
             )
-            return undefined
+
+            throw new SamLaunchRequestError(message, { code: 'MissingRuntime' })
         }
 
         // SAM CLI versions before 1.18.1 do not work correctly for Go debugging.
@@ -494,11 +547,22 @@ export class SamDebugConfigProvider implements vscode.DebugConfigurationProvider
             if (fromStore) {
                 awsCredentials = fromStore
             } else {
-                getLogger().error(
-                    `SAM debug: invalid "aws.credentials" value in launch config: ${config.aws.credentials}`
-                )
-                notifyUserInvalidCredentials(config.aws.credentials)
-                return undefined
+                const credentialsId = config.aws.credentials
+                const getHelp = localize('AWS.generic.message.getHelp', 'Get Help...')
+                // TODO: getHelp page for Cloud9.
+                const extraButtons = isCloud9()
+                    ? []
+                    : [
+                          {
+                              label: getHelp,
+                              onClick: () => vscode.env.openExternal(vscode.Uri.parse(credentialHelpUrl)),
+                          },
+                      ]
+
+                throw new SamLaunchRequestError(`Invalid credentials found in launch configuration: ${credentialsId}`, {
+                    code: 'InvalidCredentials',
+                    extraButtons,
+                })
             }
         }
 
@@ -527,7 +591,7 @@ export class SamDebugConfigProvider implements vscode.DebugConfigurationProvider
             request: 'attach',
             codeRoot: codeRoot ?? '',
             workspaceFolder: folder,
-            runtime: runtime,
+            runtime: runtime as Runtime,
             runtimeFamily: runtimeFamily,
             handlerName: handlerName,
             documentUri: documentUri,
@@ -581,16 +645,13 @@ export class SamDebugConfigProvider implements vscode.DebugConfigurationProvider
                 break
             }
             default: {
-                getLogger().error(`SAM debug: unknown runtime: ${runtime})`)
-                vscode.window.showErrorMessage(
-                    localize(
-                        'AWS.sam.debugger.invalidRuntime',
-                        '{0} SAM debug: unknown runtime: {1}',
-                        getIdeProperties().company,
-                        runtime
-                    )
+                const message = localize(
+                    'AWS.sam.debugger.invalidRuntime',
+                    'Unknown or unsupported runtime: {0}',
+                    runtime
                 )
-                return undefined
+
+                throw new ToolkitError(message, { code: 'UnsupportedRuntime' })
             }
         }
         await makeJsonFiles(launchConfig)
@@ -618,6 +679,14 @@ export class SamDebugConfigProvider implements vscode.DebugConfigurationProvider
      * Performs the EXECUTE phase of SAM run/debug.
      */
     public async invokeConfig(config: SamLaunchRequestArgs): Promise<SamLaunchRequestArgs> {
+        telemetry.record({
+            debug: !config.noDebug,
+            runtime: config.runtime as Runtime,
+            lambdaArchitecture: config.architecture,
+            lambdaPackageType: isImageLambdaConfig(config) ? 'Image' : 'Zip',
+            version: await getSamCliVersion(getSamCliContext()),
+        })
+
         await LoginManager.tryAutoConnect()
         switch (config.runtimeFamily) {
             case RuntimeFamily.NodeJS: {
@@ -642,7 +711,7 @@ export class SamDebugConfigProvider implements vscode.DebugConfigurationProvider
                 return await javaDebug.invokeJavaLambda(this.ctx, config)
             }
             default: {
-                throw Error(`unknown runtimeFamily: ${config.runtimeFamily}`)
+                throw new Error(`unknown runtimeFamily: ${config.runtimeFamily}`)
             }
         }
     }

src/shared/sam/debugger/goSamDebug.ts

@@ -21,9 +21,9 @@ import { Timeout } from '../../utilities/timeoutUtils'
 import { SystemUtilities } from '../../../shared/systemUtilities'
 import { execFileSync, SpawnOptions } from 'child_process'
 import * as nls from 'vscode-nls'
-import { showViewLogsMessage } from '../../../shared/utilities/messages'
 import { sleep } from '../../utilities/timeoutUtils'
 import globals from '../../extensionGlobals'
+import { ToolkitError } from '../../errors'
 const localize = nls.loadMessageBundle()
 
 /**
@@ -56,12 +56,12 @@ export async function invokeGoLambda(ctx: ExtContext, config: GoDebugConfigurati
     config.onWillAttachDebugger = waitForDelve
 
     if (!config.noDebug && !(await installDebugger(config.debuggerPath!))) {
-        showViewLogsMessage(
-            localize('AWS.sam.debugger.godelve.failed', 'Failed to install Delve for the lambda container.')
+        throw new ToolkitError(
+            localize('AWS.sam.debugger.godelve.failed', 'Failed to install Delve for the lambda container.'),
+            {
+                code: 'NoDelveInstallation',
+            }
         )
-
-        // Terminates the debug session by sending up a dummy config
-        return {} as GoDebugConfiguration
     }
 
     const c = (await runLambdaFunction(ctx, config, async () => {})) as GoDebugConfiguration