fix(lambda): fix user agent and headers for remote debugging (#8304)

## Problem
The AWS SDK v2 to v3 migration in PR #8183 introduced breaking changes
for LocalStack compatibility and Lambda debugging is not working
anymore:

1. **User Agent format changed:** The migration changed from
string-based user agents to AWS SDK v3's `UserAgent` pairs format
(`[name, version]` tuples), but several places still used the old string
format, breaking LocalStack's custom user agent for Lambda debugging and
part of the integration with IoT
2. **Response headers no longer accessible**: AWS SDK v3 removed access
to HTTP response headers via `$response.httpResponse.headers`, breaking
LocalStack connection detection which relies on the
`x-localstack-request-url` header

These issues were not caught most likely because LocalStack
compatibility was not tested during the SDK v3 migration.

## Solution
1. Fix user agent format for AWS SDK v3
* Migrated from string-based user agents to proper UserAgent pairs
format:
     
     * Changed from "LAMBDA-DEBUG/1.0.0 AWS-Toolkit/..." (string)
* To [['LAMBDA-DEBUG', '1.0.0'], ['AWS-Toolkit', '2.0'], ...] (pairs)
* Updated awsClientBuilderV3 to use customUserAgent instead of userAgent


2. Add response headers middleware for AWS SDK v3 
* Added `captureHeadersMiddleware` to `AWSClientBuilderV3` that attaches
HTTP response headers to the output via `$httpHeaders` property
* Updated `loginManager` to check for LocalStack connection using
`$httpHeaders` instead of the removed `$response.httpResponse.headers`

## AWS SDK v3 Breaking Changes
### User Agent Format
**SDK v2**:

```ts
// String format in client options
const options = {
    region: 'us-east-1',
    userAgent: 'LAMBDA-DEBUG/1.0.0 AWS-Toolkit/2.0.0'
}
```

**SDK v3:**

```ts
// Array of [name, version] tuples
const options = {
    region: 'us-east-1',
    customUserAgent: [
        ['LAMBDA-DEBUG', '1.0.0'],
        ['AWS-Toolkit', '2.0.0']
    ]
}
```

### Response Headers Access
**SDK v2**:

```ts
const response = await client.send(command)
// Headers directly accessible
const headers = response.$response.httpResponse.headers
const localStackUrl = headers['x-localstack-request-url']
```

**SDK v3**:

```ts
const response = await client.send(command)
// Headers NOT accessible by default - requires custom middleware
const headers = response.$httpHeaders  // undefined without middleware
```

Supersedes #8289 and #8291


---

- Treat all work as PUBLIC. Private `feature/x` branches will not be
squash-merged at release time.
- Your code changes must meet the guidelines in
[CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines).
- License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

---------

Co-authored-by: carole-lavillonniere <[email protected]>
Co-authored-by: Cristian Pallarés <[email protected]>
Co-authored-by: Laxman Reddy <[email protected]>

Author: 4 people

packages/core/src/auth/deprecated/loginManager.ts

@@ -30,7 +30,11 @@ import { isAutomation } from '../../shared/vscode/env'
 import { Credentials } from '@aws-sdk/types'
 import { ToolkitError } from '../../shared/errors'
 import * as localizedText from '../../shared/localizedText'
-import { DefaultStsClient, type GetCallerIdentityResponse } from '../../shared/clients/stsClient'
+import {
+    DefaultStsClient,
+    type GetCallerIdentityResponse,
+    type GetCallerIdentityResponseWithHeaders,
+} from '../../shared/clients/stsClient'
 import { findAsync } from '../../shared/utilities/collectionUtils'
 import { telemetry } from '../../shared/telemetry/telemetry'
 import { withTelemetryContext } from '../../shared/telemetry/util'
@@ -135,9 +139,11 @@ export class LoginManager {
         return accountId
     }
 
-    private async detectExternalConnection(callerIdentity: GetCallerIdentityResponse): Promise<void> {
-        // @ts-ignore
-        const headers = callerIdentity.$response?.httpResponse?.headers
+    private async detectExternalConnection(
+        callerIdentity: GetCallerIdentityResponse | GetCallerIdentityResponseWithHeaders
+    ): Promise<void> {
+        // SDK v3: Headers are captured via middleware and attached as $httpHeaders
+        const headers = (callerIdentity as GetCallerIdentityResponseWithHeaders).$httpHeaders
         if (headers !== undefined && localStackConnectionHeader in headers) {
             await globals.globalState.update('aws.toolkit.externalConnection', localStackConnectionString)
             telemetry.auth_localstackEndpoint.emit({ source: 'validateCredentials', result: 'Succeeded' })

packages/core/src/lambda/remoteDebugging/ldkClient.ts

@@ -17,7 +17,7 @@ import { DefaultLambdaClient } from '../../shared/clients/lambdaClient'
 import { LocalProxy } from './localProxy'
 import globals from '../../shared/extensionGlobals'
 import { getLogger } from '../../shared/logger/logger'
-import { getIoTSTClientWithAgent, getLambdaClientWithAgent, getLambdaDebugUserAgent } from './utils'
+import { getIoTSTClientWithAgent, getLambdaClientWithAgent, getLambdaDebugUserAgentPairs } from './utils'
 import { ToolkitError } from '../../shared/errors'
 import * as nls from 'vscode-nls'
 
@@ -99,7 +99,7 @@ export class LdkClient {
      */
     private getLambdaClient(region: string): DefaultLambdaClient {
         if (!this.lambdaClientCache.has(region)) {
-            this.lambdaClientCache.set(region, getLambdaClientWithAgent(region, getLambdaDebugUserAgent()))
+            this.lambdaClientCache.set(region, getLambdaClientWithAgent(region, getLambdaDebugUserAgentPairs()))
         }
         return this.lambdaClientCache.get(region)!
     }

packages/core/src/lambda/remoteDebugging/utils.ts

@@ -5,36 +5,50 @@
 
 import { IoTSecureTunnelingClient } from '@aws-sdk/client-iotsecuretunneling'
 import { DefaultLambdaClient } from '../../shared/clients/lambdaClient'
-import { getUserAgent } from '../../shared/telemetry/util'
+import { getUserAgentPairs, userAgentPairsToString } from '../../shared/telemetry/util'
 import globals from '../../shared/extensionGlobals'
+import type { UserAgent } from '@aws-sdk/types'
 
-const customUserAgentBase = 'LAMBDA-DEBUG/1.0.0'
+const customUserAgentName = 'LAMBDA-DEBUG'
+const customUserAgentVersion = '1.0.0'
 
-export function getLambdaClientWithAgent(region: string, customUserAgent?: string): DefaultLambdaClient {
+export function getLambdaClientWithAgent(region: string, customUserAgent?: UserAgent): DefaultLambdaClient {
     if (!customUserAgent) {
-        customUserAgent = getLambdaUserAgent()
+        customUserAgent = getLambdaUserAgentPairs()
     }
     return new DefaultLambdaClient(region, customUserAgent)
 }
 
-// Example user agent:
-// LAMBDA-DEBUG/1.0.0 AWS-Toolkit-For-VSCode/testPluginVersion Visual-Studio-Code/1.102.2 ClientId/11111111-1111-1111-1111-111111111111
-export function getLambdaDebugUserAgent(): string {
-    return `${customUserAgentBase} ${getLambdaUserAgent()}`
+/**
+ * Returns properly formatted UserAgent pairs for AWS SDK v3
+ */
+export function getLambdaDebugUserAgentPairs(): UserAgent {
+    return [
+        [customUserAgentName, customUserAgentVersion],
+        ...getUserAgentPairs({ includePlatform: true, includeClientId: true }),
+    ]
 }
 
-// Example user agent:
-// AWS-Toolkit-For-VSCode/testPluginVersion Visual-Studio-Code/1.102.2 ClientId/11111111-1111-1111-1111-111111111111
-export function getLambdaUserAgent(): string {
-    return `${getUserAgent({ includePlatform: true, includeClientId: true })}`
+/**
+ * Returns properly formatted UserAgent pairs for AWS SDK v3
+ */
+export function getLambdaUserAgentPairs(): UserAgent {
+    return getUserAgentPairs({ includePlatform: true, includeClientId: true })
+}
+
+/**
+ * Returns user agent string for Lambda debugging in traditional format.
+ * Example: "LAMBDA-DEBUG/1.0.0 AWS-Toolkit-For-VSCode/testPluginVersion Visual-Studio-Code/1.105.1 ClientId/11111111-1111-1111-1111-111111111111"
+ */
+export function getLambdaDebugUserAgent(): string {
+    return userAgentPairsToString(getLambdaDebugUserAgentPairs())
 }
 
 export function getIoTSTClientWithAgent(region: string): IoTSecureTunnelingClient {
-    const customUserAgent = `${customUserAgentBase} ${getUserAgent({ includePlatform: true, includeClientId: true })}`
     return globals.sdkClientBuilderV3.createAwsService({
         serviceClient: IoTSecureTunnelingClient,
         clientOptions: {
-            userAgent: [[customUserAgent]],
+            customUserAgent: getLambdaDebugUserAgentPairs(),
             region,
         },
         userAgent: false,

packages/core/src/lambda/vue/remoteInvoke/invokeLambda.ts

@@ -36,7 +36,7 @@ import { getLambdaHandlerFile } from '../../../awsService/appBuilder/utils'
 import { runUploadDirectory } from '../../commands/uploadLambda'
 import fs from '../../../shared/fs/fs'
 import { showConfirmationMessage, showMessage } from '../../../shared/utilities/messages'
-import { getLambdaClientWithAgent, getLambdaDebugUserAgent } from '../../remoteDebugging/utils'
+import { getLambdaClientWithAgent, getLambdaDebugUserAgentPairs } from '../../remoteDebugging/utils'
 import { isLocalStackConnection } from '../../../auth/utils'
 import { getRemoteDebugLayer } from '../../remoteDebugging/remoteLambdaDebugger'
 
@@ -906,7 +906,7 @@ export async function invokeRemoteLambda(
     const resource: LambdaFunctionNode = params.functionNode
     const source: string = params.source || 'AwsExplorerRemoteInvoke'
     const client = getLambdaClientWithAgent(resource.regionCode)
-    const clientDebug = getLambdaClientWithAgent(resource.regionCode, getLambdaDebugUserAgent())
+    const clientDebug = getLambdaClientWithAgent(resource.regionCode, getLambdaDebugUserAgentPairs())
 
     const Panel = VueWebview.compilePanel(RemoteInvokeWebview)
 

packages/core/src/shared/awsClientBuilderV3.ts

@@ -12,7 +12,7 @@ import {
     TokenIdentity,
     TokenIdentityProvider,
 } from '@smithy/types'
-import { getUserAgent } from './telemetry/util'
+import { getUserAgentPairs } from './telemetry/util'
 import { DevSettings } from './settings'
 import {
     BuildHandler,
@@ -37,7 +37,6 @@ import { HttpResponse, HttpRequest } from '@aws-sdk/protocol-http'
 import { ConfiguredRetryStrategy } from '@smithy/util-retry'
 import { telemetry } from './telemetry/telemetry'
 import { getRequestId, getTelemetryReason, getTelemetryReasonDesc, getTelemetryResult } from './errors'
-import { extensionVersion } from './vscode/env'
 import { getLogger } from './logger/logger'
 import { partialClone } from './utilities/collectionUtils'
 import { selectFrom } from './utilities/tsUtils'
@@ -72,7 +71,7 @@ export interface AwsCommand<InputType extends object, OutputType extends object>
 export interface AwsClientOptions {
     credentials: AwsCredentialIdentityProvider
     region: string | Provider<string>
-    userAgent: UserAgent
+    customUserAgent: UserAgent
     requestHandler: {
         metadata?: RequestHandlerMetadata
         handle: (req: any, options?: any) => Promise<RequestHandlerOutput<any>>
@@ -155,8 +154,8 @@ export class AWSClientBuilderV3 {
             opt.region = serviceOptions.region
         }
 
-        if (!opt.userAgent && userAgent) {
-            opt.userAgent = [[getUserAgent({ includePlatform: true, includeClientId: true }), extensionVersion]]
+        if (!opt.customUserAgent && userAgent) {
+            opt.customUserAgent = getUserAgentPairs({ includePlatform: true, includeClientId: true })
         }
 
         if (!opt.retryStrategy) {
@@ -196,6 +195,7 @@ export class AWSClientBuilderV3 {
         }
         const service = new serviceOptions.serviceClient(opt)
         service.middlewareStack.add(telemetryMiddleware, { step: 'deserialize' })
+        service.middlewareStack.add(captureHeadersMiddleware, { step: 'deserialize' })
         service.middlewareStack.add(loggingMiddleware, { step: 'finalizeRequest' })
         service.middlewareStack.add(getEndpointMiddleware(serviceOptions.settings), { step: 'build' })
 
@@ -254,6 +254,26 @@ function getEndpointMiddleware(settings: DevSettings = DevSettings.instance): Bu
 const keepAliveMiddleware: BuildMiddleware<any, any> = (next: BuildHandler<any, any>) => async (args: any) =>
     addKeepAliveHeader(next, args)
 
+/**
+ * Middleware that captures HTTP response headers and attaches them to the output object.
+ * This makes headers accessible via `response.$httpHeaders` for all AWS SDK v3 operations.
+ * Useful for detecting custom headers from services like LocalStack.
+ */
+const captureHeadersMiddleware: DeserializeMiddleware<any, any> =
+    (next: DeserializeHandler<any, any>) => async (args: any) => {
+        const result = await next(args)
+
+        // Extract headers from HTTP response and attach to output for easy access
+        if (HttpResponse.isInstance(result.response)) {
+            const headers = result.response.headers
+            if (headers && result.output) {
+                result.output.$httpHeaders = headers
+            }
+        }
+
+        return result
+    }
+
 export async function emitOnRequest(next: DeserializeHandler<any, any>, context: HandlerExecutionContext, args: any) {
     if (!HttpResponse.isInstance(args.request)) {
         return next(args)

packages/core/src/shared/clients/lambdaClient.ts

@@ -41,6 +41,7 @@ import { CancellationError } from '../utilities/timeoutUtils'
 import { fromSSO } from '@aws-sdk/credential-provider-sso'
 import { getIAMConnection } from '../../auth/utils'
 import { NodeHttpHandler } from '@smithy/node-http-handler'
+import type { UserAgent } from '@aws-sdk/types'
 
 export type LambdaClient = ClassToInterfaceType<DefaultLambdaClient>
 
@@ -49,7 +50,7 @@ export class DefaultLambdaClient {
 
     public constructor(
         public readonly regionCode: string,
-        public readonly userAgent: string | undefined = undefined
+        public readonly userAgent: UserAgent | undefined = undefined
     ) {
         this.defaultTimeoutInMs = 5 * 60 * 1000 // 5 minutes (SDK default is 2 minutes)
     }
@@ -322,7 +323,7 @@ export class DefaultLambdaClient {
             serviceClient: LambdaSdkClient,
             userAgent: !this.userAgent,
             clientOptions: {
-                userAgent: this.userAgent ? [[this.userAgent]] : undefined,
+                customUserAgent: this.userAgent,
                 region: this.regionCode,
                 requestHandler: new NodeHttpHandler({
                     requestTimeout: this.defaultTimeoutInMs,

packages/core/src/shared/clients/stsClient.ts

@@ -10,6 +10,11 @@ import { Credentials } from '@aws-sdk/types'
 import globals from '../extensionGlobals'
 import { ClassToInterfaceType } from '../utilities/tsUtils'
 
+// Extended response type that includes captured HTTP headers (added by global middleware)
+export interface GetCallerIdentityResponseWithHeaders extends GetCallerIdentityResponse {
+    $httpHeaders?: Record<string, string>
+}
+
 export type { GetCallerIdentityResponse }
 export type StsClient = ClassToInterfaceType<DefaultStsClient>
 
@@ -35,8 +40,9 @@ export class DefaultStsClient {
         return response
     }
 
-    public async getCallerIdentity(): Promise<GetCallerIdentityResponse> {
+    public async getCallerIdentity(): Promise<GetCallerIdentityResponseWithHeaders> {
         const sdkClient = this.createSdkClient()
+        // Note: $httpHeaders are added by global middleware in awsClientBuilderV3
         const response = await sdkClient.send(new GetCallerIdentityCommand({}))
         return response
     }

packages/core/src/shared/telemetry/util.ts

@@ -31,6 +31,7 @@ import { telemetry } from './telemetry'
 import { v5 as uuidV5 } from 'uuid'
 import { ToolkitError } from '../errors'
 import { GlobalState } from '../globalState'
+import type { UserAgent } from '@aws-sdk/types'
 
 const legacySettingsTelemetryValueDisable = 'Disable'
 const legacySettingsTelemetryValueEnable = 'Enable'
@@ -240,20 +241,46 @@ export function getUserAgent(
     opt?: { includePlatform?: boolean; includeClientId?: boolean },
     globalState = globals.globalState
 ): string {
-    const pairs = isAmazonQ()
-        ? [`AmazonQ-For-VSCode/${extensionVersion}`]
-        : [`AWS-Toolkit-For-VSCode/${extensionVersion}`]
+    return userAgentPairsToString(getUserAgentPairs(opt, globalState))
+}
+
+/**
+ * Returns a UserAgent array (AWS SDK v3 format) with proper [name, version] pairs.
+ *
+ * Omits the platform and `ClientId` pairs by default.
+ *
+ * @returns Array of [name, version] tuples for AWS SDK v3's customUserAgent option
+ */
+export function getUserAgentPairs(
+    opt?: { includePlatform?: boolean; includeClientId?: boolean },
+    globalState = globals.globalState
+): UserAgent {
+    const pairs: UserAgent = isAmazonQ()
+        ? [['AmazonQ-For-VSCode', extensionVersion]]
+        : [['AWS-Toolkit-For-VSCode', extensionVersion]]
 
     if (opt?.includePlatform) {
-        pairs.push(platformPair())
+        const platform = platformPair()
+        const [name, version] = platform.split('/')
+        if (name && version) {
+            pairs.push([name, version])
+        }
     }
 
     if (opt?.includeClientId) {
         const clientId = getClientId(globalState)
-        pairs.push(`ClientId/${clientId}`)
+        pairs.push(['ClientId', clientId])
     }
 
-    return pairs.join(' ')
+    return pairs
+}
+
+/**
+ * Converts UserAgent array format to traditional user agent string format.
+ * Example: [['LAMBDA-DEBUG', '1.0.0'], ['AWS-Toolkit', '2.0']] => "LAMBDA-DEBUG/1.0.0 AWS-Toolkit/2.0"
+ */
+export function userAgentPairsToString(pairs: UserAgent): string {
+    return pairs.map(([name, version]) => `${name}/${version}`).join(' ')
 }
 
 /**