From 47b7770bea06b99bb1ebd5758f43604d285ebea3 Mon Sep 17 00:00:00 2001 From: "mxiamxia@gmail.com" Date: Tue, 29 Jul 2025 10:35:54 -0700 Subject: [PATCH 1/2] Fix Bedrock instrumentation TextDecoder error with robust type handling - Fix "The 'list' argument must be an instance of SharedArrayBuffer, ArrayBuffer or ArrayBufferView" error in BedrockRuntimeServiceExtension.responseHook() - Add type checking to handle different response body types from AWS SDK middleware: - String: Already processed by AWS SDK (Uint8ArrayBlobAdapter) - Uint8Array: Raw binary from AWS (decode with TextDecoder) - Buffer: Node.js Buffer (convert with toString('utf8')) - Unexpected types: Log debug message and skip processing gracefully --- .../src/patches/aws/services/bedrock.ts | 20 ++- .../test/patches/aws/services/bedrock.test.ts | 155 ++++++++++++++++++ 2 files changed, 173 insertions(+), 2 deletions(-) diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/aws/services/bedrock.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/aws/services/bedrock.ts index 68fa3fe9..034ae361 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/aws/services/bedrock.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/aws/services/bedrock.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { Attributes, DiagLogger, Span, SpanKind, Tracer } from '@opentelemetry/api'; +import { Attributes, DiagLogger, Span, SpanKind, Tracer, diag } from '@opentelemetry/api'; import { AwsSdkInstrumentationConfig, NormalizedRequest, @@ -331,7 +331,23 @@ export class BedrockRuntimeServiceExtension implements ServiceExtension { responseHook(response: NormalizedResponse, span: Span, tracer: Tracer, config: AwsSdkInstrumentationConfig): void { const currentModelId = response.request.commandInput?.modelId; if (response.data?.body) { - const decodedResponseBody = new TextDecoder().decode(response.data.body); + let decodedResponseBody: string; + + if (typeof response.data.body === 'string') { + // Already converted by AWS SDK middleware by Uint8ArrayBlobAdapter + decodedResponseBody = response.data.body; + } else if (response.data.body instanceof Uint8Array) { + // Raw Uint8Array from AWS + decodedResponseBody = new TextDecoder().decode(response.data.body); + } else if (Buffer.isBuffer(response.data.body)) { + // Node.js Buffer convert with toString('utf8') + decodedResponseBody = response.data.body.toString('utf8'); + } else { + // Handle unexpected types - log and skip processing + diag.debug(`Unexpected body type in Bedrock response: ${typeof response.data.body}`, response.data.body); + return; + } + const responseBody = JSON.parse(decodedResponseBody); if (currentModelId.includes('amazon.titan')) { if (responseBody.inputTextTokenCount !== undefined) { diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/aws/services/bedrock.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/aws/services/bedrock.test.ts index ead43dde..e8690471 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/aws/services/bedrock.test.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/aws/services/bedrock.test.ts @@ -736,5 +736,160 @@ describe('BedrockRuntime', () => { expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_RESPONSE_FINISH_REASONS]).toEqual(['stop']); expect(invokeModelSpan.kind).toBe(SpanKind.CLIENT); }); + + describe('Response Body Type Handling', () => { + it('handles string response body correctly', async () => { + const modelId: string = 'anthropic.claude-3-5-sonnet-20240620-v1:0'; + const mockRequestBody: string = JSON.stringify({ + anthropic_version: 'bedrock-2023-05-31', + max_tokens: 1000, + messages: [{ role: 'user', content: [{ type: 'text', text: 'test' }] }], + }); + + // Mock response body as already converted string + const mockResponseBodyString = JSON.stringify({ + stop_reason: 'end_turn', + usage: { input_tokens: 15, output_tokens: 13 }, + }); + + nock(`https://bedrock-runtime.${region}.amazonaws.com`) + .post(`/model/${encodeURIComponent(modelId)}/invoke`) + .reply(200, mockResponseBodyString); + + await bedrock + .invokeModel({ + modelId: modelId, + body: mockRequestBody, + }) + .catch((err: any) => {}); + + const testSpans: ReadableSpan[] = getTestSpans(); + const invokeModelSpans: ReadableSpan[] = testSpans.filter((s: ReadableSpan) => { + return s.name === 'BedrockRuntime.InvokeModel'; + }); + expect(invokeModelSpans.length).toBe(1); + const invokeModelSpan = invokeModelSpans[0]; + + // Verify attributes are set correctly despite body being a string + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_INPUT_TOKENS]).toBe(15); + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(13); + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_RESPONSE_FINISH_REASONS]).toEqual(['end_turn']); + }); + + it('handles Anthropic Claude response body correctly', async () => { + const modelId: string = 'anthropic.claude-3-5-sonnet-20240620-v1:0'; + const mockRequestBody: string = JSON.stringify({ + anthropic_version: 'bedrock-2023-05-31', + max_tokens: 1000, + messages: [{ role: 'user', content: [{ type: 'text', text: 'test' }] }], + }); + + // Mock response body - use standard object format (AWS SDK will handle type conversion) + const mockResponseBodyObj = { + stop_reason: 'end_turn', + usage: { input_tokens: 20, output_tokens: 15 }, + }; + + nock(`https://bedrock-runtime.${region}.amazonaws.com`) + .post(`/model/${encodeURIComponent(modelId)}/invoke`) + .reply(200, mockResponseBodyObj); + + await bedrock + .invokeModel({ + modelId: modelId, + body: mockRequestBody, + }) + .catch((err: any) => {}); + + const testSpans: ReadableSpan[] = getTestSpans(); + const invokeModelSpans: ReadableSpan[] = testSpans.filter((s: ReadableSpan) => { + return s.name === 'BedrockRuntime.InvokeModel'; + }); + expect(invokeModelSpans.length).toBe(1); + const invokeModelSpan = invokeModelSpans[0]; + + // Verify attributes are set correctly - this tests our type handling logic works + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_INPUT_TOKENS]).toBe(20); + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(15); + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_RESPONSE_FINISH_REASONS]).toEqual(['end_turn']); + }); + + it('handles Buffer response body correctly', async () => { + const modelId: string = 'anthropic.claude-3-5-sonnet-20240620-v1:0'; + const mockRequestBody: string = JSON.stringify({ + anthropic_version: 'bedrock-2023-05-31', + max_tokens: 1000, + messages: [{ role: 'user', content: [{ type: 'text', text: 'test' }] }], + }); + + // Mock response body as Buffer + const mockResponseBodyObj = { + stop_reason: 'max_tokens', + usage: { input_tokens: 25, output_tokens: 18 }, + }; + const mockResponseBodyBuffer = Buffer.from(JSON.stringify(mockResponseBodyObj), 'utf8'); + + nock(`https://bedrock-runtime.${region}.amazonaws.com`) + .post(`/model/${encodeURIComponent(modelId)}/invoke`) + .reply(200, mockResponseBodyBuffer); + + await bedrock + .invokeModel({ + modelId: modelId, + body: mockRequestBody, + }) + .catch((err: any) => {}); + + const testSpans: ReadableSpan[] = getTestSpans(); + const invokeModelSpans: ReadableSpan[] = testSpans.filter((s: ReadableSpan) => { + return s.name === 'BedrockRuntime.InvokeModel'; + }); + expect(invokeModelSpans.length).toBe(1); + const invokeModelSpan = invokeModelSpans[0]; + + // Verify attributes are set correctly when body is Buffer + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_INPUT_TOKENS]).toBe(25); + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(18); + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_RESPONSE_FINISH_REASONS]).toEqual([ + 'max_tokens', + ]); + }); + + it('handles unexpected body type gracefully', async () => { + const modelId: string = 'anthropic.claude-3-5-sonnet-20240620-v1:0'; + const mockRequestBody: string = JSON.stringify({ + anthropic_version: 'bedrock-2023-05-31', + max_tokens: 1000, + messages: [{ role: 'user', content: [{ type: 'text', text: 'test' }] }], + }); + + // Mock response body as unexpected type - using reply function to return a number + nock(`https://bedrock-runtime.${region}.amazonaws.com`) + .post(`/model/${encodeURIComponent(modelId)}/invoke`) + .reply(200, () => 12345 as any); + + await bedrock + .invokeModel({ + modelId: modelId, + body: mockRequestBody, + }) + .catch((err: any) => {}); + + const testSpans: ReadableSpan[] = getTestSpans(); + const invokeModelSpans: ReadableSpan[] = testSpans.filter((s: ReadableSpan) => { + return s.name === 'BedrockRuntime.InvokeModel'; + }); + expect(invokeModelSpans.length).toBe(1); + const invokeModelSpan = invokeModelSpans[0]; + + // Verify that no AI attributes are set when body type is unexpected + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_OUTPUT_TOKENS]).toBeUndefined(); + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_RESPONSE_FINISH_REASONS]).toBeUndefined(); + + // Note: We can't easily test diag.debug() output in unit tests, but the important part + // is that the function returns early and doesn't crash when encountering unexpected types + }); + }); }); }); From 9159941999f334560834238c0350bc65a09269e3 Mon Sep 17 00:00:00 2001 From: "mxiamxia@gmail.com" Date: Wed, 30 Jul 2025 11:55:49 -0700 Subject: [PATCH 2/2] suppress the exception from unspported InvokeModel streaming APIs --- .../src/patches/aws/services/bedrock.ts | 26 ++-- .../test/patches/aws/services/bedrock.test.ts | 111 ++++++------------ 2 files changed, 53 insertions(+), 84 deletions(-) diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/aws/services/bedrock.ts b/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/aws/services/bedrock.ts index 034ae361..4b8c38c0 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/aws/services/bedrock.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/src/patches/aws/services/bedrock.ts @@ -331,20 +331,28 @@ export class BedrockRuntimeServiceExtension implements ServiceExtension { responseHook(response: NormalizedResponse, span: Span, tracer: Tracer, config: AwsSdkInstrumentationConfig): void { const currentModelId = response.request.commandInput?.modelId; if (response.data?.body) { - let decodedResponseBody: string; + // Check if this is a streaming response (SmithyMessageDecoderStream) + // Intend to not using instanceOf to avoid import smithy as new dep for this file + // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/bedrock-runtime/command/InvokeModelWithResponseStreamCommand + if (response.data.body.constructor?.name === 'SmithyMessageDecoderStream') { + // TODO: support InvokeModel Streaming API and Converse APIs later + diag.debug('Streaming API for invoking model is not supported', response.request.commandName); + return; + } - if (typeof response.data.body === 'string') { - // Already converted by AWS SDK middleware by Uint8ArrayBlobAdapter - decodedResponseBody = response.data.body; - } else if (response.data.body instanceof Uint8Array) { + let decodedResponseBody: string; + // For InvokeModel API which should always have reponse body with Uint8Array type + // https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/bedrock-runtime/command/InvokeModelCommand/ + if (response.data.body instanceof Uint8Array) { // Raw Uint8Array from AWS decodedResponseBody = new TextDecoder().decode(response.data.body); - } else if (Buffer.isBuffer(response.data.body)) { - // Node.js Buffer convert with toString('utf8') - decodedResponseBody = response.data.body.toString('utf8'); } else { // Handle unexpected types - log and skip processing - diag.debug(`Unexpected body type in Bedrock response: ${typeof response.data.body}`, response.data.body); + diag.debug( + `Unexpected body type in Bedrock response: ${typeof response.data.body} for commandName ${ + response.request.commandName + }` + ); return; } diff --git a/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/aws/services/bedrock.test.ts b/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/aws/services/bedrock.test.ts index e8690471..62b0d8ef 100644 --- a/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/aws/services/bedrock.test.ts +++ b/aws-distro-opentelemetry-node-autoinstrumentation/test/patches/aws/services/bedrock.test.ts @@ -738,7 +738,7 @@ describe('BedrockRuntime', () => { }); describe('Response Body Type Handling', () => { - it('handles string response body correctly', async () => { + it('handles normal Anthropic Claude response correctly', async () => { const modelId: string = 'anthropic.claude-3-5-sonnet-20240620-v1:0'; const mockRequestBody: string = JSON.stringify({ anthropic_version: 'bedrock-2023-05-31', @@ -746,45 +746,7 @@ describe('BedrockRuntime', () => { messages: [{ role: 'user', content: [{ type: 'text', text: 'test' }] }], }); - // Mock response body as already converted string - const mockResponseBodyString = JSON.stringify({ - stop_reason: 'end_turn', - usage: { input_tokens: 15, output_tokens: 13 }, - }); - - nock(`https://bedrock-runtime.${region}.amazonaws.com`) - .post(`/model/${encodeURIComponent(modelId)}/invoke`) - .reply(200, mockResponseBodyString); - - await bedrock - .invokeModel({ - modelId: modelId, - body: mockRequestBody, - }) - .catch((err: any) => {}); - - const testSpans: ReadableSpan[] = getTestSpans(); - const invokeModelSpans: ReadableSpan[] = testSpans.filter((s: ReadableSpan) => { - return s.name === 'BedrockRuntime.InvokeModel'; - }); - expect(invokeModelSpans.length).toBe(1); - const invokeModelSpan = invokeModelSpans[0]; - - // Verify attributes are set correctly despite body being a string - expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_INPUT_TOKENS]).toBe(15); - expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(13); - expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_RESPONSE_FINISH_REASONS]).toEqual(['end_turn']); - }); - - it('handles Anthropic Claude response body correctly', async () => { - const modelId: string = 'anthropic.claude-3-5-sonnet-20240620-v1:0'; - const mockRequestBody: string = JSON.stringify({ - anthropic_version: 'bedrock-2023-05-31', - max_tokens: 1000, - messages: [{ role: 'user', content: [{ type: 'text', text: 'test' }] }], - }); - - // Mock response body - use standard object format (AWS SDK will handle type conversion) + // Use standard object format - AWS SDK and instrumentation will handle the conversion const mockResponseBodyObj = { stop_reason: 'end_turn', usage: { input_tokens: 20, output_tokens: 15 }, @@ -814,7 +776,7 @@ describe('BedrockRuntime', () => { expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_RESPONSE_FINISH_REASONS]).toEqual(['end_turn']); }); - it('handles Buffer response body correctly', async () => { + it('handles unexpected body type gracefully', async () => { const modelId: string = 'anthropic.claude-3-5-sonnet-20240620-v1:0'; const mockRequestBody: string = JSON.stringify({ anthropic_version: 'bedrock-2023-05-31', @@ -822,23 +784,15 @@ describe('BedrockRuntime', () => { messages: [{ role: 'user', content: [{ type: 'text', text: 'test' }] }], }); - // Mock response body as Buffer - const mockResponseBodyObj = { - stop_reason: 'max_tokens', - usage: { input_tokens: 25, output_tokens: 18 }, - }; - const mockResponseBodyBuffer = Buffer.from(JSON.stringify(mockResponseBodyObj), 'utf8'); - + // Mock response body as unexpected type - using reply function to return a number nock(`https://bedrock-runtime.${region}.amazonaws.com`) .post(`/model/${encodeURIComponent(modelId)}/invoke`) - .reply(200, mockResponseBodyBuffer); + .reply(200, () => 12345 as any); - await bedrock - .invokeModel({ - modelId: modelId, - body: mockRequestBody, - }) - .catch((err: any) => {}); + await bedrock.invokeModel({ + modelId: modelId, + body: mockRequestBody, + }); const testSpans: ReadableSpan[] = getTestSpans(); const invokeModelSpans: ReadableSpan[] = testSpans.filter((s: ReadableSpan) => { @@ -847,15 +801,17 @@ describe('BedrockRuntime', () => { expect(invokeModelSpans.length).toBe(1); const invokeModelSpan = invokeModelSpans[0]; - // Verify attributes are set correctly when body is Buffer - expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_INPUT_TOKENS]).toBe(25); - expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_OUTPUT_TOKENS]).toBe(18); - expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_RESPONSE_FINISH_REASONS]).toEqual([ - 'max_tokens', - ]); + // Verify that no AI attributes are set when body type is unexpected + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_OUTPUT_TOKENS]).toBeUndefined(); + expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_RESPONSE_FINISH_REASONS]).toBeUndefined(); + + // Note: We can't easily test diag.debug() output in unit tests, but the important part + // is that the function returns early and doesn't crash when encountering unexpected types + // Debug message will be: "Unexpected body type in Bedrock response: number for commandName InvokeModelCommand" }); - it('handles unexpected body type gracefully', async () => { + it('handles streaming response (SmithyMessageDecoderStream) gracefully', async () => { const modelId: string = 'anthropic.claude-3-5-sonnet-20240620-v1:0'; const mockRequestBody: string = JSON.stringify({ anthropic_version: 'bedrock-2023-05-31', @@ -863,32 +819,37 @@ describe('BedrockRuntime', () => { messages: [{ role: 'user', content: [{ type: 'text', text: 'test' }] }], }); - // Mock response body as unexpected type - using reply function to return a number + // Mock response body as streaming object (constructor name matching) + const mockStreamingBody = { + constructor: { name: 'SmithyMessageDecoderStream' }, + [Symbol.asyncIterator]: function* () { + yield { chunk: { bytes: new TextEncoder().encode('{"type":"chunk"}') } }; + }, + }; + nock(`https://bedrock-runtime.${region}.amazonaws.com`) - .post(`/model/${encodeURIComponent(modelId)}/invoke`) - .reply(200, () => 12345 as any); + .post(`/model/${encodeURIComponent(modelId)}/invoke-with-response-stream`) + .reply(200, mockStreamingBody); - await bedrock - .invokeModel({ - modelId: modelId, - body: mockRequestBody, - }) - .catch((err: any) => {}); + await bedrock.invokeModelWithResponseStream({ + modelId: modelId, + body: mockRequestBody, + }); const testSpans: ReadableSpan[] = getTestSpans(); const invokeModelSpans: ReadableSpan[] = testSpans.filter((s: ReadableSpan) => { - return s.name === 'BedrockRuntime.InvokeModel'; + return s.name === 'BedrockRuntime.InvokeModelWithResponseStream'; }); expect(invokeModelSpans.length).toBe(1); const invokeModelSpan = invokeModelSpans[0]; - // Verify that no AI attributes are set when body type is unexpected + // Verify that no AI attributes are set when body is streaming (metrics not available in initial response) expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined(); expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_USAGE_OUTPUT_TOKENS]).toBeUndefined(); expect(invokeModelSpan.attributes[AwsSpanProcessingUtil.GEN_AI_RESPONSE_FINISH_REASONS]).toBeUndefined(); - // Note: We can't easily test diag.debug() output in unit tests, but the important part - // is that the function returns early and doesn't crash when encountering unexpected types + // Streaming responses should be skipped gracefully without crashing + // TODO: support InvokeModel Streaming API and Converse APIs later }); }); });