Skip to content

Commit 9eb8562

Browse files
roomoteroomote
committed
fix: Handle AWS Bedrock 20-image conversation limit
- Add image limiting functionality to prevent "too many images" errors - Automatically remove oldest images when conversation exceeds 20 images - Preserve most recent 20 images for Browser tool continuity - Add comprehensive error handling for image limit errors - Include extensive test coverage for image limiting logic Fixes #6348
1 parent cc0f9e3 commit 9eb8562

File tree

4 files changed

+745
-1
lines changed

4 files changed

+745
-1
lines changed

‎src/api/providers/__tests__/bedrock-image-limiting.spec.ts‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
// npx vitest run src/api/providers/__tests__/bedrock-image-limiting.spec.ts
2+
3+
import { describe, it, expect, vi, beforeEach } from "vitest"
4+
import { Anthropic } from "@anthropic-ai/sdk"
5+
import { AwsBedrockHandler } from "../bedrock"
6+
import { ApiHandlerOptions } from "../../../shared/api"
7+
import { AWS_BEDROCK_MAX_IMAGES_PER_CONVERSATION } from "../../transform/image-limiting"
8+
9+
// Valid base64 encoded 1x1 pixel PNG image for testing
10+
const VALID_BASE64_IMAGE =
11+
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChAGAWA0+kgAAAABJRU5ErkJggg=="
12+
13+
// Mock AWS SDK
14+
vi.mock("@aws-sdk/client-bedrock-runtime", () => {
15+
return {
16+
BedrockRuntimeClient: vi.fn().mockImplementation(() => ({
17+
send: vi.fn(),
18+
})),
19+
ConverseStreamCommand: vi.fn(),
20+
ConverseCommand: vi.fn(),
21+
}
22+
})
23+
24+
// Mock credential providers
25+
vi.mock("@aws-sdk/credential-providers", () => ({
26+
fromIni: vi.fn().mockReturnValue({
27+
accessKeyId: "test-access-key",
28+
secretAccessKey: "test-secret-key",
29+
}),
30+
}))
31+
32+
describe("AwsBedrockHandler - Image Limiting", () => {
33+
let handler: AwsBedrockHandler
34+
let mockOptions: ApiHandlerOptions
35+
36+
beforeEach(() => {
37+
mockOptions = {
38+
apiModelId: "anthropic.claude-3-5-sonnet-20241022-v2:0",
39+
awsAccessKey: "test-access-key",
40+
awsSecretKey: "test-secret-key",
41+
awsRegion: "us-east-1",
42+
}
43+
handler = new AwsBedrockHandler(mockOptions)
44+
})
45+
46+
describe("convertToBedrockConverseMessages with image limiting", () => {
47+
it("should not modify messages when under image limit", () => {
48+
const messages: Anthropic.Messages.MessageParam[] = [
49+
{
50+
role: "user",
51+
content: [
52+
{ type: "text", text: "Look at this image:" },
53+
{
54+
type: "image",
55+
source: { type: "base64", media_type: "image/png", data: VALID_BASE64_IMAGE },
56+
},
57+
],
58+
},
59+
{
60+
role: "user",
61+
content: [
62+
{ type: "text", text: "And this one:" },
63+
{
64+
type: "image",
65+
source: { type: "base64", media_type: "image/jpeg", data: VALID_BASE64_IMAGE },
66+
},
67+
],
68+
},
69+
]
70+
71+
// Access the private method for testing
72+
const result = (handler as any).convertToBedrockConverseMessages(messages, "System prompt")
73+
74+
// Should have 2 messages with images intact
75+
expect(result.messages).toHaveLength(2)
76+
77+
// Check that images are preserved
78+
const firstMessage = result.messages[0]
79+
const secondMessage = result.messages[1]
80+
81+
expect(firstMessage.content).toHaveLength(2)
82+
expect(firstMessage.content[1]).toHaveProperty("image")
83+
84+
expect(secondMessage.content).toHaveLength(2)
85+
expect(secondMessage.content[1]).toHaveProperty("image")
86+
})
87+
88+
it("should limit images when over AWS Bedrock limit", () => {
89+
const messages: Anthropic.Messages.MessageParam[] = []
90+
91+
// Create 25 messages with 1 image each (5 over the limit)
92+
for (let i = 0; i < 25; i++) {
93+
messages.push({
94+
role: "user",
95+
content: [
96+
{ type: "text", text: `Browser screenshot ${i + 1}` },
97+
{
98+
type: "image",
99+
source: { type: "base64", media_type: "image/png", data: VALID_BASE64_IMAGE },
100+
},
101+
],
102+
})
103+
}
104+
105+
// Access the private method for testing
106+
const result = (handler as any).convertToBedrockConverseMessages(messages, "System prompt")
107+
108+
// Should have 25 messages
109+
expect(result.messages).toHaveLength(25)
110+
111+
// Count actual images in the result
112+
let imageCount = 0
113+
let textPlaceholderCount = 0
114+
115+
for (const message of result.messages) {
116+
for (const block of message.content) {
117+
if (block.image) {
118+
imageCount++
119+
} else if (block.text && block.text.includes("[Image removed due to conversation limit")) {
120+
textPlaceholderCount++
121+
}
122+
}
123+
}
124+
125+
// Should have exactly 20 images and 5 text placeholders
126+
expect(imageCount).toBe(AWS_BEDROCK_MAX_IMAGES_PER_CONVERSATION)
127+
expect(textPlaceholderCount).toBe(5)
128+
})
129+
130+
it("should preserve text content when limiting images", () => {
131+
const messages: Anthropic.Messages.MessageParam[] = []
132+
133+
// Create 22 messages with mixed content (2 over the limit)
134+
for (let i = 0; i < 22; i++) {
135+
messages.push({
136+
role: "user",
137+
content: [
138+
{ type: "text", text: `Important context ${i + 1}` },
139+
{
140+
type: "image",
141+
source: { type: "base64", media_type: "image/png", data: VALID_BASE64_IMAGE },
142+
},
143+
{ type: "text", text: `Additional info ${i + 1}` },
144+
],
145+
})
146+
}
147+
148+
// Access the private method for testing
149+
const result = (handler as any).convertToBedrockConverseMessages(messages, "System prompt")
150+
151+
// All text content should be preserved
152+
for (let i = 0; i < 22; i++) {
153+
const message = result.messages[i]
154+
expect(message.content[0].text).toBe(`Important context ${i + 1}`)
155+
expect(message.content[2].text).toBe(`Additional info ${i + 1}`)
156+
}
157+
158+
// First 2 messages should have image placeholders
159+
expect(result.messages[0].content[1].text).toBe(
160+
"[Image removed due to conversation limit - Browser tool screenshot]",
161+
)
162+
expect(result.messages[1].content[1].text).toBe(
163+
"[Image removed due to conversation limit - Browser tool screenshot]",
164+
)
165+
166+
// Remaining messages should have images
167+
for (let i = 2; i < 22; i++) {
168+
expect(result.messages[i].content[1]).toHaveProperty("image")
169+
}
170+
})
171+
172+
it("should handle exactly 20 images without modification", () => {
173+
const messages: Anthropic.Messages.MessageParam[] = []
174+
175+
// Create exactly 20 messages with 1 image each
176+
for (let i = 0; i < AWS_BEDROCK_MAX_IMAGES_PER_CONVERSATION; i++) {
177+
messages.push({
178+
role: "user",
179+
content: [
180+
{ type: "text", text: `Message ${i + 1}` },
181+
{
182+
type: "image",
183+
source: { type: "base64", media_type: "image/png", data: VALID_BASE64_IMAGE },
184+
},
185+
],
186+
})
187+
}
188+
189+
// Access the private method for testing
190+
const result = (handler as any).convertToBedrockConverseMessages(messages, "System prompt")
191+
192+
// Should have 20 messages with all images intact
193+
expect(result.messages).toHaveLength(20)
194+
195+
let imageCount = 0
196+
for (const message of result.messages) {
197+
for (const block of message.content) {
198+
if (block.image) {
199+
imageCount++
200+
}
201+
}
202+
}
203+
204+
expect(imageCount).toBe(AWS_BEDROCK_MAX_IMAGES_PER_CONVERSATION)
205+
})
206+
207+
it("should handle mixed message types correctly", () => {
208+
const messages: Anthropic.Messages.MessageParam[] = [
209+
{ role: "user", content: "Text only message" },
210+
{
211+
role: "assistant",
212+
content: [{ type: "text", text: "I understand." }],
213+
},
214+
]
215+
216+
// Add 21 image messages to exceed the limit
217+
for (let i = 0; i < 21; i++) {
218+
messages.push({
219+
role: "user",
220+
content: [
221+
{
222+
type: "image",
223+
source: { type: "base64", media_type: "image/png", data: VALID_BASE64_IMAGE },
224+
},
225+
],
226+
})
227+
}
228+
229+
// Access the private method for testing
230+
const result = (handler as any).convertToBedrockConverseMessages(messages, "System prompt")
231+
232+
// Should have 23 messages total
233+
expect(result.messages).toHaveLength(23)
234+
235+
// First two messages should be unchanged (no images)
236+
expect(result.messages[0].content[0].text).toBe("Text only message")
237+
expect(result.messages[1].content[0].text).toBe("I understand.")
238+
239+
// Count images in the result
240+
let imageCount = 0
241+
let placeholderCount = 0
242+
243+
for (const message of result.messages) {
244+
for (const block of message.content) {
245+
if (block.image) {
246+
imageCount++
247+
} else if (block.text && block.text.includes("[Image removed due to conversation limit")) {
248+
placeholderCount++
249+
}
250+
}
251+
}
252+
253+
// Should have exactly 20 images and 1 placeholder
254+
expect(imageCount).toBe(20)
255+
expect(placeholderCount).toBe(1)
256+
})
257+
258+
it("should work with system message", () => {
259+
const messages: Anthropic.Messages.MessageParam[] = []
260+
261+
// Create 22 messages with images (2 over limit)
262+
for (let i = 0; i < 22; i++) {
263+
messages.push({
264+
role: "user",
265+
content: [
266+
{
267+
type: "image",
268+
source: { type: "base64", media_type: "image/png", data: VALID_BASE64_IMAGE },
269+
},
270+
],
271+
})
272+
}
273+
274+
const systemMessage = "You are a helpful assistant that can analyze images."
275+
276+
// Access the private method for testing
277+
const result = (handler as any).convertToBedrockConverseMessages(messages, systemMessage)
278+
279+
// System message should be preserved
280+
expect(result.system).toHaveLength(1)
281+
expect(result.system[0].text).toBe(systemMessage)
282+
283+
// Should have 22 messages with limited images
284+
expect(result.messages).toHaveLength(22)
285+
286+
// Count images
287+
let imageCount = 0
288+
for (const message of result.messages) {
289+
for (const block of message.content) {
290+
if (block.image) {
291+
imageCount++
292+
}
293+
}
294+
}
295+
296+
expect(imageCount).toBe(AWS_BEDROCK_MAX_IMAGES_PER_CONVERSATION)
297+
})
298+
})
299+
})

‎src/api/providers/bedrock.ts‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import { MultiPointStrategy } from "../transform/cache-strategy/multi-point-stra
3030
import { ModelInfo as CacheModelInfo } from "../transform/cache-strategy/types"
3131
import { convertToBedrockConverseMessages as sharedConverter } from "../transform/bedrock-converse-format"
3232
import { getModelParams } from "../transform/model-params"
33+
import { limitImagesInConversation, hasExceededImageLimit } from "../transform/image-limiting"
3334
import { shouldUseReasoningBudget } from "../../shared/api"
3435
import type { SingleCompletionHandler, ApiHandlerCreateMessageMetadata } from "../index"
3536

@@ -706,8 +707,20 @@ export class AwsBedrockHandler extends BaseProvider implements SingleCompletionH
706707
modelInfo?: any,
707708
conversationId?: string, // Optional conversation ID to track cache points across messages
708709
): { system: SystemContentBlock[]; messages: Message[] } {
710+
// Apply image limiting for AWS Bedrock's 20-image conversation limit
711+
const limitedMessages = limitImagesInConversation(anthropicMessages as Anthropic.Messages.MessageParam[])
712+
713+
// Log if images were removed due to the limit
714+
if (hasExceededImageLimit(anthropicMessages as Anthropic.Messages.MessageParam[])) {
715+
logger.info("Applied image limiting for AWS Bedrock conversation", {
716+
ctx: "bedrock",
717+
originalImageCount: limitedMessages.length,
718+
action: "removed_oldest_images_to_stay_within_limit",
719+
})
720+
}
721+
709722
// First convert messages using shared converter for proper image handling
710-
const convertedMessages = sharedConverter(anthropicMessages as Anthropic.Messages.MessageParam[])
723+
const convertedMessages = sharedConverter(limitedMessages)
711724

712725
// If prompt caching is disabled, return the converted messages directly
713726
if (!usePromptCache) {
@@ -1121,6 +1134,21 @@ Suggestions:
11211134
`,
11221135
logLevel: "error",
11231136
},
1137+
TOO_MANY_IMAGES: {
1138+
patterns: ["too many images", "too many images and documents", "images and documents:", "> 20"],
1139+
messageTemplate: `AWS Bedrock "too many images" error detected.
1140+
1141+
This error occurs when the conversation contains more than 20 images total. The application has automatically applied image limiting to prevent this error in future requests.
1142+
1143+
What happened:
1144+
- AWS Bedrock has a hard limit of 20 images per conversation
1145+
- Your conversation exceeded this limit (likely from Browser tool screenshots)
1146+
- The oldest images have been automatically replaced with text placeholders
1147+
- The conversation can now continue normally
1148+
1149+
No action needed - the issue has been resolved automatically.`,
1150+
logLevel: "info",
1151+
},
11241152
SERVICE_QUOTA_EXCEEDED: {
11251153
patterns: ["service quota exceeded", "service quota", "quota exceeded for model"],
11261154
messageTemplate: `Service quota exceeded. This error indicates you've reached AWS service limits.
@@ -1234,6 +1262,7 @@ Please check:
12341262
const errorTypeOrder = [
12351263
"SERVICE_QUOTA_EXCEEDED", // Most specific - check before THROTTLING
12361264
"MODEL_NOT_READY",
1265+
"TOO_MANY_IMAGES", // Check before TOO_MANY_TOKENS for specificity
12371266
"TOO_MANY_TOKENS",
12381267
"INTERNAL_SERVER_ERROR",
12391268
"ON_DEMAND_NOT_SUPPORTED",

0 commit comments

Comments
 (0)