Skip to content

Commit 7c1185e

Browse files
canrobins13mrubens
canrobins13
and
mrubens
authored
[Condense] Condense messages with an LLM rather than truncating (#3582)
Co-authored-by: Matt Rubens <[email protected]>
1 parent 4ee23b5 commit 7c1185e

File tree

33 files changed

+863
-60
lines changed

33 files changed

+863
-60
lines changed

‎.changeset/large-bags-send.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"roo-cline": patch
3+
---
4+
5+
Adds experimental feature to intelligently condense the task context

‎evals/packages/types/src/roo-code.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,7 @@ export type CommandExecutionStatus = z.infer<typeof commandExecutionStatusSchema
297297
* ExperimentId
298298
*/
299299

300-
export const experimentIds = ["powerSteering"] as const
300+
export const experimentIds = ["autoCondenseContext", "powerSteering"] as const
301301

302302
export const experimentIdsSchema = z.enum(experimentIds)
303303

@@ -308,6 +308,7 @@ export type ExperimentId = z.infer<typeof experimentIdsSchema>
308308
*/
309309

310310
const experimentsSchema = z.object({
311+
autoCondenseContext: z.boolean(),
311312
powerSteering: z.boolean(),
312313
})
313314

‎src/api/transform/__tests__/image-cleaning.test.ts‎

Lines changed: 336 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,336 @@
1+
import { ApiHandler } from "../.."
2+
import { ApiMessage } from "../../../core/task-persistence/apiMessages"
3+
import { maybeRemoveImageBlocks } from "../image-cleaning"
4+
import { ModelInfo } from "../../../shared/api"
5+
6+
describe("maybeRemoveImageBlocks", () => {
7+
// Mock ApiHandler factory function
8+
const createMockApiHandler = (supportsImages: boolean): ApiHandler => {
9+
return {
10+
getModel: jest.fn().mockReturnValue({
11+
id: "test-model",
12+
info: {
13+
supportsImages,
14+
} as ModelInfo,
15+
}),
16+
createMessage: jest.fn(),
17+
countTokens: jest.fn(),
18+
}
19+
}
20+
21+
it("should handle empty messages array", () => {
22+
const apiHandler = createMockApiHandler(true)
23+
const messages: ApiMessage[] = []
24+
25+
const result = maybeRemoveImageBlocks(messages, apiHandler)
26+
27+
expect(result).toEqual([])
28+
// No need to check if getModel was called since there are no messages to process
29+
})
30+
31+
it("should not modify messages with no image blocks", () => {
32+
const apiHandler = createMockApiHandler(true)
33+
const messages: ApiMessage[] = [
34+
{
35+
role: "user",
36+
content: "Hello, world!",
37+
},
38+
{
39+
role: "assistant",
40+
content: "Hi there!",
41+
},
42+
]
43+
44+
const result = maybeRemoveImageBlocks(messages, apiHandler)
45+
46+
expect(result).toEqual(messages)
47+
// getModel is only called when content is an array, which is not the case here
48+
})
49+
50+
it("should not modify messages with array content but no image blocks", () => {
51+
const apiHandler = createMockApiHandler(true)
52+
const messages: ApiMessage[] = [
53+
{
54+
role: "user",
55+
content: [
56+
{
57+
type: "text",
58+
text: "Hello, world!",
59+
},
60+
{
61+
type: "text",
62+
text: "How are you?",
63+
},
64+
],
65+
},
66+
]
67+
68+
const result = maybeRemoveImageBlocks(messages, apiHandler)
69+
70+
expect(result).toEqual(messages)
71+
expect(apiHandler.getModel).toHaveBeenCalled()
72+
})
73+
74+
it("should not modify image blocks when API handler supports images", () => {
75+
const apiHandler = createMockApiHandler(true)
76+
const messages: ApiMessage[] = [
77+
{
78+
role: "user",
79+
content: [
80+
{
81+
type: "text",
82+
text: "Check out this image:",
83+
},
84+
{
85+
type: "image",
86+
source: {
87+
type: "base64",
88+
media_type: "image/jpeg",
89+
data: "base64-encoded-image-data",
90+
},
91+
},
92+
],
93+
},
94+
]
95+
96+
const result = maybeRemoveImageBlocks(messages, apiHandler)
97+
98+
// Should not modify the messages since the API handler supports images
99+
expect(result).toEqual(messages)
100+
expect(apiHandler.getModel).toHaveBeenCalled()
101+
})
102+
103+
it("should convert image blocks to text descriptions when API handler doesn't support images", () => {
104+
const apiHandler = createMockApiHandler(false)
105+
const messages: ApiMessage[] = [
106+
{
107+
role: "user",
108+
content: [
109+
{
110+
type: "text",
111+
text: "Check out this image:",
112+
},
113+
{
114+
type: "image",
115+
source: {
116+
type: "base64",
117+
media_type: "image/jpeg",
118+
data: "base64-encoded-image-data",
119+
},
120+
},
121+
],
122+
},
123+
]
124+
125+
const result = maybeRemoveImageBlocks(messages, apiHandler)
126+
127+
// Should convert image blocks to text descriptions
128+
expect(result).toEqual([
129+
{
130+
role: "user",
131+
content: [
132+
{
133+
type: "text",
134+
text: "Check out this image:",
135+
},
136+
{
137+
type: "text",
138+
text: "[Referenced image in conversation]",
139+
},
140+
],
141+
},
142+
])
143+
expect(apiHandler.getModel).toHaveBeenCalled()
144+
})
145+
146+
it("should handle mixed content messages with multiple text and image blocks", () => {
147+
const apiHandler = createMockApiHandler(false)
148+
const messages: ApiMessage[] = [
149+
{
150+
role: "user",
151+
content: [
152+
{
153+
type: "text",
154+
text: "Here are some images:",
155+
},
156+
{
157+
type: "image",
158+
source: {
159+
type: "base64",
160+
media_type: "image/jpeg",
161+
data: "image-data-1",
162+
},
163+
},
164+
{
165+
type: "text",
166+
text: "And another one:",
167+
},
168+
{
169+
type: "image",
170+
source: {
171+
type: "base64",
172+
media_type: "image/png",
173+
data: "image-data-2",
174+
},
175+
},
176+
],
177+
},
178+
]
179+
180+
const result = maybeRemoveImageBlocks(messages, apiHandler)
181+
182+
// Should convert all image blocks to text descriptions
183+
expect(result).toEqual([
184+
{
185+
role: "user",
186+
content: [
187+
{
188+
type: "text",
189+
text: "Here are some images:",
190+
},
191+
{
192+
type: "text",
193+
text: "[Referenced image in conversation]",
194+
},
195+
{
196+
type: "text",
197+
text: "And another one:",
198+
},
199+
{
200+
type: "text",
201+
text: "[Referenced image in conversation]",
202+
},
203+
],
204+
},
205+
])
206+
expect(apiHandler.getModel).toHaveBeenCalled()
207+
})
208+
209+
it("should handle multiple messages with image blocks", () => {
210+
const apiHandler = createMockApiHandler(false)
211+
const messages: ApiMessage[] = [
212+
{
213+
role: "user",
214+
content: [
215+
{
216+
type: "text",
217+
text: "Here's an image:",
218+
},
219+
{
220+
type: "image",
221+
source: {
222+
type: "base64",
223+
media_type: "image/jpeg",
224+
data: "image-data-1",
225+
},
226+
},
227+
],
228+
},
229+
{
230+
role: "assistant",
231+
content: "I see the image!",
232+
},
233+
{
234+
role: "user",
235+
content: [
236+
{
237+
type: "text",
238+
text: "Here's another image:",
239+
},
240+
{
241+
type: "image",
242+
source: {
243+
type: "base64",
244+
media_type: "image/png",
245+
data: "image-data-2",
246+
},
247+
},
248+
],
249+
},
250+
]
251+
252+
const result = maybeRemoveImageBlocks(messages, apiHandler)
253+
254+
// Should convert all image blocks to text descriptions
255+
expect(result).toEqual([
256+
{
257+
role: "user",
258+
content: [
259+
{
260+
type: "text",
261+
text: "Here's an image:",
262+
},
263+
{
264+
type: "text",
265+
text: "[Referenced image in conversation]",
266+
},
267+
],
268+
},
269+
{
270+
role: "assistant",
271+
content: "I see the image!",
272+
},
273+
{
274+
role: "user",
275+
content: [
276+
{
277+
type: "text",
278+
text: "Here's another image:",
279+
},
280+
{
281+
type: "text",
282+
text: "[Referenced image in conversation]",
283+
},
284+
],
285+
},
286+
])
287+
expect(apiHandler.getModel).toHaveBeenCalled()
288+
})
289+
290+
it("should preserve additional message properties", () => {
291+
const apiHandler = createMockApiHandler(false)
292+
const messages: ApiMessage[] = [
293+
{
294+
role: "user",
295+
content: [
296+
{
297+
type: "text",
298+
text: "Here's an image:",
299+
},
300+
{
301+
type: "image",
302+
source: {
303+
type: "base64",
304+
media_type: "image/jpeg",
305+
data: "image-data",
306+
},
307+
},
308+
],
309+
ts: 1620000000000,
310+
isSummary: true,
311+
},
312+
]
313+
314+
const result = maybeRemoveImageBlocks(messages, apiHandler)
315+
316+
// Should convert image blocks to text descriptions while preserving additional properties
317+
expect(result).toEqual([
318+
{
319+
role: "user",
320+
content: [
321+
{
322+
type: "text",
323+
text: "Here's an image:",
324+
},
325+
{
326+
type: "text",
327+
text: "[Referenced image in conversation]",
328+
},
329+
],
330+
ts: 1620000000000,
331+
isSummary: true,
332+
},
333+
])
334+
expect(apiHandler.getModel).toHaveBeenCalled()
335+
})
336+
})

‎src/api/transform/image-cleaning.ts‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import { ApiHandler } from ".."
2+
import { ApiMessage } from "../../core/task-persistence/apiMessages"
3+
4+
/* Removes image blocks from messages if they are not supported by the Api Handler */
5+
export function maybeRemoveImageBlocks(messages: ApiMessage[], apiHandler: ApiHandler): ApiMessage[] {
6+
return messages.map((message) => {
7+
// Handle array content (could contain image blocks).
8+
let { content } = message
9+
if (Array.isArray(content)) {
10+
if (!apiHandler.getModel().info.supportsImages) {
11+
// Convert image blocks to text descriptions.
12+
content = content.map((block) => {
13+
if (block.type === "image") {
14+
// Convert image blocks to text descriptions.
15+
// Note: We can't access the actual image content/url due to API limitations,
16+
// but we can indicate that an image was present in the conversation.
17+
return {
18+
type: "text",
19+
text: "[Referenced image in conversation]",
20+
}
21+
}
22+
return block
23+
})
24+
}
25+
}
26+
return { ...message, content }
27+
})
28+
}

0 commit comments

Comments
 (0)