Skip to content

Commit f7d371f

Browse files
roomoteroomote
committed
feat: add image and multimedia support to Cerebras integration
- Updated all Cerebras models to support image inputs (supportsImages: true) - Modified convertToCerebrasMessages to handle multimodal content including images - Enhanced message processing to preserve image data in base64 format - Updated token usage calculation to account for image content - Added comprehensive tests for image handling in Cerebras provider Closes #7670
1 parent b50104c commit f7d371f

File tree

3 files changed

+235
-55
lines changed

3 files changed

+235
-55
lines changed

‎packages/types/src/providers/cerebras.ts‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,68 +9,68 @@ export const cerebrasModels = {
99
"qwen-3-coder-480b-free": {
1010
maxTokens: 40000,
1111
contextWindow: 64000,
12-
supportsImages: false,
12+
supportsImages: true,
1313
supportsPromptCache: false,
1414
inputPrice: 0,
1515
outputPrice: 0,
1616
description:
17-
"SOTA coding model with ~2000 tokens/s ($0 free tier)\n\n• Use this if you don't have a Cerebras subscription\n• 64K context window\n• Rate limits: 150K TPM, 1M TPH/TPD, 10 RPM, 100 RPH/RPD\n\nUpgrade for higher limits: [https://cloud.cerebras.ai/?utm=roocode](https://cloud.cerebras.ai/?utm=roocode)",
17+
"SOTA coding model with ~2000 tokens/s ($0 free tier)\n\n• Use this if you don't have a Cerebras subscription\n• 64K context window\n• Supports image inputs for multimodal tasks\n• Rate limits: 150K TPM, 1M TPH/TPD, 10 RPM, 100 RPH/RPD\n\nUpgrade for higher limits: [https://cloud.cerebras.ai/?utm=roocode](https://cloud.cerebras.ai/?utm=roocode)",
1818
},
1919
"qwen-3-coder-480b": {
2020
maxTokens: 40000,
2121
contextWindow: 128000,
22-
supportsImages: false,
22+
supportsImages: true,
2323
supportsPromptCache: false,
2424
inputPrice: 0,
2525
outputPrice: 0,
2626
description:
27-
"SOTA coding model with ~2000 tokens/s ($50/$250 paid tiers)\n\n• Use this if you have a Cerebras subscription\n• 131K context window with higher rate limits",
27+
"SOTA coding model with ~2000 tokens/s ($50/$250 paid tiers)\n\n• Use this if you have a Cerebras subscription\n• 131K context window with higher rate limits\n• Supports image inputs for multimodal tasks",
2828
},
2929
"qwen-3-235b-a22b-instruct-2507": {
3030
maxTokens: 64000,
3131
contextWindow: 64000,
32-
supportsImages: false,
32+
supportsImages: true,
3333
supportsPromptCache: false,
3434
inputPrice: 0,
3535
outputPrice: 0,
36-
description: "Intelligent model with ~1400 tokens/s",
36+
description: "Intelligent model with ~1400 tokens/s\n• Supports image inputs for multimodal tasks",
3737
},
3838
"llama-3.3-70b": {
3939
maxTokens: 64000,
4040
contextWindow: 64000,
41-
supportsImages: false,
41+
supportsImages: true,
4242
supportsPromptCache: false,
4343
inputPrice: 0,
4444
outputPrice: 0,
45-
description: "Powerful model with ~2600 tokens/s",
45+
description: "Powerful model with ~2600 tokens/s\n• Supports image inputs for multimodal tasks",
4646
},
4747
"qwen-3-32b": {
4848
maxTokens: 64000,
4949
contextWindow: 64000,
50-
supportsImages: false,
50+
supportsImages: true,
5151
supportsPromptCache: false,
5252
inputPrice: 0,
5353
outputPrice: 0,
54-
description: "SOTA coding performance with ~2500 tokens/s",
54+
description: "SOTA coding performance with ~2500 tokens/s\n• Supports image inputs for multimodal tasks",
5555
},
5656
"qwen-3-235b-a22b-thinking-2507": {
5757
maxTokens: 40000,
5858
contextWindow: 65000,
59-
supportsImages: false,
59+
supportsImages: true,
6060
supportsPromptCache: false,
6161
inputPrice: 0,
6262
outputPrice: 0,
63-
description: "SOTA performance with ~1500 tokens/s",
63+
description: "SOTA performance with ~1500 tokens/s\n• Supports image inputs for multimodal tasks",
6464
supportsReasoningEffort: true,
6565
},
6666
"gpt-oss-120b": {
6767
maxTokens: 8000,
6868
contextWindow: 64000,
69-
supportsImages: false,
69+
supportsImages: true,
7070
supportsPromptCache: false,
7171
inputPrice: 0,
7272
outputPrice: 0,
7373
description:
74-
"OpenAI GPT OSS model with ~2800 tokens/s\n\n• 64K context window\n• Excels at efficient reasoning across science, math, and coding",
74+
"OpenAI GPT OSS model with ~2800 tokens/s\n\n• 64K context window\n• Supports image inputs for multimodal tasks\n• Excels at efficient reasoning across science, math, and coding",
7575
},
7676
} as const satisfies Record<string, ModelInfo>

‎src/api/providers/__tests__/cerebras.spec.ts‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,133 @@ describe("CerebrasHandler", () => {
7777
})
7878
})
7979

80+
describe("createMessage with images", () => {
81+
it("should handle messages with image content", async () => {
82+
const mockFetch = vi.fn().mockResolvedValue({
83+
ok: true,
84+
body: {
85+
getReader: () => ({
86+
read: vi
87+
.fn()
88+
.mockResolvedValueOnce({
89+
done: false,
90+
value: new TextEncoder().encode(
91+
'data: {"choices":[{"delta":{"content":"Image analysis:"}}]}\n',
92+
),
93+
})
94+
.mockResolvedValueOnce({
95+
done: false,
96+
value: new TextEncoder().encode(
97+
'data: {"choices":[{"delta":{"content":" I can see the image"}}]}\n',
98+
),
99+
})
100+
.mockResolvedValueOnce({
101+
done: false,
102+
value: new TextEncoder().encode(
103+
'data: {"usage":{"prompt_tokens":100,"completion_tokens":10}}\n',
104+
),
105+
})
106+
.mockResolvedValueOnce({ done: true }),
107+
releaseLock: vi.fn(),
108+
}),
109+
},
110+
})
111+
global.fetch = mockFetch
112+
113+
const messages = [
114+
{
115+
role: "user" as const,
116+
content: [
117+
{ type: "text" as const, text: "What's in this image?" },
118+
{
119+
type: "image" as const,
120+
source: {
121+
type: "base64" as const,
122+
media_type: "image/png" as const,
123+
data: "base64encodedimagedata",
124+
},
125+
},
126+
],
127+
},
128+
]
129+
130+
const stream = handler.createMessage("System prompt", messages)
131+
const chunks = []
132+
for await (const chunk of stream) {
133+
chunks.push(chunk)
134+
}
135+
136+
// Verify the request was made with image content
137+
expect(mockFetch).toHaveBeenCalledWith(
138+
"https://api.cerebras.ai/v1/chat/completions",
139+
expect.objectContaining({
140+
body: expect.stringContaining("image_url"),
141+
}),
142+
)
143+
144+
// Verify we got the expected response chunks
145+
expect(chunks).toContainEqual({ type: "text", text: "Image analysis:" })
146+
expect(chunks).toContainEqual({ type: "text", text: " I can see the image" })
147+
expect(chunks).toContainEqual({ type: "usage", inputTokens: 100, outputTokens: 10 })
148+
})
149+
150+
it("should handle mixed text and image content", async () => {
151+
const mockFetch = vi.fn().mockResolvedValue({
152+
ok: true,
153+
body: {
154+
getReader: () => ({
155+
read: vi
156+
.fn()
157+
.mockResolvedValueOnce({
158+
done: false,
159+
value: new TextEncoder().encode(
160+
'data: {"choices":[{"delta":{"content":"Response"}}]}\n',
161+
),
162+
})
163+
.mockResolvedValueOnce({ done: true }),
164+
releaseLock: vi.fn(),
165+
}),
166+
},
167+
})
168+
global.fetch = mockFetch
169+
170+
const messages = [
171+
{
172+
role: "user" as const,
173+
content: [
174+
{ type: "text" as const, text: "Analyze this:" },
175+
{
176+
type: "image" as const,
177+
source: {
178+
type: "base64" as const,
179+
media_type: "image/jpeg" as const,
180+
data: "base64data",
181+
},
182+
},
183+
{ type: "text" as const, text: "What do you see?" },
184+
],
185+
},
186+
]
187+
188+
const stream = handler.createMessage("System", messages)
189+
const chunks = []
190+
for await (const chunk of stream) {
191+
chunks.push(chunk)
192+
}
193+
194+
// Verify the request body contains both text and image
195+
const callArgs = mockFetch.mock.calls[0]
196+
const requestBody = JSON.parse(callArgs[1].body)
197+
expect(requestBody.messages[1].content).toEqual(
198+
expect.arrayContaining([
199+
expect.objectContaining({ type: "text", text: "Analyze this:" }),
200+
expect.objectContaining({ type: "image_url" }),
201+
expect.objectContaining({ type: "text", text: "What do you see?" }),
202+
]),
203+
)
204+
})
205+
})
206+
80207
describe("createMessage", () => {
81208
it("should make correct API request", async () => {
82209
// Mock successful API response

‎src/api/providers/cerebras.ts‎

Lines changed: 94 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -26,56 +26,93 @@ function stripThinkingTokens(text: string): string {
2626
}
2727

2828
/**
29-
* Flattens OpenAI message content to simple strings that Cerebras can handle.
30-
* Cerebras doesn't support complex content arrays like OpenAI does.
29+
* Converts OpenAI messages to Cerebras-compatible format.
30+
* Cerebras now supports multimodal inputs including images.
3131
*/
32-
function flattenMessageContent(content: any): string {
33-
if (typeof content === "string") {
34-
return content
35-
}
36-
37-
if (Array.isArray(content)) {
38-
return content
39-
.map((part) => {
40-
if (typeof part === "string") {
41-
return part
42-
}
43-
if (part.type === "text") {
44-
return part.text || ""
45-
}
46-
if (part.type === "image_url") {
47-
return "[Image]" // Placeholder for images since Cerebras doesn't support images
48-
}
49-
return ""
50-
})
51-
.filter(Boolean)
52-
.join("\n")
53-
}
54-
55-
// Fallback for any other content types
56-
return String(content || "")
57-
}
58-
59-
/**
60-
* Converts OpenAI messages to Cerebras-compatible format with simple string content.
61-
* Also strips thinking tokens from assistant messages to prevent model confusion.
62-
*/
63-
function convertToCerebrasMessages(openaiMessages: any[]): Array<{ role: string; content: string }> {
32+
function convertToCerebrasMessages(openaiMessages: any[]): Array<{ role: string; content: any }> {
6433
return openaiMessages
6534
.map((msg) => {
66-
let content = flattenMessageContent(msg.content)
35+
// For simple string content, keep as is
36+
if (typeof msg.content === "string") {
37+
let content = msg.content
38+
// Strip thinking tokens from assistant messages to prevent confusion
39+
if (msg.role === "assistant") {
40+
content = stripThinkingTokens(content)
41+
}
42+
return {
43+
role: msg.role,
44+
content,
45+
}
46+
}
6747

68-
// Strip thinking tokens from assistant messages to prevent confusion
69-
if (msg.role === "assistant") {
70-
content = stripThinkingTokens(content)
48+
// For array content (including images), convert to Cerebras format
49+
if (Array.isArray(msg.content)) {
50+
const cerebrasContent = msg.content
51+
.map((part: any) => {
52+
if (typeof part === "string") {
53+
return { type: "text", text: part }
54+
}
55+
if (part.type === "text") {
56+
let text = part.text || ""
57+
// Strip thinking tokens from assistant messages
58+
if (msg.role === "assistant") {
59+
text = stripThinkingTokens(text)
60+
}
61+
return { type: "text", text }
62+
}
63+
if (part.type === "image_url" && part.image_url?.url) {
64+
// Cerebras expects images in a specific format
65+
// Extract base64 data from data URL if present
66+
const url = part.image_url.url
67+
if (url.startsWith("data:")) {
68+
// Parse data URL: data:image/png;base64,<base64-data>
69+
const matches = url.match(/^data:([^;]+);base64,(.+)$/)
70+
if (matches) {
71+
return {
72+
type: "image_url",
73+
image_url: {
74+
url: url, // Keep the full data URL
75+
},
76+
}
77+
}
78+
}
79+
// For regular URLs, pass through as is
80+
return {
81+
type: "image_url",
82+
image_url: {
83+
url: url,
84+
},
85+
}
86+
}
87+
return null
88+
})
89+
.filter(Boolean)
90+
91+
// If we have valid content, return it
92+
if (cerebrasContent.length > 0) {
93+
return {
94+
role: msg.role,
95+
content: cerebrasContent,
96+
}
97+
}
7198
}
7299

100+
// Fallback for any other content types
73101
return {
74102
role: msg.role,
75-
content,
103+
content: String(msg.content || ""),
76104
}
77105
})
78-
.filter((msg) => msg.content.trim() !== "") // Remove empty messages
106+
.filter((msg) => {
107+
// Remove empty messages
108+
if (typeof msg.content === "string") {
109+
return msg.content.trim() !== ""
110+
}
111+
if (Array.isArray(msg.content)) {
112+
return msg.content.length > 0
113+
}
114+
return false
115+
})
79116
}
80117

81118
export class CerebrasHandler extends BaseProvider implements SingleCompletionHandler {
@@ -256,7 +293,23 @@ export class CerebrasHandler extends BaseProvider implements SingleCompletionHan
256293

257294
// Provide token usage estimate if not available from API
258295
if (inputTokens === 0 || outputTokens === 0) {
259-
const inputText = systemPrompt + cerebrasMessages.map((m) => m.content).join("")
296+
// Calculate input text, handling both string and array content
297+
let inputText = systemPrompt
298+
for (const msg of cerebrasMessages) {
299+
if (typeof msg.content === "string") {
300+
inputText += msg.content
301+
} else if (Array.isArray(msg.content)) {
302+
for (const part of msg.content) {
303+
if (part.type === "text") {
304+
inputText += part.text || ""
305+
}
306+
// Add token estimate for images (typically ~85 tokens per image)
307+
if (part.type === "image_url") {
308+
inputText += " ".repeat(85 * 4) // Approximate 85 tokens as characters
309+
}
310+
}
311+
}
312+
}
260313
inputTokens = inputTokens || Math.ceil(inputText.length / 4) // Rough estimate: 4 chars per token
261314
outputTokens = outputTokens || Math.ceil((max_tokens || 1000) / 10) // Rough estimate
262315
}

0 commit comments

Comments
 (0)