Skip to content

Commit f487380

Browse files
roomoteroomote
committed
fix: handle empty responses from Docker Model Runner and similar APIs
- Add fallback to yield empty text chunk when no content is received - Prevents "The language model did not provide any assistant messages" error - Fixes issue with Docker Model Runner integration - Add tests for empty response handling Fixes #8226
1 parent 0e1b23d commit f487380

File tree

3 files changed

+273
-0
lines changed

3 files changed

+273
-0
lines changed

‎src/api/providers/__tests__/base-openai-compatible.spec.ts‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
import { describe, it, expect, vi, beforeEach } from "vitest"
2+
import { Anthropic } from "@anthropic-ai/sdk"
3+
import OpenAI from "openai"
4+
5+
import { BaseOpenAiCompatibleProvider } from "../base-openai-compatible-provider"
6+
import { type ModelInfo } from "@roo-code/types"
7+
8+
// Create a concrete implementation for testing
9+
class TestOpenAiCompatibleProvider extends BaseOpenAiCompatibleProvider<"test-model"> {
10+
constructor(options: any) {
11+
super({
12+
providerName: "TestProvider",
13+
baseURL: "https://test.api.com/v1",
14+
defaultProviderModelId: "test-model",
15+
providerModels: {
16+
"test-model": {
17+
maxTokens: 4096,
18+
contextWindow: 8192,
19+
supportsPromptCache: false,
20+
supportsComputerUse: false,
21+
supportsStreaming: true,
22+
inputPrice: 0,
23+
outputPrice: 0,
24+
} as ModelInfo,
25+
},
26+
...options,
27+
})
28+
}
29+
}
30+
31+
describe("BaseOpenAiCompatibleProvider", () => {
32+
let provider: TestOpenAiCompatibleProvider
33+
let mockCreate: ReturnType<typeof vi.fn>
34+
35+
beforeEach(() => {
36+
mockCreate = vi.fn()
37+
vi.spyOn(OpenAI.Chat.Completions.prototype, "create").mockImplementation(mockCreate)
38+
})
39+
40+
describe("createMessage", () => {
41+
it("should handle empty response from API gracefully", async () => {
42+
provider = new TestOpenAiCompatibleProvider({
43+
apiKey: "test-key",
44+
})
45+
46+
// Mock an empty stream response (no content in chunks)
47+
const mockStream = {
48+
async *[Symbol.asyncIterator]() {
49+
// Yield chunks with no content
50+
yield {
51+
choices: [
52+
{
53+
delta: {
54+
// No content field
55+
},
56+
},
57+
],
58+
}
59+
// Yield usage information
60+
yield {
61+
choices: [
62+
{
63+
delta: {},
64+
},
65+
],
66+
usage: {
67+
prompt_tokens: 100,
68+
completion_tokens: 0,
69+
},
70+
}
71+
},
72+
}
73+
74+
mockCreate.mockResolvedValue(mockStream)
75+
76+
const systemPrompt = "You are a helpful assistant"
77+
const messages: Anthropic.Messages.MessageParam[] = [
78+
{
79+
role: "user",
80+
content: "Hello",
81+
},
82+
]
83+
84+
const stream = provider.createMessage(systemPrompt, messages)
85+
const chunks = []
86+
87+
for await (const chunk of stream) {
88+
chunks.push(chunk)
89+
}
90+
91+
// Should have at least one text chunk (even if empty) and one usage chunk
92+
expect(chunks).toHaveLength(2)
93+
94+
// Should have a usage chunk
95+
const usageChunk = chunks.find((c) => c.type === "usage")
96+
expect(usageChunk).toEqual({
97+
type: "usage",
98+
inputTokens: 100,
99+
outputTokens: 0,
100+
})
101+
102+
// Should have an empty text chunk (added by our fix)
103+
const textChunk = chunks.find((c) => c.type === "text")
104+
expect(textChunk).toEqual({
105+
type: "text",
106+
text: "",
107+
})
108+
})
109+
110+
it("should handle normal response with content", async () => {
111+
provider = new TestOpenAiCompatibleProvider({
112+
apiKey: "test-key",
113+
})
114+
115+
// Mock a normal stream response with content
116+
const mockStream = {
117+
async *[Symbol.asyncIterator]() {
118+
yield {
119+
choices: [
120+
{
121+
delta: {
122+
content: "Hello, ",
123+
},
124+
},
125+
],
126+
}
127+
yield {
128+
choices: [
129+
{
130+
delta: {
131+
content: "world!",
132+
},
133+
},
134+
],
135+
}
136+
yield {
137+
choices: [
138+
{
139+
delta: {},
140+
},
141+
],
142+
usage: {
143+
prompt_tokens: 100,
144+
completion_tokens: 10,
145+
},
146+
}
147+
},
148+
}
149+
150+
mockCreate.mockResolvedValue(mockStream)
151+
152+
const systemPrompt = "You are a helpful assistant"
153+
const messages: Anthropic.Messages.MessageParam[] = [
154+
{
155+
role: "user",
156+
content: "Hello",
157+
},
158+
]
159+
160+
const stream = provider.createMessage(systemPrompt, messages)
161+
const chunks = []
162+
163+
for await (const chunk of stream) {
164+
chunks.push(chunk)
165+
}
166+
167+
// Should have text chunks and usage chunk
168+
expect(chunks).toHaveLength(3)
169+
170+
// First two chunks should be text
171+
expect(chunks[0]).toEqual({
172+
type: "text",
173+
text: "Hello, ",
174+
})
175+
expect(chunks[1]).toEqual({
176+
type: "text",
177+
text: "world!",
178+
})
179+
180+
// Last chunk should be usage information
181+
expect(chunks[2]).toEqual({
182+
type: "usage",
183+
inputTokens: 100,
184+
outputTokens: 10,
185+
})
186+
})
187+
188+
it("should handle response with only usage and no content", async () => {
189+
provider = new TestOpenAiCompatibleProvider({
190+
apiKey: "test-key",
191+
})
192+
193+
// Mock a stream response with only usage, no content
194+
const mockStream = {
195+
async *[Symbol.asyncIterator]() {
196+
yield {
197+
choices: [
198+
{
199+
delta: {},
200+
},
201+
],
202+
usage: {
203+
prompt_tokens: 50,
204+
completion_tokens: 0,
205+
},
206+
}
207+
},
208+
}
209+
210+
mockCreate.mockResolvedValue(mockStream)
211+
212+
const systemPrompt = "You are a helpful assistant"
213+
const messages: Anthropic.Messages.MessageParam[] = [
214+
{
215+
role: "user",
216+
content: "Test",
217+
},
218+
]
219+
220+
const stream = provider.createMessage(systemPrompt, messages)
221+
const chunks = []
222+
223+
for await (const chunk of stream) {
224+
chunks.push(chunk)
225+
}
226+
227+
// Should have empty text chunk and usage chunk
228+
expect(chunks).toHaveLength(2)
229+
230+
// Should have a usage chunk
231+
const usageChunk = chunks.find((c) => c.type === "usage")
232+
expect(usageChunk).toEqual({
233+
type: "usage",
234+
inputTokens: 50,
235+
outputTokens: 0,
236+
})
237+
238+
// Should have an empty text chunk (added by our fix)
239+
const textChunk = chunks.find((c) => c.type === "text")
240+
expect(textChunk).toEqual({
241+
type: "text",
242+
text: "",
243+
})
244+
})
245+
})
246+
})

‎src/api/providers/base-openai-compatible-provider.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,11 +98,13 @@ export abstract class BaseOpenAiCompatibleProvider<ModelName extends string>
9898
metadata?: ApiHandlerCreateMessageMetadata,
9999
): ApiStream {
100100
const stream = await this.createStream(systemPrompt, messages, metadata)
101+
let hasContent = false
101102

102103
for await (const chunk of stream) {
103104
const delta = chunk.choices[0]?.delta
104105

105106
if (delta?.content) {
107+
hasContent = true
106108
yield {
107109
type: "text",
108110
text: delta.content,
@@ -117,6 +119,16 @@ export abstract class BaseOpenAiCompatibleProvider<ModelName extends string>
117119
}
118120
}
119121
}
122+
123+
// If no content was received, yield an empty text chunk to prevent
124+
// "The language model did not provide any assistant messages" error
125+
// This can happen with some Docker Model Runner configurations
126+
if (!hasContent) {
127+
yield {
128+
type: "text",
129+
text: "",
130+
}
131+
}
120132
}
121133

122134
async completePrompt(prompt: string): Promise<string> {

‎src/api/providers/lm-studio.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
8080
}
8181

8282
let assistantText = ""
83+
let hasContent = false
8384

8485
try {
8586
const params: OpenAI.Chat.ChatCompletionCreateParamsStreaming & { draft_model?: string } = {
@@ -113,6 +114,7 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
113114
const delta = chunk.choices[0]?.delta
114115

115116
if (delta?.content) {
117+
hasContent = true
116118
assistantText += delta.content
117119
for (const processedChunk of matcher.update(delta.content)) {
118120
yield processedChunk
@@ -121,9 +123,22 @@ export class LmStudioHandler extends BaseProvider implements SingleCompletionHan
121123
}
122124

123125
for (const processedChunk of matcher.final()) {
126+
if (processedChunk.text) {
127+
hasContent = true
128+
}
124129
yield processedChunk
125130
}
126131

132+
// If no content was received, yield an empty text chunk to prevent
133+
// "The language model did not provide any assistant messages" error
134+
// This can happen with some model configurations
135+
if (!hasContent) {
136+
yield {
137+
type: "text",
138+
text: "",
139+
}
140+
}
141+
127142
let outputTokens = 0
128143
try {
129144
outputTokens = await this.countTokens([{ type: "text", text: assistantText }])

0 commit comments

Comments
 (0)