Skip to content

Commit edde377

Browse files
roomoteroomote
committed
fix: prevent token burning when Roo gets stuck in error loops
- Add maximum guidance request limit (3 attempts) to prevent infinite loops - Track guidance count separately from mistake count - Abort task when guidance limit is exceeded with clear error message - Add resetConsecutiveMistakeCounts() method to reset both counters - Update tools to use the new reset method - Add logging for debugging token burning issues Fixes #8148
1 parent 87b45de commit edde377

File tree

4 files changed

+326
-7
lines changed

4 files changed

+326
-7
lines changed

‎src/core/task/Task.ts‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
261261
// Tool Use
262262
consecutiveMistakeCount: number = 0
263263
consecutiveMistakeLimit: number
264+
consecutiveMistakeGuidanceCount: number = 0 // Track how many times we've asked for guidance
265+
maxConsecutiveMistakeGuidance: number = 3 // Maximum times to ask for guidance before aborting
264266
consecutiveMistakeCountForApplyDiff: Map<string, number> = new Map()
265267
toolUsage: ToolUsage = {}
266268

@@ -1725,10 +1727,35 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
17251727
}
17261728

17271729
if (this.consecutiveMistakeLimit > 0 && this.consecutiveMistakeCount >= this.consecutiveMistakeLimit) {
1728-
const { response, text, images } = await this.ask(
1729-
"mistake_limit_reached",
1730-
t("common:errors.mistake_limit_guidance"),
1731-
)
1730+
// Check if we've asked for guidance too many times
1731+
if (this.consecutiveMistakeGuidanceCount >= this.maxConsecutiveMistakeGuidance) {
1732+
// We've asked for guidance too many times, abort to prevent token burning
1733+
await this.say(
1734+
"error",
1735+
`I've been unable to proceed despite multiple attempts and guidance. The task appears to be stuck in a loop. To prevent excessive token usage, I'm stopping here. Please review the conversation and consider:\n\n1. Providing more specific instructions\n2. Breaking down the task into smaller steps\n3. Checking if there are any environmental issues preventing progress`,
1736+
)
1737+
1738+
// Track token burning event in telemetry
1739+
// Use captureConsecutiveMistakeError with additional logging for now
1740+
TelemetryService.instance.captureConsecutiveMistakeError(this.taskId)
1741+
console.error(
1742+
`[Task#${this.taskId}] Token burning detected - Guidance count: ${this.consecutiveMistakeGuidanceCount}, Mistake count: ${this.consecutiveMistakeCount}`,
1743+
)
1744+
1745+
// Return true to end the task loop
1746+
return true
1747+
}
1748+
1749+
// Increment guidance count before asking
1750+
this.consecutiveMistakeGuidanceCount++
1751+
1752+
// Add exponential backoff message if we've asked before
1753+
let guidanceMessage = t("common:errors.mistake_limit_guidance")
1754+
if (this.consecutiveMistakeGuidanceCount > 1) {
1755+
guidanceMessage = `${guidanceMessage}\n\n(Attempt ${this.consecutiveMistakeGuidanceCount}/${this.maxConsecutiveMistakeGuidance} - I'm having difficulty making progress)`
1756+
}
1757+
1758+
const { response, text, images } = await this.ask("mistake_limit_reached", guidanceMessage)
17321759

17331760
if (response === "messageResponse") {
17341761
currentUserContent.push(
@@ -1740,10 +1767,11 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
17401767

17411768
await this.say("user_feedback", text, images)
17421769

1743-
// Track consecutive mistake errors in telemetry.
1770+
// Track consecutive mistake errors in telemetry
17441771
TelemetryService.instance.captureConsecutiveMistakeError(this.taskId)
17451772
}
17461773

1774+
// Reset mistake count but keep guidance count
17471775
this.consecutiveMistakeCount = 0
17481776
}
17491777

@@ -2304,6 +2332,11 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
23042332
if (!didToolUse) {
23052333
this.userMessageContent.push({ type: "text", text: formatResponse.noToolsUsed() })
23062334
this.consecutiveMistakeCount++
2335+
2336+
// Log when we're incrementing mistake count for debugging
2337+
console.log(
2338+
`[Task#${this.taskId}] Consecutive mistake count: ${this.consecutiveMistakeCount}/${this.consecutiveMistakeLimit}, Guidance count: ${this.consecutiveMistakeGuidanceCount}/${this.maxConsecutiveMistakeGuidance}`,
2339+
)
23072340
}
23082341

23092342
if (this.userMessageContent.length > 0) {
@@ -2932,4 +2965,14 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
29322965
console.error(`[Task] Queue processing error:`, e)
29332966
}
29342967
}
2968+
2969+
/**
2970+
* Reset consecutive mistake tracking counters.
2971+
* This should be called when a tool executes successfully to indicate
2972+
* that Roo is making progress and not stuck in a loop.
2973+
*/
2974+
public resetConsecutiveMistakeCounts(): void {
2975+
this.consecutiveMistakeCount = 0
2976+
this.consecutiveMistakeGuidanceCount = 0
2977+
}
29352978
}

‎src/core/task/__tests__/Task.tokenBurning.spec.ts‎

Lines changed: 276 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,276 @@
1+
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"
2+
import { Task } from "../Task"
3+
import { ClineProvider } from "../../webview/ClineProvider"
4+
import { ApiHandler } from "../../../api"
5+
import { TelemetryService } from "@roo-code/telemetry"
6+
import { DEFAULT_CONSECUTIVE_MISTAKE_LIMIT } from "@roo-code/types"
7+
8+
// Mock TelemetryService before any imports that might use it
9+
vi.mock("@roo-code/telemetry", () => {
10+
const mockTelemetryService = {
11+
captureConsecutiveMistakeError: vi.fn(),
12+
captureEvent: vi.fn(),
13+
captureTaskCreated: vi.fn(),
14+
captureTaskRestarted: vi.fn(),
15+
captureConversationMessage: vi.fn(),
16+
}
17+
18+
return {
19+
TelemetryService: {
20+
instance: mockTelemetryService,
21+
initialize: vi.fn(),
22+
},
23+
}
24+
})
25+
26+
describe("Task - Token Burning Prevention", () => {
27+
let mockProvider: any
28+
let mockApiHandler: any
29+
let task: Task
30+
31+
beforeEach(() => {
32+
// Mock provider
33+
mockProvider = {
34+
context: {
35+
globalStorageUri: { fsPath: "/test/storage" },
36+
extensionUri: { fsPath: "/test/extension" },
37+
},
38+
getState: vi.fn().mockResolvedValue({
39+
mode: "code",
40+
apiConfiguration: {
41+
apiProvider: "anthropic",
42+
apiKey: "test-key",
43+
},
44+
}),
45+
postStateToWebview: vi.fn(),
46+
postMessageToWebview: vi.fn(),
47+
updateTaskHistory: vi.fn(),
48+
log: vi.fn(),
49+
} as any
50+
51+
// Mock API handler
52+
mockApiHandler = {
53+
getModel: vi.fn().mockReturnValue({
54+
id: "claude-3-opus",
55+
info: {
56+
contextWindow: 200000,
57+
supportsComputerUse: false,
58+
},
59+
}),
60+
createMessage: vi.fn(),
61+
} as any
62+
})
63+
64+
afterEach(() => {
65+
vi.clearAllMocks()
66+
})
67+
68+
describe("Consecutive Mistake Guidance Limit", () => {
69+
it("should initialize with default values", () => {
70+
task = new Task({
71+
provider: mockProvider as ClineProvider,
72+
apiConfiguration: { apiProvider: "anthropic", apiKey: "test" },
73+
task: "test task",
74+
})
75+
76+
expect(task.consecutiveMistakeCount).toBe(0)
77+
expect(task.consecutiveMistakeLimit).toBe(DEFAULT_CONSECUTIVE_MISTAKE_LIMIT)
78+
expect(task.consecutiveMistakeGuidanceCount).toBe(0)
79+
expect(task.maxConsecutiveMistakeGuidance).toBe(3)
80+
})
81+
82+
it("should reset both counters when resetConsecutiveMistakeCounts is called", () => {
83+
task = new Task({
84+
provider: mockProvider as ClineProvider,
85+
apiConfiguration: { apiProvider: "anthropic", apiKey: "test" },
86+
task: "test task",
87+
})
88+
89+
// Set some values
90+
task.consecutiveMistakeCount = 5
91+
task.consecutiveMistakeGuidanceCount = 2
92+
93+
// Reset
94+
task.resetConsecutiveMistakeCounts()
95+
96+
// Both should be reset
97+
expect(task.consecutiveMistakeCount).toBe(0)
98+
expect(task.consecutiveMistakeGuidanceCount).toBe(0)
99+
})
100+
101+
it("should increment guidance count when asking for user guidance", async () => {
102+
task = new Task({
103+
provider: mockProvider as ClineProvider,
104+
apiConfiguration: { apiProvider: "anthropic", apiKey: "test" },
105+
consecutiveMistakeLimit: 3,
106+
task: "test task",
107+
})
108+
109+
// Mock the ask method to simulate user providing feedback
110+
task.ask = vi.fn().mockResolvedValue({
111+
response: "messageResponse",
112+
text: "Try a different approach",
113+
images: undefined,
114+
})
115+
116+
// Mock the say method
117+
task.say = vi.fn().mockResolvedValue(undefined)
118+
119+
// Set mistake count to trigger guidance request
120+
task.consecutiveMistakeCount = 3
121+
task.consecutiveMistakeGuidanceCount = 0
122+
123+
// Create a mock recursivelyMakeClineRequests that simulates the guidance flow
124+
const mockUserContent: any[] = []
125+
const stack = [{ userContent: mockUserContent, includeFileDetails: false }]
126+
127+
// Simulate the part of recursivelyMakeClineRequests that handles mistakes
128+
if (task.consecutiveMistakeLimit > 0 && task.consecutiveMistakeCount >= task.consecutiveMistakeLimit) {
129+
if (task.consecutiveMistakeGuidanceCount >= task.maxConsecutiveMistakeGuidance) {
130+
// Should not reach here in this test
131+
expect(true).toBe(false)
132+
}
133+
134+
task.consecutiveMistakeGuidanceCount++
135+
136+
const guidanceMessage =
137+
task.consecutiveMistakeGuidanceCount > 1
138+
? `I've been making too many mistakes. Could you provide some guidance or corrections to help me proceed?\n\n(Attempt ${task.consecutiveMistakeGuidanceCount}/${task.maxConsecutiveMistakeGuidance} - I'm having difficulty making progress)`
139+
: "I've been making too many mistakes. Could you provide some guidance or corrections to help me proceed?"
140+
141+
await task.ask("mistake_limit_reached", guidanceMessage)
142+
task.consecutiveMistakeCount = 0
143+
}
144+
145+
expect(task.consecutiveMistakeGuidanceCount).toBe(1)
146+
expect(task.consecutiveMistakeCount).toBe(0)
147+
})
148+
149+
it("should abort task when guidance limit is exceeded", async () => {
150+
task = new Task({
151+
provider: mockProvider as ClineProvider,
152+
apiConfiguration: { apiProvider: "anthropic", apiKey: "test" },
153+
consecutiveMistakeLimit: 3,
154+
task: "test task",
155+
})
156+
157+
// Mock the say method
158+
task.say = vi.fn().mockResolvedValue(undefined)
159+
160+
// Set counters to exceed limit
161+
task.consecutiveMistakeCount = 3
162+
task.consecutiveMistakeGuidanceCount = 3 // Already at max
163+
164+
// Simulate the check in recursivelyMakeClineRequests
165+
let shouldAbort = false
166+
if (task.consecutiveMistakeLimit > 0 && task.consecutiveMistakeCount >= task.consecutiveMistakeLimit) {
167+
if (task.consecutiveMistakeGuidanceCount >= task.maxConsecutiveMistakeGuidance) {
168+
await task.say(
169+
"error",
170+
`I've been unable to proceed despite multiple attempts and guidance. The task appears to be stuck in a loop. To prevent excessive token usage, I'm stopping here. Please review the conversation and consider:\n\n1. Providing more specific instructions\n2. Breaking down the task into smaller steps\n3. Checking if there are any environmental issues preventing progress`,
171+
)
172+
173+
// In the real code, this would capture telemetry
174+
TelemetryService.instance.captureConsecutiveMistakeError(task.taskId)
175+
176+
shouldAbort = true
177+
}
178+
}
179+
180+
expect(shouldAbort).toBe(true)
181+
expect(task.say).toHaveBeenCalledWith(
182+
"error",
183+
expect.stringContaining("unable to proceed despite multiple attempts"),
184+
)
185+
expect(TelemetryService.instance.captureConsecutiveMistakeError).toHaveBeenCalledWith(task.taskId)
186+
})
187+
188+
it("should show attempt count in guidance message after first attempt", async () => {
189+
task = new Task({
190+
provider: mockProvider as ClineProvider,
191+
apiConfiguration: { apiProvider: "anthropic", apiKey: "test" },
192+
consecutiveMistakeLimit: 3,
193+
task: "test task",
194+
})
195+
196+
// Mock the ask method
197+
task.ask = vi.fn().mockResolvedValue({
198+
response: "messageResponse",
199+
text: "Try again",
200+
images: undefined,
201+
})
202+
203+
// Set guidance count to simulate second attempt
204+
task.consecutiveMistakeGuidanceCount = 1
205+
task.consecutiveMistakeCount = 3
206+
207+
// Simulate generating the guidance message
208+
let guidanceMessage =
209+
"I've been making too many mistakes. Could you provide some guidance or corrections to help me proceed?"
210+
if (task.consecutiveMistakeGuidanceCount > 0) {
211+
task.consecutiveMistakeGuidanceCount++ // Increment before showing
212+
guidanceMessage = `${guidanceMessage}\n\n(Attempt ${task.consecutiveMistakeGuidanceCount}/${task.maxConsecutiveMistakeGuidance} - I'm having difficulty making progress)`
213+
} else {
214+
task.consecutiveMistakeGuidanceCount++
215+
}
216+
217+
await task.ask("mistake_limit_reached", guidanceMessage)
218+
219+
expect(task.ask).toHaveBeenCalledWith(
220+
"mistake_limit_reached",
221+
expect.stringContaining("(Attempt 2/3 - I'm having difficulty making progress)"),
222+
)
223+
})
224+
225+
it("should log debug information when incrementing mistake count", () => {
226+
const consoleSpy = vi.spyOn(console, "log").mockImplementation(() => {})
227+
228+
task = new Task({
229+
provider: mockProvider as ClineProvider,
230+
apiConfiguration: { apiProvider: "anthropic", apiKey: "test" },
231+
consecutiveMistakeLimit: 3,
232+
task: "test task",
233+
})
234+
235+
// Simulate incrementing mistake count with logging
236+
task.consecutiveMistakeCount++
237+
console.log(
238+
`[Task#${task.taskId}] Consecutive mistake count: ${task.consecutiveMistakeCount}/${task.consecutiveMistakeLimit}, Guidance count: ${task.consecutiveMistakeGuidanceCount}/${task.maxConsecutiveMistakeGuidance}`,
239+
)
240+
241+
expect(consoleSpy).toHaveBeenCalledWith(
242+
expect.stringContaining("Consecutive mistake count: 1/3, Guidance count: 0/3"),
243+
)
244+
245+
consoleSpy.mockRestore()
246+
})
247+
})
248+
249+
describe("Token Burning Detection", () => {
250+
it("should log error when token burning is detected", async () => {
251+
const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {})
252+
253+
task = new Task({
254+
provider: mockProvider as ClineProvider,
255+
apiConfiguration: { apiProvider: "anthropic", apiKey: "test" },
256+
task: "test task",
257+
})
258+
259+
// Set counters to trigger token burning detection
260+
task.consecutiveMistakeGuidanceCount = 3
261+
task.consecutiveMistakeCount = 5
262+
263+
// Simulate token burning detection
264+
TelemetryService.instance.captureConsecutiveMistakeError(task.taskId)
265+
console.error(
266+
`[Task#${task.taskId}] Token burning detected - Guidance count: ${task.consecutiveMistakeGuidanceCount}, Mistake count: ${task.consecutiveMistakeCount}`,
267+
)
268+
269+
expect(consoleErrorSpy).toHaveBeenCalledWith(
270+
expect.stringContaining("Token burning detected - Guidance count: 3, Mistake count: 5"),
271+
)
272+
273+
consoleErrorSpy.mockRestore()
274+
})
275+
})
276+
})

‎src/core/tools/executeCommandTool.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ export async function executeCommandTool(
5151
return
5252
}
5353

54-
task.consecutiveMistakeCount = 0
54+
task.resetConsecutiveMistakeCounts()
5555

5656
command = unescapeHtmlEntities(command) // Unescape HTML entities.
5757
const didApprove = await askApproval("command", command)

‎src/core/tools/writeToFileTool.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ export async function writeToFileTool(
159159
return
160160
}
161161

162-
cline.consecutiveMistakeCount = 0
162+
cline.resetConsecutiveMistakeCounts()
163163

164164
// Check if preventFocusDisruption experiment is enabled
165165
const provider = cline.providerRef.deref()

0 commit comments

Comments
 (0)