Skip to content

Commit b9bf8ff

Browse files
committed
feat: improve tool error guidance with contextual suggestions
- Created ToolErrorGuidance class to analyze error patterns - Enhanced error messages based on specific failure types (file not found, permissions, etc.) - Added pattern detection for common tool usage issues - Integrated contextual guidance into Task.ts consecutive mistake handling - Added comprehensive test coverage for the new guidance system Fixes #7936
1 parent 08d7f80 commit b9bf8ff

File tree

3 files changed

+538
-4
lines changed

3 files changed

+538
-4
lines changed

src/core/task/Task.ts

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ import { Gpt5Metadata, ClineMessageWithMetadata } from "./types"
114114
import { MessageQueueService } from "../message-queue/MessageQueueService"
115115

116116
import { AutoApprovalHandler } from "./AutoApprovalHandler"
117+
import { ToolErrorGuidance } from "../tools/errorGuidance"
117118

118119
const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes
119120
const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds
@@ -263,6 +264,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
263264
consecutiveMistakeLimit: number
264265
consecutiveMistakeCountForApplyDiff: Map<string, number> = new Map()
265266
toolUsage: ToolUsage = {}
267+
toolErrorHistory: Array<{ toolName: ToolName; error?: string; timestamp: number }> = []
266268

267269
// Checkpoints
268270
enableCheckpoints: boolean
@@ -1725,10 +1727,37 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
17251727
}
17261728

17271729
if (this.consecutiveMistakeLimit > 0 && this.consecutiveMistakeCount >= this.consecutiveMistakeLimit) {
1728-
const { response, text, images } = await this.ask(
1729-
"mistake_limit_reached",
1730-
t("common:errors.mistake_limit_guidance"),
1731-
)
1730+
// Build error patterns from the tool error history
1731+
const toolErrorMap = new Map<ToolName, { count: number; lastError?: string }>()
1732+
const recentTools: ToolName[] = []
1733+
1734+
// Process error history to build patterns
1735+
for (const error of this.toolErrorHistory) {
1736+
recentTools.push(error.toolName)
1737+
const existing = toolErrorMap.get(error.toolName) || { count: 0 }
1738+
toolErrorMap.set(error.toolName, {
1739+
count: existing.count + 1,
1740+
lastError: error.error || existing.lastError,
1741+
})
1742+
}
1743+
1744+
// Build error patterns using the ToolErrorGuidance helper
1745+
const errorPatterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrorMap)
1746+
1747+
// Create guidance context
1748+
const guidanceContext = {
1749+
recentTools,
1750+
errorPatterns,
1751+
consecutiveMistakeCount: this.consecutiveMistakeCount,
1752+
}
1753+
1754+
// Get contextual guidance
1755+
const contextualGuidance = ToolErrorGuidance.getContextualGuidance(guidanceContext)
1756+
1757+
// Format the guidance message
1758+
let guidanceMessage = ToolErrorGuidance.formatGuidanceMessage(contextualGuidance)
1759+
1760+
const { response, text, images } = await this.ask("mistake_limit_reached", guidanceMessage)
17321761

17331762
if (response === "messageResponse") {
17341763
currentUserContent.push(
@@ -1745,6 +1774,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
17451774
}
17461775

17471776
this.consecutiveMistakeCount = 0
1777+
// Clear error history after providing guidance
1778+
this.toolErrorHistory = []
17481779
}
17491780

17501781
// In this Cline request loop, we need to check if this task instance
@@ -2828,6 +2859,18 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
28282859

28292860
this.toolUsage[toolName].failures++
28302861

2862+
// Track error in history for contextual guidance
2863+
this.toolErrorHistory.push({
2864+
toolName,
2865+
error,
2866+
timestamp: Date.now(),
2867+
})
2868+
2869+
// Keep only recent errors (last 20)
2870+
if (this.toolErrorHistory.length > 20) {
2871+
this.toolErrorHistory = this.toolErrorHistory.slice(-20)
2872+
}
2873+
28312874
if (error) {
28322875
this.emit(RooCodeEventName.TaskToolFailed, this.taskId, toolName, error)
28332876
}
Lines changed: 304 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,304 @@
1+
import { describe, it, expect } from "vitest"
2+
import { ToolErrorGuidance, ToolErrorPattern, GuidanceContext } from "../errorGuidance"
3+
4+
describe("ToolErrorGuidance", () => {
5+
describe("getContextualGuidance", () => {
6+
it("should return generic guidance when no error patterns are provided", () => {
7+
const context: GuidanceContext = {
8+
recentTools: [],
9+
errorPatterns: [],
10+
consecutiveMistakeCount: 3,
11+
}
12+
13+
const result = ToolErrorGuidance.getContextualGuidance(context)
14+
15+
expect(result).toHaveLength(3)
16+
expect(result[0]).toContain("Try breaking down the task")
17+
})
18+
19+
it("should detect file not found errors", () => {
20+
const context: GuidanceContext = {
21+
recentTools: ["read_file", "read_file"],
22+
errorPatterns: [
23+
{
24+
toolName: "read_file",
25+
errorType: "file_not_found",
26+
count: 2,
27+
lastError: "File not found: src/test.ts",
28+
},
29+
],
30+
consecutiveMistakeCount: 3,
31+
}
32+
33+
const result = ToolErrorGuidance.getContextualGuidance(context)
34+
35+
// Check for file operation guidance
36+
expect(result.some((s: string) => s.toLowerCase().includes("file") || s.includes("'list_files'"))).toBe(
37+
true,
38+
)
39+
})
40+
41+
it("should detect missing parameter errors", () => {
42+
const context: GuidanceContext = {
43+
recentTools: ["write_to_file", "apply_diff"],
44+
errorPatterns: [
45+
{
46+
toolName: "write_to_file",
47+
errorType: "missing_param",
48+
count: 1,
49+
lastError: "Missing required parameter: content",
50+
},
51+
{
52+
toolName: "apply_diff",
53+
errorType: "missing_param",
54+
count: 1,
55+
lastError: "Required parameter path is missing",
56+
},
57+
],
58+
consecutiveMistakeCount: 3,
59+
}
60+
61+
const result = ToolErrorGuidance.getContextualGuidance(context)
62+
63+
// Should return some guidance
64+
expect(result).toBeTruthy()
65+
expect(result.length).toBeGreaterThan(0)
66+
expect(result.length).toBeLessThanOrEqual(3)
67+
})
68+
69+
it("should detect permission errors", () => {
70+
const context: GuidanceContext = {
71+
recentTools: ["write_to_file", "execute_command"],
72+
errorPatterns: [
73+
{
74+
toolName: "write_to_file",
75+
errorType: "permission_denied",
76+
count: 1,
77+
lastError: "Permission denied",
78+
},
79+
{
80+
toolName: "execute_command",
81+
errorType: "permission_denied",
82+
count: 1,
83+
lastError: "Access denied",
84+
},
85+
],
86+
consecutiveMistakeCount: 3,
87+
}
88+
89+
const result = ToolErrorGuidance.getContextualGuidance(context)
90+
91+
// Should return some guidance
92+
expect(result).toBeTruthy()
93+
expect(result.length).toBeGreaterThan(0)
94+
expect(result.length).toBeLessThanOrEqual(3)
95+
})
96+
97+
it("should detect repeated failures", () => {
98+
const context: GuidanceContext = {
99+
recentTools: ["read_file", "read_file", "read_file", "read_file", "read_file"],
100+
errorPatterns: [
101+
{
102+
toolName: "read_file",
103+
errorType: "repeated_failure",
104+
count: 5,
105+
lastError: "Some error",
106+
},
107+
],
108+
consecutiveMistakeCount: 5,
109+
}
110+
111+
const result = ToolErrorGuidance.getContextualGuidance(context)
112+
113+
expect(result.some((s: string) => s.includes("breaking down the task"))).toBe(true)
114+
})
115+
116+
it("should detect search operation issues", () => {
117+
const context: GuidanceContext = {
118+
recentTools: ["search_files", "list_files", "search_files"],
119+
errorPatterns: [],
120+
consecutiveMistakeCount: 3,
121+
}
122+
123+
const result = ToolErrorGuidance.getContextualGuidance(context)
124+
125+
expect(result.some((s: string) => s.includes("search patterns") || s.includes("project structure"))).toBe(
126+
true,
127+
)
128+
})
129+
130+
it("should detect code modification issues", () => {
131+
const context: GuidanceContext = {
132+
recentTools: ["apply_diff", "write_to_file", "apply_diff"],
133+
errorPatterns: [],
134+
consecutiveMistakeCount: 3,
135+
}
136+
137+
const result = ToolErrorGuidance.getContextualGuidance(context)
138+
139+
expect(
140+
result.some(
141+
(s: string) => s.includes("Read the file first") || s.includes("smaller, targeted changes"),
142+
),
143+
).toBe(true)
144+
})
145+
146+
it("should limit suggestions to 3", () => {
147+
const context: GuidanceContext = {
148+
recentTools: [
149+
"read_file",
150+
"write_to_file",
151+
"apply_diff",
152+
"execute_command",
153+
"search_files",
154+
"list_files",
155+
],
156+
errorPatterns: [
157+
{
158+
toolName: "read_file",
159+
errorType: "file_not_found",
160+
count: 2,
161+
lastError: "File not found",
162+
},
163+
{
164+
toolName: "write_to_file",
165+
errorType: "permission_denied",
166+
count: 1,
167+
lastError: "Permission denied",
168+
},
169+
{
170+
toolName: "apply_diff",
171+
errorType: "missing_param",
172+
count: 1,
173+
lastError: "Missing parameter",
174+
},
175+
],
176+
consecutiveMistakeCount: 5,
177+
}
178+
179+
const result = ToolErrorGuidance.getContextualGuidance(context)
180+
181+
expect(result.length).toBeLessThanOrEqual(3)
182+
})
183+
184+
it("should handle mixed error patterns", () => {
185+
const context: GuidanceContext = {
186+
recentTools: ["read_file", "write_to_file", "apply_diff"],
187+
errorPatterns: [
188+
{
189+
toolName: "read_file",
190+
errorType: "file_not_found",
191+
count: 1,
192+
lastError: "File not found: config.json",
193+
},
194+
{
195+
toolName: "write_to_file",
196+
errorType: "permission_denied",
197+
count: 1,
198+
lastError: "Permission denied",
199+
},
200+
],
201+
consecutiveMistakeCount: 3,
202+
}
203+
204+
const result = ToolErrorGuidance.getContextualGuidance(context)
205+
206+
expect(result).toBeTruthy()
207+
expect(result.length).toBeGreaterThan(0)
208+
expect(result.length).toBeLessThanOrEqual(3)
209+
})
210+
})
211+
212+
describe("formatGuidanceMessage", () => {
213+
it("should format guidance messages properly", () => {
214+
const guidance = [
215+
"Try breaking down the task into smaller steps",
216+
"Use list_files to verify directory structure",
217+
]
218+
219+
const result = ToolErrorGuidance.formatGuidanceMessage(guidance)
220+
221+
expect(result).toContain("struggling with tool usage")
222+
expect(result).toContain("1.")
223+
expect(result).toContain("2.")
224+
})
225+
226+
it("should return default message for empty guidance", () => {
227+
const result = ToolErrorGuidance.formatGuidanceMessage([])
228+
229+
expect(result).toContain("failure in the model's thought process")
230+
})
231+
})
232+
233+
describe("buildErrorPatterns", () => {
234+
it("should correctly identify file not found pattern", () => {
235+
const recentTools = ["read_file" as any]
236+
const toolErrors = new Map([["read_file" as any, { count: 1, lastError: "ENOENT: no such file" }]])
237+
238+
const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors)
239+
240+
expect(patterns).toHaveLength(1)
241+
expect(patterns[0].errorType).toBe("file_not_found")
242+
expect(patterns[0].count).toBe(1)
243+
})
244+
245+
it("should correctly identify missing parameter pattern", () => {
246+
const recentTools = ["write_to_file" as any]
247+
const toolErrors = new Map([
248+
["write_to_file" as any, { count: 1, lastError: "Missing required parameter: content" }],
249+
])
250+
251+
const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors)
252+
253+
expect(patterns).toHaveLength(1)
254+
expect(patterns[0].errorType).toBe("missing_param")
255+
})
256+
257+
it("should correctly identify permission denied pattern", () => {
258+
const recentTools = ["execute_command" as any]
259+
const toolErrors = new Map([["execute_command" as any, { count: 1, lastError: "permission denied" }]])
260+
261+
const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors)
262+
263+
expect(patterns).toHaveLength(1)
264+
expect(patterns[0].errorType).toBe("permission_denied")
265+
})
266+
267+
it("should correctly identify invalid format pattern", () => {
268+
const recentTools = ["write_to_file" as any]
269+
const toolErrors = new Map([["write_to_file" as any, { count: 1, lastError: "Invalid JSON format" }]])
270+
271+
const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors)
272+
273+
expect(patterns).toHaveLength(1)
274+
expect(patterns[0].errorType).toBe("invalid_format")
275+
})
276+
277+
it("should default to repeated_failure for unknown errors", () => {
278+
const recentTools = ["read_file" as any]
279+
const toolErrors = new Map([["read_file" as any, { count: 3, lastError: "Unknown error" }]])
280+
281+
const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors)
282+
283+
expect(patterns).toHaveLength(1)
284+
expect(patterns[0].errorType).toBe("repeated_failure")
285+
expect(patterns[0].count).toBe(3)
286+
})
287+
288+
it("should handle multiple tools with errors", () => {
289+
const recentTools = ["read_file" as any, "write_to_file" as any, "apply_diff" as any]
290+
const toolErrors = new Map([
291+
["read_file" as any, { count: 2, lastError: "File not found" }],
292+
["write_to_file" as any, { count: 1, lastError: "Permission denied" }],
293+
["apply_diff" as any, { count: 1, lastError: "Missing required parameter" }],
294+
])
295+
296+
const patterns = ToolErrorGuidance.buildErrorPatterns(recentTools, toolErrors)
297+
298+
expect(patterns).toHaveLength(3)
299+
expect(patterns.find((p) => p.toolName === "read_file")?.errorType).toBe("file_not_found")
300+
expect(patterns.find((p) => p.toolName === "write_to_file")?.errorType).toBe("permission_denied")
301+
expect(patterns.find((p) => p.toolName === "apply_diff")?.errorType).toBe("missing_param")
302+
})
303+
})
304+
})

0 commit comments

Comments
 (0)