fix: improve error messages for OpenAI-compatible providers

- Add specific guidance when OpenAI-compatible models fail to use tools
- Provide clearer instructions about XML tool format requirements
- Help users understand common issues with tool usage formatting
- Add tests for the new error message variations

Fixes #7226

Author: roomote

src/core/prompts/__tests__/responses-openai-compatible.spec.ts

@@ -18,17 +18,22 @@ describe("formatResponse.noToolsUsed", () => {
 		expect(result).not.toContain("OpenAI Compatible")
 	})
 
-	it("should include OpenAI Compatible specific hints when apiProvider is openai-compatible", () => {
-		const result = formatResponse.noToolsUsed("openai-compatible")
-
-		expect(result).toContain("[ERROR] You did not use a tool in your previous response!")
-		expect(result).toContain("# Important Note for OpenAI Compatible Models")
-		expect(result).toContain("Your model appears to not be using the required XML tool format")
-		expect(result).toContain("Use XML tags for ALL tool invocations")
-		expect(result).toContain("Place tool uses at the END of your message")
-		expect(result).toContain("Use only ONE tool per message")
-		expect(result).toContain("Follow the exact XML format shown below")
-		expect(result).toContain("# Reminder: Instructions for Tool Use")
+	it("should include OpenAI Compatible specific hints for OpenAI-compatible providers", () => {
+		// Test with various OpenAI-compatible providers
+		const openAICompatibleProviders = ["openai", "openai-native", "fireworks", "groq", "ollama"]
+
+		for (const provider of openAICompatibleProviders) {
+			const result = formatResponse.noToolsUsed(provider)
+
+			expect(result).toContain("[ERROR] You did not use a tool in your previous response!")
+			expect(result).toContain("# Important Note for OpenAI Compatible Models")
+			expect(result).toContain("Your model appears to not be using the required XML tool format")
+			expect(result).toContain("Use XML tags for ALL tool invocations")
+			expect(result).toContain("Place tool uses at the END of your message")
+			expect(result).toContain("Use only ONE tool per message")
+			expect(result).toContain("Follow the exact XML format shown below")
+			expect(result).toContain("# Reminder: Instructions for Tool Use")
+		}
 	})
 
 	it("should maintain the same structure with Next Steps section", () => {

src/core/prompts/responses.ts

@@ -21,8 +21,27 @@ export const formatResponse = {
 	noToolsUsed: (apiProvider?: string) => {
 		const baseMessage = `[ERROR] You did not use a tool in your previous response! Please retry with a tool use.`
 
+		// List of providers that use OpenAI-compatible APIs
+		const openAICompatibleProviders = [
+			"openai",
+			"openai-native",
+			"fireworks",
+			"groq",
+			"sambanova",
+			"chutes",
+			"roo",
+			"zai",
+			"io-intelligence",
+			"deepseek",
+			"moonshot",
+			"doubao",
+			"litellm",
+			"lmstudio",
+			"ollama",
+		]
+
 		let providerSpecificHint = ""
-		if (apiProvider === "openai-compatible") {
+		if (apiProvider && openAICompatibleProviders.includes(apiProvider)) {
 			providerSpecificHint = `
 
 # Important Note for OpenAI Compatible Models

src/core/task/Task.ts

@@ -1539,14 +1539,33 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 			}
 
 			if (this.consecutiveMistakeLimit > 0 && this.consecutiveMistakeCount >= this.consecutiveMistakeLimit) {
-				// Provide more specific guidance for OpenAI Compatible providers
-				const isOpenAICompatible = this.apiConfiguration.apiProvider === "openai-compatible"
+				// Provide more specific guidance for OpenAI-style API providers
+				const openAICompatibleProviders = [
+					"openai",
+					"openai-native",
+					"fireworks",
+					"groq",
+					"sambanova",
+					"chutes",
+					"roo",
+					"zai",
+					"io-intelligence",
+					"deepseek",
+					"moonshot",
+					"doubao",
+					"litellm",
+					"lmstudio",
+					"ollama",
+				]
+				const isOpenAICompatible =
+					this.apiConfiguration.apiProvider &&
+					openAICompatibleProviders.includes(this.apiConfiguration.apiProvider)
 				const modelId = getModelId(this.apiConfiguration)
 
 				let guidanceMessage = t("common:errors.mistake_limit_guidance")
 
 				if (isOpenAICompatible) {
-					guidanceMessage = `The model appears to be having difficulty with tool usage. This often happens with OpenAI Compatible providers when the model doesn't properly format tool calls using XML tags.
+					guidanceMessage = `The model appears to be having difficulty with tool usage. This often happens with OpenAI-compatible API providers when the model doesn't properly format tool calls using XML tags.
 
 Common issues with ${modelId || "this model"}:
 1. The model may not be following the XML tool format correctly
@@ -1557,7 +1576,7 @@ Try these solutions:
 • Break down your request into smaller, more specific steps
 • Be more explicit about what you want to accomplish
 • Try a different model that better supports tool usage
-• Ensure your OpenAI Compatible endpoint is properly configured`
+• Ensure your API endpoint is properly configured`
 				}
 
 				const { response, text, images } = await this.ask("mistake_limit_reached", guidanceMessage)