Fix bug where OpenRouter/Cline providers generation endpoint failed (RooCodeInc#2262)

saoudrizwan · web-flow · commit 8cc775958486 · 2025-03-14T17:01:48.000-07:00
* Fix openrouter/cline provider cost endpoint bug

* Increase generation endpoint timeout
diff --git a/src/api/index.ts b/src/api/index.ts
@@ -9,7 +9,7 @@ import { OllamaHandler } from "./providers/ollama"
 import { LmStudioHandler } from "./providers/lmstudio"
 import { GeminiHandler } from "./providers/gemini"
 import { OpenAiNativeHandler } from "./providers/openai-native"
-import { ApiStream } from "./transform/stream"
+import { ApiStream, ApiStreamUsageChunk } from "./transform/stream"
 import { DeepSeekHandler } from "./providers/deepseek"
 import { RequestyHandler } from "./providers/requesty"
 import { TogetherHandler } from "./providers/together"
@@ -25,6 +25,7 @@ import { SambanovaHandler } from "./providers/sambanova"
 export interface ApiHandler {
 	createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream
 	getModel(): { id: string; info: ModelInfo }
+	getApiStreamUsage?(): Promise<ApiStreamUsageChunk | undefined>
 }
 
 export interface SingleCompletionHandler {
diff --git a/src/api/providers/cline.ts b/src/api/providers/cline.ts
@@ -2,13 +2,15 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
 import { ApiHandler } from "../"
 import { ApiHandlerOptions, ModelInfo, openRouterDefaultModelId, openRouterDefaultModelInfo } from "../../shared/api"
-import { streamOpenRouterFormatRequest } from "../transform/openrouter-stream"
-import { ApiStream } from "../transform/stream"
+import { createOpenRouterStream } from "../transform/openrouter-stream"
+import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
 import axios from "axios"
+import { OpenRouterErrorResponse } from "./types"
 
 export class ClineHandler implements ApiHandler {
 	private options: ApiHandlerOptions
 	private client: OpenAI
+	lastGenerationId?: string
 
 	constructor(options: ApiHandlerOptions) {
 		this.options = options
@@ -19,36 +21,78 @@ export class ClineHandler implements ApiHandler {
 	}
 
 	async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
-		const model = this.getModel()
-		const genId = yield* streamOpenRouterFormatRequest(
+		this.lastGenerationId = undefined
+
+		const stream = await createOpenRouterStream(
 			this.client,
 			systemPrompt,
 			messages,
-			model,
+			this.getModel(),
 			this.options.o3MiniReasoningEffort,
 			this.options.thinkingBudgetTokens,
 		)
 
-		try {
-			const response = await axios.get(`https://api.cline.bot/v1/generation?id=${genId}`, {
-				headers: {
-					Authorization: `Bearer ${this.options.clineApiKey}`,
-				},
-				timeout: 5_000, // this request hangs sometimes
-			})
-
-			const generation = response.data
-			console.log("cline generation details:", generation)
-			yield {
-				type: "usage",
-				inputTokens: generation?.native_tokens_prompt || 0,
-				outputTokens: generation?.native_tokens_completion || 0,
-				totalCost: generation?.total_cost || 0,
+		for await (const chunk of stream) {
+			// openrouter returns an error object instead of the openai sdk throwing an error
+			if ("error" in chunk) {
+				const error = chunk.error as OpenRouterErrorResponse["error"]
+				console.error(`Cline API Error: ${error?.code} - ${error?.message}`)
+				// Include metadata in the error message if available
+				const metadataStr = error.metadata ? `\nMetadata: ${JSON.stringify(error.metadata, null, 2)}` : ""
+				throw new Error(`Cline API Error ${error.code}: ${error.message}${metadataStr}`)
+			}
+
+			if (!this.lastGenerationId && chunk.id) {
+				this.lastGenerationId = chunk.id
+			}
+
+			const delta = chunk.choices[0]?.delta
+			if (delta?.content) {
+				yield {
+					type: "text",
+					text: delta.content,
+				}
+			}
+
+			// Reasoning tokens are returned separately from the content
+			if ("reasoning" in delta && delta.reasoning) {
+				yield {
+					type: "reasoning",
+					// @ts-ignore-next-line
+					reasoning: delta.reasoning,
+				}
+			}
+		}
+
+		const apiStreamUsage = await this.getApiStreamUsage()
+		if (apiStreamUsage) {
+			yield apiStreamUsage
+		}
+	}
+
+	async getApiStreamUsage(): Promise<ApiStreamUsageChunk | undefined> {
+		if (this.lastGenerationId) {
+			try {
+				const response = await axios.get(`https://api.cline.bot/v1/generation?id=${this.lastGenerationId}`, {
+					headers: {
+						Authorization: `Bearer ${this.options.clineApiKey}`,
+					},
+					timeout: 15_000, // this request hangs sometimes
+				})
+
+				const generation = response.data
+				return {
+					type: "usage",
+					inputTokens: generation?.native_tokens_prompt || 0,
+					outputTokens: generation?.native_tokens_completion || 0,
+					totalCost: generation?.total_cost || 0,
+				}
+			} catch (error) {
+				// ignore if fails
+				console.error("Error fetching cline generation details:", error)
 			}
-		} catch (error) {
-			// ignore if fails
-			console.error("Error fetching cline generation details:", error)
 		}
+		return undefined
 	}
 
 	getModel(): { id: string; info: ModelInfo } {
diff --git a/src/api/providers/openrouter.ts b/src/api/providers/openrouter.ts
@@ -2,17 +2,17 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import axios from "axios"
 import delay from "delay"
 import OpenAI from "openai"
-import { withRetry } from "../retry"
 import { ApiHandler } from "../"
 import { ApiHandlerOptions, ModelInfo, openRouterDefaultModelId, openRouterDefaultModelInfo } from "../../shared/api"
-import { streamOpenRouterFormatRequest } from "../transform/openrouter-stream"
-import { ApiStream } from "../transform/stream"
-import { convertToR1Format } from "../transform/r1-format"
+import { withRetry } from "../retry"
+import { createOpenRouterStream } from "../transform/openrouter-stream"
+import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
 import { OpenRouterErrorResponse } from "./types"
 
 export class OpenRouterHandler implements ApiHandler {
 	private options: ApiHandlerOptions
 	private client: OpenAI
+	lastGenerationId?: string
 
 	constructor(options: ApiHandlerOptions) {
 		this.options = options
@@ -28,23 +28,63 @@ export class OpenRouterHandler implements ApiHandler {
 
 	@withRetry()
 	async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
-		const model = this.getModel()
-		const genId = yield* streamOpenRouterFormatRequest(
+		this.lastGenerationId = undefined
+
+		const stream = await createOpenRouterStream(
 			this.client,
 			systemPrompt,
 			messages,
-			model,
+			this.getModel(),
 			this.options.o3MiniReasoningEffort,
 			this.options.thinkingBudgetTokens,
 		)
 
-		if (genId) {
+		for await (const chunk of stream) {
+			// openrouter returns an error object instead of the openai sdk throwing an error
+			if ("error" in chunk) {
+				const error = chunk.error as OpenRouterErrorResponse["error"]
+				console.error(`OpenRouter API Error: ${error?.code} - ${error?.message}`)
+				// Include metadata in the error message if available
+				const metadataStr = error.metadata ? `\nMetadata: ${JSON.stringify(error.metadata, null, 2)}` : ""
+				throw new Error(`OpenRouter API Error ${error.code}: ${error.message}${metadataStr}`)
+			}
+
+			if (!this.lastGenerationId && chunk.id) {
+				this.lastGenerationId = chunk.id
+			}
+
+			const delta = chunk.choices[0]?.delta
+			if (delta?.content) {
+				yield {
+					type: "text",
+					text: delta.content,
+				}
+			}
+
+			// Reasoning tokens are returned separately from the content
+			if ("reasoning" in delta && delta.reasoning) {
+				yield {
+					type: "reasoning",
+					// @ts-ignore-next-line
+					reasoning: delta.reasoning,
+				}
+			}
+		}
+
+		const apiStreamUsage = await this.getApiStreamUsage()
+		if (apiStreamUsage) {
+			yield apiStreamUsage
+		}
+	}
+
+	async getApiStreamUsage(): Promise<ApiStreamUsageChunk | undefined> {
+		if (this.lastGenerationId) {
 			await delay(500) // FIXME: necessary delay to ensure generation endpoint is ready
 			try {
-				const generationIterator = this.fetchGenerationDetails(genId)
+				const generationIterator = this.fetchGenerationDetails(this.lastGenerationId)
 				const generation = (await generationIterator.next()).value
 				// console.log("OpenRouter generation details:", generation)
-				yield {
+				return {
 					type: "usage",
 					// cacheWriteTokens: 0,
 					// cacheReadTokens: 0,
@@ -58,6 +98,7 @@ export class OpenRouterHandler implements ApiHandler {
 				console.error("Error fetching OpenRouter generation details:", error)
 			}
 		}
+		return undefined
 	}
 
 	@withRetry({ maxRetries: 4, baseDelay: 250, maxDelay: 1000, retryAllErrors: true })
@@ -68,7 +109,7 @@ export class OpenRouterHandler implements ApiHandler {
 				headers: {
 					Authorization: `Bearer ${this.options.openRouterApiKey}`,
 				},
-				timeout: 5_000, // this request hangs sometimes
+				timeout: 15_000, // this request hangs sometimes
 			})
 			yield response.data?.data
 		} catch (error) {
diff --git a/src/api/transform/openrouter-stream.ts b/src/api/transform/openrouter-stream.ts
@@ -6,14 +6,14 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
 import { OpenRouterErrorResponse } from "../providers/types"
 
-export async function* streamOpenRouterFormatRequest(
+export async function createOpenRouterStream(
 	client: OpenAI,
 	systemPrompt: string,
 	messages: Anthropic.Messages.MessageParam[],
 	model: { id: string; info: ModelInfo },
 	o3MiniReasoningEffort?: string,
 	thinkingBudgetTokens?: number,
-): AsyncGenerator<ApiStreamChunk, string | undefined, unknown> {
+) {
 	// Convert Anthropic messages to OpenAI format
 	let openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
 		{ role: "system", content: systemPrompt },
@@ -147,39 +147,5 @@ export async function* streamOpenRouterFormatRequest(
 		...(reasoning ? { reasoning } : {}),
 	})
 
-	let genId: string | undefined
-
-	for await (const chunk of stream) {
-		// openrouter returns an error object instead of the openai sdk throwing an error
-		if ("error" in chunk) {
-			const error = chunk.error as OpenRouterErrorResponse["error"]
-			console.error(`OpenRouter API Error: ${error?.code} - ${error?.message}`)
-			// Include metadata in the error message if available
-			const metadataStr = error.metadata ? `\nMetadata: ${JSON.stringify(error.metadata, null, 2)}` : ""
-			throw new Error(`OpenRouter API Error ${error.code}: ${error.message}${metadataStr}`)
-		}
-
-		if (!genId && chunk.id) {
-			genId = chunk.id
-		}
-
-		const delta = chunk.choices[0]?.delta
-		if (delta?.content) {
-			yield {
-				type: "text",
-				text: delta.content,
-			}
-		}
-
-		// Reasoning tokens are returned separately from the content
-		if ("reasoning" in delta && delta.reasoning) {
-			yield {
-				type: "reasoning",
-				// @ts-ignore-next-line
-				reasoning: delta.reasoning,
-			}
-		}
-	}
-
-	return genId
+	return stream
 }
diff --git a/src/core/Cline.ts b/src/core/Cline.ts
@@ -3218,13 +3218,15 @@ export class Cline {
 			let assistantMessage = ""
 			let reasoningMessage = ""
 			this.isStreaming = true
+			let didReceiveUsageChunk = false
 			try {
 				for await (const chunk of stream) {
 					if (!chunk) {
 						continue
 					}
 					switch (chunk.type) {
 						case "usage":
+							didReceiveUsageChunk = true
 							inputTokens += chunk.inputTokens
 							outputTokens += chunk.outputTokens
 							cacheWriteTokens += chunk.cacheWriteTokens ?? 0
@@ -3294,6 +3296,23 @@ export class Cline {
 				this.isStreaming = false
 			}
 
+			// OpenRouter/Cline may not return token usage as part of the stream (since it may abort early), so we fetch after the stream is finished
+			// (updateApiReq below will update the api_req_started message with the usage details. we do this async so it updates the api_req_started message in the background)
+			if (!didReceiveUsageChunk) {
+				this.api.getApiStreamUsage?.().then(async (apiStreamUsage) => {
+					if (apiStreamUsage) {
+						inputTokens += apiStreamUsage.inputTokens
+						outputTokens += apiStreamUsage.outputTokens
+						cacheWriteTokens += apiStreamUsage.cacheWriteTokens ?? 0
+						cacheReadTokens += apiStreamUsage.cacheReadTokens ?? 0
+						totalCost = apiStreamUsage.totalCost
+					}
+					updateApiReqMsg()
+					await this.saveClineMessages()
+					await this.providerRef.deref()?.postStateToWebview()
+				})
+			}
+
 			// need to call here in case the stream was aborted
 			if (this.abort) {
 				throw new Error("Cline instance aborted")
