feat: add API error telemetry to OpenRouter provider (#9953)

Co-authored-by: Roo Code <[email protected]>

Author: daniel-lxs

packages/cloud/src/TelemetryClient.ts

@@ -69,6 +69,10 @@ abstract class BaseTelemetryClient implements TelemetryClient {
 
 	public abstract capture(event: TelemetryEvent): Promise<void>
 
+	public captureException(_error: Error, _additionalProperties?: Record<string, unknown>): void {
+		// No-op - exception capture is only supported by PostHog
+	}
+
 	public setProvider(provider: TelemetryPropertiesProvider): void {
 		this.providerRef = new WeakRef(provider)
 	}

packages/telemetry/src/BaseTelemetryClient.ts

@@ -59,6 +59,8 @@ export abstract class BaseTelemetryClient implements TelemetryClient {
 
 	public abstract capture(event: TelemetryEvent): Promise<void>
 
+	public abstract captureException(error: Error, additionalProperties?: Record<string, unknown>): void
+
 	public setProvider(provider: TelemetryPropertiesProvider): void {
 		this.providerRef = new WeakRef(provider)
 	}

packages/telemetry/src/PostHogTelemetryClient.ts

@@ -61,6 +61,22 @@ export class PostHogTelemetryClient extends BaseTelemetryClient {
 		})
 	}
 
+	public override captureException(error: Error, additionalProperties?: Record<string, unknown>): void {
+		if (!this.isTelemetryEnabled()) {
+			if (this.debug) {
+				console.info(`[PostHogTelemetryClient#captureException] Skipping exception: ${error.message}`)
+			}
+
+			return
+		}
+
+		if (this.debug) {
+			console.info(`[PostHogTelemetryClient#captureException] ${error.message}`)
+		}
+
+		this.client.captureException(error, this.distinctId, additionalProperties)
+	}
+
 	/**
 	 * Updates the telemetry state based on user preferences and VSCode settings.
 	 * Only enables telemetry if both VSCode global telemetry is enabled and

packages/telemetry/src/TelemetryService.ts

@@ -65,6 +65,19 @@ export class TelemetryService {
 		this.clients.forEach((client) => client.capture({ event: eventName, properties }))
 	}
 
+	/**
+	 * Captures an exception using PostHog's error tracking
+	 * @param error The error to capture
+	 * @param additionalProperties Additional properties to include with the exception
+	 */
+	public captureException(error: Error, additionalProperties?: Record<string, unknown>): void {
+		if (!this.isReady) {
+			return
+		}
+
+		this.clients.forEach((client) => client.captureException(error, additionalProperties))
+	}
+
 	public captureTaskCreated(taskId: string): void {
 		this.captureEvent(TelemetryEventName.TASK_CREATED, { taskId })
 	}

packages/types/src/telemetry.ts

@@ -262,7 +262,44 @@ export interface TelemetryClient {
 
 	setProvider(provider: TelemetryPropertiesProvider): void
 	capture(options: TelemetryEvent): Promise<void>
+	captureException(error: Error, additionalProperties?: Record<string, unknown>): void
 	updateTelemetryState(isOptedIn: boolean): void
 	isTelemetryEnabled(): boolean
 	shutdown(): Promise<void>
 }
+
+/**
+ * Expected API error codes that should not be reported to telemetry.
+ * These are normal/expected errors that users can't do much about.
+ */
+export const EXPECTED_API_ERROR_CODES = new Set([
+	429, // Rate limit - expected when hitting API limits
+])
+
+/**
+ * Helper to check if an API error should be reported to telemetry.
+ * Filters out expected errors like rate limits.
+ * @param errorCode - The HTTP error code (if available)
+ * @returns true if the error should be reported, false if it should be filtered out
+ */
+export function shouldReportApiErrorToTelemetry(errorCode?: number): boolean {
+	if (errorCode === undefined) return true
+	return !EXPECTED_API_ERROR_CODES.has(errorCode)
+}
+
+/**
+ * Generic API provider error class for structured error tracking via PostHog.
+ * Can be reused by any API provider.
+ */
+export class ApiProviderError extends Error {
+	constructor(
+		message: string,
+		public readonly provider: string,
+		public readonly modelId: string,
+		public readonly operation: string,
+		public readonly errorCode?: number,
+	) {
+		super(message)
+		this.name = "ApiProviderError"
+	}
+}

src/api/providers/__tests__/openrouter.spec.ts

@@ -9,10 +9,21 @@ import OpenAI from "openai"
 import { OpenRouterHandler } from "../openrouter"
 import { ApiHandlerOptions } from "../../../shared/api"
 import { Package } from "../../../shared/package"
+import { ApiProviderError } from "@roo-code/types"
 
 // Mock dependencies
 vitest.mock("openai")
 vitest.mock("delay", () => ({ default: vitest.fn(() => Promise.resolve()) }))
+
+// Mock TelemetryService
+const mockCaptureException = vitest.fn()
+vitest.mock("@roo-code/telemetry", () => ({
+	TelemetryService: {
+		instance: {
+			captureException: (...args: unknown[]) => mockCaptureException(...args),
+		},
+	},
+}))
 vitest.mock("../fetchers/modelCache", () => ({
 	getModels: vitest.fn().mockImplementation(() => {
 		return Promise.resolve({
@@ -267,7 +278,7 @@ describe("OpenRouterHandler", () => {
 			)
 		})
 
-		it("handles API errors", async () => {
+		it("handles API errors and captures telemetry", async () => {
 			const handler = new OpenRouterHandler(mockOptions)
 			const mockStream = {
 				async *[Symbol.asyncIterator]() {
@@ -282,6 +293,52 @@ describe("OpenRouterHandler", () => {
 
 			const generator = handler.createMessage("test", [])
 			await expect(generator.next()).rejects.toThrow("OpenRouter API Error 500: API Error")
+
+			// Verify telemetry was captured
+			expect(mockCaptureException).toHaveBeenCalledWith(expect.any(ApiProviderError), {
+				provider: "OpenRouter",
+				modelId: mockOptions.openRouterModelId,
+				operation: "createMessage",
+				errorCode: 500,
+			})
+		})
+
+		it("captures telemetry when createMessage throws an exception", async () => {
+			const handler = new OpenRouterHandler(mockOptions)
+			const mockCreate = vitest.fn().mockRejectedValue(new Error("Connection failed"))
+			;(OpenAI as any).prototype.chat = {
+				completions: { create: mockCreate },
+			} as any
+
+			const generator = handler.createMessage("test", [])
+			await expect(generator.next()).rejects.toThrow()
+
+			// Verify telemetry was captured
+			expect(mockCaptureException).toHaveBeenCalledWith(expect.any(ApiProviderError), {
+				provider: "OpenRouter",
+				modelId: mockOptions.openRouterModelId,
+				operation: "createMessage",
+			})
+		})
+
+		it("does NOT capture telemetry for 429 rate limit errors", async () => {
+			const handler = new OpenRouterHandler(mockOptions)
+			const mockStream = {
+				async *[Symbol.asyncIterator]() {
+					yield { error: { message: "Rate limit exceeded", code: 429 } }
+				},
+			}
+
+			const mockCreate = vitest.fn().mockResolvedValue(mockStream)
+			;(OpenAI as any).prototype.chat = {
+				completions: { create: mockCreate },
+			} as any
+
+			const generator = handler.createMessage("test", [])
+			await expect(generator.next()).rejects.toThrow("OpenRouter API Error 429: Rate limit exceeded")
+
+			// Verify telemetry was NOT captured for 429 errors
+			expect(mockCaptureException).not.toHaveBeenCalled()
 		})
 
 		it("yields tool_call_end events when finish_reason is tool_calls", async () => {
@@ -384,7 +441,7 @@ describe("OpenRouterHandler", () => {
 			)
 		})
 
-		it("handles API errors", async () => {
+		it("handles API errors and captures telemetry", async () => {
 			const handler = new OpenRouterHandler(mockOptions)
 			const mockError = {
 				error: {
@@ -399,16 +456,53 @@ describe("OpenRouterHandler", () => {
 			} as any
 
 			await expect(handler.completePrompt("test prompt")).rejects.toThrow("OpenRouter API Error 500: API Error")
+
+			// Verify telemetry was captured
+			expect(mockCaptureException).toHaveBeenCalledWith(expect.any(ApiProviderError), {
+				provider: "OpenRouter",
+				modelId: mockOptions.openRouterModelId,
+				operation: "completePrompt",
+				errorCode: 500,
+			})
 		})
 
-		it("handles unexpected errors", async () => {
+		it("handles unexpected errors and captures telemetry", async () => {
 			const handler = new OpenRouterHandler(mockOptions)
 			const mockCreate = vitest.fn().mockRejectedValue(new Error("Unexpected error"))
 			;(OpenAI as any).prototype.chat = {
 				completions: { create: mockCreate },
 			} as any
 
 			await expect(handler.completePrompt("test prompt")).rejects.toThrow("Unexpected error")
+
+			// Verify telemetry was captured
+			expect(mockCaptureException).toHaveBeenCalledWith(expect.any(ApiProviderError), {
+				provider: "OpenRouter",
+				modelId: mockOptions.openRouterModelId,
+				operation: "completePrompt",
+			})
+		})
+
+		it("does NOT capture telemetry for 429 rate limit errors", async () => {
+			const handler = new OpenRouterHandler(mockOptions)
+			const mockError = {
+				error: {
+					message: "Rate limit exceeded",
+					code: 429,
+				},
+			}
+
+			const mockCreate = vitest.fn().mockResolvedValue(mockError)
+			;(OpenAI as any).prototype.chat = {
+				completions: { create: mockCreate },
+			} as any
+
+			await expect(handler.completePrompt("test prompt")).rejects.toThrow(
+				"OpenRouter API Error 429: Rate limit exceeded",
+			)
+
+			// Verify telemetry was NOT captured for 429 errors
+			expect(mockCaptureException).not.toHaveBeenCalled()
 		})
 	})
 })

src/api/providers/openrouter.ts

@@ -7,7 +7,10 @@ import {
 	OPENROUTER_DEFAULT_PROVIDER_NAME,
 	OPEN_ROUTER_PROMPT_CACHING_MODELS,
 	DEEP_SEEK_DEFAULT_TEMPERATURE,
+	shouldReportApiErrorToTelemetry,
+	ApiProviderError,
 } from "@roo-code/types"
+import { TelemetryService } from "@roo-code/telemetry"
 
 import { NativeToolCallParser } from "../../core/assistant-message/NativeToolCallParser"
 
@@ -224,6 +227,15 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 		try {
 			stream = await this.client.chat.completions.create(completionParams, requestOptions)
 		} catch (error) {
+			TelemetryService.instance.captureException(
+				new ApiProviderError(
+					error instanceof Error ? error.message : String(error),
+					this.providerName,
+					modelId,
+					"createMessage",
+				),
+				{ provider: this.providerName, modelId, operation: "createMessage" },
+			)
 			throw handleOpenAIError(error, this.providerName)
 		}
 
@@ -248,6 +260,18 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 			if ("error" in chunk) {
 				const error = chunk.error as { message?: string; code?: number }
 				console.error(`OpenRouter API Error: ${error?.code} - ${error?.message}`)
+				if (shouldReportApiErrorToTelemetry(error?.code)) {
+					TelemetryService.instance.captureException(
+						new ApiProviderError(
+							error?.message ?? "Unknown error",
+							this.providerName,
+							modelId,
+							"createMessage",
+							error?.code,
+						),
+						{ provider: this.providerName, modelId, operation: "createMessage", errorCode: error?.code },
+					)
+				}
 				throw new Error(`OpenRouter API Error ${error?.code}: ${error?.message}`)
 			}
 
@@ -442,11 +466,32 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 		try {
 			response = await this.client.chat.completions.create(completionParams, requestOptions)
 		} catch (error) {
+			TelemetryService.instance.captureException(
+				new ApiProviderError(
+					error instanceof Error ? error.message : String(error),
+					this.providerName,
+					modelId,
+					"completePrompt",
+				),
+				{ provider: this.providerName, modelId, operation: "completePrompt" },
+			)
 			throw handleOpenAIError(error, this.providerName)
 		}
 
 		if ("error" in response) {
 			const error = response.error as { message?: string; code?: number }
+			if (shouldReportApiErrorToTelemetry(error?.code)) {
+				TelemetryService.instance.captureException(
+					new ApiProviderError(
+						error?.message ?? "Unknown error",
+						this.providerName,
+						modelId,
+						"completePrompt",
+						error?.code,
+					),
+					{ provider: this.providerName, modelId, operation: "completePrompt", errorCode: error?.code },
+				)
+			}
 			throw new Error(`OpenRouter API Error ${error?.code}: ${error?.message}`)
 		}
 

src/core/assistant-message/presentAssistantMessage.ts

@@ -42,8 +42,6 @@ import { Task } from "../task/Task"
 import { codebaseSearchTool } from "../tools/CodebaseSearchTool"
 import { experiments, EXPERIMENT_IDS } from "../../shared/experiments"
 import { applyDiffTool as applyDiffToolClass } from "../tools/ApplyDiffTool"
-import { isNativeProtocol } from "@roo-code/types"
-import { resolveToolProtocol } from "../../utils/resolveToolProtocol"
 
 /**
  * Processes and presents assistant message content to the user interface.

src/core/task/Task.ts

@@ -849,6 +849,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 			role: "user",
 			content: this.userMessageContent,
 		}
+
 		const userMessageWithTs = { ...userMessage, ts: Date.now() }
 		this.apiConversationHistory.push(userMessageWithTs as ApiMessage)