feat(cloud): Add telemetry retry queue for network resilience (#7597)

* feat(cloud): Add telemetry retry queue for network resilience

- Implement RetryQueue class with workspace-scoped persistence
- Queue failed telemetry events for automatic retry
- Retry events every 60 seconds with fresh auth tokens
- FIFO eviction when queue reaches 100 events
- Persist queue across VS Code restarts

This ensures telemetry data isn't lost during network failures or temporary server issues.
Migrated from RooCodeInc/Roo-Code-Cloud#744

* fix: address PR review feedback for retry queue

- Fix retry order to use consistent FIFO processing
- Add retry limit enforcement with max retries check
- Add configurable request timeout (default 30s)
- Add comprehensive tests for retryAll() method
- Add request-max-retries-exceeded event
- Fix timeout test to avoid timing issues

* fix: resolve TypeScript errors in RetryQueue tests

* fix(cloud): Address PR feedback for telemetry retry queue

- Handle HTTP error status codes (500s, 401/403, 429) as failures that trigger retry
- Remove queuing of backfill operations since they're user-initiated
- Fix race condition in concurrent retry processing with isProcessing flag
- Add specialized retry logic for 429 with Retry-After header support
- Clean up unnecessary comments
- Add comprehensive tests for new status code handling
- Add temporary debug logs with emojis for testing

* refactor: address PR feedback for telemetry retry queue

- Remove unused X-Organization-Id header from auth header provider
- Simplify enqueue() API by removing operation parameter
- Fix error retry logic: only retry 5xx, 429, and network failures
- Stop retrying 4xx client errors (400, 401, 403, 404, 422)
- Implement queue-wide pause for 429 rate limiting
- Add auth state management integration:
  - Pause queue when not in active-session
  - Clear queue on logout or user change
  - Preserve queue when same user logs back in
- Remove debug comments
- Fix ESLint no-case-declarations error with proper block scope
- Update tests for all new behaviors

Author: daniel-lxs

packages/cloud/src/CloudService.ts

@@ -24,6 +24,7 @@ import { StaticSettingsService } from "./StaticSettingsService.js"
 import { CloudTelemetryClient as TelemetryClient } from "./TelemetryClient.js"
 import { CloudShareService } from "./CloudShareService.js"
 import { CloudAPI } from "./CloudAPI.js"
+import { RetryQueue } from "./retry-queue/index.js"
 
 type AuthStateChangedPayload = CloudServiceEvents["auth-state-changed"][0]
 type AuthUserInfoPayload = CloudServiceEvents["user-info"][0]
@@ -75,13 +76,21 @@ export class CloudService extends EventEmitter<CloudServiceEvents> implements Di
 		return this._cloudAPI
 	}
 
+	private _retryQueue: RetryQueue | null = null
+
+	public get retryQueue() {
+		return this._retryQueue
+	}
+
 	private constructor(context: ExtensionContext, log?: (...args: unknown[]) => void) {
 		super()
 
 		this.context = context
 		this.log = log || console.log
 
 		this.authStateListener = (data: AuthStateChangedPayload) => {
+			// Handle retry queue based on auth state changes
+			this.handleAuthStateChangeForRetryQueue(data)
 			this.emit("auth-state-changed", data)
 		}
 
@@ -131,7 +140,24 @@ export class CloudService extends EventEmitter<CloudServiceEvents> implements Di
 
 			this._cloudAPI = new CloudAPI(this._authService, this.log)
 
-			this._telemetryClient = new TelemetryClient(this._authService, this._settingsService)
+			// Initialize retry queue with auth header provider
+			this._retryQueue = new RetryQueue(
+				this.context,
+				undefined, // Use default config
+				this.log,
+				() => {
+					// Provide fresh auth headers for retries
+					const sessionToken = this._authService?.getSessionToken()
+					if (sessionToken) {
+						return {
+							Authorization: `Bearer ${sessionToken}`,
+						}
+					}
+					return undefined
+				},
+			)
+
+			this._telemetryClient = new TelemetryClient(this._authService, this._settingsService, this._retryQueue)
 
 			this._shareService = new CloudShareService(this._cloudAPI, this._settingsService, this.log)
 
@@ -303,6 +329,10 @@ export class CloudService extends EventEmitter<CloudServiceEvents> implements Di
 			this.settingsService.dispose()
 		}
 
+		if (this._retryQueue) {
+			this._retryQueue.dispose()
+		}
+
 		this.isInitialized = false
 	}
 
@@ -365,4 +395,67 @@ export class CloudService extends EventEmitter<CloudServiceEvents> implements Di
 	static isEnabled(): boolean {
 		return !!this._instance?.isAuthenticated()
 	}
+
+	/**
+	 * Handle auth state changes for the retry queue
+	 * - Pause queue when not in 'active-session' state
+	 * - Clear queue when user logs out or logs in as different user
+	 * - Resume queue when returning to active-session with same user
+	 */
+	private handleAuthStateChangeForRetryQueue(data: AuthStateChangedPayload): void {
+		if (!this._retryQueue) {
+			return
+		}
+
+		const newState = data.state
+		const userInfo = this.getUserInfo()
+		const newUserId = userInfo?.id
+
+		this.log(`[CloudService] Auth state changed to: ${newState}, user: ${newUserId}`)
+
+		// Handle different auth states
+		switch (newState) {
+			case "active-session": {
+				// Check if user changed (different user logged in)
+				const wasCleared = this._retryQueue.clearIfUserChanged(newUserId)
+
+				if (!wasCleared) {
+					// Same user or first login, resume the queue
+					this._retryQueue.resume()
+					this.log("[CloudService] Resuming retry queue for active session")
+				} else {
+					// Different user, queue was cleared, but we can resume processing
+					this._retryQueue.resume()
+					this.log("[CloudService] Retry queue cleared for new user, resuming processing")
+				}
+				break
+			}
+
+			case "logged-out":
+				// User is logged out, clear the queue
+				this._retryQueue.clearIfUserChanged(undefined)
+				this._retryQueue.pause()
+				this.log("[CloudService] Pausing and clearing retry queue for logged-out state")
+				break
+
+			case "initializing":
+			case "attempting-session":
+				// Transitional states, pause the queue but don't clear
+				this._retryQueue.pause()
+				this.log(`[CloudService] Pausing retry queue during ${newState}`)
+				break
+
+			case "inactive-session":
+				// Session is inactive (possibly expired), pause but don't clear
+				// The queue might resume if the session becomes active again
+				this._retryQueue.pause()
+				this.log("[CloudService] Pausing retry queue for inactive session")
+				break
+
+			default:
+				// Unknown state, pause as a safety measure
+				this._retryQueue.pause()
+				this.log(`[CloudService] Pausing retry queue for unknown state: ${newState}`)
+		}
+	}
 }

packages/cloud/src/TelemetryClient.ts

@@ -11,6 +11,7 @@ import {
 } from "@roo-code/types"
 
 import { getRooCodeApiUrl } from "./config.js"
+import type { RetryQueue } from "./retry-queue/index.js"
 
 abstract class BaseTelemetryClient implements TelemetryClient {
 	protected providerRef: WeakRef<TelemetryPropertiesProvider> | null = null
@@ -82,21 +83,21 @@ abstract class BaseTelemetryClient implements TelemetryClient {
 }
 
 export class CloudTelemetryClient extends BaseTelemetryClient {
+	private retryQueue: RetryQueue | null = null
+
 	constructor(
 		private authService: AuthService,
 		private settingsService: SettingsService,
-		debug = false,
+		retryQueue?: RetryQueue,
 	) {
-		super(
-			{
-				type: "exclude",
-				events: [TelemetryEventName.TASK_CONVERSATION_MESSAGE],
-			},
-			debug,
-		)
+		super({
+			type: "exclude",
+			events: [TelemetryEventName.TASK_CONVERSATION_MESSAGE],
+		})
+		this.retryQueue = retryQueue || null
 	}
 
-	private async fetch(path: string, options: RequestInit) {
+	private async fetch(path: string, options: RequestInit, allowQueueing = true) {
 		if (!this.authService.isAuthenticated()) {
 			return
 		}
@@ -108,18 +109,46 @@ export class CloudTelemetryClient extends BaseTelemetryClient {
 			return
 		}
 
-		const response = await fetch(`${getRooCodeApiUrl()}/api/${path}`, {
+		const url = `${getRooCodeApiUrl()}/api/${path}`
+		const fetchOptions: RequestInit = {
 			...options,
 			headers: {
 				Authorization: `Bearer ${token}`,
 				"Content-Type": "application/json",
 			},
-		})
+		}
 
-		if (!response.ok) {
-			console.error(
-				`[TelemetryClient#fetch] ${options.method} ${path} -> ${response.status} ${response.statusText}`,
-			)
+		try {
+			const response = await fetch(url, fetchOptions)
+
+			if (!response.ok) {
+				console.error(
+					`[TelemetryClient#fetch] ${options.method} ${path} -> ${response.status} ${response.statusText}`,
+				)
+
+				// Queue for retry on server errors (5xx) or rate limiting (429)
+				// Do NOT retry on client errors (4xx) except 429 - they won't succeed
+				if (this.retryQueue && allowQueueing && (response.status >= 500 || response.status === 429)) {
+					await this.retryQueue.enqueue(url, fetchOptions, "telemetry")
+				}
+			}
+
+			return response
+		} catch (error) {
+			console.error(`[TelemetryClient#fetch] Network error for ${options.method} ${path}: ${error}`)
+
+			// Queue for retry on network failures (typically TypeError with "fetch failed" message)
+			// These are transient network issues that may succeed on retry
+			if (
+				this.retryQueue &&
+				allowQueueing &&
+				error instanceof TypeError &&
+				error.message.includes("fetch failed")
+			) {
+				await this.retryQueue.enqueue(url, fetchOptions, "telemetry")
+			}
+
+			throw error
 		}
 	}
 
@@ -158,6 +187,7 @@ export class CloudTelemetryClient extends BaseTelemetryClient {
 			})
 		} catch (error) {
 			console.error(`[TelemetryClient#capture] Error sending telemetry event: ${error}`)
+			// Error is already queued for retry in the fetch method
 		}
 	}
 
@@ -199,22 +229,26 @@ export class CloudTelemetryClient extends BaseTelemetryClient {
 				)
 			}
 
-			// Custom fetch for multipart - don't set Content-Type header (let browser set it)
-			const response = await fetch(`${getRooCodeApiUrl()}/api/events/backfill`, {
+			const url = `${getRooCodeApiUrl()}/api/events/backfill`
+			const fetchOptions: RequestInit = {
 				method: "POST",
 				headers: {
 					Authorization: `Bearer ${token}`,
-					// Note: No Content-Type header - browser will set multipart/form-data with boundary
 				},
 				body: formData,
-			})
+			}
 
-			if (!response.ok) {
-				console.error(
-					`[TelemetryClient#backfillMessages] POST events/backfill -> ${response.status} ${response.statusText}`,
-				)
-			} else if (this.debug) {
-				console.info(`[TelemetryClient#backfillMessages] Successfully uploaded messages for task ${taskId}`)
+			try {
+				const response = await fetch(url, fetchOptions)
+
+				if (!response.ok) {
+					console.error(
+						`[TelemetryClient#backfillMessages] POST events/backfill -> ${response.status} ${response.statusText}`,
+					)
+				}
+			} catch (fetchError) {
+				console.error(`[TelemetryClient#backfillMessages] Network error: ${fetchError}`)
+				throw fetchError
 			}
 		} catch (error) {
 			console.error(`[TelemetryClient#backfillMessages] Error uploading messages: ${error}`)

packages/cloud/src/__tests__/TelemetryClient.test.ts

@@ -641,28 +641,6 @@ describe("TelemetryClient", () => {
 			)
 		})
 
-		it("should log debug information when debug is enabled", async () => {
-			const client = new TelemetryClient(mockAuthService, mockSettingsService, true)
-
-			const messages = [
-				{
-					ts: 1,
-					type: "say" as const,
-					say: "text" as const,
-					text: "test message",
-				},
-			]
-
-			await client.backfillMessages(messages, "test-task-id")
-
-			expect(console.info).toHaveBeenCalledWith(
-				"[TelemetryClient#backfillMessages] Uploading 1 messages for task test-task-id",
-			)
-			expect(console.info).toHaveBeenCalledWith(
-				"[TelemetryClient#backfillMessages] Successfully uploaded messages for task test-task-id",
-			)
-		})
-
 		it("should handle empty messages array", async () => {
 			const client = new TelemetryClient(mockAuthService, mockSettingsService)
 

packages/cloud/src/index.ts

@@ -3,3 +3,6 @@ export * from "./config.js"
 export { CloudService } from "./CloudService.js"
 
 export { BridgeOrchestrator } from "./bridge/index.js"
+
+export { RetryQueue } from "./retry-queue/index.js"
+export type { QueuedRequest, QueueStats, RetryQueueConfig, RetryQueueEvents } from "./retry-queue/index.js"