Add isCacheAvailable to OpenRouter provider

Author: cte

src/api/providers/gemini.ts

@@ -54,7 +54,6 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 
 		let uncachedContent: Content[] | undefined = undefined
 		let cachedContent: string | undefined = undefined
-		let cacheWriteTokens: number | undefined = undefined
 
 		// The minimum input token count for context caching is 4,096.
 		// For a basic approximation we assume 4 characters per token.
@@ -67,6 +66,8 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 			cacheKey &&
 			contentsLength > 4 * CONTEXT_CACHE_TOKEN_MINIMUM
 
+		let cacheWrite = false
+
 		if (isCacheAvailable) {
 			const cacheEntry = this.contentCaches.get<CacheEntry>(cacheKey)
 
@@ -97,9 +98,8 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 
 						if (name) {
 							this.contentCaches.set<CacheEntry>(cacheKey, { key: name, count: contents.length })
-							cacheWriteTokens = usageMetadata?.totalTokenCount ?? 0
 							console.log(
-								`[GeminiHandler] cached ${contents.length} messages (${cacheWriteTokens} tokens) in ${Date.now() - timestamp}ms`,
+								`[GeminiHandler] cached ${contents.length} messages (${usageMetadata?.totalTokenCount ?? "-"} tokens) in ${Date.now() - timestamp}ms`,
 							)
 						}
 					})
@@ -109,6 +109,8 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 					.finally(() => {
 						this.isCacheBusy = false
 					})
+
+				cacheWrite = true
 			}
 		}
 
@@ -146,27 +148,24 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 		if (lastUsageMetadata) {
 			const inputTokens = lastUsageMetadata.promptTokenCount ?? 0
 			const outputTokens = lastUsageMetadata.candidatesTokenCount ?? 0
+			const cacheWriteTokens = cacheWrite ? inputTokens : 0
 			const cacheReadTokens = lastUsageMetadata.cachedContentTokenCount
 			const reasoningTokens = lastUsageMetadata.thoughtsTokenCount
 
-			const totalCost = isCacheUsed
-				? this.calculateCost({
-						info,
-						inputTokens,
-						outputTokens,
-						cacheWriteTokens,
-						cacheReadTokens,
-					})
-				: undefined
-
 			yield {
 				type: "usage",
 				inputTokens,
 				outputTokens,
 				cacheWriteTokens,
 				cacheReadTokens,
 				reasoningTokens,
-				totalCost,
+				totalCost: this.calculateCost({
+					info,
+					inputTokens,
+					outputTokens,
+					cacheWriteTokens,
+					cacheReadTokens,
+				}),
 			}
 		}
 	}
@@ -250,8 +249,8 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 		info,
 		inputTokens,
 		outputTokens,
-		cacheWriteTokens,
-		cacheReadTokens,
+		cacheWriteTokens = 0,
+		cacheReadTokens = 0,
 	}: {
 		info: ModelInfo
 		inputTokens: number
@@ -281,21 +280,32 @@ export class GeminiHandler extends BaseProvider implements SingleCompletionHandl
 			}
 		}
 
-		let inputTokensCost = inputPrice * (inputTokens / 1_000_000)
-		let outputTokensCost = outputPrice * (outputTokens / 1_000_000)
-		let cacheWriteCost = 0
-		let cacheReadCost = 0
+		// Subtract the cached input tokens from the total input tokens.
+		const uncachedInputTokens = inputTokens - cacheReadTokens
 
-		if (cacheWriteTokens) {
-			cacheWriteCost = cacheWritesPrice * (cacheWriteTokens / 1_000_000) * (CACHE_TTL / 60)
+		let cacheWriteCost =
+			cacheWriteTokens > 0 ? cacheWritesPrice * (cacheWriteTokens / 1_000_000) * (CACHE_TTL / 60) : 0
+		let cacheReadCost = cacheReadTokens > 0 ? cacheReadsPrice * (cacheReadTokens / 1_000_000) : 0
+
+		const inputTokensCost = inputPrice * (uncachedInputTokens / 1_000_000)
+		const outputTokensCost = outputPrice * (outputTokens / 1_000_000)
+		const totalCost = inputTokensCost + outputTokensCost + cacheWriteCost + cacheReadCost
+
+		const trace: Record<string, { price: number; tokens: number; cost: number }> = {
+			input: { price: inputPrice, tokens: uncachedInputTokens, cost: inputTokensCost },
+			output: { price: outputPrice, tokens: outputTokens, cost: outputTokensCost },
 		}
 
-		if (cacheReadTokens) {
-			const uncachedReadTokens = inputTokens - cacheReadTokens
-			cacheReadCost = cacheReadsPrice * (cacheReadTokens / 1_000_000)
-			inputTokensCost = inputPrice * (uncachedReadTokens / 1_000_000)
+		if (cacheWriteTokens > 0) {
+			trace.cacheWrite = { price: cacheWritesPrice, tokens: cacheWriteTokens, cost: cacheWriteCost }
 		}
 
-		return inputTokensCost + outputTokensCost + cacheWriteCost + cacheReadCost
+		if (cacheReadTokens > 0) {
+			trace.cacheRead = { price: cacheReadsPrice, tokens: cacheReadTokens, cost: cacheReadCost }
+		}
+
+		// console.log(`[GeminiHandler] calculateCost -> ${totalCost}`, trace)
+
+		return totalCost
 	}
 }

src/api/providers/openrouter.ts

@@ -65,7 +65,7 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 		systemPrompt: string,
 		messages: Anthropic.Messages.MessageParam[],
 	): AsyncGenerator<ApiStreamChunk> {
-		let { id: modelId, maxTokens, thinking, temperature, topP, reasoningEffort, info } = this.getModel()
+		let { id: modelId, maxTokens, thinking, temperature, topP, reasoningEffort, promptCache } = this.getModel()
 
 		// Convert Anthropic messages to OpenAI format.
 		let openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
@@ -78,11 +78,13 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 			openAiMessages = convertToR1Format([{ role: "user", content: systemPrompt }, ...messages])
 		}
 
+		const isCacheAvailable = promptCache.supported && (!promptCache.optional || this.options.promptCachingEnabled)
+
 		// Prompt caching: https://openrouter.ai/docs/prompt-caching
 		// Now with Gemini support: https://openrouter.ai/docs/features/prompt-caching
 		// Note that we don't check the `ModelInfo` object because it is cached
 		// in the settings for OpenRouter and the value could be stale.
-		if (PROMPT_CACHING_MODELS.has(modelId)) {
+		if (isCacheAvailable) {
 			openAiMessages[0] = {
 				role: "system",
 				// @ts-ignore-next-line
@@ -195,6 +197,10 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 			info,
 			...getModelParams({ options: this.options, model: info, defaultTemperature }),
 			topP,
+			promptCache: {
+				supported: PROMPT_CACHING_MODELS.has(id),
+				optional: PROMPT_CACHING_MODELS.has(id),
+			},
 		}
 	}