Gemini OpenRouter caching

Author: cte

src/api/providers/anthropic-vertex.ts

@@ -3,13 +3,14 @@ import { AnthropicVertex } from "@anthropic-ai/vertex-sdk"
 import { GoogleAuth, JWTInput } from "google-auth-library"
 
 import { ApiHandlerOptions, ModelInfo, vertexDefaultModelId, VertexModelId, vertexModels } from "../../shared/api"
-import { ApiStream } from "../transform/stream"
 import { safeJsonParse } from "../../shared/safeJsonParse"
 
+import { ApiStream } from "../transform/stream"
+import { addCacheBreakpoints } from "../transform/caching/vertex"
+
 import { getModelParams, SingleCompletionHandler } from "../index"
-import { BaseProvider } from "./base-provider"
 import { ANTHROPIC_DEFAULT_MAX_TOKENS } from "./constants"
-import { formatMessageForCache } from "../transform/vertex-caching"
+import { BaseProvider } from "./base-provider"
 
 // https://docs.anthropic.com/en/api/claude-on-vertex-ai
 export class AnthropicVertexHandler extends BaseProvider implements SingleCompletionHandler {
@@ -57,16 +58,6 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple
 			thinking,
 		} = this.getModel()
 
-		// Find indices of user messages that we want to cache
-		// We only cache the last two user messages to stay within the 4-block limit
-		// (1 block for system + 1 block each for last two user messages = 3 total)
-		const userMsgIndices = supportsPromptCache
-			? messages.reduce((acc, msg, i) => (msg.role === "user" ? [...acc, i] : acc), [] as number[])
-			: []
-
-		const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
-		const secondLastMsgUserIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
-
 		/**
 		 * Vertex API has specific limitations for prompt caching:
 		 * 1. Maximum of 4 blocks can have cache_control
@@ -89,12 +80,7 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple
 			system: supportsPromptCache
 				? [{ text: systemPrompt, type: "text" as const, cache_control: { type: "ephemeral" } }]
 				: systemPrompt,
-			messages: messages.map((message, index) => {
-				// Only cache the last two user messages.
-				const shouldCache =
-					supportsPromptCache && (index === lastUserMsgIndex || index === secondLastMsgUserIndex)
-				return formatMessageForCache(message, shouldCache)
-			}),
+			messages: supportsPromptCache ? addCacheBreakpoints(messages) : messages,
 			stream: true,
 		}
 

src/api/providers/glama.ts

@@ -3,9 +3,11 @@ import axios from "axios"
 import OpenAI from "openai"
 
 import { ApiHandlerOptions, glamaDefaultModelId, glamaDefaultModelInfo } from "../../shared/api"
+
 import { ApiStream } from "../transform/stream"
 import { convertToOpenAiMessages } from "../transform/openai-format"
-import { addCacheControlDirectives } from "../transform/caching"
+import { addCacheBreakpoints } from "../transform/caching/anthropic"
+
 import { SingleCompletionHandler } from "../index"
 import { RouterProvider } from "./router-provider"
 
@@ -37,7 +39,7 @@ export class GlamaHandler extends RouterProvider implements SingleCompletionHand
 		]
 
 		if (modelId.startsWith("anthropic/claude-3")) {
-			addCacheControlDirectives(systemPrompt, openAiMessages)
+			addCacheBreakpoints(systemPrompt, openAiMessages)
 		}
 
 		// Required by Anthropic; other providers default to max tokens allowed.

src/api/providers/openrouter.ts

@@ -11,9 +11,12 @@ import {
 	OPTIONAL_PROMPT_CACHING_MODELS,
 	REASONING_MODELS,
 } from "../../shared/api"
+
 import { convertToOpenAiMessages } from "../transform/openai-format"
 import { ApiStreamChunk } from "../transform/stream"
 import { convertToR1Format } from "../transform/r1-format"
+import { addCacheBreakpoints as addAnthropicCacheBreakpoints } from "../transform/caching/anthropic"
+import { addCacheBreakpoints as addGeminiCacheBreakpoints } from "../transform/caching/gemini"
 
 import { getModelParams, SingleCompletionHandler } from "../index"
 import { DEFAULT_HEADERS, DEEP_SEEK_DEFAULT_TEMPERATURE } from "./constants"
@@ -93,42 +96,11 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 
 		const isCacheAvailable = promptCache.supported && (!promptCache.optional || this.options.promptCachingEnabled)
 
-		// Prompt caching: https://openrouter.ai/docs/prompt-caching
-		// Now with Gemini support: https://openrouter.ai/docs/features/prompt-caching
-		// Note that we don't check the `ModelInfo` object because it is cached
-		// in the settings for OpenRouter and the value could be stale.
+		// https://openrouter.ai/docs/features/prompt-caching
 		if (isCacheAvailable) {
-			openAiMessages[0] = {
-				role: "system",
-				// @ts-ignore-next-line
-				content: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }],
-			}
-
-			// Add cache_control to the last two user messages
-			// (note: this works because we only ever add one user message at a time, but if we added multiple we'd need to mark the user message before the last assistant message)
-			const lastTwoUserMessages = openAiMessages.filter((msg) => msg.role === "user").slice(-2)
-
-			lastTwoUserMessages.forEach((msg) => {
-				if (typeof msg.content === "string") {
-					msg.content = [{ type: "text", text: msg.content }]
-				}
-
-				if (Array.isArray(msg.content)) {
-					// NOTE: This is fine since env details will always be added
-					// at the end. But if it wasn't there, and the user added a
-					// image_url type message, it would pop a text part before
-					// it and then move it after to the end.
-					let lastTextPart = msg.content.filter((part) => part.type === "text").pop()
-
-					if (!lastTextPart) {
-						lastTextPart = { type: "text", text: "..." }
-						msg.content.push(lastTextPart)
-					}
-
-					// @ts-ignore-next-line
-					lastTextPart["cache_control"] = { type: "ephemeral" }
-				}
-			})
+			modelId.startsWith("google")
+				? addGeminiCacheBreakpoints(systemPrompt, openAiMessages, 10) // Pass frequency
+				: addAnthropicCacheBreakpoints(systemPrompt, openAiMessages)
 		}
 
 		// https://openrouter.ai/docs/transforms

src/api/providers/unbound.ts

@@ -2,9 +2,11 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
 
 import { ApiHandlerOptions, unboundDefaultModelId, unboundDefaultModelInfo } from "../../shared/api"
+
 import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
 import { convertToOpenAiMessages } from "../transform/openai-format"
-import { addCacheControlDirectives } from "../transform/caching"
+import { addCacheBreakpoints } from "../transform/caching/anthropic"
+
 import { SingleCompletionHandler } from "../index"
 import { RouterProvider } from "./router-provider"
 
@@ -39,7 +41,7 @@ export class UnboundHandler extends RouterProvider implements SingleCompletionHa
 		]
 
 		if (modelId.startsWith("anthropic/claude-3")) {
-			addCacheControlDirectives(systemPrompt, openAiMessages)
+			addCacheBreakpoints(systemPrompt, openAiMessages)
 		}
 
 		// Required by Anthropic; other providers default to max tokens allowed.