refactor: unify cache control with centralized breakpoints and universal provider options (#11426)

Author: hannesrudolph

src/api/index.ts

@@ -88,6 +88,13 @@ export interface ApiHandlerCreateMessageMetadata {
 	 * Only applies to providers that support function calling restrictions (e.g., Gemini).
 	 */
 	allowedFunctionNames?: string[]
+	/** Provider-specific options for tool definitions (e.g. cache control). */
+	toolProviderOptions?: Record<string, Record<string, unknown>>
+	/** Provider-specific options for the system prompt (e.g. cache control).
+	 * Cache-aware providers use this to inject the system prompt as a cached
+	 * system message, since AI SDK v6 does not support providerOptions on the
+	 * `system` string parameter. */
+	systemProviderOptions?: Record<string, Record<string, unknown>>
 }
 
 export interface ApiHandler {

src/api/providers/__tests__/anthropic.spec.ts

@@ -399,7 +399,7 @@ describe("AnthropicHandler", () => {
 			expect(endChunk).toBeDefined()
 		})
 
-		it("should pass system prompt via system param with systemProviderOptions for cache control", async () => {
+		it("should pass system prompt via system param when no systemProviderOptions", async () => {
 			setupStreamTextMock([{ type: "text-delta", text: "test" }])
 
 			const stream = handler.createMessage(systemPrompt, [
@@ -410,16 +410,37 @@ describe("AnthropicHandler", () => {
 				// Consume
 			}
 
-			// Verify streamText was called with system + systemProviderOptions (not as a message)
+			// Without systemProviderOptions, system prompt is passed via the system parameter
 			const callArgs = mockStreamText.mock.calls[0]![0]
 			expect(callArgs.system).toBe(systemPrompt)
-			expect(callArgs.systemProviderOptions).toEqual({
-				anthropic: { cacheControl: { type: "ephemeral" } },
-			})
 			// System prompt should NOT be in the messages array
 			const systemMessages = callArgs.messages.filter((m: any) => m.role === "system")
 			expect(systemMessages).toHaveLength(0)
 		})
+
+		it("should inject system prompt as cached system message when systemProviderOptions provided", async () => {
+			setupStreamTextMock([{ type: "text-delta", text: "test" }])
+
+			const cacheOpts = { anthropic: { cacheControl: { type: "ephemeral" } } }
+			const stream = handler.createMessage(
+				systemPrompt,
+				[{ role: "user", content: [{ type: "text" as const, text: "test" }] }],
+				{ taskId: "test-task", systemProviderOptions: cacheOpts },
+			)
+
+			for await (const _chunk of stream) {
+				// Consume
+			}
+
+			// With systemProviderOptions, system prompt is injected as messages[0]
+			const callArgs = mockStreamText.mock.calls[0]![0]
+			expect(callArgs.system).toBeUndefined()
+			// System prompt should be the first message with providerOptions
+			const systemMessages = callArgs.messages.filter((m: any) => m.role === "system")
+			expect(systemMessages).toHaveLength(1)
+			expect(systemMessages[0].content).toBe(systemPrompt)
+			expect(systemMessages[0].providerOptions).toEqual(cacheOpts)
+		})
 	})
 
 	describe("completePrompt", () => {

src/api/providers/__tests__/minimax.spec.ts

@@ -338,16 +338,12 @@ describe("MiniMaxHandler", () => {
 
 			expect(mockMergeEnvironmentDetailsForMiniMax).toHaveBeenCalledWith(messages)
 			const callArgs = mockStreamText.mock.calls[0]?.[0]
+			// Cache control is now applied centrally in Task.ts, not per-provider
 			expect(callArgs.messages).toEqual(
 				expect.arrayContaining([
 					expect.objectContaining({
 						role: "user",
 						content: [{ type: "text", text: "Merged message" }],
-						providerOptions: {
-							anthropic: {
-								cacheControl: { type: "ephemeral" },
-							},
-						},
 					}),
 				]),
 			)

src/api/providers/anthropic-vertex.ts

@@ -26,6 +26,7 @@ import {
 	handleAiSdkError,
 	yieldResponseMessage,
 } from "../transform/ai-sdk"
+import { applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints"
 import { calculateApiCostAnthropic } from "../../shared/cost"
 
 import { DEFAULT_HEADERS } from "./constants"
@@ -96,6 +97,7 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple
 		// Convert tools to AI SDK format
 		const openAiTools = this.convertToolsForOpenAI(metadata?.tools)
 		const aiSdkTools = convertToolsForAiSdk(openAiTools) as ToolSet | undefined
+		applyToolCacheOptions(aiSdkTools as Parameters<typeof applyToolCacheOptions>[0], metadata?.toolProviderOptions)
 
 		// Build Anthropic provider options
 		const anthropicProviderOptions: Record<string, unknown> = {}
@@ -119,45 +121,18 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple
 			anthropicProviderOptions.disableParallelToolUse = true
 		}
 
-		/**
-		 * Vertex API has specific limitations for prompt caching:
-		 * 1. Maximum of 4 blocks can have cache_control
-		 * 2. Only text blocks can be cached (images and other content types cannot)
-		 * 3. Cache control can only be applied to user messages, not assistant messages
-		 *
-		 * Our caching strategy:
-		 * - Cache the system prompt (1 block)
-		 * - Cache the last text block of the second-to-last user message (1 block)
-		 * - Cache the last text block of the last user message (1 block)
-		 * This ensures we stay under the 4-block limit while maintaining effective caching
-		 * for the most relevant context.
-		 */
-		const cacheProviderOption = { anthropic: { cacheControl: { type: "ephemeral" as const } } }
-
-		const userMsgIndices = messages.reduce(
-			(acc, msg, index) => ("role" in msg && msg.role === "user" ? [...acc, index] : acc),
-			[] as number[],
+		// Breakpoint 1: System prompt caching — inject as cached system message
+		const effectiveSystemPrompt = applySystemPromptCaching(
+			systemPrompt,
+			aiSdkMessages,
+			metadata?.systemProviderOptions,
 		)
 
-		const targetIndices = new Set<number>()
-		const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
-		const secondLastUserMsgIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
-
-		if (lastUserMsgIndex >= 0) targetIndices.add(lastUserMsgIndex)
-		if (secondLastUserMsgIndex >= 0) targetIndices.add(secondLastUserMsgIndex)
-
-		if (targetIndices.size > 0) {
-			this.applyCacheControlToAiSdkMessages(messages as ModelMessage[], targetIndices, cacheProviderOption)
-		}
-
 		// Build streamText request
 		// Cast providerOptions to any to bypass strict JSONObject typing — the AI SDK accepts the correct runtime values
 		const requestOptions: Parameters<typeof streamText>[0] = {
 			model: this.provider(modelConfig.id),
-			system: systemPrompt,
-			...({
-				systemProviderOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
-			} as Record<string, unknown>),
+			system: effectiveSystemPrompt,
 			messages: aiSdkMessages,
 			temperature: modelConfig.temperature,
 			maxOutputTokens: modelConfig.maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS,
@@ -216,12 +191,19 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple
 		const inputTokens = usage.inputTokens ?? 0
 		const outputTokens = usage.outputTokens ?? 0
 
-		// Extract cache metrics from Anthropic's providerMetadata
+		// Extract cache metrics from Anthropic's providerMetadata.
+		// In @ai-sdk/anthropic v3.0.38+, cacheReadInputTokens may only exist at
+		// usage.cache_read_input_tokens rather than the top-level property.
 		const anthropicMeta = providerMetadata?.anthropic as
-			| { cacheCreationInputTokens?: number; cacheReadInputTokens?: number }
+			| {
+					cacheCreationInputTokens?: number
+					cacheReadInputTokens?: number
+					usage?: { cache_read_input_tokens?: number }
+			  }
 			| undefined
 		const cacheWriteTokens = anthropicMeta?.cacheCreationInputTokens ?? 0
-		const cacheReadTokens = anthropicMeta?.cacheReadInputTokens ?? 0
+		const cacheReadTokens =
+			anthropicMeta?.cacheReadInputTokens ?? anthropicMeta?.usage?.cache_read_input_tokens ?? 0
 
 		const { totalCost } = calculateApiCostAnthropic(
 			info,
@@ -241,29 +223,6 @@ export class AnthropicVertexHandler extends BaseProvider implements SingleComple
 		}
 	}
 
-	/**
-	 * Apply cacheControl providerOptions to the correct AI SDK messages by walking
-	 * the original Anthropic messages and converted AI SDK messages in parallel.
-	 *
-	 * convertToAiSdkMessages() can split a single Anthropic user message (containing
-	 * tool_results + text) into 2 AI SDK messages (tool role + user role). This method
-	 * accounts for that split so cache control lands on the right message.
-	 */
-	private applyCacheControlToAiSdkMessages(
-		aiSdkMessages: { role: string; providerOptions?: Record<string, Record<string, unknown>> }[],
-		targetIndices: Set<number>,
-		cacheProviderOption: Record<string, Record<string, unknown>>,
-	): void {
-		for (const idx of targetIndices) {
-			if (idx >= 0 && idx < aiSdkMessages.length) {
-				aiSdkMessages[idx].providerOptions = {
-					...aiSdkMessages[idx].providerOptions,
-					...cacheProviderOption,
-				}
-			}
-		}
-	}
-
 	getModel() {
 		const modelId = this.options.apiModelId
 		let id = modelId && modelId in vertexModels ? (modelId as VertexModelId) : vertexDefaultModelId

src/api/providers/anthropic.ts

@@ -24,6 +24,7 @@ import {
 	handleAiSdkError,
 	yieldResponseMessage,
 } from "../transform/ai-sdk"
+import { applyToolCacheOptions, applySystemPromptCaching } from "../transform/cache-breakpoints"
 import { calculateApiCostAnthropic } from "../../shared/cost"
 
 import { DEFAULT_HEADERS } from "./constants"
@@ -82,6 +83,7 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
 		// Convert tools to AI SDK format
 		const openAiTools = this.convertToolsForOpenAI(metadata?.tools)
 		const aiSdkTools = convertToolsForAiSdk(openAiTools) as ToolSet | undefined
+		applyToolCacheOptions(aiSdkTools as Parameters<typeof applyToolCacheOptions>[0], metadata?.toolProviderOptions)
 
 		// Build Anthropic provider options
 		const anthropicProviderOptions: Record<string, unknown> = {}
@@ -105,34 +107,20 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
 			anthropicProviderOptions.disableParallelToolUse = true
 		}
 
-		// Apply cache control to user messages
-		// Strategy: cache the last 2 user messages (write-to-cache + read-from-cache)
-		const cacheProviderOption = { anthropic: { cacheControl: { type: "ephemeral" as const } } }
-
-		const userMsgIndices = messages.reduce(
-			(acc, msg, index) => ("role" in msg && msg.role === "user" ? [...acc, index] : acc),
-			[] as number[],
+		// Breakpoint 1: System prompt caching — inject as cached system message
+		// AI SDK v6 does not support providerOptions on the system string parameter,
+		// so cache-aware providers convert it to a system message with providerOptions.
+		const effectiveSystemPrompt = applySystemPromptCaching(
+			systemPrompt,
+			aiSdkMessages,
+			metadata?.systemProviderOptions,
 		)
 
-		const targetIndices = new Set<number>()
-		const lastUserMsgIndex = userMsgIndices[userMsgIndices.length - 1] ?? -1
-		const secondLastUserMsgIndex = userMsgIndices[userMsgIndices.length - 2] ?? -1
-
-		if (lastUserMsgIndex >= 0) targetIndices.add(lastUserMsgIndex)
-		if (secondLastUserMsgIndex >= 0) targetIndices.add(secondLastUserMsgIndex)
-
-		if (targetIndices.size > 0) {
-			this.applyCacheControlToAiSdkMessages(messages as ModelMessage[], targetIndices, cacheProviderOption)
-		}
-
 		// Build streamText request
 		// Cast providerOptions to any to bypass strict JSONObject typing — the AI SDK accepts the correct runtime values
 		const requestOptions: Parameters<typeof streamText>[0] = {
 			model: this.provider(modelConfig.id),
-			system: systemPrompt,
-			...({
-				systemProviderOptions: { anthropic: { cacheControl: { type: "ephemeral" } } },
-			} as Record<string, unknown>),
+			system: effectiveSystemPrompt,
 			messages: aiSdkMessages,
 			temperature: modelConfig.temperature,
 			maxOutputTokens: modelConfig.maxTokens ?? ANTHROPIC_DEFAULT_MAX_TOKENS,
@@ -191,12 +179,19 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
 		const inputTokens = usage.inputTokens ?? 0
 		const outputTokens = usage.outputTokens ?? 0
 
-		// Extract cache metrics from Anthropic's providerMetadata
+		// Extract cache metrics from Anthropic's providerMetadata.
+		// In @ai-sdk/anthropic v3.0.38+, cacheReadInputTokens may only exist at
+		// usage.cache_read_input_tokens rather than the top-level property.
 		const anthropicMeta = providerMetadata?.anthropic as
-			| { cacheCreationInputTokens?: number; cacheReadInputTokens?: number }
+			| {
+					cacheCreationInputTokens?: number
+					cacheReadInputTokens?: number
+					usage?: { cache_read_input_tokens?: number }
+			  }
 			| undefined
 		const cacheWriteTokens = anthropicMeta?.cacheCreationInputTokens ?? 0
-		const cacheReadTokens = anthropicMeta?.cacheReadInputTokens ?? 0
+		const cacheReadTokens =
+			anthropicMeta?.cacheReadInputTokens ?? anthropicMeta?.usage?.cache_read_input_tokens ?? 0
 
 		const { totalCost } = calculateApiCostAnthropic(
 			info,
@@ -216,29 +211,6 @@ export class AnthropicHandler extends BaseProvider implements SingleCompletionHa
 		}
 	}
 
-	/**
-	 * Apply cacheControl providerOptions to the correct AI SDK messages by walking
-	 * the original Anthropic messages and converted AI SDK messages in parallel.
-	 *
-	 * convertToAiSdkMessages() can split a single Anthropic user message (containing
-	 * tool_results + text) into 2 AI SDK messages (tool role + user role). This method
-	 * accounts for that split so cache control lands on the right message.
-	 */
-	private applyCacheControlToAiSdkMessages(
-		aiSdkMessages: { role: string; providerOptions?: Record<string, Record<string, unknown>> }[],
-		targetIndices: Set<number>,
-		cacheProviderOption: Record<string, Record<string, unknown>>,
-	): void {
-		for (const idx of targetIndices) {
-			if (idx >= 0 && idx < aiSdkMessages.length) {
-				aiSdkMessages[idx].providerOptions = {
-					...aiSdkMessages[idx].providerOptions,
-					...cacheProviderOption,
-				}
-			}
-		}
-	}
-
 	getModel() {
 		const modelId = this.options.apiModelId
 		let id = modelId && modelId in anthropicModels ? (modelId as AnthropicModelId) : anthropicDefaultModelId

src/api/providers/azure.ts

@@ -13,6 +13,7 @@ import {
 	mapToolChoice,
 	handleAiSdkError,
 } from "../transform/ai-sdk"
+import { applyToolCacheOptions } from "../transform/cache-breakpoints"
 import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
 import { getModelParams } from "../transform/model-params"
 
@@ -144,11 +145,12 @@ export class AzureHandler extends BaseProvider implements SingleCompletionHandle
 		// Convert tools to OpenAI format first, then to AI SDK format
 		const openAiTools = this.convertToolsForOpenAI(metadata?.tools)
 		const aiSdkTools = convertToolsForAiSdk(openAiTools) as ToolSet | undefined
+		applyToolCacheOptions(aiSdkTools as Parameters<typeof applyToolCacheOptions>[0], metadata?.toolProviderOptions)
 
 		// Build the request options
 		const requestOptions: Parameters<typeof streamText>[0] = {
 			model: languageModel,
-			system: systemPrompt,
+			system: systemPrompt || undefined,
 			messages: aiSdkMessages,
 			temperature: this.options.modelTemperature ?? temperature ?? AZURE_DEFAULT_TEMPERATURE,
 			maxOutputTokens: this.getMaxOutputTokens(),

src/api/providers/baseten.ts

@@ -13,6 +13,7 @@ import {
 	mapToolChoice,
 	handleAiSdkError,
 } from "../transform/ai-sdk"
+import { applyToolCacheOptions } from "../transform/cache-breakpoints"
 import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
 import { getModelParams } from "../transform/model-params"
 
@@ -105,10 +106,11 @@ export class BasetenHandler extends BaseProvider implements SingleCompletionHand
 
 		const openAiTools = this.convertToolsForOpenAI(metadata?.tools)
 		const aiSdkTools = convertToolsForAiSdk(openAiTools) as ToolSet | undefined
+		applyToolCacheOptions(aiSdkTools as Parameters<typeof applyToolCacheOptions>[0], metadata?.toolProviderOptions)
 
 		const requestOptions: Parameters<typeof streamText>[0] = {
 			model: languageModel,
-			system: systemPrompt,
+			system: systemPrompt || undefined,
 			messages: aiSdkMessages,
 			temperature: this.options.modelTemperature ?? temperature ?? BASETEN_DEFAULT_TEMPERATURE,
 			maxOutputTokens: this.getMaxOutputTokens(),