Use a hardcoded list

Author: cte

src/api/providers/fetchers/__tests__/openrouter.test.ts

@@ -4,7 +4,7 @@ import path from "path"
 
 import { back as nockBack } from "nock"
 
-import { getOpenRouterModels } from "../openrouter"
+import { getOpenRouterModels, modelsSupportingPromptCache } from "../openrouter"
 
 nockBack.fixtures = path.join(__dirname, "fixtures")
 nockBack.setMode("dryrun")
@@ -16,40 +16,19 @@ describe("OpenRouter API", () => {
 
 			const models = await getOpenRouterModels()
 
-			const modelsSupportingPromptCache = Object.entries(models)
-				.filter(([_, model]) => model.supportsPromptCache)
-				.map(([id, _]) => id)
-				.sort()
+			expect(
+				Object.entries(models)
+					.filter(([_, model]) => model.supportsPromptCache)
+					.map(([id, _]) => id)
+					.sort(),
+			).toEqual(Array.from(modelsSupportingPromptCache).sort())
 
-			expect(modelsSupportingPromptCache).toEqual([
-				"anthropic/claude-3-haiku",
-				"anthropic/claude-3-haiku:beta",
-				"anthropic/claude-3-opus",
-				"anthropic/claude-3-opus:beta",
-				"anthropic/claude-3-sonnet",
-				"anthropic/claude-3-sonnet:beta",
-				"anthropic/claude-3.5-haiku",
-				"anthropic/claude-3.5-haiku-20241022",
-				"anthropic/claude-3.5-haiku-20241022:beta",
-				"anthropic/claude-3.5-haiku:beta",
-				"anthropic/claude-3.5-sonnet",
-				"anthropic/claude-3.5-sonnet-20240620",
-				"anthropic/claude-3.5-sonnet-20240620:beta",
-				"anthropic/claude-3.5-sonnet:beta",
-				"anthropic/claude-3.7-sonnet",
-				"anthropic/claude-3.7-sonnet:beta",
-				"anthropic/claude-3.7-sonnet:thinking",
-				"google/gemini-2.0-flash-001",
-				"google/gemini-flash-1.5",
-				"google/gemini-flash-1.5-8b",
-			])
-
-			const modelsSupportingComputerUse = Object.entries(models)
-				.filter(([_, model]) => model.supportsComputerUse)
-				.map(([id, _]) => id)
-				.sort()
-
-			expect(modelsSupportingComputerUse).toEqual([
+			expect(
+				Object.entries(models)
+					.filter(([_, model]) => model.supportsComputerUse)
+					.map(([id, _]) => id)
+					.sort(),
+			).toEqual([
 				"anthropic/claude-3.5-sonnet",
 				"anthropic/claude-3.5-sonnet:beta",
 				"anthropic/claude-3.7-sonnet",

src/api/providers/fetchers/openrouter.ts

@@ -62,7 +62,8 @@ export async function getOpenRouterModels(options?: ApiHandlerOptions) {
 				? parseApiPrice(rawModel.pricing?.input_cache_read)
 				: undefined
 
-			const supportsPromptCache = !!cacheWritesPrice && !!cacheWritesPrice
+			// Disable prompt caching for Gemini models for now.
+			const supportsPromptCache = !!cacheWritesPrice && !!cacheWritesPrice && !rawModel.id.startsWith("google")
 
 			const modelInfo: ModelInfo = {
 				maxTokens: rawModel.top_provider?.max_completion_tokens,
@@ -109,3 +110,26 @@ export async function getOpenRouterModels(options?: ApiHandlerOptions) {
 
 	return models
 }
+
+export const modelsSupportingPromptCache = new Set([
+	"anthropic/claude-3-haiku",
+	"anthropic/claude-3-haiku:beta",
+	"anthropic/claude-3-opus",
+	"anthropic/claude-3-opus:beta",
+	"anthropic/claude-3-sonnet",
+	"anthropic/claude-3-sonnet:beta",
+	"anthropic/claude-3.5-haiku",
+	"anthropic/claude-3.5-haiku-20241022",
+	"anthropic/claude-3.5-haiku-20241022:beta",
+	"anthropic/claude-3.5-haiku:beta",
+	"anthropic/claude-3.5-sonnet",
+	"anthropic/claude-3.5-sonnet-20240620",
+	"anthropic/claude-3.5-sonnet-20240620:beta",
+	"anthropic/claude-3.5-sonnet:beta",
+	"anthropic/claude-3.7-sonnet",
+	"anthropic/claude-3.7-sonnet:beta",
+	"anthropic/claude-3.7-sonnet:thinking",
+	// "google/gemini-2.0-flash-001",
+	// "google/gemini-flash-1.5",
+	// "google/gemini-flash-1.5-8b",
+])

src/api/providers/openrouter.ts

@@ -10,6 +10,7 @@ import { convertToR1Format } from "../transform/r1-format"
 import { DEFAULT_HEADERS, DEEP_SEEK_DEFAULT_TEMPERATURE } from "./constants"
 import { getModelParams, SingleCompletionHandler } from ".."
 import { BaseProvider } from "./base-provider"
+import { modelsSupportingPromptCache } from "./fetchers/openrouter"
 
 const OPENROUTER_DEFAULT_PROVIDER_NAME = "[default]"
 
@@ -76,8 +77,8 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 		// Prompt caching: https://openrouter.ai/docs/prompt-caching
 		// Now with Gemini support: https://openrouter.ai/docs/features/prompt-caching
 		// Note that we don't check the `ModelInfo` object because it is cached
-		// in the settings for OpenRouter.
-		if (this.isPromptCacheSupported({ id: modelId, ...info })) {
+		// in the settings for OpenRouter and the value could be stale.
+		if (modelsSupportingPromptCache.has(modelId)) {
 			openAiMessages[0] = {
 				role: "system",
 				// @ts-ignore-next-line
@@ -181,7 +182,6 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 
 		let id = modelId ?? openRouterDefaultModelId
 		const info = modelInfo ?? openRouterDefaultModelInfo
-		const supportsPromptCache = modelInfo?.supportsPromptCache
 		const isDeepSeekR1 = id.startsWith("deepseek/deepseek-r1") || modelId === "perplexity/sonar-reasoning"
 		const defaultTemperature = isDeepSeekR1 ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0
 		const topP = isDeepSeekR1 ? 0.95 : undefined
@@ -190,7 +190,6 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 			id,
 			info,
 			...getModelParams({ options: this.options, model: info, defaultTemperature }),
-			supportsPromptCache,
 			topP,
 		}
 	}
@@ -217,17 +216,4 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
 		const completion = response as OpenAI.Chat.ChatCompletion
 		return completion.choices[0]?.message?.content || ""
 	}
-
-	private isPromptCacheSupported(model: ModelInfo & { id: string }) {
-		if (!model.supportsPromptCache) {
-			return false
-		}
-
-		return (
-			model.id.startsWith("anthropic/claude-3.7-sonnet") ||
-			model.id.startsWith("anthropic/claude-3.5-sonnet") ||
-			model.id.startsWith("anthropic/claude-3-opus") ||
-			model.id.startsWith("anthropic/claude-3-haiku")
-		)
-	}
 }