Skip to content

Commit d1f8764

Browse files
committed
Use a hardcoded list
1 parent 45b9778 commit d1f8764

File tree

3 files changed

+41
-52
lines changed

3 files changed

+41
-52
lines changed

src/api/providers/fetchers/__tests__/openrouter.test.ts

Lines changed: 13 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import path from "path"
44

55
import { back as nockBack } from "nock"
66

7-
import { getOpenRouterModels } from "../openrouter"
7+
import { getOpenRouterModels, modelsSupportingPromptCache } from "../openrouter"
88

99
nockBack.fixtures = path.join(__dirname, "fixtures")
1010
nockBack.setMode("dryrun")
@@ -16,40 +16,19 @@ describe("OpenRouter API", () => {
1616

1717
const models = await getOpenRouterModels()
1818

19-
const modelsSupportingPromptCache = Object.entries(models)
20-
.filter(([_, model]) => model.supportsPromptCache)
21-
.map(([id, _]) => id)
22-
.sort()
19+
expect(
20+
Object.entries(models)
21+
.filter(([_, model]) => model.supportsPromptCache)
22+
.map(([id, _]) => id)
23+
.sort(),
24+
).toEqual(Array.from(modelsSupportingPromptCache).sort())
2325

24-
expect(modelsSupportingPromptCache).toEqual([
25-
"anthropic/claude-3-haiku",
26-
"anthropic/claude-3-haiku:beta",
27-
"anthropic/claude-3-opus",
28-
"anthropic/claude-3-opus:beta",
29-
"anthropic/claude-3-sonnet",
30-
"anthropic/claude-3-sonnet:beta",
31-
"anthropic/claude-3.5-haiku",
32-
"anthropic/claude-3.5-haiku-20241022",
33-
"anthropic/claude-3.5-haiku-20241022:beta",
34-
"anthropic/claude-3.5-haiku:beta",
35-
"anthropic/claude-3.5-sonnet",
36-
"anthropic/claude-3.5-sonnet-20240620",
37-
"anthropic/claude-3.5-sonnet-20240620:beta",
38-
"anthropic/claude-3.5-sonnet:beta",
39-
"anthropic/claude-3.7-sonnet",
40-
"anthropic/claude-3.7-sonnet:beta",
41-
"anthropic/claude-3.7-sonnet:thinking",
42-
"google/gemini-2.0-flash-001",
43-
"google/gemini-flash-1.5",
44-
"google/gemini-flash-1.5-8b",
45-
])
46-
47-
const modelsSupportingComputerUse = Object.entries(models)
48-
.filter(([_, model]) => model.supportsComputerUse)
49-
.map(([id, _]) => id)
50-
.sort()
51-
52-
expect(modelsSupportingComputerUse).toEqual([
26+
expect(
27+
Object.entries(models)
28+
.filter(([_, model]) => model.supportsComputerUse)
29+
.map(([id, _]) => id)
30+
.sort(),
31+
).toEqual([
5332
"anthropic/claude-3.5-sonnet",
5433
"anthropic/claude-3.5-sonnet:beta",
5534
"anthropic/claude-3.7-sonnet",

src/api/providers/fetchers/openrouter.ts

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ export async function getOpenRouterModels(options?: ApiHandlerOptions) {
6262
? parseApiPrice(rawModel.pricing?.input_cache_read)
6363
: undefined
6464

65-
const supportsPromptCache = !!cacheWritesPrice && !!cacheWritesPrice
65+
// Disable prompt caching for Gemini models for now.
66+
const supportsPromptCache = !!cacheWritesPrice && !!cacheWritesPrice && !rawModel.id.startsWith("google")
6667

6768
const modelInfo: ModelInfo = {
6869
maxTokens: rawModel.top_provider?.max_completion_tokens,
@@ -109,3 +110,26 @@ export async function getOpenRouterModels(options?: ApiHandlerOptions) {
109110

110111
return models
111112
}
113+
114+
export const modelsSupportingPromptCache = new Set([
115+
"anthropic/claude-3-haiku",
116+
"anthropic/claude-3-haiku:beta",
117+
"anthropic/claude-3-opus",
118+
"anthropic/claude-3-opus:beta",
119+
"anthropic/claude-3-sonnet",
120+
"anthropic/claude-3-sonnet:beta",
121+
"anthropic/claude-3.5-haiku",
122+
"anthropic/claude-3.5-haiku-20241022",
123+
"anthropic/claude-3.5-haiku-20241022:beta",
124+
"anthropic/claude-3.5-haiku:beta",
125+
"anthropic/claude-3.5-sonnet",
126+
"anthropic/claude-3.5-sonnet-20240620",
127+
"anthropic/claude-3.5-sonnet-20240620:beta",
128+
"anthropic/claude-3.5-sonnet:beta",
129+
"anthropic/claude-3.7-sonnet",
130+
"anthropic/claude-3.7-sonnet:beta",
131+
"anthropic/claude-3.7-sonnet:thinking",
132+
// "google/gemini-2.0-flash-001",
133+
// "google/gemini-flash-1.5",
134+
// "google/gemini-flash-1.5-8b",
135+
])

src/api/providers/openrouter.ts

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import { convertToR1Format } from "../transform/r1-format"
1010
import { DEFAULT_HEADERS, DEEP_SEEK_DEFAULT_TEMPERATURE } from "./constants"
1111
import { getModelParams, SingleCompletionHandler } from ".."
1212
import { BaseProvider } from "./base-provider"
13+
import { modelsSupportingPromptCache } from "./fetchers/openrouter"
1314

1415
const OPENROUTER_DEFAULT_PROVIDER_NAME = "[default]"
1516

@@ -76,8 +77,8 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
7677
// Prompt caching: https://openrouter.ai/docs/prompt-caching
7778
// Now with Gemini support: https://openrouter.ai/docs/features/prompt-caching
7879
// Note that we don't check the `ModelInfo` object because it is cached
79-
// in the settings for OpenRouter.
80-
if (this.isPromptCacheSupported({ id: modelId, ...info })) {
80+
// in the settings for OpenRouter and the value could be stale.
81+
if (modelsSupportingPromptCache.has(modelId)) {
8182
openAiMessages[0] = {
8283
role: "system",
8384
// @ts-ignore-next-line
@@ -181,7 +182,6 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
181182

182183
let id = modelId ?? openRouterDefaultModelId
183184
const info = modelInfo ?? openRouterDefaultModelInfo
184-
const supportsPromptCache = modelInfo?.supportsPromptCache
185185
const isDeepSeekR1 = id.startsWith("deepseek/deepseek-r1") || modelId === "perplexity/sonar-reasoning"
186186
const defaultTemperature = isDeepSeekR1 ? DEEP_SEEK_DEFAULT_TEMPERATURE : 0
187187
const topP = isDeepSeekR1 ? 0.95 : undefined
@@ -190,7 +190,6 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
190190
id,
191191
info,
192192
...getModelParams({ options: this.options, model: info, defaultTemperature }),
193-
supportsPromptCache,
194193
topP,
195194
}
196195
}
@@ -217,17 +216,4 @@ export class OpenRouterHandler extends BaseProvider implements SingleCompletionH
217216
const completion = response as OpenAI.Chat.ChatCompletion
218217
return completion.choices[0]?.message?.content || ""
219218
}
220-
221-
private isPromptCacheSupported(model: ModelInfo & { id: string }) {
222-
if (!model.supportsPromptCache) {
223-
return false
224-
}
225-
226-
return (
227-
model.id.startsWith("anthropic/claude-3.7-sonnet") ||
228-
model.id.startsWith("anthropic/claude-3.5-sonnet") ||
229-
model.id.startsWith("anthropic/claude-3-opus") ||
230-
model.id.startsWith("anthropic/claude-3-haiku")
231-
)
232-
}
233219
}

0 commit comments

Comments
 (0)