Skip to content

Commit 6ad11ba

Browse files
0xToshii0xToshii
authored
Systematic selection of gemini models w/ caching (RooCodeInc#3343)
* no more updating gemini models * changeset
1 parent 39c7da3 commit 6ad11ba

File tree

3 files changed

+71
-69
lines changed

3 files changed

+71
-69
lines changed

‎.changeset/selfish-feet-camp.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"claude-dev": minor
3+
---
4+
5+
updated OR/cline provider to automate gemini models caching

‎src/api/transform/openrouter-stream.ts‎

Lines changed: 60 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ export async function createOpenRouterStream(
2121

2222
// prompt caching: https://openrouter.ai/docs/prompt-caching
2323
// this was initially specifically for claude models (some models may 'support prompt caching' automatically without this)
24-
// includes custom support for gemini which does not have iterative caching
24+
// handles direct model.id match logic
2525
switch (model.id) {
2626
case "anthropic/claude-3.7-sonnet":
2727
case "anthropic/claude-3.7-sonnet:beta":
@@ -71,77 +71,76 @@ export async function createOpenRouterStream(
7171
}
7272
})
7373
break
74-
case "google/gemini-2.5-pro-preview-03-25":
75-
case "google/gemini-2.0-flash-001":
76-
case "google/gemini-flash-1.5":
77-
case "google/gemini-pro-1.5":
78-
// gemini only uses the last breakpoint for caching, so the others will be ignored
79-
openAiMessages[0] = {
80-
role: "system",
81-
content: [
82-
{
83-
type: "text",
84-
text: systemPrompt,
85-
// @ts-ignore-next-line
86-
cache_control: { type: "ephemeral" },
87-
},
88-
],
89-
}
74+
default:
75+
break
76+
}
77+
78+
// handles gemini caching logic
79+
if (model.id.startsWith("google/") && model.info.supportsPromptCache) {
80+
// gemini only uses the last breakpoint for caching, so the others will be ignored
81+
openAiMessages[0] = {
82+
role: "system",
83+
content: [
84+
{
85+
type: "text",
86+
text: systemPrompt,
87+
// @ts-ignore-next-line
88+
cache_control: { type: "ephemeral" },
89+
},
90+
],
91+
}
9092

91-
// for safety, but this should always be the case
92-
if (openAiMessages.length >= 2) {
93-
const msg = openAiMessages[1]
93+
// for safety, but this should always be the case
94+
if (openAiMessages.length >= 2) {
95+
const msg = openAiMessages[1]
9496

95-
if (msg) {
96-
if (typeof msg.content === "string") {
97-
msg.content = [{ type: "text", text: msg.content }]
98-
}
99-
if (Array.isArray(msg.content)) {
100-
// NOTE: this is fine since env details will always be added at the end. but if it weren't there, and the user added a image_url type message, it would pop a text part before it and then move it after to the end.
101-
let lastTextPart = msg.content.filter((part) => part.type === "text").pop()
102-
103-
if (!lastTextPart) {
104-
lastTextPart = { type: "text", text: "..." }
105-
msg.content.push(lastTextPart)
106-
}
107-
// @ts-ignore-next-line
108-
lastTextPart["cache_control"] = { type: "ephemeral" }
97+
if (msg) {
98+
if (typeof msg.content === "string") {
99+
msg.content = [{ type: "text", text: msg.content }]
100+
}
101+
if (Array.isArray(msg.content)) {
102+
// NOTE: this is fine since env details will always be added at the end. but if it weren't there, and the user added a image_url type message, it would pop a text part before it and then move it after to the end.
103+
let lastTextPart = msg.content.filter((part) => part.type === "text").pop()
104+
105+
if (!lastTextPart) {
106+
lastTextPart = { type: "text", text: "..." }
107+
msg.content.push(lastTextPart)
109108
}
109+
// @ts-ignore-next-line
110+
lastTextPart["cache_control"] = { type: "ephemeral" }
110111
}
111112
}
113+
}
112114

113-
// it doesn't make sense to alter breakpoints at all with the gemini cache implementation at this time
114-
/*const GEMINI_CACHE_USER_MESSAGE_INTERVAL = 4 // add new breakpoint every 4 turns
115-
const userMessages = openAiMessages.filter((msg) => msg.role === "user")
115+
// it doesn't make sense to alter breakpoints at all with the gemini cache implementation at this time
116+
/*const GEMINI_CACHE_USER_MESSAGE_INTERVAL = 4 // add new breakpoint every 4 turns
117+
const userMessages = openAiMessages.filter((msg) => msg.role === "user")
116118
117-
const userMessageCount = userMessages.length
118-
const targetUserMessageNumber =
119-
Math.floor(userMessageCount / GEMINI_CACHE_USER_MESSAGE_INTERVAL) * GEMINI_CACHE_USER_MESSAGE_INTERVAL
119+
const userMessageCount = userMessages.length
120+
const targetUserMessageNumber =
121+
Math.floor(userMessageCount / GEMINI_CACHE_USER_MESSAGE_INTERVAL) * GEMINI_CACHE_USER_MESSAGE_INTERVAL
120122
121-
if (targetUserMessageNumber > 0) {
122-
// otherwise dont need to add a breakpoint
123-
const msg = userMessages[targetUserMessageNumber - 1]
123+
if (targetUserMessageNumber > 0) {
124+
// otherwise dont need to add a breakpoint
125+
const msg = userMessages[targetUserMessageNumber - 1]
124126
125-
if (msg) {
126-
if (typeof msg.content === "string") {
127-
msg.content = [{ type: "text", text: msg.content }]
128-
}
129-
if (Array.isArray(msg.content)) {
130-
// NOTE: this is fine since env details will always be added at the end. but if it weren't there, and the user added a image_url type message, it would pop a text part before it and then move it after to the end.
131-
let lastTextPart = msg.content.filter((part) => part.type === "text").pop()
132-
133-
if (!lastTextPart) {
134-
lastTextPart = { type: "text", text: "..." }
135-
msg.content.push(lastTextPart)
136-
}
137-
// @ts-ignore-next-line
138-
lastTextPart["cache_control"] = { type: "ephemeral" }
127+
if (msg) {
128+
if (typeof msg.content === "string") {
129+
msg.content = [{ type: "text", text: msg.content }]
130+
}
131+
if (Array.isArray(msg.content)) {
132+
// NOTE: this is fine since env details will always be added at the end. but if it weren't there, and the user added a image_url type message, it would pop a text part before it and then move it after to the end.
133+
let lastTextPart = msg.content.filter((part) => part.type === "text").pop()
134+
135+
if (!lastTextPart) {
136+
lastTextPart = { type: "text", text: "..." }
137+
msg.content.push(lastTextPart)
139138
}
139+
// @ts-ignore-next-line
140+
lastTextPart["cache_control"] = { type: "ephemeral" }
140141
}
141-
}*/
142-
break
143-
default:
144-
break
142+
}
143+
}*/
145144
}
146145

147146
// Not sure how openrouter defaults max tokens when no value is provided, but the anthropic api requires this value and since they offer both 4096 and 8192 variants, we should ensure 8192.

‎src/core/controller/index.ts‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1301,14 +1301,6 @@ Here is the project's README to help you get started:\n\n${mcpDetails.readmeCont
13011301
modelInfo.cacheWritesPrice = 0.14
13021302
modelInfo.cacheReadsPrice = 0.014
13031303
break
1304-
case "google/gemini-2.5-pro-preview-03-25":
1305-
case "google/gemini-2.0-flash-001":
1306-
case "google/gemini-flash-1.5":
1307-
case "google/gemini-pro-1.5":
1308-
modelInfo.supportsPromptCache = true
1309-
modelInfo.cacheWritesPrice = parsePrice(rawModel.pricing?.input_cache_write)
1310-
modelInfo.cacheReadsPrice = parsePrice(rawModel.pricing?.input_cache_read)
1311-
break
13121304
default:
13131305
if (rawModel.id.startsWith("openai/")) {
13141306
modelInfo.cacheReadsPrice = parsePrice(rawModel.pricing?.input_cache_read)
@@ -1317,6 +1309,12 @@ Here is the project's README to help you get started:\n\n${mcpDetails.readmeCont
13171309
modelInfo.cacheWritesPrice = parsePrice(rawModel.pricing?.input_cache_write)
13181310
// openrouter charges no cache write pricing for openAI models
13191311
}
1312+
} else if (rawModel.id.startsWith("google/")) {
1313+
modelInfo.cacheReadsPrice = parsePrice(rawModel.pricing?.input_cache_read)
1314+
if (modelInfo.cacheReadsPrice) {
1315+
modelInfo.supportsPromptCache = true
1316+
modelInfo.cacheWritesPrice = parsePrice(rawModel.pricing?.input_cache_write)
1317+
}
13201318
}
13211319
break
13221320
}

0 commit comments

Comments
 (0)