Skip to content

Commit 9b5ee27

Browse files
mrubensmrubens
authored
Merge pull request #1587 from dleen/cache
feat: Add prompt caching to OpenAI-compatible custom models
2 parents 6fb61a2 + 10c1e7d commit 9b5ee27

File tree

3 files changed

+153
-3
lines changed

3 files changed

+153
-3
lines changed

‎.changeset/thin-fans-deliver.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"roo-cline": patch
3+
---
4+
5+
Add prompt caching to OpenAI-compatible custom model info

‎src/api/providers/openai.ts‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
7272
}
7373

7474
if (this.options.openAiStreamingEnabled ?? true) {
75-
const systemMessage: OpenAI.Chat.ChatCompletionSystemMessageParam = {
75+
let systemMessage: OpenAI.Chat.ChatCompletionSystemMessageParam = {
7676
role: "system",
7777
content: systemPrompt,
7878
}
@@ -83,7 +83,42 @@ export class OpenAiHandler extends BaseProvider implements SingleCompletionHandl
8383
} else if (ark) {
8484
convertedMessages = [systemMessage, ...convertToSimpleMessages(messages)]
8585
} else {
86+
if (modelInfo.supportsPromptCache) {
87+
systemMessage = {
88+
role: "system",
89+
content: [
90+
{
91+
type: "text",
92+
text: systemPrompt,
93+
// @ts-ignore-next-line
94+
cache_control: { type: "ephemeral" },
95+
},
96+
],
97+
}
98+
}
8699
convertedMessages = [systemMessage, ...convertToOpenAiMessages(messages)]
100+
if (modelInfo.supportsPromptCache) {
101+
// Note: the following logic is copied from openrouter:
102+
// Add cache_control to the last two user messages
103+
// (note: this works because we only ever add one user message at a time, but if we added multiple we'd need to mark the user message before the last assistant message)
104+
const lastTwoUserMessages = convertedMessages.filter((msg) => msg.role === "user").slice(-2)
105+
lastTwoUserMessages.forEach((msg) => {
106+
if (typeof msg.content === "string") {
107+
msg.content = [{ type: "text", text: msg.content }]
108+
}
109+
if (Array.isArray(msg.content)) {
110+
// NOTE: this is fine since env details will always be added at the end. but if it weren't there, and the user added a image_url type message, it would pop a text part before it and then move it after to the end.
111+
let lastTextPart = msg.content.filter((part) => part.type === "text").pop()
112+
113+
if (!lastTextPart) {
114+
lastTextPart = { type: "text", text: "..." }
115+
msg.content.push(lastTextPart)
116+
}
117+
// @ts-ignore-next-line
118+
lastTextPart["cache_control"] = { type: "ephemeral" }
119+
}
120+
})
121+
}
87122
}
88123

89124
const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {

‎webview-ui/src/components/settings/ApiOptions.tsx‎

Lines changed: 112 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -819,7 +819,7 @@ const ApiOptions = ({
819819
style={{ fontSize: "12px" }}
820820
/>
821821
</div>
822-
<div className="text-sm text-vscode-descriptionForeground">
822+
<div className="text-sm text-vscode-descriptionForeground pt-1">
823823
Is this model capable of processing and understanding images?
824824
</div>
825825
</div>
@@ -842,11 +842,34 @@ const ApiOptions = ({
842842
style={{ fontSize: "12px" }}
843843
/>
844844
</div>
845-
<div className="text-sm text-vscode-descriptionForeground [pt">
845+
<div className="text-sm text-vscode-descriptionForeground pt-1">
846846
Is this model capable of interacting with a browser? (e.g. Claude 3.7 Sonnet).
847847
</div>
848848
</div>
849849

850+
<div>
851+
<div className="flex items-center gap-1">
852+
<Checkbox
853+
checked={apiConfiguration?.openAiCustomModelInfo?.supportsPromptCache ?? false}
854+
onChange={handleInputChange("openAiCustomModelInfo", (checked) => {
855+
return {
856+
...(apiConfiguration?.openAiCustomModelInfo || openAiModelInfoSaneDefaults),
857+
supportsPromptCache: checked,
858+
}
859+
})}>
860+
<span className="font-medium">Prompt Caching</span>
861+
</Checkbox>
862+
<i
863+
className="codicon codicon-info text-vscode-descriptionForeground"
864+
title="Enable if the model supports prompt caching. This can improve performance and reduce costs."
865+
style={{ fontSize: "12px" }}
866+
/>
867+
</div>
868+
<div className="text-sm text-vscode-descriptionForeground pt-1">
869+
Is this model capable of caching prompts?
870+
</div>
871+
</div>
872+
850873
<div>
851874
<VSCodeTextField
852875
value={
@@ -933,6 +956,93 @@ const ApiOptions = ({
933956
</VSCodeTextField>
934957
</div>
935958

959+
{apiConfiguration?.openAiCustomModelInfo?.supportsPromptCache && (
960+
<>
961+
<div>
962+
<VSCodeTextField
963+
value={
964+
apiConfiguration?.openAiCustomModelInfo?.cacheReadsPrice?.toString() ?? "0"
965+
}
966+
type="text"
967+
style={{
968+
borderColor: (() => {
969+
const value = apiConfiguration?.openAiCustomModelInfo?.cacheReadsPrice
970+
971+
if (!value && value !== 0) {
972+
return "var(--vscode-input-border)"
973+
}
974+
975+
return value >= 0
976+
? "var(--vscode-charts-green)"
977+
: "var(--vscode-errorForeground)"
978+
})(),
979+
}}
980+
onChange={handleInputChange("openAiCustomModelInfo", (e) => {
981+
const value = (e.target as HTMLInputElement).value
982+
const parsed = parseFloat(value)
983+
984+
return {
985+
...(apiConfiguration?.openAiCustomModelInfo ??
986+
openAiModelInfoSaneDefaults),
987+
cacheReadsPrice: isNaN(parsed) ? 0 : parsed,
988+
}
989+
})}
990+
placeholder="e.g. 0.0001"
991+
className="w-full">
992+
<div className="flex items-center gap-1">
993+
<span className="font-medium">Cache Reads Price</span>
994+
<i
995+
className="codicon codicon-info text-vscode-descriptionForeground"
996+
title="Cost per million tokens for reading from the cache. This is the price charged when a cached response is retrieved."
997+
style={{ fontSize: "12px" }}
998+
/>
999+
</div>
1000+
</VSCodeTextField>
1001+
</div>
1002+
<div>
1003+
<VSCodeTextField
1004+
value={
1005+
apiConfiguration?.openAiCustomModelInfo?.cacheWritesPrice?.toString() ?? "0"
1006+
}
1007+
type="text"
1008+
style={{
1009+
borderColor: (() => {
1010+
const value = apiConfiguration?.openAiCustomModelInfo?.cacheWritesPrice
1011+
1012+
if (!value && value !== 0) {
1013+
return "var(--vscode-input-border)"
1014+
}
1015+
1016+
return value >= 0
1017+
? "var(--vscode-charts-green)"
1018+
: "var(--vscode-errorForeground)"
1019+
})(),
1020+
}}
1021+
onChange={handleInputChange("openAiCustomModelInfo", (e) => {
1022+
const value = (e.target as HTMLInputElement).value
1023+
const parsed = parseFloat(value)
1024+
1025+
return {
1026+
...(apiConfiguration?.openAiCustomModelInfo ??
1027+
openAiModelInfoSaneDefaults),
1028+
cacheWritesPrice: isNaN(parsed) ? 0 : parsed,
1029+
}
1030+
})}
1031+
placeholder="e.g. 0.00005"
1032+
className="w-full">
1033+
<div className="flex items-center gap-1">
1034+
<span className="font-medium">Cache Writes Price</span>
1035+
<i
1036+
className="codicon codicon-info text-vscode-descriptionForeground"
1037+
title="Cost per million tokens for writing to the cache. This is the price charged when a prompt is cached for the first time."
1038+
style={{ fontSize: "12px" }}
1039+
/>
1040+
</div>
1041+
</VSCodeTextField>
1042+
</div>
1043+
</>
1044+
)}
1045+
9361046
<Button
9371047
variant="secondary"
9381048
onClick={() =>

0 commit comments

Comments
 (0)