Cache every 10 messages

Author: cte

src/api/transform/caching/__tests__/gemini.test.ts

@@ -80,8 +80,22 @@ describe("addCacheBreakpoints", () => {
 			})),
 		]
 
+		expect(messages.length).toEqual(frequency * 2 + 1)
+
 		addCacheBreakpoints(systemPrompt, messages, frequency)
 
+		const indices = []
+
+		for (let i = 0; i < messages.length; i++) {
+			const content = messages[i].content?.[0]
+
+			if (typeof content === "object" && "cache_control" in content) {
+				indices.push(i)
+			}
+		}
+
+		expect(indices).toEqual([0, 5, 10])
+
 		// Check Nth user message (index frequency)
 		expect(messages[frequency].content).toEqual([
 			{ type: "text", text: `User message ${frequency}`, cache_control: { type: "ephemeral" } },

src/api/transform/caching/gemini.ts

@@ -3,7 +3,7 @@ import OpenAI from "openai"
 export function addCacheBreakpoints(
 	systemPrompt: string,
 	messages: OpenAI.Chat.ChatCompletionMessageParam[],
-	frequency: number = 5,
+	frequency: number = 10,
 ) {
 	// *Always* cache the system prompt.
 	messages[0] = {