Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .changeset/fix-system-cache-control.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
"@openrouter/ai-sdk-provider": patch
---

Fix system message cache control to use block-level format

When cache control is specified on a system message via `providerOptions`, the content is now converted to array format with `cache_control` on the text block, matching the existing behavior for user messages. This ensures consistent Anthropic prompt caching behavior across all message types.

Before (message-level cache_control):
```json
{ "role": "system", "content": "...", "cache_control": { "type": "ephemeral" } }
```

After (block-level cache_control):
```json
{ "role": "system", "content": [{ "type": "text", "text": "...", "cache_control": { "type": "ephemeral" } }] }
```

Fixes #389
107 changes: 107 additions & 0 deletions e2e/issues/issue-389-system-cache-control.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/**
* Regression test for GitHub issue #389
* https://github.com/OpenRouterTeam/ai-sdk-provider/issues/389
*
* Issue: "Anthropic prompt caching not applied when `system` is a string
* in AI SDK (`ModelMessage[]`); only block content works"
*
* The user reported that prompt caching does not work when a system message
* is provided as a plain string with cache_control at the message level via
* providerOptions. Caching only worked when content was an array of text
* blocks with cache_control on each block.
*
* The fix converts system message content to array format with block-level
* cache_control when cache control is present, matching the existing behavior
* for user messages.
*/
import { generateText } from 'ai';
import { describe, expect, it, vi } from 'vitest';
import { createOpenRouter } from '@/src';

vi.setConfig({
testTimeout: 120_000,
});

describe('Issue #389: System message cache control with string content', () => {
const openrouter = createOpenRouter({
apiKey: process.env.OPENROUTER_API_KEY,
baseUrl: `${process.env.OPENROUTER_API_BASE}/api/v1`,
});

const model = openrouter('anthropic/claude-sonnet-4');

const longSystemPrompt = `You are a helpful assistant. Here is some context that should be cached:
${Array(50)
.fill(
'This is padding text to ensure the prompt meets the minimum token threshold for Anthropic prompt caching. ' +
'Prompt caching requires a minimum number of tokens in the prompt prefix. ' +
'This text is repeated multiple times to reach that threshold. ',
)
.join('\n')}
Remember to be helpful and concise in your responses.`;

it('should trigger cache write on first request with system message cache control', async () => {
const response = await generateText({
model,
messages: [
{
role: 'system',
content: longSystemPrompt,
providerOptions: {
anthropic: {
cacheControl: { type: 'ephemeral' },
},
},
},
{
role: 'user',
content: 'What is 2+2? Answer with just the number.',
},
],
});

expect(response.text).toBeDefined();
expect(response.text.length).toBeGreaterThan(0);
expect(response.finishReason).toBeDefined();
});

it('should trigger cache read on second request with system message cache control', async () => {
const makeRequest = () =>
generateText({
model,
messages: [
{
role: 'system',
content: longSystemPrompt,
providerOptions: {
anthropic: {
cacheControl: { type: 'ephemeral' },
},
},
},
{
role: 'user',
content: 'What is 2+2? Answer with just the number.',
},
],
});

await makeRequest();

const response = await makeRequest();

const openrouterMetadata = response.providerMetadata?.openrouter as {
usage?: {
promptTokensDetails?: { cachedTokens?: number };
};
};

const cachedTokens =
openrouterMetadata?.usage?.promptTokensDetails?.cachedTokens;

expect(cachedTokens).toBeDefined();
expect(cachedTokens).toBeGreaterThan(0);
});
});
61 changes: 57 additions & 4 deletions src/chat/convert-to-openrouter-chat-messages.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ describe('user messages', () => {
});

describe('cache control', () => {
it('should pass cache control from system message provider metadata', () => {
it('should convert system message to array content with block-level cache control', () => {
const result = convertToOpenRouterChatMessages([
{
role: 'system',
Expand All @@ -246,10 +246,58 @@ describe('cache control', () => {
]);

expect(result).toEqual([
{
role: 'system',
content: [
{
type: 'text',
text: 'System prompt',
cache_control: { type: 'ephemeral' },
},
],
},
]);
});

it('should keep system message as string when no cache control is present', () => {
const result = convertToOpenRouterChatMessages([
{
role: 'system',
content: 'System prompt',
cache_control: { type: 'ephemeral' },
},
]);

expect(result).toEqual([
{
role: 'system',
content: 'System prompt',
},
]);
});

it('should convert system message to array content with openrouter namespace cache control', () => {
const result = convertToOpenRouterChatMessages([
{
role: 'system',
content: 'System prompt',
providerOptions: {
openrouter: {
cacheControl: { type: 'ephemeral' },
},
},
},
]);

expect(result).toEqual([
{
role: 'system',
content: [
{
type: 'text',
text: 'System prompt',
cache_control: { type: 'ephemeral' },
},
],
},
]);
});
Expand Down Expand Up @@ -677,8 +725,13 @@ describe('cache control', () => {
expect(result).toEqual([
{
role: 'system',
content: 'System prompt',
cache_control: { type: 'ephemeral' },
content: [
{
type: 'text',
text: 'System prompt',
cache_control: { type: 'ephemeral' },
},
],
},
]);
});
Expand Down
12 changes: 10 additions & 2 deletions src/chat/convert-to-openrouter-chat-messages.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,18 @@ export function convertToOpenRouterChatMessages(
for (const { role, content, providerOptions } of prompt) {
switch (role) {
case 'system': {
const cacheControl = getCacheControl(providerOptions);
messages.push({
role: 'system',
content,
cache_control: getCacheControl(providerOptions),
content: cacheControl
? [
{
type: 'text' as const,
text: content,
cache_control: cacheControl,
},
]
: content,
});
break;
}
Expand Down
2 changes: 1 addition & 1 deletion src/types/openrouter-chat-completions-input.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export type ChatCompletionMessageParam =

export interface ChatCompletionSystemMessageParam {
role: 'system';
content: string;
content: string | Array<ChatCompletionContentPartText>;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd argue to rid this union and just use an array for this

cache_control?: OpenRouterCacheControl;
}

Expand Down