From 5084b2695dc31d7d85db2db214202c7b82ef3892 Mon Sep 17 00:00:00 2001 From: Elliott Lawson Date: Wed, 9 Jul 2025 22:38:13 -0400 Subject: [PATCH] fix: prevent tool result cache accumulation across multiple tool calls Apply tool_result_cache_type only to the last tool result message across all messages instead of applying it to each tool result during creation. This prevents cache accumulation when there are multiple tool call rounds. --- src/Providers/Anthropic/Handlers/Stream.php | 6 - src/Providers/Anthropic/Handlers/Text.php | 5 - src/Providers/Anthropic/Maps/MessageMap.php | 24 +- .../Anthropic/ToolResultCachingTest.php | 247 ++++++++++++++++++ 4 files changed, 270 insertions(+), 12 deletions(-) create mode 100644 tests/Providers/Anthropic/ToolResultCachingTest.php diff --git a/src/Providers/Anthropic/Handlers/Stream.php b/src/Providers/Anthropic/Handlers/Stream.php index 1943ab3c6..3ebf40cc1 100644 --- a/src/Providers/Anthropic/Handlers/Stream.php +++ b/src/Providers/Anthropic/Handlers/Stream.php @@ -617,12 +617,6 @@ protected function addMessagesToRequest(Request $request, array $toolResults, ?a $message = new ToolResultMessage($toolResults); - // Apply tool result caching if configured - $tool_result_cache_type = $request->providerOptions('tool_result_cache_type'); - if ($tool_result_cache_type) { - $message->withProviderOptions(['cacheType' => $tool_result_cache_type]); - } - $request->addMessage($message); } diff --git a/src/Providers/Anthropic/Handlers/Text.php b/src/Providers/Anthropic/Handlers/Text.php index cd61a2981..abade742a 100644 --- a/src/Providers/Anthropic/Handlers/Text.php +++ b/src/Providers/Anthropic/Handlers/Text.php @@ -104,11 +104,6 @@ protected function handleToolCalls(): Response $toolResults = $this->callTools($this->request->tools(), $this->tempResponse->toolCalls); $message = new ToolResultMessage($toolResults); - // Apply tool result caching if configured - if ($tool_result_cache_type = $this->request->providerOptions('tool_result_cache_type')) { - $message->withProviderOptions(['cacheType' => $tool_result_cache_type]); - } - $this->request->addMessage($message); $this->addStep($toolResults); diff --git a/src/Providers/Anthropic/Maps/MessageMap.php b/src/Providers/Anthropic/Maps/MessageMap.php index 52e85c60e..dadc3c8fb 100644 --- a/src/Providers/Anthropic/Maps/MessageMap.php +++ b/src/Providers/Anthropic/Maps/MessageMap.php @@ -30,10 +30,32 @@ public static function map(array $messages, array $requestProviderOptions = []): throw new PrismException('Anthropic does not support SystemMessages in the messages array. Use withSystemPrompt or withSystemPrompts instead.'); } - return array_map( + $mappedMessages = array_map( fn (Message $message): array => self::mapMessage($message, $requestProviderOptions), $messages ); + + if (isset($requestProviderOptions['tool_result_cache_type'])) { + $lastToolResultIndex = null; + + for ($i = count($mappedMessages) - 1; $i >= 0; $i--) { + if ($mappedMessages[$i]['role'] === 'user' && + isset($mappedMessages[$i]['content'][0]['type']) && + $mappedMessages[$i]['content'][0]['type'] === 'tool_result') { + $lastToolResultIndex = $i; + break; + } + } + + if ($lastToolResultIndex !== null) { + $lastContent = &$mappedMessages[$lastToolResultIndex]['content']; + $lastContent[count($lastContent) - 1]['cache_control'] = [ + 'type' => $requestProviderOptions['tool_result_cache_type'], + ]; + } + } + + return $mappedMessages; } /** diff --git a/tests/Providers/Anthropic/ToolResultCachingTest.php b/tests/Providers/Anthropic/ToolResultCachingTest.php new file mode 100644 index 000000000..5c8453797 --- /dev/null +++ b/tests/Providers/Anthropic/ToolResultCachingTest.php @@ -0,0 +1,247 @@ +set('prism.providers.anthropic.api_key', env('ANTHROPIC_API_KEY', 'sk-1234')); +}); + +it('applies tool_result_cache_type only to the last tool result message across all messages', function (): void { + // Create test messages simulating multiple tool call rounds + $messages = [ + new UserMessage('What time is the tigers game today and should I wear a coat?'), + new AssistantMessage('', toolCalls: [ + new ToolCall( + id: 'call_1', + name: 'search', + arguments: ['query' => 'Detroit Tigers baseball game time today'] + ), + ]), + new ToolResultMessage([ + new ToolResult( + toolCallId: 'call_1', + toolName: 'search', + args: ['query' => 'Detroit Tigers baseball game time today'], + result: 'The tigers game is at 3pm in detroit' + ), + ]), + new AssistantMessage('', toolCalls: [ + new ToolCall( + id: 'call_2', + name: 'weather', + arguments: ['city' => 'Detroit'] + ), + ]), + new ToolResultMessage([ + new ToolResult( + toolCallId: 'call_2', + toolName: 'weather', + args: ['city' => 'Detroit'], + result: 'The weather will be 75° and sunny' + ), + ]), + new AssistantMessage('The Tigers game is at 3pm today. The weather will be 75° and sunny, so you won\'t need a coat!'), + ]; + + // Map the messages with provider options + $mappedMessages = MessageMap::map( + $messages, + ['tool_result_cache_type' => 'ephemeral'] + ); + + // Verify that only the last tool result message has cache_control + $toolResultMessages = array_filter($mappedMessages, fn ($message): bool => $message['role'] === 'user' && + isset($message['content'][0]['type']) && + $message['content'][0]['type'] === 'tool_result'); + + expect(count($toolResultMessages))->toBe(2); + + // Get the tool result messages by their indices + $toolResultIndices = array_keys($toolResultMessages); + $firstToolResultIndex = $toolResultIndices[0]; + $lastToolResultIndex = $toolResultIndices[1]; + + // First tool result should NOT have cache_control + $firstToolResult = $mappedMessages[$firstToolResultIndex]; + expect($firstToolResult['content'][0])->not->toHaveKey('cache_control'); + + // Last tool result SHOULD have cache_control + $lastToolResult = $mappedMessages[$lastToolResultIndex]; + expect($lastToolResult['content'][0])->toHaveKey('cache_control'); + expect($lastToolResult['content'][0]['cache_control'])->toBe(['type' => 'ephemeral']); +}); + +it('handles single tool result message with cache_control', function (): void { + $messages = [ + new UserMessage('What is the weather?'), + new AssistantMessage('', toolCalls: [ + new ToolCall( + id: 'call_1', + name: 'weather', + arguments: ['city' => 'Detroit'] + ), + ]), + new ToolResultMessage([ + new ToolResult( + toolCallId: 'call_1', + toolName: 'weather', + args: ['city' => 'Detroit'], + result: 'The weather will be 75° and sunny' + ), + ]), + ]; + + // Map the messages with provider options + $mappedMessages = MessageMap::map( + $messages, + ['tool_result_cache_type' => 'ephemeral'] + ); + + // Find the tool result message + $toolResultMessage = null; + foreach ($mappedMessages as $message) { + if ($message['role'] === 'user' && + isset($message['content'][0]['type']) && + $message['content'][0]['type'] === 'tool_result') { + $toolResultMessage = $message; + break; + } + } + + // The single tool result should have cache_control + expect($toolResultMessage)->not->toBeNull(); + expect($toolResultMessage['content'][0])->toHaveKey('cache_control'); + expect($toolResultMessage['content'][0]['cache_control'])->toBe(['type' => 'ephemeral']); +}); + +it('does not apply cache_control when tool_result_cache_type is not set', function (): void { + $messages = [ + new UserMessage('What is the weather?'), + new AssistantMessage('', toolCalls: [ + new ToolCall( + id: 'call_1', + name: 'weather', + arguments: ['city' => 'Detroit'] + ), + ]), + new ToolResultMessage([ + new ToolResult( + toolCallId: 'call_1', + toolName: 'weather', + args: ['city' => 'Detroit'], + result: 'The weather will be 75° and sunny' + ), + ]), + ]; + + // Map the messages without provider options + $mappedMessages = MessageMap::map($messages); + + // Find the tool result message + $toolResultMessage = null; + foreach ($mappedMessages as $message) { + if ($message['role'] === 'user' && + isset($message['content'][0]['type']) && + $message['content'][0]['type'] === 'tool_result') { + $toolResultMessage = $message; + break; + } + } + + // Should not have cache_control + expect($toolResultMessage)->not->toBeNull(); + expect($toolResultMessage['content'][0])->not->toHaveKey('cache_control'); +}); + +it('sends only one cache block when request has multiple tool results in full lifecycle', function (): void { + Prism::fake(); + + // Simulate a request that already has multiple tool call rounds in history + $request = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-latest') + ->withMessages([ + new UserMessage('What time is the game and weather?'), + new AssistantMessage('', toolCalls: [ + new ToolCall( + id: 'call_1', + name: 'search', + arguments: ['query' => 'game time'] + ), + ]), + new ToolResultMessage([ + new ToolResult( + toolCallId: 'call_1', + toolName: 'search', + args: ['query' => 'game time'], + result: '3pm' + ), + ]), + new AssistantMessage('', toolCalls: [ + new ToolCall( + id: 'call_2', + name: 'weather', + arguments: ['city' => 'Detroit'] + ), + ]), + new ToolResultMessage([ + new ToolResult( + toolCallId: 'call_2', + toolName: 'weather', + args: ['city' => 'Detroit'], + result: 'sunny' + ), + ]), + ]) + ->withProviderOptions(['tool_result_cache_type' => 'ephemeral']); + + // Get the actual payload that would be sent + $payload = Text::buildHttpRequestPayload($request->toRequest()); + + // Count cache blocks in the payload + $cacheBlocks = 0; + foreach ($payload['messages'] as $message) { + foreach ($message['content'] as $content) { + if (isset($content['cache_control'])) { + $cacheBlocks++; + } + } + } + + expect($cacheBlocks)->toBe(1); + + // Find the last tool result message + $lastToolResultIndex = null; + for ($i = count($payload['messages']) - 1; $i >= 0; $i--) { + if ($payload['messages'][$i]['role'] === 'user' && + isset($payload['messages'][$i]['content'][0]['type']) && + $payload['messages'][$i]['content'][0]['type'] === 'tool_result') { + $lastToolResultIndex = $i; + break; + } + } + + // Verify the cache is on the last tool result + expect($lastToolResultIndex)->not->toBeNull(); + expect($payload['messages'][$lastToolResultIndex]['content'][0])->toHaveKey('cache_control'); + expect($payload['messages'][$lastToolResultIndex]['content'][0]['cache_control'])->toBe(['type' => 'ephemeral']); + + // Verify earlier tool results don't have cache + for ($i = 0; $i < $lastToolResultIndex; $i++) { + if ($payload['messages'][$i]['role'] === 'user' && + isset($payload['messages'][$i]['content'][0]['type']) && + $payload['messages'][$i]['content'][0]['type'] === 'tool_result') { + expect($payload['messages'][$i]['content'][0])->not->toHaveKey('cache_control'); + } + } +});