diff --git a/docs/core-concepts/tools-function-calling.md b/docs/core-concepts/tools-function-calling.md index f8a6f55d2..5d9cd6920 100644 --- a/docs/core-concepts/tools-function-calling.md +++ b/docs/core-concepts/tools-function-calling.md @@ -305,6 +305,56 @@ use Prism\Prism\Facades\Tool; $tool = Tool::make(CurrentWeatherTool::class); ``` +## Client-Executed Tools + +Sometimes you need tools that are executed by the client (e.g., frontend application) rather than on the server. Client-executed tools are defined without a handler function - simply omit the `using()` call: + +```php +use Prism\Prism\Facades\Tool; + +$clientTool = Tool::as('browser_action') + ->for('Perform an action in the user\'s browser') + ->withStringParameter('action', 'The action to perform'); + // Note: No using() call - this tool will be executed by the client +``` + +When the AI calls a client-executed tool, Prism will: +1. Stop execution and return control to your application +2. Set the response's `finishReason` to `FinishReason::ToolCalls` +3. Include the tool calls in the response for your client to execute + +### Handling Client-Executed Tools + +```php +use Prism\Prism\Facades\Prism; +use Prism\Prism\Enums\FinishReason; + +$response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-latest') + ->withTools([$clientTool]) + ->withMaxSteps(3) + ->withPrompt('Click the submit button') + ->asText(); + +``` + +### Streaming with Client-Executed Tools + +When streaming, client-executed tools emit a `ToolCallEvent` but no `ToolResultEvent`: + +```php + +$response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-latest') + ->withTools([$clientTool]) + ->withMaxSteps(3) + ->withPrompt('Click the submit button') + ->asStream(); +``` + +> [!NOTE] +> Client-executed tools are useful for scenarios like browser automation, UI interactions, or any operation that must run on the user's device rather than the server. + ## Tool Choice Options You can control how the AI uses tools with the `withToolChoice` method: diff --git a/src/Concerns/CallsTools.php b/src/Concerns/CallsTools.php index 1555f515a..c351e7c51 100644 --- a/src/Concerns/CallsTools.php +++ b/src/Concerns/CallsTools.php @@ -6,6 +6,7 @@ use Illuminate\Support\ItemNotFoundException; use Illuminate\Support\MultipleItemsFoundException; +use JsonException; use Prism\Prism\Exceptions\PrismException; use Prism\Prism\Tool; use Prism\Prism\ValueObjects\ToolCall; @@ -19,6 +20,8 @@ trait CallsTools * @param Tool[] $tools * @param ToolCall[] $toolCalls * @return ToolResult[] + * + * @throws PrismException|JsonException */ protected function callTools(array $tools, array $toolCalls): array { @@ -53,12 +56,25 @@ function (ToolCall $toolCall) use ($tools): ToolResult { } }, - $toolCalls + array_filter($toolCalls, fn (ToolCall $toolCall): bool => ! $this->resolveTool($toolCall->name, $tools)->isClientExecuted()) ); } /** * @param Tool[] $tools + * @param ToolCall[] $toolCalls + * + * @throws PrismException + */ + protected function hasDeferredTools(array $tools, array $toolCalls): bool + { + return array_any($toolCalls, fn (ToolCall $toolCall): bool => $this->resolveTool($toolCall->name, $tools)->isClientExecuted()); + } + + /** + * @param Tool[] $tools + * + * @throws PrismException */ protected function resolveTool(string $name, array $tools): Tool { diff --git a/src/Enums/StreamEventType.php b/src/Enums/StreamEventType.php index b0af6d1c8..187d80dbe 100644 --- a/src/Enums/StreamEventType.php +++ b/src/Enums/StreamEventType.php @@ -21,4 +21,6 @@ enum StreamEventType: string case Artifact = 'artifact'; case Error = 'error'; case StreamEnd = 'stream_end'; + case StepStart = 'step_start'; + case StepFinish = 'step_finish'; } diff --git a/src/Events/Broadcasting/StepFinishBroadcast.php b/src/Events/Broadcasting/StepFinishBroadcast.php new file mode 100644 index 000000000..abc2c58b1 --- /dev/null +++ b/src/Events/Broadcasting/StepFinishBroadcast.php @@ -0,0 +1,7 @@ +processEvent($event); if ($streamEvent instanceof Generator) { - yield from $streamEvent; + foreach ($streamEvent as $event) { + yield $event; + } } elseif ($streamEvent instanceof StreamEvent) { yield $streamEvent; } } if ($this->state->hasToolCalls()) { - yield from $this->handleToolCalls($request, $depth); + foreach ($this->handleToolCalls($request, $depth) as $item) { + yield $item; + } return; } + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + yield $this->emitStreamEndEvent(); } @@ -115,8 +127,9 @@ protected function processEvent(array $event): StreamEvent|Generator|null /** * @param array $event + * @return Generator */ - protected function handleMessageStart(array $event): ?StreamStartEvent + protected function handleMessageStart(array $event): Generator { $message = $event['message'] ?? []; $this->state->withMessageId($message['id'] ?? EventID::generate()); @@ -132,18 +145,25 @@ protected function handleMessageStart(array $event): ?StreamStartEvent } // Only emit StreamStartEvent once per streaming session - if (! $this->state->shouldEmitStreamStart()) { - return null; + if ($this->state->shouldEmitStreamStart()) { + $this->state->markStreamStarted(); + + yield new StreamStartEvent( + id: EventID::generate(), + timestamp: time(), + model: $message['model'] ?? 'unknown', + provider: 'anthropic' + ); } - $this->state->markStreamStarted(); + if ($this->state->shouldEmitStepStart()) { + $this->state->markStepStarted(); - return new StreamStartEvent( - id: EventID::generate(), - timestamp: time(), - model: $message['model'] ?? 'unknown', - provider: 'anthropic' - ); + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } } /** @@ -475,9 +495,18 @@ protected function handleToolCalls(Request $request, int $depth): Generator // Execute tools and emit results $toolResults = []; + $hasDeferred = false; foreach ($toolCalls as $toolCall) { try { $tool = $this->resolveTool($toolCall->name, $request->tools()); + + // Skip deferred tools - frontend will provide results + if ($tool->isClientExecuted()) { + $hasDeferred = true; + + continue; + } + $output = call_user_func_array($tool->handle(...), $toolCall->arguments()); if (is_string($output)) { @@ -531,6 +560,23 @@ protected function handleToolCalls(Request $request, int $depth): Generator } } + // skip calling llm if there are pending deferred tools + if ($hasDeferred) { + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + + yield new StreamEndEvent( + id: EventID::generate(), + timestamp: time(), + finishReason: FinishReason::ToolCalls + ); + + return; + } + // Add messages to request for next turn if ($toolResults !== []) { $request->addMessage(new AssistantMessage( @@ -544,6 +590,13 @@ protected function handleToolCalls(Request $request, int $depth): Generator $request->addMessage(new ToolResultMessage($toolResults)); + // Emit step finish after tool calls + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + // Continue streaming if within step limit $depth++; if ($depth < $request->maxSteps()) { diff --git a/src/Providers/Anthropic/Handlers/Structured.php b/src/Providers/Anthropic/Handlers/Structured.php index 42f1a6b38..2755c11b7 100644 --- a/src/Providers/Anthropic/Handlers/Structured.php +++ b/src/Providers/Anthropic/Handlers/Structured.php @@ -172,7 +172,7 @@ protected function executeCustomToolsAndContinue(array $toolCalls, Response $tem $this->request->addMessage($message); $this->addStep($toolCalls, $tempResponse, $toolResults); - if ($this->canContinue()) { + if (! $this->hasDeferredTools($this->request->tools(), $toolCalls) && $this->canContinue()) { return $this->handle(); } diff --git a/src/Providers/Anthropic/Handlers/Text.php b/src/Providers/Anthropic/Handlers/Text.php index eec98d845..2ac9b78ac 100644 --- a/src/Providers/Anthropic/Handlers/Text.php +++ b/src/Providers/Anthropic/Handlers/Text.php @@ -113,7 +113,7 @@ protected function handleToolCalls(): Response $this->addStep($toolResults); - if ($this->responseBuilder->steps->count() < $this->request->maxSteps()) { + if (! $this->hasDeferredTools($this->request->tools(), $this->tempResponse->toolCalls) && $this->responseBuilder->steps->count() < $this->request->maxSteps()) { return $this->handle(); } diff --git a/src/Providers/DeepSeek/Handlers/Stream.php b/src/Providers/DeepSeek/Handlers/Stream.php index 688efce04..bf1338553 100644 --- a/src/Providers/DeepSeek/Handlers/Stream.php +++ b/src/Providers/DeepSeek/Handlers/Stream.php @@ -21,6 +21,8 @@ use Prism\Prism\Providers\DeepSeek\Maps\ToolMap; use Prism\Prism\Streaming\EventID; use Prism\Prism\Streaming\Events\ArtifactEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -96,6 +98,15 @@ protected function processStream(Response $response, Request $request, int $dept ); } + if ($this->state->shouldEmitStepStart()) { + $this->state->markStepStarted(); + + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } + if ($this->hasToolCalls($data)) { $toolCalls = $this->extractToolCalls($data, $toolCalls); @@ -214,6 +225,12 @@ protected function processStream(Response $response, Request $request, int $dept return; } + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + yield new StreamEndEvent( id: EventID::generate(), timestamp: time(), @@ -378,9 +395,32 @@ protected function handleToolCalls(Request $request, string $text, array $toolCa } } + // skip calling llm if there are pending deferred tools + if ($this->hasDeferredTools($request->tools(), $mappedToolCalls)) { + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + + yield new StreamEndEvent( + id: EventID::generate(), + timestamp: time(), + finishReason: FinishReason::ToolCalls + ); + + return; + } + $request->addMessage(new AssistantMessage($text, $mappedToolCalls)); $request->addMessage(new ToolResultMessage($toolResults)); + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + $this->state->resetTextState(); $this->state->withMessageId(EventID::generate()); diff --git a/src/Providers/DeepSeek/Handlers/Text.php b/src/Providers/DeepSeek/Handlers/Text.php index 8645f13f2..2dae46cff 100644 --- a/src/Providers/DeepSeek/Handlers/Text.php +++ b/src/Providers/DeepSeek/Handlers/Text.php @@ -73,7 +73,10 @@ protected function handleToolCalls(array $data, Request $request): TextResponse $this->addStep($data, $request, $toolResults); - if ($this->shouldContinue($request)) { + if (! $this->hasDeferredTools($request->tools(), ToolCallMap::map(data_get($data, 'choices.0.message.tool_calls', []))) + && + $this->shouldContinue($request) + ) { return $this->handle($request); } diff --git a/src/Providers/Gemini/Handlers/Stream.php b/src/Providers/Gemini/Handlers/Stream.php index 87a402d31..7af29561a 100644 --- a/src/Providers/Gemini/Handlers/Stream.php +++ b/src/Providers/Gemini/Handlers/Stream.php @@ -18,6 +18,8 @@ use Prism\Prism\Providers\Gemini\Maps\ToolMap; use Prism\Prism\Streaming\EventID; use Prism\Prism\Streaming\Events\ArtifactEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -100,6 +102,16 @@ protected function processStream(Response $response, Request $request, int $dept $this->state->markStreamStarted(); } + // Emit step start event once per step + if ($this->state->shouldEmitStepStart()) { + $this->state->markStepStarted(); + + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } + // Update usage data from each chunk $this->state->withUsage($this->extractUsage($data, $request)); @@ -219,6 +231,13 @@ protected function processStream(Response $response, Request $request, int $dept return; } + // Emit step finish before stream end + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + yield new StreamEndEvent( id: EventID::generate(), timestamp: time(), @@ -286,6 +305,7 @@ protected function handleToolCalls( array $data = [] ): Generator { $mappedToolCalls = []; + $hasDeferred = false; // Convert tool calls to ToolCall objects foreach ($this->state->toolCalls() as $toolCallData) { @@ -297,6 +317,14 @@ protected function handleToolCalls( foreach ($mappedToolCalls as $toolCall) { try { $tool = $this->resolveTool($toolCall->name, $request->tools()); + + // Skip deferred tools - frontend will provide results + if ($tool->isClientExecuted()) { + $hasDeferred = true; + + continue; + } + $output = call_user_func_array($tool->handle(...), $toolCall->arguments()); if (is_string($output)) { @@ -352,10 +380,35 @@ protected function handleToolCalls( } } + // skip calling llm if there are pending deferred tools + if ($hasDeferred) { + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + + yield new StreamEndEvent( + id: EventID::generate(), + timestamp: time(), + finishReason: FinishReason::ToolCalls + ); + + return; + } + + // Add messages for next turn and continue streaming if ($toolResults !== []) { $request->addMessage(new AssistantMessage($this->state->currentText(), $mappedToolCalls)); $request->addMessage(new ToolResultMessage($toolResults)); + // Emit step finish after tool calls + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + $depth++; if ($depth < $request->maxSteps()) { $previousUsage = $this->state->usage(); diff --git a/src/Providers/Gemini/Handlers/Structured.php b/src/Providers/Gemini/Handlers/Structured.php index b0053a3d4..9238d9176 100644 --- a/src/Providers/Gemini/Handlers/Structured.php +++ b/src/Providers/Gemini/Handlers/Structured.php @@ -209,7 +209,7 @@ protected function handleToolCalls(array $data, Request $request): StructuredRes $this->addStep($data, $request, FinishReason::ToolCalls, $toolResults); - if ($this->shouldContinue($request)) { + if (! $this->hasDeferredTools($request->tools(), ToolCallMap::map(data_get($data, 'candidates.0.content.parts', []))) && $this->shouldContinue($request)) { return $this->handle($request); } diff --git a/src/Providers/Gemini/Handlers/Text.php b/src/Providers/Gemini/Handlers/Text.php index 1384c6f4f..03eb28b23 100644 --- a/src/Providers/Gemini/Handlers/Text.php +++ b/src/Providers/Gemini/Handlers/Text.php @@ -156,7 +156,7 @@ protected function handleToolCalls(array $data, Request $request): TextResponse $this->addStep($data, $request, FinishReason::ToolCalls, $toolResults); - if ($this->shouldContinue($request)) { + if (! $this->hasDeferredTools($request->tools(), ToolCallMap::map(data_get($data, 'candidates.0.content.parts', []))) && $this->shouldContinue($request)) { return $this->handle($request); } diff --git a/src/Providers/Groq/Handlers/Stream.php b/src/Providers/Groq/Handlers/Stream.php index e51777efe..b39a9f4be 100644 --- a/src/Providers/Groq/Handlers/Stream.php +++ b/src/Providers/Groq/Handlers/Stream.php @@ -22,6 +22,8 @@ use Prism\Prism\Streaming\EventID; use Prism\Prism\Streaming\Events\ArtifactEvent; use Prism\Prism\Streaming\Events\ErrorEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -96,6 +98,16 @@ protected function processStream(Response $response, Request $request, int $dept ); } + // Emit step start event once per step + if ($this->state->shouldEmitStepStart()) { + $this->state->markStepStarted(); + + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } + if ($this->hasError($data)) { yield from $this->handleErrors($data, $request); @@ -168,6 +180,13 @@ protected function processStream(Response $response, Request $request, int $dept } } + // Emit step finish before stream end + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + yield new StreamEndEvent( id: EventID::generate(), timestamp: time(), @@ -272,9 +291,33 @@ protected function handleToolCalls( } } + // skip calling llm if there are pending deferred tools + if ($this->hasDeferredTools($request->tools(), $mappedToolCalls)) { + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + + yield new StreamEndEvent( + id: EventID::generate(), + timestamp: time(), + finishReason: FinishReason::ToolCalls + ); + + return; + } + $request->addMessage(new AssistantMessage($text, $mappedToolCalls)); $request->addMessage(new ToolResultMessage($toolResults)); + // Emit step finish after tool calls + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + // Reset text state for next response $this->state->resetTextState(); $this->state->withMessageId(EventID::generate()); diff --git a/src/Providers/Groq/Handlers/Text.php b/src/Providers/Groq/Handlers/Text.php index 84e539bff..37c7ac299 100644 --- a/src/Providers/Groq/Handlers/Text.php +++ b/src/Providers/Groq/Handlers/Text.php @@ -95,7 +95,7 @@ protected function handleToolCalls(array $data, Request $request, ClientResponse $this->addStep($data, $request, $clientResponse, FinishReason::ToolCalls, $toolResults); - if ($this->shouldContinue($request)) { + if (! $this->hasDeferredTools($request->tools(), $this->mapToolCalls(data_get($data, 'choices.0.message.tool_calls', []) ?? [])) && $this->shouldContinue($request)) { return $this->handle($request); } diff --git a/src/Providers/Mistral/Handlers/Stream.php b/src/Providers/Mistral/Handlers/Stream.php index f206db6ae..9eabe6a06 100644 --- a/src/Providers/Mistral/Handlers/Stream.php +++ b/src/Providers/Mistral/Handlers/Stream.php @@ -21,6 +21,8 @@ use Prism\Prism\Providers\Mistral\Maps\ToolMap; use Prism\Prism\Streaming\EventID; use Prism\Prism\Streaming\Events\ArtifactEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -95,6 +97,15 @@ protected function processStream(Response $response, Request $request, int $dept ); } + if ($this->state->shouldEmitStepStart()) { + $this->state->markStepStarted(); + + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } + if ($this->hasToolCalls($data)) { $toolCalls = $this->extractToolCalls($data, $toolCalls); @@ -173,6 +184,12 @@ protected function processStream(Response $response, Request $request, int $dept } } + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + yield new StreamEndEvent( id: EventID::generate(), timestamp: time(), @@ -268,9 +285,32 @@ protected function handleToolCalls( } } + // skip calling llm if there are pending deferred tools + if ($this->hasDeferredTools($request->tools(), $mappedToolCalls)) { + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + + yield new StreamEndEvent( + id: EventID::generate(), + timestamp: time(), + finishReason: FinishReason::ToolCalls + ); + + return; + } + $request->addMessage(new AssistantMessage($text, $mappedToolCalls)); $request->addMessage(new ToolResultMessage($toolResults)); + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + $this->state->resetTextState(); $this->state->withMessageId(EventID::generate()); diff --git a/src/Providers/Mistral/Handlers/Text.php b/src/Providers/Mistral/Handlers/Text.php index a5582f0fd..c22953f23 100644 --- a/src/Providers/Mistral/Handlers/Text.php +++ b/src/Providers/Mistral/Handlers/Text.php @@ -81,7 +81,7 @@ protected function handleToolCalls(array $data, Request $request, ClientResponse $this->addStep($data, $request, $clientResponse, $toolResults); - if ($this->shouldContinue($request)) { + if (! $this->hasDeferredTools($request->tools(), $this->mapToolCalls(data_get($data, 'choices.0.message.tool_calls', []))) && $this->shouldContinue($request)) { return $this->handle($request); } diff --git a/src/Providers/Ollama/Handlers/Stream.php b/src/Providers/Ollama/Handlers/Stream.php index b481413a0..6aecea86d 100644 --- a/src/Providers/Ollama/Handlers/Stream.php +++ b/src/Providers/Ollama/Handlers/Stream.php @@ -18,6 +18,8 @@ use Prism\Prism\Providers\Ollama\ValueObjects\OllamaStreamState; use Prism\Prism\Streaming\EventID; use Prism\Prism\Streaming\Events\ArtifactEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -91,6 +93,16 @@ protected function processStream(Response $response, Request $request, int $dept $this->state->markStreamStarted()->withMessageId(EventID::generate()); } + // Emit step start event once per step + if ($this->state->shouldEmitStepStart()) { + $this->state->markStepStarted(); + + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } + // Accumulate token counts $this->state->addPromptTokens((int) data_get($data, 'prompt_eval_count', 0)); $this->state->addCompletionTokens((int) data_get($data, 'eval_count', 0)); @@ -186,6 +198,13 @@ protected function processStream(Response $response, Request $request, int $dept ); } + // Emit step finish before stream end + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + // Emit stream end event with usage yield new StreamEndEvent( id: EventID::generate(), @@ -288,10 +307,34 @@ protected function handleToolCalls( } } + // skip calling llm if there are pending deferred tools + if ($this->hasDeferredTools($request->tools(), $mappedToolCalls)) { + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + + yield new StreamEndEvent( + id: EventID::generate(), + timestamp: time(), + finishReason: FinishReason::ToolCalls + ); + + return; + } + // Add messages for next turn $request->addMessage(new AssistantMessage($text, $mappedToolCalls)); $request->addMessage(new ToolResultMessage($toolResults)); + // Emit step finish after tool calls + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + // Continue streaming if within step limit $depth++; if ($depth < $request->maxSteps()) { diff --git a/src/Providers/Ollama/Handlers/Text.php b/src/Providers/Ollama/Handlers/Text.php index 04a63b3ee..b50a193dc 100644 --- a/src/Providers/Ollama/Handlers/Text.php +++ b/src/Providers/Ollama/Handlers/Text.php @@ -105,7 +105,7 @@ protected function handleToolCalls(array $data, Request $request): Response $this->addStep($data, $request, $toolResults); - if ($this->shouldContinue($request)) { + if (! $this->hasDeferredTools($request->tools(), $this->mapToolCalls(data_get($data, 'message.tool_calls', []))) && $this->shouldContinue($request)) { return $this->handle($request); } @@ -133,10 +133,18 @@ protected function shouldContinue(Request $request): bool */ protected function addStep(array $data, Request $request, array $toolResults = []): void { + $toolCalls = $this->mapToolCalls(data_get($data, 'message.tool_calls', []) ?? []); + + // Ollama sends done_reason: "stop" even when there are tool calls + // Override finish reason to ToolCalls when tool calls are present + $finishReason = $toolCalls === [] + ? $this->mapFinishReason($data) + : FinishReason::ToolCalls; + $this->responseBuilder->addStep(new Step( text: data_get($data, 'message.content') ?? '', - finishReason: $this->mapFinishReason($data), - toolCalls: $this->mapToolCalls(data_get($data, 'message.tool_calls', []) ?? []), + finishReason: $finishReason, + toolCalls: $toolCalls, toolResults: $toolResults, providerToolCalls: [], usage: new Usage( diff --git a/src/Providers/OpenAI/Handlers/Stream.php b/src/Providers/OpenAI/Handlers/Stream.php index 41bd80d52..083ba8c69 100644 --- a/src/Providers/OpenAI/Handlers/Stream.php +++ b/src/Providers/OpenAI/Handlers/Stream.php @@ -22,6 +22,8 @@ use Prism\Prism\Streaming\EventID; use Prism\Prism\Streaming\Events\ArtifactEvent; use Prism\Prism\Streaming\Events\ProviderToolEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -107,9 +109,28 @@ protected function processStream(Response $response, Request $request, int $dept $this->state->markStreamStarted(); + // Emit step start after stream start + if ($this->state->shouldEmitStepStart()) { + $this->state->markStepStarted(); + + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } + continue; } + if ($this->state->shouldEmitStepStart()) { + $this->state->markStepStarted(); + + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } + if ($this->hasReasoningSummaryDelta($data)) { $reasoningDelta = $this->extractReasoningSummaryDelta($data); @@ -261,6 +282,12 @@ protected function processStream(Response $response, Request $request, int $dept return; } + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + yield new StreamEndEvent( id: EventID::generate(), timestamp: time(), @@ -395,9 +422,33 @@ protected function handleToolCalls(Request $request, int $depth): Generator } } + // skip calling llm if there are pending deferred tools + if ($this->hasDeferredTools($request->tools(), $mappedToolCalls)) { + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + + yield new StreamEndEvent( + id: EventID::generate(), + timestamp: time(), + finishReason: FinishReason::ToolCalls + ); + + return; + } + $request->addMessage(new AssistantMessage($this->state->currentText(), $mappedToolCalls)); $request->addMessage(new ToolResultMessage($toolResults)); + // Emit step finish after tool calls + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + $depth++; if ($depth < $request->maxSteps()) { diff --git a/src/Providers/OpenAI/Handlers/Structured.php b/src/Providers/OpenAI/Handlers/Structured.php index cc6a62190..cef595fc8 100644 --- a/src/Providers/OpenAI/Handlers/Structured.php +++ b/src/Providers/OpenAI/Handlers/Structured.php @@ -100,7 +100,7 @@ protected function handleToolCalls(array $data, Request $request, ClientResponse $this->addStep($data, $request, $clientResponse, $toolResults); - if ($this->shouldContinue($request)) { + if (! $this->hasDeferredTools($request->tools(), ToolCallMap::map($this->extractFunctionCalls($data))) && $this->shouldContinue($request)) { return $this->handle($request); } diff --git a/src/Providers/OpenAI/Handlers/Text.php b/src/Providers/OpenAI/Handlers/Text.php index 3f1026add..a9ef7549c 100644 --- a/src/Providers/OpenAI/Handlers/Text.php +++ b/src/Providers/OpenAI/Handlers/Text.php @@ -100,7 +100,10 @@ protected function handleToolCalls(array $data, Request $request, ClientResponse $this->addStep($data, $request, $clientResponse, $toolResults); - if ($this->shouldContinue($request)) { + if (! $this->hasDeferredTools($request->tools(), ToolCallMap::map(array_filter(data_get($data, 'output', []), fn (array $output): bool => $output['type'] === 'function_call'))) + && + $this->shouldContinue($request) + ) { return $this->handle($request); } diff --git a/src/Providers/OpenRouter/Handlers/Stream.php b/src/Providers/OpenRouter/Handlers/Stream.php index fa52f68de..fcd9227d1 100644 --- a/src/Providers/OpenRouter/Handlers/Stream.php +++ b/src/Providers/OpenRouter/Handlers/Stream.php @@ -17,6 +17,8 @@ use Prism\Prism\Providers\OpenRouter\Maps\MessageMap; use Prism\Prism\Streaming\EventID; use Prism\Prism\Streaming\Events\ArtifactEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -93,6 +95,15 @@ protected function processStream(Response $response, Request $request, int $dept ); } + if ($this->state->shouldEmitStepStart()) { + $this->state->markStepStarted(); + + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } + if ($this->hasToolCalls($data)) { $toolCalls = $this->extractToolCalls($data, $toolCalls); @@ -225,6 +236,12 @@ protected function processStream(Response $response, Request $request, int $dept } } + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + yield new StreamEndEvent( id: EventID::generate(), timestamp: time(), @@ -382,6 +399,23 @@ protected function handleToolCalls( } } + // skip calling llm if there are pending deferred tools + if ($this->hasDeferredTools($request->tools(), $mappedToolCalls)) { + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + + yield new StreamEndEvent( + id: EventID::generate(), + timestamp: time(), + finishReason: FinishReason::ToolCalls + ); + + return; + } + $request->addMessage(new AssistantMessage($text, $mappedToolCalls)); $request->addMessage(new ToolResultMessage($toolResults)); @@ -391,6 +425,12 @@ protected function handleToolCalls( $depth++; + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + if ($depth < $request->maxSteps()) { $nextResponse = $this->sendRequest($request); yield from $this->processStream($nextResponse, $request, $depth); diff --git a/src/Providers/OpenRouter/Handlers/Text.php b/src/Providers/OpenRouter/Handlers/Text.php index baf39954f..454b44b28 100644 --- a/src/Providers/OpenRouter/Handlers/Text.php +++ b/src/Providers/OpenRouter/Handlers/Text.php @@ -72,7 +72,7 @@ protected function handleToolCalls(array $data, Request $request): TextResponse $this->addStep($data, $request, $toolResults); - if ($this->shouldContinue($request)) { + if (! $this->hasDeferredTools($request->tools(), ToolCallMap::map(data_get($data, 'choices.0.message.tool_calls', []))) && $this->shouldContinue($request)) { return $this->handle($request); } diff --git a/src/Providers/XAI/Handlers/Stream.php b/src/Providers/XAI/Handlers/Stream.php index 672d1ca3c..60d5af225 100644 --- a/src/Providers/XAI/Handlers/Stream.php +++ b/src/Providers/XAI/Handlers/Stream.php @@ -21,6 +21,8 @@ use Prism\Prism\Providers\XAI\Maps\ToolMap; use Prism\Prism\Streaming\EventID; use Prism\Prism\Streaming\Events\ArtifactEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -100,6 +102,15 @@ protected function processStream(Response $response, Request $request, int $dept ); } + if ($this->state->shouldEmitStepStart()) { + $this->state->markStepStarted(); + + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } + $thinkingContent = $this->extractThinking($data, $request); if ($thinkingContent !== '' && $thinkingContent !== '0') { @@ -199,6 +210,12 @@ protected function processStream(Response $response, Request $request, int $dept ); } + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + yield new StreamEndEvent( id: EventID::generate(), timestamp: time(), @@ -365,9 +382,32 @@ protected function handleToolCalls( } } + // skip calling llm if there are pending deferred tools + if ($this->hasDeferredTools($request->tools(), $mappedToolCalls)) { + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + + yield new StreamEndEvent( + id: EventID::generate(), + timestamp: time(), + finishReason: FinishReason::ToolCalls + ); + + return; + } + $request->addMessage(new AssistantMessage($text, $mappedToolCalls)); $request->addMessage(new ToolResultMessage($toolResults)); + $this->state->markStepFinished(); + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + $this->state ->resetTextState() ->withMessageId(EventID::generate()); diff --git a/src/Providers/XAI/Handlers/Text.php b/src/Providers/XAI/Handlers/Text.php index 5191a41d4..318da0b8b 100644 --- a/src/Providers/XAI/Handlers/Text.php +++ b/src/Providers/XAI/Handlers/Text.php @@ -83,7 +83,7 @@ protected function handleToolCalls(array $data, Request $request): TextResponse $this->addStep($data, $request, $toolResults); - if ($this->shouldContinue($request)) { + if (! $this->hasDeferredTools($request->tools(), $toolCalls) && $this->shouldContinue($request)) { return $this->handle($request); } diff --git a/src/Streaming/Adapters/BroadcastAdapter.php b/src/Streaming/Adapters/BroadcastAdapter.php index c07797033..511d19305 100644 --- a/src/Streaming/Adapters/BroadcastAdapter.php +++ b/src/Streaming/Adapters/BroadcastAdapter.php @@ -12,6 +12,8 @@ use Prism\Prism\Events\Broadcasting\ArtifactBroadcast; use Prism\Prism\Events\Broadcasting\ErrorBroadcast; use Prism\Prism\Events\Broadcasting\ProviderToolEventBroadcast; +use Prism\Prism\Events\Broadcasting\StepFinishBroadcast; +use Prism\Prism\Events\Broadcasting\StepStartBroadcast; use Prism\Prism\Events\Broadcasting\StreamEndBroadcast; use Prism\Prism\Events\Broadcasting\StreamStartBroadcast; use Prism\Prism\Events\Broadcasting\TextCompleteBroadcast; @@ -25,6 +27,8 @@ use Prism\Prism\Streaming\Events\ArtifactEvent; use Prism\Prism\Streaming\Events\ErrorEvent; use Prism\Prism\Streaming\Events\ProviderToolEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -69,6 +73,7 @@ protected function broadcastEvent(StreamEvent $event): ShouldBroadcast { return match ($event::class) { StreamStartEvent::class => new StreamStartBroadcast($event, $this->channels), + StepStartEvent::class => new StepStartBroadcast($event, $this->channels), TextStartEvent::class => new TextStartBroadcast($event, $this->channels), TextDeltaEvent::class => new TextDeltaBroadcast($event, $this->channels), TextCompleteEvent::class => new TextCompleteBroadcast($event, $this->channels), @@ -80,6 +85,7 @@ protected function broadcastEvent(StreamEvent $event): ShouldBroadcast ArtifactEvent::class => new ArtifactBroadcast($event, $this->channels), ProviderToolEvent::class => new ProviderToolEventBroadcast($event, $this->channels), ErrorEvent::class => new ErrorBroadcast($event, $this->channels), + StepFinishEvent::class => new StepFinishBroadcast($event, $this->channels), StreamEndEvent::class => new StreamEndBroadcast($event, $this->channels), default => throw new InvalidArgumentException('Unsupported event type for broadcasting: '.$event::class), }; diff --git a/src/Streaming/Adapters/DataProtocolAdapter.php b/src/Streaming/Adapters/DataProtocolAdapter.php index 022530210..a19109812 100644 --- a/src/Streaming/Adapters/DataProtocolAdapter.php +++ b/src/Streaming/Adapters/DataProtocolAdapter.php @@ -10,6 +10,8 @@ use Prism\Prism\Streaming\Events\ArtifactEvent; use Prism\Prism\Streaming\Events\ErrorEvent; use Prism\Prism\Streaming\Events\ProviderToolEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -126,6 +128,7 @@ protected function handleEventConversion(StreamEvent $event): ?string { $data = match ($event::class) { StreamStartEvent::class => $this->handleStreamStart($event), + StepStartEvent::class => $this->handleStepStart($event), TextStartEvent::class => $this->handleTextStart($event), TextDeltaEvent::class => $this->handleTextDelta($event), TextCompleteEvent::class => $this->handleTextComplete($event), @@ -136,6 +139,7 @@ protected function handleEventConversion(StreamEvent $event): ?string ToolResultEvent::class => $this->handleToolResult($event), ArtifactEvent::class => $this->handleArtifact($event), ProviderToolEvent::class => $this->handleProviderTool($event), + StepFinishEvent::class => $this->handleStepFinish($event), StreamEndEvent::class => $this->handleStreamEnd($event), ErrorEvent::class => $this->handleError($event), default => $this->handleDefault($event), @@ -387,4 +391,24 @@ protected function handleDefault(StreamEvent $event): array 'data' => $event->toArray(), ]; } + + /** + * @return array + */ + protected function handleStepStart(StepStartEvent $event): array + { + return [ + 'type' => 'start-step', + ]; + } + + /** + * @return array + */ + protected function handleStepFinish(StepFinishEvent $event): array + { + return [ + 'type' => 'finish-step', + ]; + } } diff --git a/src/Streaming/Events/StepFinishEvent.php b/src/Streaming/Events/StepFinishEvent.php new file mode 100644 index 000000000..18b6fb322 --- /dev/null +++ b/src/Streaming/Events/StepFinishEvent.php @@ -0,0 +1,30 @@ + $this->id, + 'timestamp' => $this->timestamp, + ]; + } +} diff --git a/src/Streaming/Events/StepStartEvent.php b/src/Streaming/Events/StepStartEvent.php new file mode 100644 index 000000000..aebf59a7b --- /dev/null +++ b/src/Streaming/Events/StepStartEvent.php @@ -0,0 +1,30 @@ + $this->id, + 'timestamp' => $this->timestamp, + ]; + } +} diff --git a/src/Streaming/StreamState.php b/src/Streaming/StreamState.php index abdaf2217..bbe320f97 100644 --- a/src/Streaming/StreamState.php +++ b/src/Streaming/StreamState.php @@ -16,6 +16,8 @@ class StreamState protected bool $streamStarted = false; + protected bool $stepStarted = false; + protected bool $textStarted = false; protected bool $thinkingStarted = false; @@ -95,6 +97,20 @@ public function markStreamStarted(): self return $this; } + public function markStepStarted(): self + { + $this->stepStarted = true; + + return $this; + } + + public function markStepFinished(): self + { + $this->stepStarted = false; + + return $this; + } + public function markTextStarted(): self { $this->textStarted = true; @@ -335,6 +351,11 @@ public function shouldEmitStreamStart(): bool return ! $this->streamStarted; } + public function shouldEmitStepStart(): bool + { + return ! $this->stepStarted; + } + public function shouldEmitTextStart(): bool { return ! $this->textStarted; diff --git a/src/Testing/PrismFake.php b/src/Testing/PrismFake.php index c9437cb8c..b3ba44a51 100644 --- a/src/Testing/PrismFake.php +++ b/src/Testing/PrismFake.php @@ -20,6 +20,8 @@ use Prism\Prism\Moderation\Response as ModerationResponse; use Prism\Prism\Providers\Provider; use Prism\Prism\Streaming\EventID; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -244,7 +246,15 @@ protected function streamEventsFromTextResponse(TextResponse $response, TextRequ provider: 'fake' ); + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + if ($response->steps->isNotEmpty()) { + $stepIndex = 0; + $totalSteps = $response->steps->count(); + foreach ($response->steps as $step) { if ($step->text !== '') { yield new TextStartEvent( @@ -287,6 +297,20 @@ protected function streamEventsFromTextResponse(TextResponse $response, TextRequ success: true ); } + + $stepIndex++; + + // If this step has tool calls/results and there are more steps, end current step and start new one + if (($step->toolCalls !== [] || $step->toolResults !== []) && $stepIndex < $totalSteps) { + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + yield new StepStartEvent( + id: EventID::generate(), + timestamp: time() + ); + } } } elseif ($response->text !== '') { yield new TextStartEvent( @@ -309,6 +333,11 @@ protected function streamEventsFromTextResponse(TextResponse $response, TextRequ ); } + yield new StepFinishEvent( + id: EventID::generate(), + timestamp: time() + ); + yield new StreamEndEvent( id: EventID::generate(), timestamp: time(), diff --git a/src/Tool.php b/src/Tool.php index fdebfe6c9..03914c231 100644 --- a/src/Tool.php +++ b/src/Tool.php @@ -38,7 +38,7 @@ class Tool /** @var array */ protected array $requiredParameters = []; - /** @var Closure():mixed|callable():mixed */ + /** @var Closure():mixed|callable():mixed|null */ protected $fn; /** @var null|false|Closure(Throwable,array):string */ @@ -231,6 +231,11 @@ public function hasParameters(): bool return (bool) count($this->parameters); } + public function isClientExecuted(): bool + { + return $this->fn === null; + } + /** * @return null|false|Closure(Throwable,array):string */ @@ -246,6 +251,10 @@ public function failedHandler(): null|false|Closure */ public function handle(...$args): string|ToolOutput { + if ($this->fn === null) { + throw PrismException::toolHandlerNotDefined($this->name); + } + try { $value = call_user_func($this->fn, ...$args); diff --git a/tests/Fixtures/anthropic/stream-with-client-executed-tool-1.sse b/tests/Fixtures/anthropic/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..e6fcba676 --- /dev/null +++ b/tests/Fixtures/anthropic/stream-with-client-executed-tool-1.sse @@ -0,0 +1,31 @@ +event: message_start +data: {"type":"message_start","message":{"id":"msg_client_executed_test","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":100,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":1}}} + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"I'll use the client tool to help you."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: content_block_start +data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_client_tool_stream","name":"client_tool","input":{}}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}} + +event: content_block_delta +data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"input\": \"test input\"}"}} + +event: content_block_stop +data: {"type":"content_block_stop","index":1} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":50}} + +event: message_stop +data: {"type":"message_stop"} + + diff --git a/tests/Fixtures/anthropic/structured-with-client-executed-tool-1.json b/tests/Fixtures/anthropic/structured-with-client-executed-tool-1.json new file mode 100644 index 000000000..980813079 --- /dev/null +++ b/tests/Fixtures/anthropic/structured-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"model":"claude-sonnet-4-20250514","id":"msg_client_executed_structured","type":"message","role":"assistant","content":[{"type":"text","text":"I'll use the client tool to help you with that request."},{"type":"tool_use","id":"toolu_client_structured","name":"client_tool","input":{"input":"test input"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":200,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":50}} + diff --git a/tests/Fixtures/anthropic/text-with-client-executed-tool-1.json b/tests/Fixtures/anthropic/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..9d409fb32 --- /dev/null +++ b/tests/Fixtures/anthropic/text-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"id":"msg_01ClientExecutedTest","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[{"type":"text","text":"I'll use the client tool to help you with that."},{"type":"tool_use","id":"toolu_client_tool_123","name":"client_tool","input":{"input":"test input"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":100,"output_tokens":50}} + diff --git a/tests/Fixtures/deepseek/stream-with-client-executed-tool-1.sse b/tests/Fixtures/deepseek/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..5dc224e2c --- /dev/null +++ b/tests/Fixtures/deepseek/stream-with-client-executed-tool-1.sse @@ -0,0 +1,9 @@ +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"delta":{"role":"assistant","content":"","tool_calls":[{"index":0,"id":"call_client_tool_stream","type":"function","function":{"name":"client_tool","arguments":""}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"input\": \"test input\"}"}}]},"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + +data: [DONE] + + diff --git a/tests/Fixtures/deepseek/text-with-client-executed-tool-1.json b/tests/Fixtures/deepseek/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..cc22dab36 --- /dev/null +++ b/tests/Fixtures/deepseek/text-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"id":"client-executed-test","object":"chat.completion","created":1737244481,"model":"deepseek-chat","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"index":0,"id":"call_client_tool_123","type":"function","function":{"name":"client_tool","arguments":"{\"input\":\"test input\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150},"system_fingerprint":"fp_test"} + diff --git a/tests/Fixtures/gemini/stream-with-client-executed-tool-1.json b/tests/Fixtures/gemini/stream-with-client-executed-tool-1.json new file mode 100644 index 000000000..972a0996c --- /dev/null +++ b/tests/Fixtures/gemini/stream-with-client-executed-tool-1.json @@ -0,0 +1,3 @@ +data: {"candidates": [{"content": {"parts": [{"functionCall": {"name": "client_tool","args": {"input": "test input"}}}],"role": "model"},"finishReason": "STOP","index": 0}],"usageMetadata": {"promptTokenCount": 100,"candidatesTokenCount": 50,"totalTokenCount": 150,"promptTokensDetails": [{"modality": "TEXT","tokenCount": 100}]},"modelVersion": "gemini-1.5-flash"} + + diff --git a/tests/Fixtures/gemini/structured-with-client-executed-tool-1.json b/tests/Fixtures/gemini/structured-with-client-executed-tool-1.json new file mode 100644 index 000000000..c5a633d1c --- /dev/null +++ b/tests/Fixtures/gemini/structured-with-client-executed-tool-1.json @@ -0,0 +1,40 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "client_tool", + "args": { + "input": "test input" + } + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.00003298009687569 + } + ], + "usageMetadata": { + "promptTokenCount": 200, + "candidatesTokenCount": 50, + "totalTokenCount": 250, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 200 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 50 + } + ] + }, + "modelVersion": "gemini-2.0-flash" +} + diff --git a/tests/Fixtures/gemini/text-with-client-executed-tool-1.json b/tests/Fixtures/gemini/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..5fb57e5b9 --- /dev/null +++ b/tests/Fixtures/gemini/text-with-client-executed-tool-1.json @@ -0,0 +1,40 @@ +{ + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "client_tool", + "args": { + "input": "test input" + } + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.00003298009687569 + } + ], + "usageMetadata": { + "promptTokenCount": 100, + "candidatesTokenCount": 50, + "totalTokenCount": 150, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 100 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 50 + } + ] + }, + "modelVersion": "gemini-1.5-flash" +} + diff --git a/tests/Fixtures/groq/stream-with-client-executed-tool-1.sse b/tests/Fixtures/groq/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..c750092fc --- /dev/null +++ b/tests/Fixtures/groq/stream-with-client-executed-tool-1.sse @@ -0,0 +1,9 @@ +data: {"id":"chatcmpl-client-executed-stream","object":"chat.completion.chunk","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_client_tool_stream","type":"function","function":{"name":"client_tool","arguments":""}}]},"logprobs":null,"finish_reason":null}],"x_groq":{"id":"req_test"}} + +data: {"id":"chatcmpl-client-executed-stream","object":"chat.completion.chunk","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"input\": \"test input\"}"}}]},"logprobs":null,"finish_reason":null}],"x_groq":{"id":"req_test"}} + +data: {"id":"chatcmpl-client-executed-stream","object":"chat.completion.chunk","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"queue_time":0.1,"prompt_tokens":100,"prompt_time":0.01,"completion_tokens":50,"completion_time":0.1,"total_tokens":150,"total_time":0.2},"x_groq":{"id":"req_test"}} + +data: [DONE] + + diff --git a/tests/Fixtures/groq/text-with-client-executed-tool-1.json b/tests/Fixtures/groq/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..eecec7bb8 --- /dev/null +++ b/tests/Fixtures/groq/text-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"id":"chatcmpl-client-executed","object":"chat.completion","created":1740311145,"model":"llama-3.3-70b-versatile","choices":[{"index":0,"message":{"role":"assistant","tool_calls":[{"id":"call_client_tool","type":"function","function":{"name":"client_tool","arguments":"{\"input\": \"test input\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"queue_time":0.1,"prompt_tokens":100,"prompt_time":0.01,"completion_tokens":50,"completion_time":0.1,"total_tokens":150,"total_time":0.2},"system_fingerprint":"fp_test"} + diff --git a/tests/Fixtures/mistral/stream-with-client-executed-tool-1.sse b/tests/Fixtures/mistral/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..756442567 --- /dev/null +++ b/tests/Fixtures/mistral/stream-with-client-executed-tool-1.sse @@ -0,0 +1,6 @@ +data: {"id":"client-executed-test","object":"chat.completion.chunk","created":1759185828,"model":"mistral-large-latest","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + +data: {"id":"client-executed-test","object":"chat.completion.chunk","created":1759185828,"model":"mistral-large-latest","choices":[{"index":0,"delta":{"tool_calls":[{"id":"client_tool_stream","function":{"name":"client_tool","arguments":"{\"input\": \"test input\"}"},"index":0}]},"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"total_tokens":150,"completion_tokens":50}} + +data: [DONE] + diff --git a/tests/Fixtures/mistral/text-with-client-executed-tool-1.json b/tests/Fixtures/mistral/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..21a812489 --- /dev/null +++ b/tests/Fixtures/mistral/text-with-client-executed-tool-1.json @@ -0,0 +1,32 @@ +{ + "id": "client_executed_test", + "object": "chat.completion", + "created": 1728462827, + "model": "mistral-large-latest", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "client_tool_123", + "type": "function", + "function": { + "name": "client_tool", + "arguments": "{\"input\": \"test input\"}" + } + } + ] + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 100, + "total_tokens": 150, + "completion_tokens": 50 + } +} + diff --git a/tests/Fixtures/ollama/stream-with-client-executed-tool-1.sse b/tests/Fixtures/ollama/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..044f3d26e --- /dev/null +++ b/tests/Fixtures/ollama/stream-with-client-executed-tool-1.sse @@ -0,0 +1,3 @@ +{"model":"qwen2.5:14b","created_at":"2025-06-09T18:55:26.684517Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"client_tool","arguments":{"input":"test input"}}}]},"done_reason":"stop","done":true,"total_duration":8210142000,"load_duration":22224542,"prompt_eval_count":100,"prompt_eval_duration":269880958,"eval_count":50,"eval_duration":7916594250} + + diff --git a/tests/Fixtures/ollama/text-with-client-executed-tool-1.json b/tests/Fixtures/ollama/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..d5d619328 --- /dev/null +++ b/tests/Fixtures/ollama/text-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"model":"qwen2.5:14b","created_at":"2025-06-09T18:55:26.684517Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"client_tool","arguments":{"input":"test input"}}}]},"done_reason":"stop","done":true,"total_duration":8210142000,"load_duration":22224542,"prompt_eval_count":100,"prompt_eval_duration":269880958,"eval_count":50,"eval_duration":7916594250} + diff --git a/tests/Fixtures/openai/stream-with-client-executed-tool-1.json b/tests/Fixtures/openai/stream-with-client-executed-tool-1.json new file mode 100644 index 000000000..272daa16c --- /dev/null +++ b/tests/Fixtures/openai/stream-with-client-executed-tool-1.json @@ -0,0 +1,22 @@ +event: response.created +data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_client_executed_stream","object":"response","created_at":1750705330,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","description":"A tool that executes on the client","name":"client_tool","parameters":{"type":"object","properties":{"input":{"description":"Input parameter","type":"string"}},"required":["input"]},"strict":true}],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + +event: response.in_progress +data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_client_executed_stream","object":"response","created_at":1750705330,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","description":"A tool that executes on the client","name":"client_tool","parameters":{"type":"object","properties":{"input":{"description":"Input parameter","type":"string"}},"required":["input"]},"strict":true}],"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}}} + +event: response.output_item.added +data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"fc_client_tool_stream","type":"function_call","status":"in_progress","arguments":"","call_id":"call_client_tool_stream","name":"client_tool"}} + +event: response.function_call_arguments.delta +data: {"type":"response.function_call_arguments.delta","sequence_number":3,"item_id":"fc_client_tool_stream","output_index":0,"delta":"{\"input\":\"test input\"}"} + +event: response.function_call_arguments.done +data: {"type":"response.function_call_arguments.done","sequence_number":4,"item_id":"fc_client_tool_stream","output_index":0,"arguments":"{\"input\":\"test input\"}"} + +event: response.output_item.done +data: {"type":"response.output_item.done","sequence_number":5,"output_index":0,"item":{"id":"fc_client_tool_stream","type":"function_call","status":"completed","arguments":"{\"input\":\"test input\"}","call_id":"call_client_tool_stream","name":"client_tool"}} + +event: response.completed +data: {"type":"response.completed","sequence_number":6,"response":{"id":"resp_client_executed_stream","object":"response","created_at":1750705330,"status":"completed","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":2048,"model":"gpt-4o-2024-08-06","output":[{"id":"fc_client_tool_stream","type":"function_call","status":"completed","arguments":"{\"input\":\"test input\"}","call_id":"call_client_tool_stream","name":"client_tool"}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"}},"tool_choice":"auto","tools":[{"type":"function","description":"A tool that executes on the client","name":"client_tool","parameters":{"type":"object","properties":{"input":{"description":"Input parameter","type":"string"}},"required":["input"]},"strict":true}],"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":100,"input_tokens_details":{"cached_tokens":0},"output_tokens":50,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":150},"user":null,"metadata":{}}} + + diff --git a/tests/Fixtures/openai/structured-with-client-executed-tool-1.json b/tests/Fixtures/openai/structured-with-client-executed-tool-1.json new file mode 100644 index 000000000..8aff9433b --- /dev/null +++ b/tests/Fixtures/openai/structured-with-client-executed-tool-1.json @@ -0,0 +1,31 @@ +{ + "id": "resp_structured_client_executed", + "object": "response", + "created_at": 1741989983, + "status": "completed", + "model": "gpt-4o-2024-08-06", + "output": [ + { + "id": "fc_client_tool_structured", + "type": "function_call", + "status": "completed", + "arguments": "{\"input\": \"test input\"}", + "call_id": "call_client_tool_structured", + "name": "client_tool" + } + ], + "usage": { + "input_tokens": 200, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 50, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 250 + }, + "service_tier": "default", + "system_fingerprint": "fp_test" +} + diff --git a/tests/Fixtures/openai/text-with-client-executed-tool-1.json b/tests/Fixtures/openai/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..a62212d13 --- /dev/null +++ b/tests/Fixtures/openai/text-with-client-executed-tool-1.json @@ -0,0 +1,31 @@ +{ + "id": "resp_client_executed_test", + "object": "response", + "created_at": 1741989983, + "status": "completed", + "model": "gpt-4o-2024-08-06", + "output": [ + { + "id": "fc_client_tool_123", + "type": "function_call", + "status": "completed", + "arguments": "{\"input\": \"test input\"}", + "call_id": "call_client_tool_123", + "name": "client_tool" + } + ], + "usage": { + "input_tokens": 100, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 50, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 150 + }, + "service_tier": "default", + "system_fingerprint": "fp_test" +} + diff --git a/tests/Fixtures/openrouter/stream-with-client-executed-tool-1.sse b/tests/Fixtures/openrouter/stream-with-client-executed-tool-1.sse new file mode 100644 index 000000000..e0a4d5e6c --- /dev/null +++ b/tests/Fixtures/openrouter/stream-with-client-executed-tool-1.sse @@ -0,0 +1,9 @@ +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"call_client_tool_stream","type":"function","function":{"name":"client_tool","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"input\": \"test input\"}"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + +data: [DONE] + + diff --git a/tests/Fixtures/openrouter/text-with-client-executed-tool-1.json b/tests/Fixtures/openrouter/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..5afde1944 --- /dev/null +++ b/tests/Fixtures/openrouter/text-with-client-executed-tool-1.json @@ -0,0 +1,2 @@ +{"id":"gen-client-executed","object":"chat.completion","created":1737243487,"model":"openai/gpt-4-turbo","choices":[{"index":0,"message":{"role":"assistant","content":"","tool_calls":[{"id":"call_client_tool","type":"function","function":{"name":"client_tool","arguments":"{\"input\":\"test input\"}"}}]},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + diff --git a/tests/Fixtures/xai/stream-with-client-executed-tool-1.json b/tests/Fixtures/xai/stream-with-client-executed-tool-1.json new file mode 100644 index 000000000..63313a45e --- /dev/null +++ b/tests/Fixtures/xai/stream-with-client-executed-tool-1.json @@ -0,0 +1,9 @@ +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1731129810,"model":"grok-4","choices":[{"index":0,"delta":{"role":"assistant","tool_calls":[{"index":0,"id":"0","type":"function","function":{"name":"client_tool","arguments":""}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1731129810,"model":"grok-4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"input\":\"test input\"}"}}]},"logprobs":null,"finish_reason":null}]} + +data: {"id":"chatcmpl-client-executed","object":"chat.completion.chunk","created":1731129810,"model":"grok-4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":100,"completion_tokens":50,"total_tokens":150}} + +data: [DONE] + + diff --git a/tests/Fixtures/xai/text-with-client-executed-tool-1.json b/tests/Fixtures/xai/text-with-client-executed-tool-1.json new file mode 100644 index 000000000..064707bc0 --- /dev/null +++ b/tests/Fixtures/xai/text-with-client-executed-tool-1.json @@ -0,0 +1,34 @@ +{ + "id": "client-executed-test", + "object": "chat.completion", + "created": 1731129810, + "model": "grok-beta", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "0", + "function": { + "name": "client_tool", + "arguments": "{\"input\":\"test input\"}" + }, + "type": "function" + } + ], + "refusal": null + }, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 100, + "completion_tokens": 50, + "total_tokens": 150 + }, + "system_fingerprint": "fp_test" +} + diff --git a/tests/Providers/Anthropic/AnthropicTextTest.php b/tests/Providers/Anthropic/AnthropicTextTest.php index c04791bfe..e335bb4fa 100644 --- a/tests/Providers/Anthropic/AnthropicTextTest.php +++ b/tests/Providers/Anthropic/AnthropicTextTest.php @@ -523,6 +523,28 @@ }); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/messages', 'anthropic/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-20240620') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(\Prism\Prism\Enums\FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + describe('exceptions', function (): void { it('throws a RateLimitException if the Anthropic responds with a 429', function (): void { Http::fake([ diff --git a/tests/Providers/Anthropic/StreamTest.php b/tests/Providers/Anthropic/StreamTest.php index 7391619cd..dffd6ae1f 100644 --- a/tests/Providers/Anthropic/StreamTest.php +++ b/tests/Providers/Anthropic/StreamTest.php @@ -17,6 +17,8 @@ use Prism\Prism\Facades\Tool; use Prism\Prism\Streaming\Events\CitationEvent; use Prism\Prism\Streaming\Events\ProviderToolEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; @@ -655,6 +657,40 @@ })->throws(PrismRequestTooLargeException::class); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('v1/messages', 'anthropic/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using('anthropic', 'claude-3-5-sonnet-20240620') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(\Prism\Prism\Enums\FinishReason::ToolCalls); + }); +}); + describe('basic stream events', function (): void { it('can generate text with a basic stream', function (): void { FixtureResponse::fakeStreamResponses('v1/messages', 'anthropic/stream-basic-text'); @@ -687,3 +723,78 @@ }); }); }); + +describe('step events', function (): void { + it('emits step start and step finish events', function (): void { + FixtureResponse::fakeStreamResponses('v1/messages', 'anthropic/stream-basic-text'); + + $response = Prism::text() + ->using('anthropic', 'claude-3-7-sonnet-20250219') + ->withPrompt('Who are you?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // Check for StepStartEvent after StreamStartEvent + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + expect($stepStartEvents)->toHaveCount(1); + + // Check for StepFinishEvent before StreamEndEvent + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + expect($stepFinishEvents)->toHaveCount(1); + + // Verify order: StreamStart -> StepStart -> ... -> StepFinish -> StreamEnd + $eventTypes = array_map(get_class(...), $events); + $streamStartIndex = array_search(StreamStartEvent::class, $eventTypes); + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + $streamEndIndex = array_search(StreamEndEvent::class, $eventTypes); + + expect($streamStartIndex)->toBeLessThan($stepStartIndex); + expect($stepStartIndex)->toBeLessThan($stepFinishIndex); + expect($stepFinishIndex)->toBeLessThan($streamEndIndex); + }); + + it('emits multiple step events with tool calls', function (): void { + FixtureResponse::fakeStreamResponses('v1/messages', 'anthropic/stream-with-tools'); + + $tools = [ + Tool::as('weather') + ->for('useful when you need to search for current weather conditions') + ->withStringParameter('city', 'The city that you want the weather for') + ->using(fn (string $city): string => "The weather will be 75° and sunny in {$city}"), + Tool::as('search') + ->for('useful for searching current events or data') + ->withStringParameter('query', 'The detailed search query') + ->using(fn (string $query): string => "Search results for: {$query}"), + ]; + + $response = Prism::text() + ->using(Provider::Anthropic, 'claude-3-7-sonnet-20250219') + ->withTools($tools) + ->withMaxSteps(3) + ->withPrompt('What is the weather in Detroit?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // With tool calls, we should have multiple step start/finish pairs + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + + // At least 2 steps: one for tool call, one for final response + expect(count($stepStartEvents))->toBeGreaterThanOrEqual(2); + expect(count($stepFinishEvents))->toBeGreaterThanOrEqual(2); + + // Verify step start/finish pairs are balanced + expect(count($stepStartEvents))->toBe(count($stepFinishEvents)); + }); +}); diff --git a/tests/Providers/Anthropic/StructuredWithToolsTest.php b/tests/Providers/Anthropic/StructuredWithToolsTest.php index 9e904d1ce..8cd38e0a4 100644 --- a/tests/Providers/Anthropic/StructuredWithToolsTest.php +++ b/tests/Providers/Anthropic/StructuredWithToolsTest.php @@ -200,6 +200,36 @@ expect($response->toolResults)->toBeArray(); }); + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('*', 'anthropic/structured-with-client-executed-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::structured() + ->using(Provider::Anthropic, 'claude-sonnet-4-0') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withProviderOptions(['use_tool_calling' => true]) + ->withPrompt('Use the client tool') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + it('includes strict field in tool definition when specified', function (): void { Prism::fake(); diff --git a/tests/Providers/DeepSeek/StreamTest.php b/tests/Providers/DeepSeek/StreamTest.php index 3137a5899..4c164fda2 100644 --- a/tests/Providers/DeepSeek/StreamTest.php +++ b/tests/Providers/DeepSeek/StreamTest.php @@ -10,7 +10,10 @@ use Prism\Prism\Enums\Provider; use Prism\Prism\Facades\Prism; use Prism\Prism\Facades\Tool; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; +use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; @@ -133,6 +136,40 @@ }); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('chat/completions', 'deepseek/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using(Provider::DeepSeek, 'deepseek-chat') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('handles max_tokens parameter correctly', function (): void { FixtureResponse::fakeStreamResponses('chat/completions', 'deepseek/stream-max-tokens'); @@ -210,3 +247,76 @@ ->and($thinkingContent)->toContain('answer') ->and($regularContent)->toContain('32'); }); + +it('emits step start and step finish events', function (): void { + FixtureResponse::fakeStreamResponses('chat/completions', 'deepseek/stream-basic-text'); + + $response = Prism::text() + ->using(Provider::DeepSeek, 'deepseek-chat') + ->withPrompt('Who are you?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // Check for StepStartEvent after StreamStartEvent + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + expect($stepStartEvents)->toHaveCount(1); + + // Check for StepFinishEvent before StreamEndEvent + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + expect($stepFinishEvents)->toHaveCount(1); + + // Verify order: StreamStart -> StepStart -> ... -> StepFinish -> StreamEnd + $eventTypes = array_map(get_class(...), $events); + $streamStartIndex = array_search(StreamStartEvent::class, $eventTypes); + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + $streamEndIndex = array_search(StreamEndEvent::class, $eventTypes); + + expect($streamStartIndex)->toBeLessThan($stepStartIndex); + expect($stepStartIndex)->toBeLessThan($stepFinishIndex); + expect($stepFinishIndex)->toBeLessThan($streamEndIndex); +}); + +it('emits multiple step events with tool calls', function (): void { + FixtureResponse::fakeStreamResponses('chat/completions', 'deepseek/stream-with-tools'); + + $tools = [ + Tool::as('get_weather') + ->for('useful when you need to search for current weather conditions') + ->withStringParameter('city', 'The city that you want the weather for') + ->using(fn (string $city): string => "The weather will be 75° and sunny in {$city}"), + Tool::as('search') + ->for('useful for searching current events or data') + ->withStringParameter('query', 'The detailed search query') + ->using(fn (string $query): string => "Search results for: {$query}"), + ]; + + $response = Prism::text() + ->using(Provider::DeepSeek, 'deepseek-chat') + ->withTools($tools) + ->withMaxSteps(4) + ->withPrompt('What is the weather in Detroit?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // With tool calls, we should have multiple step start/finish pairs + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + + // At least 2 steps: one for tool call, one for final response + expect(count($stepStartEvents))->toBeGreaterThanOrEqual(2); + expect(count($stepFinishEvents))->toBeGreaterThanOrEqual(2); + + // Verify step start/finish pairs are balanced + expect(count($stepStartEvents))->toBe(count($stepFinishEvents)); +}); diff --git a/tests/Providers/DeepSeek/TextTest.php b/tests/Providers/DeepSeek/TextTest.php index e6fd3ccb6..2da324961 100644 --- a/tests/Providers/DeepSeek/TextTest.php +++ b/tests/Providers/DeepSeek/TextTest.php @@ -76,6 +76,28 @@ expect($response->finishReason)->toBe(FinishReason::Stop); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'deepseek/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using(Provider::DeepSeek, 'deepseek-chat') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->generate(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + it('can generate text using multiple tools and multiple steps', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'deepseek/generate-text-with-multiple-tools'); diff --git a/tests/Providers/Gemini/GeminiStreamTest.php b/tests/Providers/Gemini/GeminiStreamTest.php index 702538cd1..d30f2014e 100644 --- a/tests/Providers/Gemini/GeminiStreamTest.php +++ b/tests/Providers/Gemini/GeminiStreamTest.php @@ -10,7 +10,10 @@ use Prism\Prism\Enums\Provider; use Prism\Prism\Facades\Prism; use Prism\Prism\Facades\Tool; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; +use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; @@ -229,6 +232,40 @@ }); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using(Provider::Gemini, 'gemini-1.5-flash') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('yields ToolCall events before ToolResult events', function (): void { FixtureResponse::fakeResponseSequence('*', 'gemini/stream-with-tools'); @@ -287,3 +324,72 @@ $firstToolResultEvent = array_values($toolResultEvents)[0]; expect($firstToolResultEvent->toolResult)->not->toBeNull(); }); + +it('emits step start and step finish events', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/stream-basic-text'); + + $response = Prism::text() + ->using(Provider::Gemini, 'gemini-2.0-flash') + ->withPrompt('Explain how AI works') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // Check for StepStartEvent after StreamStartEvent + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + expect($stepStartEvents)->toHaveCount(1); + + // Check for StepFinishEvent before StreamEndEvent + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + expect($stepFinishEvents)->toHaveCount(1); + + // Verify order: StreamStart -> StepStart -> ... -> StepFinish -> StreamEnd + $eventTypes = array_map(get_class(...), $events); + $streamStartIndex = array_search(StreamStartEvent::class, $eventTypes); + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + $streamEndIndex = array_search(StreamEndEvent::class, $eventTypes); + + expect($streamStartIndex)->toBeLessThan($stepStartIndex); + expect($stepStartIndex)->toBeLessThan($stepFinishIndex); + expect($stepFinishIndex)->toBeLessThan($streamEndIndex); +}); + +it('emits multiple step events with tool calls', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/stream-with-tools'); + + $tools = [ + Tool::as('weather') + ->for('useful when you need to search for current weather conditions') + ->withStringParameter('city', 'The city that you want the weather for') + ->using(fn (string $city): string => "The weather will be 75° and sunny in {$city}"), + ]; + + $response = Prism::text() + ->using(Provider::Gemini, 'gemini-2.5-flash') + ->withTools($tools) + ->withMaxSteps(3) + ->withPrompt('What is the weather in San Francisco?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // With tool calls, we should have multiple step start/finish pairs + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + + // At least 2 steps: one for tool call, one for final response + expect(count($stepStartEvents))->toBeGreaterThanOrEqual(2); + expect(count($stepFinishEvents))->toBeGreaterThanOrEqual(2); + + // Verify step start/finish pairs are balanced + expect(count($stepStartEvents))->toBe(count($stepFinishEvents)); +}); diff --git a/tests/Providers/Gemini/GeminiTextTest.php b/tests/Providers/Gemini/GeminiTextTest.php index fd929f290..76620426c 100644 --- a/tests/Providers/Gemini/GeminiTextTest.php +++ b/tests/Providers/Gemini/GeminiTextTest.php @@ -154,6 +154,29 @@ }); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/text-with-client-executed-tool'); + + $tool = (new Tool) + ->as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using(Provider::Gemini, 'gemini-1.5-flash') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + describe('Image support with Gemini', function (): void { it('can send images from path', function (): void { FixtureResponse::fakeResponseSequence('*', 'gemini/image-detection'); diff --git a/tests/Providers/Gemini/StructuredWithToolsTest.php b/tests/Providers/Gemini/StructuredWithToolsTest.php index 36d94de36..bd167bd27 100644 --- a/tests/Providers/Gemini/StructuredWithToolsTest.php +++ b/tests/Providers/Gemini/StructuredWithToolsTest.php @@ -120,6 +120,35 @@ expect($finalStep->structured)->toBeArray(); }); + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('*', 'gemini/structured-with-client-executed-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::structured() + ->using(Provider::Gemini, 'gemini-2.0-flash') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + it('returns structured output immediately when no tool calls needed', function (): void { FixtureResponse::fakeResponseSequence('*', 'gemini/structured-without-tool-calls'); diff --git a/tests/Providers/Groq/GroqTextTest.php b/tests/Providers/Groq/GroqTextTest.php index 98ba516f4..3ecba36a7 100644 --- a/tests/Providers/Groq/GroqTextTest.php +++ b/tests/Providers/Groq/GroqTextTest.php @@ -107,6 +107,26 @@ ); }); + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'groq/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using('groq', 'llama-3.3-70b-versatile') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->generate(); + + expect($response->finishReason)->toBe(\Prism\Prism\Enums\FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + it('handles specific tool choice', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'groq/generate-text-with-required-tool-call'); diff --git a/tests/Providers/Groq/StreamTest.php b/tests/Providers/Groq/StreamTest.php index 9336cab2b..a074227a6 100644 --- a/tests/Providers/Groq/StreamTest.php +++ b/tests/Providers/Groq/StreamTest.php @@ -11,7 +11,10 @@ use Prism\Prism\Exceptions\PrismStreamDecodeException; use Prism\Prism\Facades\Prism; use Prism\Prism\Facades\Tool; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; +use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; @@ -120,6 +123,40 @@ expect($streamEndEvents)->toHaveCount(1); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('openai/v1/chat/completions', 'groq/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using(Provider::Groq, 'llama-3.1-70b-versatile') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('handles maximum tool call depth exceeded', function (): void { FixtureResponse::fakeStreamResponses('openai/v1/chat/completions', 'groq/stream-with-tools'); @@ -366,3 +403,76 @@ expect($streamEndEvent->usage->promptTokens)->toBe(7); expect($streamEndEvent->usage->completionTokens)->toBe(50); }); + +it('emits step start and step finish events', function (): void { + FixtureResponse::fakeStreamResponses('openai/v1/chat/completions', 'groq/stream-basic-text'); + + $response = Prism::text() + ->using(Provider::Groq, 'llama-3.1-70b-versatile') + ->withPrompt('Who are you?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // Check for StepStartEvent after StreamStartEvent + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + expect($stepStartEvents)->toHaveCount(1); + + // Check for StepFinishEvent before StreamEndEvent + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + expect($stepFinishEvents)->toHaveCount(1); + + // Verify order: StreamStart -> StepStart -> ... -> StepFinish -> StreamEnd + $eventTypes = array_map(get_class(...), $events); + $streamStartIndex = array_search(StreamStartEvent::class, $eventTypes); + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + $streamEndIndex = array_search(StreamEndEvent::class, $eventTypes); + + expect($streamStartIndex)->toBeLessThan($stepStartIndex); + expect($stepStartIndex)->toBeLessThan($stepFinishIndex); + expect($stepFinishIndex)->toBeLessThan($streamEndIndex); +}); + +it('emits multiple step events with tool calls', function (): void { + FixtureResponse::fakeStreamResponses('openai/v1/chat/completions', 'groq/stream-with-tools'); + + $tools = [ + Tool::as('weather') + ->for('useful when you need to search for current weather conditions') + ->withStringParameter('city', 'The city that you want the weather for') + ->using(fn (string $city): string => "The weather will be 75° and sunny in {$city}"), + Tool::as('search') + ->for('useful for searching current events or data') + ->withStringParameter('query', 'The detailed search query') + ->using(fn (string $query): string => "Search results for: {$query}"), + ]; + + $response = Prism::text() + ->using(Provider::Groq, 'llama-3.1-70b-versatile') + ->withTools($tools) + ->withMaxSteps(4) + ->withPrompt('What is the weather in Detroit?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // With tool calls, we should have multiple step start/finish pairs + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + + // At least 2 steps: one for tool call, one for final response + expect(count($stepStartEvents))->toBeGreaterThanOrEqual(2); + expect(count($stepFinishEvents))->toBeGreaterThanOrEqual(2); + + // Verify step start/finish pairs are balanced + expect(count($stepStartEvents))->toBe(count($stepFinishEvents)); +}); diff --git a/tests/Providers/Mistral/MistralTextTest.php b/tests/Providers/Mistral/MistralTextTest.php index 03aa42fd0..09d6df205 100644 --- a/tests/Providers/Mistral/MistralTextTest.php +++ b/tests/Providers/Mistral/MistralTextTest.php @@ -106,6 +106,26 @@ ); }); + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'mistral/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using('mistral', 'mistral-large-latest') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->generate(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + it('handles specific tool choice', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'mistral/generate-text-with-required-tool-call'); diff --git a/tests/Providers/Mistral/StreamTest.php b/tests/Providers/Mistral/StreamTest.php index a504bc111..8b1ae1754 100644 --- a/tests/Providers/Mistral/StreamTest.php +++ b/tests/Providers/Mistral/StreamTest.php @@ -11,7 +11,10 @@ use Prism\Prism\Exceptions\PrismStreamDecodeException; use Prism\Prism\Facades\Prism; use Prism\Prism\Facades\Tool; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; +use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\ToolCallEvent; @@ -119,6 +122,40 @@ expect($streamEndEvents)->toHaveCount(1); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'mistral/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using(Provider::Mistral, 'mistral-large-latest') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('handles maximum tool call depth exceeded', function (): void { FixtureResponse::fakeStreamResponses('v1/chat/completions', 'mistral/stream-with-tools-1'); @@ -353,3 +390,76 @@ expect($streamEndEvent->usage->promptTokens)->toBe(7); expect($streamEndEvent->usage->completionTokens)->toBe(13); }); + +it('emits step start and step finish events', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'mistral/stream-basic-text-1'); + + $response = Prism::text() + ->using(Provider::Mistral, 'mistral-small-latest') + ->withPrompt('Who are you?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // Check for StepStartEvent after StreamStartEvent + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + expect($stepStartEvents)->toHaveCount(1); + + // Check for StepFinishEvent before StreamEndEvent + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + expect($stepFinishEvents)->toHaveCount(1); + + // Verify order: StreamStart -> StepStart -> ... -> StepFinish -> StreamEnd + $eventTypes = array_map(get_class(...), $events); + $streamStartIndex = array_search(StreamStartEvent::class, $eventTypes); + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + $streamEndIndex = array_search(StreamEndEvent::class, $eventTypes); + + expect($streamStartIndex)->toBeLessThan($stepStartIndex); + expect($stepStartIndex)->toBeLessThan($stepFinishIndex); + expect($stepFinishIndex)->toBeLessThan($streamEndIndex); +}); + +it('emits multiple step events with tool calls', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'mistral/stream-with-tools-1'); + + $tools = [ + Tool::as('weather') + ->for('useful when you need to search for current weather conditions') + ->withStringParameter('city', 'The city that you want the weather for') + ->using(fn (string $city): string => "The weather will be 75° and sunny in {$city}"), + Tool::as('search') + ->for('useful for searching current events or data') + ->withStringParameter('query', 'The detailed search query') + ->using(fn (string $query): string => "Search results for: {$query}"), + ]; + + $response = Prism::text() + ->using(Provider::Mistral, 'mistral-large-latest') + ->withTools($tools) + ->withMaxSteps(4) + ->withPrompt('What is the weather in Detroit?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // With tool calls, we should have multiple step start/finish pairs + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + + // At least 2 steps: one for tool call, one for final response + expect(count($stepStartEvents))->toBeGreaterThanOrEqual(2); + expect(count($stepFinishEvents))->toBeGreaterThanOrEqual(2); + + // Verify step start/finish pairs are balanced + expect(count($stepStartEvents))->toBe(count($stepFinishEvents)); +}); diff --git a/tests/Providers/Ollama/StreamTest.php b/tests/Providers/Ollama/StreamTest.php index ce4dfed74..7121c26c7 100644 --- a/tests/Providers/Ollama/StreamTest.php +++ b/tests/Providers/Ollama/StreamTest.php @@ -10,7 +10,10 @@ use Prism\Prism\Exceptions\PrismRateLimitedException; use Prism\Prism\Facades\Prism; use Prism\Prism\Facades\Tool; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; +use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; @@ -123,6 +126,40 @@ expect($streamEndEvents)->toHaveCount(1); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('api/chat', 'ollama/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using('ollama', 'qwen2.5:14b') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('throws a PrismRateLimitedException with a 429 response code', function (): void { Http::fake([ '*' => Http::response( @@ -287,3 +324,76 @@ expect($finalText)->toContain('Here is the answer:'); expect($lastFinishReason)->toBe(FinishReason::Stop); }); + +it('emits step start and step finish events', function (): void { + FixtureResponse::fakeStreamResponses('api/chat', 'ollama/stream-basic-text'); + + $response = Prism::text() + ->using('ollama', 'granite3-dense:8b') + ->withPrompt('Who are you?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // Check for StepStartEvent after StreamStartEvent + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + expect($stepStartEvents)->toHaveCount(1); + + // Check for StepFinishEvent before StreamEndEvent + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + expect($stepFinishEvents)->toHaveCount(1); + + // Verify order: StreamStart -> StepStart -> ... -> StepFinish -> StreamEnd + $eventTypes = array_map(get_class(...), $events); + $streamStartIndex = array_search(StreamStartEvent::class, $eventTypes); + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + $streamEndIndex = array_search(StreamEndEvent::class, $eventTypes); + + expect($streamStartIndex)->toBeLessThan($stepStartIndex); + expect($stepStartIndex)->toBeLessThan($stepFinishIndex); + expect($stepFinishIndex)->toBeLessThan($streamEndIndex); +}); + +it('emits multiple step events with tool calls', function (): void { + FixtureResponse::fakeStreamResponses('api/chat', 'ollama/stream-with-tools'); + + $tools = [ + Tool::as('weather') + ->for('useful when you need to search for current weather conditions') + ->withStringParameter('city', 'The city that you want the weather for') + ->using(fn (string $city): string => "The weather will be 75° and sunny in {$city}"), + Tool::as('search') + ->for('useful for searching current events or data') + ->withStringParameter('query', 'The detailed search query') + ->using(fn (string $query): string => 'The tigers game is at 3pm today'), + ]; + + $response = Prism::text() + ->using('ollama', 'qwen3:14b') + ->withTools($tools) + ->withMaxSteps(6) + ->withPrompt('What is the weather in Detroit?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // With tool calls, we should have multiple step start/finish pairs + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + + // At least 2 steps: one for tool call, one for final response + expect(count($stepStartEvents))->toBeGreaterThanOrEqual(2); + expect(count($stepFinishEvents))->toBeGreaterThanOrEqual(2); + + // Verify step start/finish pairs are balanced + expect(count($stepStartEvents))->toBe(count($stepFinishEvents)); +}); diff --git a/tests/Providers/Ollama/TextTest.php b/tests/Providers/Ollama/TextTest.php index 9b355d7c5..ed667555c 100644 --- a/tests/Providers/Ollama/TextTest.php +++ b/tests/Providers/Ollama/TextTest.php @@ -104,6 +104,28 @@ }); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('api/chat', 'ollama/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using('ollama', 'qwen2.5:14b') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(\Prism\Prism\Enums\FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + describe('Thinking parameter', function (): void { it('includes think parameter when thinking is enabled', function (): void { FixtureResponse::fakeResponseSequence('api/chat', 'ollama/text-with-thinking-enabled'); diff --git a/tests/Providers/OpenAI/StreamTest.php b/tests/Providers/OpenAI/StreamTest.php index 222150ebe..a0c5bec48 100644 --- a/tests/Providers/OpenAI/StreamTest.php +++ b/tests/Providers/OpenAI/StreamTest.php @@ -11,7 +11,10 @@ use Prism\Prism\Facades\Prism; use Prism\Prism\Facades\Tool; use Prism\Prism\Streaming\Events\ProviderToolEvent; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; +use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; @@ -458,6 +461,40 @@ Http::assertSent(fn (Request $request): bool => $request->data()['parallel_tool_calls'] === false); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using('openai', 'gpt-4o') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(\Prism\Prism\Enums\FinishReason::ToolCalls); + }); +}); + it('emits usage information', function (): void { FixtureResponse::fakeResponseSequence('v1/responses', 'openai/stream-basic-text-responses'); @@ -648,3 +685,74 @@ return true; }); }); + +it('emits step start and step finish events', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/stream-basic-text-responses'); + + $response = Prism::text() + ->using('openai', 'gpt-4o') + ->withPrompt('Who are you?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // Check for StepStartEvent after StreamStartEvent + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + expect($stepStartEvents)->toHaveCount(1); + + // Check for StepFinishEvent before StreamEndEvent + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + expect($stepFinishEvents)->toHaveCount(1); + + // Verify order: StreamStart -> StepStart -> ... -> StepFinish -> StreamEnd + $eventTypes = array_map(get_class(...), $events); + $streamStartIndex = array_search(StreamStartEvent::class, $eventTypes); + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + $streamEndIndex = array_search(StreamEndEvent::class, $eventTypes); + + expect($streamStartIndex)->toBeLessThan($stepStartIndex); + expect($stepStartIndex)->toBeLessThan($stepFinishIndex); + expect($stepFinishIndex)->toBeLessThan($streamEndIndex); +}); + +it('emits multiple step events with tool calls', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/stream-with-tools-responses'); + + $tools = [ + Tool::as('weather') + ->for('useful when you need to search for current weather conditions') + ->withStringParameter('city', 'The city that you want the weather for') + ->using(fn (string $city): string => "The weather will be 75° and sunny in {$city}"), + Tool::as('search') + ->for('useful for searching current events or data') + ->withStringParameter('query', 'The detailed search query') + ->using(fn (string $query): string => "Search results for: {$query}"), + ]; + + $response = Prism::text() + ->using('openai', 'gpt-4o') + ->withTools($tools) + ->withMaxSteps(4) + ->withPrompt('What is the weather in Detroit?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // With tool calls, we should have multiple step start/finish pairs + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + + // At least 2 steps: one for tool call, one for final response + expect(count($stepStartEvents))->toBeGreaterThanOrEqual(2) + ->and(count($stepFinishEvents))->toBeGreaterThanOrEqual(2) + ->and(count($stepStartEvents))->toBe(count($stepFinishEvents)); +}); diff --git a/tests/Providers/OpenAI/StructuredWithToolsTest.php b/tests/Providers/OpenAI/StructuredWithToolsTest.php index 7afd8910f..f0661be6a 100644 --- a/tests/Providers/OpenAI/StructuredWithToolsTest.php +++ b/tests/Providers/OpenAI/StructuredWithToolsTest.php @@ -148,6 +148,35 @@ expect($response->steps)->toHaveCount(1); }); + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/structured-with-client-executed-tool'); + + $schema = new ObjectSchema( + 'output', + 'the output object', + [new StringSchema('result', 'The result', true)], + ['result'] + ); + + $tool = (new Tool) + ->as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::structured() + ->using(Provider::OpenAI, 'gpt-4o') + ->withSchema($schema) + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStructured(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); + it('handles tool orchestration correctly with multiple tool types', function (): void { FixtureResponse::fakeResponseSequence('v1/responses', 'openai/structured-with-tool-orchestration'); diff --git a/tests/Providers/OpenAI/TextTest.php b/tests/Providers/OpenAI/TextTest.php index b7fc8658d..25fe3e1c6 100644 --- a/tests/Providers/OpenAI/TextTest.php +++ b/tests/Providers/OpenAI/TextTest.php @@ -317,6 +317,28 @@ }); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/responses', 'openai/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using('openai', 'gpt-4o') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + it('sets usage correctly with automatic caching', function (): void { FixtureResponse::fakeResponseSequence( 'v1/responses', diff --git a/tests/Providers/OpenRouter/StreamTest.php b/tests/Providers/OpenRouter/StreamTest.php index 0556091f4..4c56e219e 100644 --- a/tests/Providers/OpenRouter/StreamTest.php +++ b/tests/Providers/OpenRouter/StreamTest.php @@ -8,7 +8,10 @@ use Prism\Prism\Enums\Provider; use Prism\Prism\Facades\Prism; use Prism\Prism\Facades\Tool; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; +use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextCompleteEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; @@ -50,15 +53,15 @@ expect($events[0]->provider)->toBe('openrouter'); // Check we have TextStartEvent - $textStartEvents = array_filter($events, fn (\Prism\Prism\Streaming\Events\StreamEvent $e): bool => $e instanceof TextStartEvent); + $textStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof TextStartEvent); expect($textStartEvents)->toHaveCount(1); // Check we have TextDeltaEvents - $textDeltaEvents = array_filter($events, fn (\Prism\Prism\Streaming\Events\StreamEvent $e): bool => $e instanceof TextDeltaEvent); + $textDeltaEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof TextDeltaEvent); expect($textDeltaEvents)->not->toBeEmpty(); // Check we have TextCompleteEvent - $textCompleteEvents = array_filter($events, fn (\Prism\Prism\Streaming\Events\StreamEvent $e): bool => $e instanceof TextCompleteEvent); + $textCompleteEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof TextCompleteEvent); expect($textCompleteEvents)->toHaveCount(1); // Check last event is StreamEndEvent @@ -241,10 +244,44 @@ expect($toolResultEvents)->toHaveCount(1); expect($text)->toContain('The current time is '.$currentTime); - $streamEndEvents = array_filter($events, fn (\Prism\Prism\Streaming\Events\StreamEvent $e): bool => $e instanceof StreamEndEvent); + $streamEndEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StreamEndEvent); expect($streamEndEvents)->not->toBeEmpty(); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'openrouter/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('can handle reasoning/thinking tokens in streaming', function (): void { FixtureResponse::fakeStreamResponses('v1/chat/completions', 'openrouter/stream-text-with-reasoning'); @@ -272,7 +309,7 @@ expect($events)->not->toBeEmpty(); // Check for ThinkingStartEvent - $thinkingStartEvents = array_filter($events, fn (\Prism\Prism\Streaming\Events\StreamEvent $e): bool => $e instanceof ThinkingStartEvent); + $thinkingStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof ThinkingStartEvent); expect($thinkingStartEvents)->toHaveCount(1); // Check for ThinkingEvent @@ -283,10 +320,104 @@ expect($text)->toBe('The answer to 2 + 2 is 4.'); // Check for usage with reasoning tokens - $streamEndEvents = array_filter($events, fn (\Prism\Prism\Streaming\Events\StreamEvent $e): bool => $e instanceof StreamEndEvent); + $streamEndEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StreamEndEvent); expect($streamEndEvents)->toHaveCount(1); $streamEndEvent = array_values($streamEndEvents)[0]; expect($streamEndEvent->usage)->not->toBeNull(); expect($streamEndEvent->usage->thoughtTokens)->toBe(12); }); + +it('emits step start and step finish events', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'openrouter/stream-text-with-a-prompt'); + + $response = Prism::text() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withPrompt('Who are you?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // Check for StepStartEvent after StreamStartEvent + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + expect($stepStartEvents)->toHaveCount(1); + + // Check for StepFinishEvent before StreamEndEvent + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + expect($stepFinishEvents)->toHaveCount(1); + + // Verify order: StreamStart -> StepStart -> ... -> StepFinish -> StreamEnd + $eventTypes = array_map(get_class(...), $events); + $streamStartIndex = array_search(StreamStartEvent::class, $eventTypes); + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + $streamEndIndex = array_search(StreamEndEvent::class, $eventTypes); + + expect($streamStartIndex)->toBeLessThan($stepStartIndex); + expect($stepStartIndex)->toBeLessThan($stepFinishIndex); + expect($stepFinishIndex)->toBeLessThan($streamEndIndex); +}); + +it('emits multiple step events with tool calls', function (): void { + FixtureResponse::fakeStreamResponses('v1/chat/completions', 'openrouter/stream-text-with-tools'); + + $weatherTool = Tool::as('weather') + ->for('Get weather for a city') + ->withStringParameter('city', 'The city name') + ->using(fn (string $city): string => "The weather in {$city} is 75°F and sunny"); + + $response = Prism::text() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withTools([$weatherTool]) + ->withMaxSteps(3) + ->withPrompt('What is the weather in San Francisco?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // Extract step events + $stepStartEvents = array_values(array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent)); + $stepFinishEvents = array_values(array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent)); + + // Should have 2 steps: tool call step + final response step + expect($stepStartEvents)->toHaveCount(2); + expect($stepFinishEvents)->toHaveCount(2); + + // Verify step start/finish pairs are balanced + expect(count($stepStartEvents))->toBe(count($stepFinishEvents)); + + // Verify event ordering using indices + $getIndices = fn (string $class): array => array_keys(array_filter($events, fn (StreamEvent $e): bool => $e instanceof $class)); + + $streamStartIdx = $getIndices(StreamStartEvent::class)[0]; + $stepStartIndices = $getIndices(StepStartEvent::class); + $stepFinishIndices = $getIndices(StepFinishEvent::class); + $toolCallIndices = $getIndices(ToolCallEvent::class); + $toolResultIndices = $getIndices(ToolResultEvent::class); + $streamEndIdx = $getIndices(StreamEndEvent::class)[0]; + + // Verify overall structure: StreamStart -> Steps -> StreamEnd + expect($streamStartIdx)->toBeLessThan($stepStartIndices[0]); + expect($stepFinishIndices[count($stepFinishIndices) - 1])->toBeLessThan($streamEndIdx); + + // Verify each step has proper start/finish ordering + foreach ($stepStartIndices as $i => $startIdx) { + expect($startIdx)->toBeLessThan($stepFinishIndices[$i], "Step $i: start should come before finish"); + } + + // Verify tool call happens within first step (before first step finish) + expect($toolCallIndices[0])->toBeGreaterThan($stepStartIndices[0]); + expect($toolCallIndices[0])->toBeLessThan($stepFinishIndices[0]); + + // Verify tool result happens after tool call but before second step starts + expect($toolResultIndices[0])->toBeGreaterThan($toolCallIndices[0]); + expect($toolResultIndices[0])->toBeLessThan($stepStartIndices[1]); +}); diff --git a/tests/Providers/OpenRouter/TextTest.php b/tests/Providers/OpenRouter/TextTest.php index 32a751716..ce09d4bb0 100644 --- a/tests/Providers/OpenRouter/TextTest.php +++ b/tests/Providers/OpenRouter/TextTest.php @@ -127,6 +127,28 @@ expect($response->finishReason)->toBe(FinishReason::Stop); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using(Provider::OpenRouter, 'openai/gpt-4-turbo') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->generate(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + it('forwards advanced provider options to openrouter', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'openrouter/generate-text-with-a-prompt'); diff --git a/tests/Providers/XAI/StreamTest.php b/tests/Providers/XAI/StreamTest.php index 15d7f8bb6..417f96c02 100644 --- a/tests/Providers/XAI/StreamTest.php +++ b/tests/Providers/XAI/StreamTest.php @@ -9,7 +9,10 @@ use Prism\Prism\Enums\FinishReason; use Prism\Prism\Facades\Prism; use Prism\Prism\Facades\Tool; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; +use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; use Prism\Prism\Streaming\Events\ThinkingEvent; @@ -130,6 +133,40 @@ }); }); +describe('client-executed tools', function (): void { + it('stops streaming when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'xai/stream-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using('xai', 'grok-4') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asStream(); + + $events = []; + $toolCallFound = false; + + foreach ($response as $event) { + $events[] = $event; + + if ($event instanceof ToolCallEvent) { + $toolCallFound = true; + } + } + + expect($toolCallFound)->toBeTrue(); + + $lastEvent = end($events); + expect($lastEvent)->toBeInstanceOf(StreamEndEvent::class); + expect($lastEvent->finishReason)->toBe(FinishReason::ToolCalls); + }); +}); + it('handles max_tokens parameter correctly', function (): void { FixtureResponse::fakeResponseSequence('v1/chat/completions', 'xai/stream-basic-text-responses'); @@ -447,3 +484,72 @@ && $body['model'] === 'grok-4'; }); }); + +it('emits step start and step finish events', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'xai/stream-basic-text-responses'); + + $response = Prism::text() + ->using('xai', 'grok-4') + ->withPrompt('Who are you?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // Check for StepStartEvent after StreamStartEvent + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + expect($stepStartEvents)->toHaveCount(1); + + // Check for StepFinishEvent before StreamEndEvent + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + expect($stepFinishEvents)->toHaveCount(1); + + // Verify order: StreamStart -> StepStart -> ... -> StepFinish -> StreamEnd + $eventTypes = array_map(get_class(...), $events); + $streamStartIndex = array_search(StreamStartEvent::class, $eventTypes); + $stepStartIndex = array_search(StepStartEvent::class, $eventTypes); + $stepFinishIndex = array_search(StepFinishEvent::class, $eventTypes); + $streamEndIndex = array_search(StreamEndEvent::class, $eventTypes); + + expect($streamStartIndex)->toBeLessThan($stepStartIndex); + expect($stepStartIndex)->toBeLessThan($stepFinishIndex); + expect($stepFinishIndex)->toBeLessThan($streamEndIndex); +}); + +it('emits multiple step events with tool calls', function (): void { + FixtureResponse::fakeResponseSequence('v1/chat/completions', 'xai/stream-with-tools-responses'); + + $tools = [ + Tool::as('get_weather') + ->for('useful when you need to search for current weather conditions') + ->withStringParameter('city', 'The city that you want the weather for') + ->using(fn (string $city): string => "The weather will be 75° and sunny in {$city}"), + ]; + + $response = Prism::text() + ->using('xai', 'grok-4') + ->withTools($tools) + ->withMaxSteps(4) + ->withPrompt('What is the weather in Detroit?') + ->asStream(); + + $events = []; + + foreach ($response as $event) { + $events[] = $event; + } + + // With tool calls, we should have multiple step start/finish pairs + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + + // At least 2 steps: one for tool call, one for final response + expect(count($stepStartEvents))->toBeGreaterThanOrEqual(2); + expect(count($stepFinishEvents))->toBeGreaterThanOrEqual(2); + + // Verify step start/finish pairs are balanced + expect(count($stepStartEvents))->toBe(count($stepFinishEvents)); +}); diff --git a/tests/Providers/XAI/XAITextTest.php b/tests/Providers/XAI/XAITextTest.php index 5665b2c65..52a19a6a5 100644 --- a/tests/Providers/XAI/XAITextTest.php +++ b/tests/Providers/XAI/XAITextTest.php @@ -142,6 +142,28 @@ }); }); +describe('client-executed tools', function (): void { + it('stops execution when client-executed tool is called', function (): void { + FixtureResponse::fakeResponseSequence('chat/completions', 'xai/text-with-client-executed-tool'); + + $tool = Tool::as('client_tool') + ->for('A tool that executes on the client') + ->withStringParameter('input', 'Input parameter'); + + $response = Prism::text() + ->using(Provider::XAI, 'grok-beta') + ->withTools([$tool]) + ->withMaxSteps(3) + ->withPrompt('Use the client tool') + ->asText(); + + expect($response->finishReason)->toBe(FinishReason::ToolCalls); + expect($response->toolCalls)->toHaveCount(1); + expect($response->toolCalls[0]->name)->toBe('client_tool'); + expect($response->steps)->toHaveCount(1); + }); +}); + describe('Image support with XAI', function (): void { it('can send images from path', function (): void { FixtureResponse::fakeResponseSequence('chat/completions', 'xai/image-detection'); diff --git a/tests/Streaming/PrismStreamIntegrationTest.php b/tests/Streaming/PrismStreamIntegrationTest.php index 21a73b0e9..191bb7898 100644 --- a/tests/Streaming/PrismStreamIntegrationTest.php +++ b/tests/Streaming/PrismStreamIntegrationTest.php @@ -5,7 +5,10 @@ use Illuminate\Broadcasting\PrivateChannel; use Prism\Prism\Enums\FinishReason; use Prism\Prism\Facades\Prism; +use Prism\Prism\Streaming\Events\StepFinishEvent; +use Prism\Prism\Streaming\Events\StepStartEvent; use Prism\Prism\Streaming\Events\StreamEndEvent; +use Prism\Prism\Streaming\Events\StreamEvent; use Prism\Prism\Streaming\Events\StreamStartEvent; use Prism\Prism\Streaming\Events\TextCompleteEvent; use Prism\Prism\Streaming\Events\TextDeltaEvent; @@ -33,13 +36,15 @@ $eventArray = iterator_to_array($events); - expect(count($eventArray))->toBeGreaterThanOrEqual(5); + expect(count($eventArray))->toBeGreaterThanOrEqual(7); // StreamStart, StepStart, TextStart, TextDelta(s), TextComplete, StepFinish, StreamEnd expect($eventArray[0])->toBeInstanceOf(StreamStartEvent::class); - expect($eventArray[1])->toBeInstanceOf(TextStartEvent::class); + expect($eventArray[1])->toBeInstanceOf(StepStartEvent::class); + expect($eventArray[2])->toBeInstanceOf(TextStartEvent::class); $lastIndex = count($eventArray) - 1; expect($eventArray[$lastIndex])->toBeInstanceOf(StreamEndEvent::class); - expect($eventArray[$lastIndex - 1])->toBeInstanceOf(TextCompleteEvent::class); + expect($eventArray[$lastIndex - 1])->toBeInstanceOf(StepFinishEvent::class); + expect($eventArray[$lastIndex - 2])->toBeInstanceOf(TextCompleteEvent::class); }); it('asStream yields text delta events with chunked content', function (): void { @@ -181,9 +186,12 @@ $eventArray = iterator_to_array($events); - expect($eventArray)->toHaveCount(2); + // StreamStart, StepStart, StepFinish, StreamEnd (no text events for empty response) + expect($eventArray)->toHaveCount(4); expect($eventArray[0])->toBeInstanceOf(StreamStartEvent::class); - expect($eventArray[1])->toBeInstanceOf(StreamEndEvent::class); + expect($eventArray[1])->toBeInstanceOf(StepStartEvent::class); + expect($eventArray[2])->toBeInstanceOf(StepFinishEvent::class); + expect($eventArray[3])->toBeInstanceOf(StreamEndEvent::class); }); it('asStream handles multi-step responses with text and tool calls', function (): void { @@ -801,3 +809,206 @@ expect($response1)->toBeInstanceOf(StreamedResponse::class); expect($response2)->toBeInstanceOf(StreamedResponse::class); }); + +describe('step events', function (): void { + it('emits step start and finish events for simple text response', function (): void { + Prism::fake([ + TextResponseFake::make()->withText('Hello World'), + ]); + + $events = iterator_to_array( + Prism::text() + ->using('openai', 'gpt-4') + ->withPrompt('Test') + ->asStream() + ); + + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + + // Single response = 1 step + expect($stepStartEvents)->toHaveCount(1); + expect($stepFinishEvents)->toHaveCount(1); + }); + + it('emits step events in correct order relative to stream events', function (): void { + Prism::fake([ + TextResponseFake::make()->withText('Test message'), + ]); + + $events = iterator_to_array( + Prism::text() + ->using('anthropic', 'claude-3-sonnet') + ->withPrompt('Test') + ->asStream() + ); + + $eventTypes = array_map(fn (StreamEvent $e): string => $e::class, $events); + + $streamStartIdx = array_search(StreamStartEvent::class, $eventTypes); + $stepStartIdx = array_search(StepStartEvent::class, $eventTypes); + $stepFinishIdx = array_search(StepFinishEvent::class, $eventTypes); + $streamEndIdx = array_search(StreamEndEvent::class, $eventTypes); + + // Order: StreamStart -> StepStart -> ... -> StepFinish -> StreamEnd + expect($streamStartIdx)->toBeLessThan($stepStartIdx); + expect($stepStartIdx)->toBeLessThan($stepFinishIdx); + expect($stepFinishIdx)->toBeLessThan($streamEndIdx); + }); + + it('emits multiple step events for multi-step tool call conversation', function (): void { + $toolCall = new ToolCall('tool-1', 'calculator', ['a' => 1, 'b' => 2]); + $toolResult = new ToolResult('tool-1', 'calculator', ['a' => 1, 'b' => 2], ['result' => 3]); + + Prism::fake([ + TextResponseFake::make()->withSteps(collect([ + TextStepFake::make() + ->withText('Let me calculate') + ->withToolCalls([$toolCall]), + TextStepFake::make() + ->withToolResults([$toolResult]), + TextStepFake::make() + ->withText('The answer is 3'), + ])), + ]); + + $events = iterator_to_array( + Prism::text() + ->using('openai', 'gpt-4') + ->withPrompt('What is 1 + 2?') + ->asStream() + ); + + $stepStartEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent); + $stepFinishEvents = array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent); + + // Multiple steps = multiple step events + expect(count($stepStartEvents))->toBeGreaterThanOrEqual(2); + expect(count($stepFinishEvents))->toBeGreaterThanOrEqual(2); + + // Start and finish counts should match + expect(count($stepStartEvents))->toBe(count($stepFinishEvents)); + }); + + it('maintains step start/finish pairing for each step', function (): void { + $toolCall = new ToolCall('tool-1', 'search', ['q' => 'test']); + $toolResult = new ToolResult('tool-1', 'search', ['q' => 'test'], ['found' => true]); + + Prism::fake([ + TextResponseFake::make()->withSteps(collect([ + TextStepFake::make()->withToolCalls([$toolCall]), + TextStepFake::make()->withToolResults([$toolResult]), + TextStepFake::make()->withText('Done'), + ])), + ]); + + $events = iterator_to_array( + Prism::text() + ->using('anthropic', 'claude-3-sonnet') + ->withPrompt('Search') + ->asStream() + ); + + // Get indices of step events + $stepStartIndices = array_keys(array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepStartEvent)); + $stepFinishIndices = array_keys(array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent)); + + // Re-index + $stepStartIndices = array_values($stepStartIndices); + $stepFinishIndices = array_values($stepFinishIndices); + + // Each step start should be followed by its corresponding finish + foreach ($stepStartIndices as $i => $startIdx) { + if (isset($stepFinishIndices[$i])) { + expect($startIdx)->toBeLessThan($stepFinishIndices[$i], + "Step $i: start index ($startIdx) should be less than finish index ({$stepFinishIndices[$i]})"); + } + } + }); + + it('places tool events within step boundaries', function (): void { + $toolCall = new ToolCall('tool-1', 'weather', ['city' => 'NYC']); + $toolResult = new ToolResult('tool-1', 'weather', ['city' => 'NYC'], ['temp' => 72]); + + Prism::fake([ + TextResponseFake::make()->withSteps(collect([ + TextStepFake::make() + ->withText('Checking weather') + ->withToolCalls([$toolCall]), + TextStepFake::make() + ->withToolResults([$toolResult]), + TextStepFake::make() + ->withText('It is 72 degrees'), + ])), + ]); + + $events = iterator_to_array( + Prism::text() + ->using('openai', 'gpt-4') + ->withPrompt('Weather in NYC?') + ->asStream() + ); + + $getFirstIndex = fn (string $class): ?int => array_key_first( + array_filter($events, fn (StreamEvent $e): bool => $e instanceof $class) + ); + + $stepStartIdx = $getFirstIndex(StepStartEvent::class); + $toolCallIdx = $getFirstIndex(ToolCallEvent::class); + $stepFinishIndices = array_keys(array_filter($events, fn (StreamEvent $e): bool => $e instanceof StepFinishEvent)); + + // Tool call should occur after first step start + expect($toolCallIdx)->toBeGreaterThan($stepStartIdx); + + // Tool call should occur before first step finish + expect($toolCallIdx)->toBeLessThan($stepFinishIndices[0]); + }); + + it('step events have valid id and timestamp', function (): void { + Prism::fake([ + TextResponseFake::make()->withText('Test'), + ]); + + $events = iterator_to_array( + Prism::text() + ->using('openai', 'gpt-4') + ->withPrompt('Test') + ->asStream() + ); + + $stepEvents = array_filter( + $events, + fn (StreamEvent $e): bool => $e instanceof StepStartEvent || $e instanceof StepFinishEvent + ); + + foreach ($stepEvents as $event) { + expect($event->id)->toBeString()->not->toBeEmpty(); + expect($event->timestamp)->toBeInt()->toBeGreaterThan(0); + } + }); + + it('step events can be converted to array', function (): void { + Prism::fake([ + TextResponseFake::make()->withText('Test'), + ]); + + $events = iterator_to_array( + Prism::text() + ->using('openai', 'gpt-4') + ->withPrompt('Test') + ->asStream() + ); + + $stepEvents = array_filter( + $events, + fn (StreamEvent $e): bool => $e instanceof StepStartEvent || $e instanceof StepFinishEvent + ); + + foreach ($stepEvents as $event) { + $array = $event->toArray(); + expect($array)->toBeArray(); + expect($array)->toHaveKey('id'); + expect($array)->toHaveKey('timestamp'); + } + }); +}); diff --git a/tests/ToolTest.php b/tests/ToolTest.php index f7865ec8b..61abf4a67 100644 --- a/tests/ToolTest.php +++ b/tests/ToolTest.php @@ -181,3 +181,15 @@ public function __invoke(string $query): string $searchTool->handle('What time is the event?'); }); + +it('can throw a prism exception when handle is called on a tool without a handler', function (): void { + $tool = (new Tool) + ->as('client_tool') + ->for('A tool without a handler') + ->withParameter(new StringSchema('query', 'the search query')); + + $this->expectException(PrismException::class); + $this->expectExceptionMessage('Tool (client_tool) has no handler defined'); + + $tool->handle('test'); +});