Fixes after refactoring

Author: ddebowczyk

evals/LLMModes/run.php

@@ -50,24 +50,25 @@
 //    'gemini-oai',
 //];
 $modes = [
-//    OutputMode::Text,
-//    OutputMode::MdJson,
+    OutputMode::Text,
+    OutputMode::MdJson,
     OutputMode::Json,
     OutputMode::JsonSchema,
     OutputMode::Tools,
-//    OutputMode::Unrestricted
+    OutputMode::Unrestricted,
 ];
 $stream = [
     false,
-//    true,
+    true,
 ];
 
+$executor = (new RunInference($data));
+//    ->withDebugPreset('on')
+//    ->wiretap(fn($e) => $e->printLog());
+
 $experiment = new Experiment(
     cases: InferenceCases::only($presets, $modes, $stream),
-    //cases: InferenceCases::all(),
-    executor: (new RunInference($data))
-        ->withDebugPreset('on'),
-        //->wiretap(fn($e) => $e->printLog()),
+    executor: $executor,
     processors: [
         new CompanyEval(
             key: 'execution.is_correct',

examples/B02_LLMAdvanced/CustomLLMDriver/run.php

@@ -25,19 +25,26 @@
 // we will use existing, bundled driver as an example, but you can provide any class that implements
 // a required interface (CanHandleInference)
 
-Inference::registerDriver('custom-driver', fn($config, $httpClient, $events) => new class($config, $httpClient, $events) extends OpenAIDriver {
-    public function handle(InferenceRequest $request): HttpClientResponse {
-        // some extra functionality to demonstrate our driver is being used
-        echo ">>> Handling request...\n";
-        return parent::handle($request);
+Inference::registerDriver(
+    name: 'custom-driver',
+    driver: fn($config, $httpClient, $events) => new class($config, $httpClient, $events) extends OpenAIDriver {
+        public function handle(InferenceRequest $request): HttpClientResponse {
+            // some extra functionality to demonstrate our driver is being used
+            echo ">>> Handling request...\n";
+            return parent::handle($request);
+        }
     }
-});
+);
 
 // Create instance of LLM client initialized with custom parameters
 $config = new LLMConfig(
-    apiUrl  : 'https://api.openai.com/v1',
-    apiKey  : Env::get('OPENAI_API_KEY'),
-    endpoint: '/chat/completions', defaultModel: 'gpt-4o-mini', defaultMaxTokens: 128, httpClientPreset: 'guzzle', driver: 'custom-driver',
+    apiUrl: 'https://api.openai.com/v1',
+    apiKey: Env::get('OPENAI_API_KEY'),
+    endpoint: '/chat/completions',
+    defaultModel: 'gpt-4o-mini',
+    defaultMaxTokens: 128,
+    httpClientPreset: 'guzzle',
+    driver: 'custom-driver',
 );
 
 $answer = (new Inference)

examples/B02_LLMAdvanced/ReasoningContent/run.php

@@ -16,27 +16,30 @@
 require 'examples/boot.php';
 
 use Cognesy\Polyglot\LLM\Inference;
+use Cognesy\Utils\Str;
 
 // EXAMPLE 1: regular API, allows to customize inference options
-//$response = (new Inference)
-////    ->wiretap(fn($e) => $e->print())
-//    ->using('deepseek-r')
-//    ->withMessages([['role' => 'user', 'content' => 'What is the capital of France. Answer with just a name.']])
-//    ->withMaxTokens(256)
-//    ->response();
-//
-//echo "\nCASE #1: Sync response\n";
-//echo "USER: What is capital of France\n";
-//echo "ASSISTANT: {$response->content()}\n";
-//echo "REASONING: {$response->reasoningContent()}\n";
-//assert($response->content() !== '');
-//assert(Str::contains($response->content(), 'Paris'));
-//assert($response->reasoningContent() !== '');
+$response = (new Inference)
+    //->withDebugPreset('on')
+    //->wiretap(fn($e) => $e->print())
+    ->using('deepseek-r')
+    ->withMessages([['role' => 'user', 'content' => 'What is the capital of France. Answer with just a name.']])
+    ->withMaxTokens(256)
+    ->response();
+
+echo "\nCASE #1: Sync response\n";
+echo "USER: What is capital of France\n";
+echo "ASSISTANT: {$response->content()}\n";
+echo "REASONING: {$response->reasoningContent()}\n";
+assert($response->content() !== '');
+assert(Str::contains($response->content(), 'Paris'));
+assert($response->reasoningContent() !== '');
 
 
 // EXAMPLE 2: streaming response
 $stream = (new Inference)
-    ->withDebugPreset('on')
+    //->withDebugPreset('on')
+    //->wiretap(fn($e) => $e->print())
     ->using('deepseek-r') // optional, default is set in /config/llm.php
     ->with(
         messages: [['role' => 'user', 'content' => 'What is capital of Brasil. Answer with just a name.']],
@@ -53,8 +56,8 @@
 }
 echo "\n";
 echo "REASONING: {$stream->final()->reasoningContent()}\n";
-//assert($stream->final()->reasoningContent() !== '');
-//assert($stream->final()->content() !== '');
-//assert(Str::contains($stream->final()->content(), 'Brasília'));
+assert($stream->final()->reasoningContent() !== '');
+assert($stream->final()->content() !== '');
+assert(Str::contains($stream->final()->content(), 'Brasília'));
 ?>
 ```

packages/instructor/src/Core/RequestHandler.php

@@ -68,7 +68,9 @@ public function responseFor(StructuredOutputRequest $request) : LLMResponse {
                 OutputMode::Tools => $llmResponse->toolCalls()->first()?->argsAsJson()
                     ?? $llmResponse->content() // fallback if no tool calls - some LLMs return just a string
                     ?? '',
-                default => Json::fromString($llmResponse->content())->toString(), // OutputMode::MdJson, OutputMode::Json, OutputMode::JsonSchema
+                // for OutputMode::MdJson, OutputMode::Json, OutputMode::JsonSchema try extracting JSON from content
+                // and replacing original content with it
+                default => Json::fromString($llmResponse->content())->toString(),
             });
             $partialResponses = [];
             $processingResult = $this->processResponse($request, $llmResponse, $partialResponses);

packages/polyglot/src/LLM/Contracts/CanHandleInference.php

@@ -9,8 +9,35 @@
 
 interface CanHandleInference
 {
+    /**
+     * Handles an inference request and returns a response.
+     *
+     * @param InferenceRequest $request The inference request to handle.
+     * @return HttpClientResponse The response from the HTTP client.
+     */
     public function handle(InferenceRequest $request) : HttpClientResponse;
+
+    /**
+     * Converts the response data into an LLMResponse object.
+     *
+     * @param array $data The response data to convert.
+     * @return LLMResponse|null The converted LLMResponse object or null if conversion fails.
+     */
     public function fromResponse(array $data): ?LLMResponse;
+
+    /**
+     * Converts a stream response into a PartialLLMResponse object.
+     *
+     * @param array $data The stream response data to convert.
+     * @return PartialLLMResponse|null The converted PartialLLMResponse object or null if conversion fails.
+     */
     public function fromStreamResponse(array $data) : ?PartialLLMResponse;
+
+    /**
+     * Converts a string of stream data into a string or false if there's no more data.
+     *
+     * @param string $data The stream data to convert.
+     * @return string|bool The converted data, or false if no more data is available.
+     */
     public function fromStreamData(string $data): string|bool;
 }

packages/polyglot/src/LLM/Data/CachedContext.php

@@ -45,4 +45,13 @@ public function isEmpty() : bool {
             && empty($this->toolChoice)
             && empty($this->responseFormat);
     }
+
+    public function clone() : self {
+        return new self(
+            messages: $this->messages->toArray(),
+            tools: $this->tools,
+            toolChoice: $this->toolChoice,
+            responseFormat: $this->responseFormat,
+        );
+    }
 }

packages/polyglot/src/LLM/Data/LLMResponse.php

@@ -12,6 +12,8 @@
 class LLMResponse
 {
     private mixed $value = null;
+    private array $partialResponses = [];
+    private bool $isPartial = false;
 
     public function __construct(
         private string     $content = '',
@@ -32,8 +34,16 @@ public function __construct(
      * @param PartialLLMResponse[] $partialResponses
      * @return LLMResponse
      */
-    public static function fromPartialResponses(array $partialResponses) : LLMResponse {
-        return (new self)->makeFromPartialResponses($partialResponses);
+    public static function fromPartialResponses(array $partialResponses = []) : self {
+        $newResponse = new self();
+        foreach ($partialResponses as $partialResponse) {
+            if ($partialResponse === null) {
+                continue;
+            }
+            $newResponse->applyPartialResponse($partialResponse);
+        }
+        $newResponse->toolCalls = ToolCalls::fromArray(self::makeTools($partialResponses));
+        return $newResponse;
     }
 
     // PUBLIC ////////////////////////////////////////////////
@@ -150,6 +160,10 @@ public function finishReason() : LLMFinishReason {
         return LLMFinishReason::fromText($this->finishReason);
     }
 
+    public function hasFinishReason() : bool {
+        return $this->finishReason !== '';
+    }
+
     public function responseData() : array {
         return $this->responseData;
     }
@@ -161,61 +175,98 @@ public function toArray() : array {
             'finishReason' => $this->finishReason,
             'toolCalls' => $this->toolCalls?->toArray() ?? [],
             'usage' => $this->usage->toArray(),
+            // raw response data
             'responseData' => $this->responseData,
         ];
     }
 
+    public function clone() : self {
+        return new self(
+            content: $this->content,
+            finishReason: $this->finishReason,
+            toolCalls: $this->toolCalls?->clone(),
+            reasoningContent: $this->reasoningContent,
+            usage: $this->usage?->clone(),
+            responseData: $this->responseData,
+        );
+    }
+
     // INTERNAL //////////////////////////////////////////////
 
     /**
-     * @param PartialLLMResponse[] $partialResponses
-     * @return LLMResponse
+     * Apply a partial response to the current response.
+     * This will accumulate content, reasoning content, usage,
+     * and response data.
+     *
+     * @param PartialLLMResponse $partialResponse
      */
-    private function makeFromPartialResponses(array $partialResponses = []) : self {
-        if (empty($partialResponses)) {
-            return $this;
-        }
-
-        $content = '';
-        $reasoningContent = '';
-        foreach($partialResponses as $partialResponse) {
-            if ($partialResponse === null) {
-                continue;
-            }
-            $content .= $partialResponse->contentDelta;
-            $reasoningContent .= $partialResponse->reasoningContentDelta;
+    private function applyPartialResponse(PartialLLMResponse $partialResponse) : void {
+        $this->content .= $partialResponse->contentDelta ?? '';
+        $this->reasoningContent .= $partialResponse->reasoningContentDelta ?? '';
+        $this->finishReason = $partialResponse->finishReason ?? $this->finishReason;
+        $this->usage()->accumulate($partialResponse->usage);
+        if (!empty($partialResponse->responseData)) {
             $this->responseData[] = $partialResponse->responseData;
-            $this->usage()->accumulate($partialResponse->usage);
-            $this->finishReason = $partialResponse->finishReason;
-        }
-        $this->content = $content;
-        $this->reasoningContent = $reasoningContent;
-
-        $tools = $this->makeTools($partialResponses);
-        if (!empty($tools)) {
-            $this->toolCalls = ToolCalls::fromArray($tools);
         }
-        return $this;
     }
 
-    private function makeTools(array $partialResponses): array {
+    /**
+     * Make a list of tool calls from the partial responses.
+     *
+     * @param PartialLLMResponse[] $partialResponses
+     * @return array
+     */
+    private static function makeTools(array $partialResponses): array {
         $tools = [];
         $currentTool = '';
         foreach ($partialResponses as $partialResponse) {
             if ($partialResponse === null) {
                 continue;
             }
-            if (('' !== ($partialResponse->toolName ?? ''))
-                && ($currentTool !== ($partialResponse->toolName ?? ''))) {
-                $currentTool = $partialResponse->toolName ?? '';
+            // if the tool name changes, start a new tool call
+            if ($partialResponse->hasToolName()
+                && ($currentTool !== ($partialResponse->toolName()))) {
+                $currentTool = $partialResponse->toolName();
                 $tools[$currentTool] = '';
             }
+            // append the tool arguments to it
             if ('' !== $currentTool) {
-                if (('' !== ($partialResponse->toolArgs ?? ''))) {
-                    $tools[$currentTool] .= $partialResponse->toolArgs ?? '';
+                if ($partialResponse->hasToolArgs()) {
+                    $tools[$currentTool] .= $partialResponse->toolArgs();
                 }
             }
         }
         return $tools;
     }
 }
+
+//    /**
+//     * @param PartialLLMResponse[] $partialResponses
+//     * @return LLMResponse
+//     */
+//    private function makeFromPartialResponses(array $partialResponses = []) : self {
+//        if (empty($partialResponses)) {
+//            return $this;
+//        }
+//
+//        $content = '';
+//        $reasoningContent = '';
+//        foreach($partialResponses as $partialResponse) {
+//            if ($partialResponse === null) {
+//                continue;
+//            }
+//            $content .= $partialResponse->contentDelta;
+//            $reasoningContent .= $partialResponse->reasoningContentDelta;
+//            $this->responseData[] = $partialResponse->responseData;
+//            $this->usage()->accumulate($partialResponse->usage);
+//            $this->finishReason = $partialResponse->finishReason;
+//        }
+//        $this->content = $content;
+//        $this->reasoningContent = $reasoningContent;
+//
+//        $tools = self::makeTools($partialResponses);
+//        if (!empty($tools)) {
+//            $this->toolCalls = ToolCalls::fromArray($tools);
+//        }
+//        return $this;
+//    }

packages/polyglot/src/LLM/Data/PartialLLMResponse.php

@@ -78,6 +78,15 @@ public function usage() : Usage {
         return $this->usage ?? new Usage();
     }
 
+    public function hasToolArgs() : bool {
+        // do not change to not empty, as it will return true for '0'
+        return '' !== ($this->toolArgs ?? '');
+    }
+
+    public function hasToolName() : string {
+        return '' !== ($this->toolName ?? '');
+    }
+
     public function toArray() : array {
         return [
             'content_delta' => $this->contentDelta,
@@ -90,4 +99,12 @@ public function toArray() : array {
             'response_data' => $this->responseData,
         ];
     }
+
+    public function toolName() : string {
+        return $this->toolName ?? '';
+    }
+
+    public function toolArgs() : string {
+        return $this->toolArgs ?? '';
+    }
 }