Fixes in example execution

ddebowczyk · ddebowczyk · commit c5d346932ed9 · 2026-03-13T08:31:25.000+01:00
diff --git a/examples/A04_APISupport/Meta/run.php b/examples/A04_APISupport/Meta/run.php
@@ -2,6 +2,7 @@
 title: 'Meta'
 docname: 'meta'
 id: 'c351'
+skip: true
 ---
 ## Overview
 
diff --git a/examples/A04_APISupport/Minimaxi/run.php b/examples/A04_APISupport/Minimaxi/run.php
@@ -2,6 +2,7 @@
 title: 'Minimaxi'
 docname: 'minimaxi'
 id: '1d6f'
+skip: true
 ---
 ## Overview
 
diff --git a/examples/A04_APISupport/Perplexity/run.php b/examples/A04_APISupport/Perplexity/run.php
@@ -37,7 +37,7 @@ class User {
 
 $structuredOutput = new StructuredOutput(
     StructuredOutputRuntime::fromProvider(LLMProvider::using('perplexity'))
-        ->withOutputMode(OutputMode::Json)
+        ->withOutputMode(OutputMode::MdJson)
 );
 
 $user = $structuredOutput->with(
diff --git a/examples/A05_Extras/CustomExtractor/run.php b/examples/A05_Extras/CustomExtractor/run.php
@@ -32,6 +32,7 @@
 use Cognesy\Instructor\Extraction\Data\ExtractionInput;
 use Cognesy\Instructor\Extraction\Exceptions\ExtractionException;
 use Cognesy\Instructor\Extraction\Extractors\DirectJsonExtractor;
+use Cognesy\Instructor\Extraction\ResponseExtractor;
 use Cognesy\Instructor\StructuredOutput;
 use Cognesy\Instructor\StructuredOutputRuntime;
 use Cognesy\Polyglot\Inference\LLMProvider;
@@ -115,10 +116,10 @@ class Person {
 // DirectJson is tried first (will fail), then XmlJsonExtractor (will succeed)
 $person = new StructuredOutput(
         StructuredOutputRuntime::fromProvider(LLMProvider::using('openai'))
-            ->withExtractors([
+            ->withExtractor(ResponseExtractor::fromExtractors(
                 new DirectJsonExtractor(),
                 new XmlJsonExtractor('json'),
-            ])
+            ))
     )
     ->withResponseClass(Person::class)
     ->withMessages("Extract: Alice Johnson, 28 years old, lives in San Francisco")
@@ -180,10 +181,10 @@ class Person {
 // Custom extractors work for streaming too
 $stream = new StructuredOutput(
         StructuredOutputRuntime::fromProvider(LLMProvider::using('openai'))
-            ->withExtractors([
+            ->withExtractor(ResponseExtractor::fromExtractors(
                 new DirectJsonExtractor(),
                 new XmlJsonExtractor('json'),
-            ])
+            ))
     )
     ->withResponseClass(Person::class)
     ->withMessages("Extract person data...")
diff --git a/examples/A05_Extras/CustomExtractor/run_streaming.php b/examples/A05_Extras/CustomExtractor/run_streaming.php
@@ -5,6 +5,7 @@
 use Cognesy\Instructor\Extraction\Data\ExtractionInput;
 use Cognesy\Instructor\Extraction\Exceptions\ExtractionException;
 use Cognesy\Instructor\Extraction\Extractors\DirectJsonExtractor;
+use Cognesy\Instructor\Extraction\ResponseExtractor;
 use Cognesy\Instructor\StructuredOutput;
 use Cognesy\Instructor\StructuredOutputRuntime;
 use Cognesy\Polyglot\Inference\LLMProvider;
@@ -65,10 +66,10 @@ class Person {
 
 $stream = new StructuredOutput(
         StructuredOutputRuntime::fromProvider(LLMProvider::using('openai'))
-            ->withExtractors([
+            ->withExtractor(ResponseExtractor::fromExtractors(
                 new DirectJsonExtractor(),
                 new XmlJsonExtractor('json'),
-            ])
+            ))
     )
     ->withResponseClass(Person::class)
     ->withMessages("Extract person data...")
diff --git a/examples/A05_Extras/TranslateUIFields/run.php b/examples/A05_Extras/TranslateUIFields/run.php
@@ -20,7 +20,6 @@
 
 use Cognesy\Instructor\StructuredOutput;
 use Cognesy\Instructor\StructuredOutputRuntime;
-use Cognesy\Instructor\Validation\Validators\SymfonyValidator;
 use Cognesy\Polyglot\Inference\LLMProvider;
 
 class TextElementModel
@@ -40,7 +39,7 @@ public function __construct(
 $transformedModel = new StructuredOutput(
         StructuredOutputRuntime::fromProvider(LLMProvider::using('openai'))
             ->withMaxRetries(2)
-            ->withValidators([SymfonyValidator::class])
+            ->withValidator(new \Cognesy\Instructor\Validation\Validators\SymfonyValidator())
     )
     ->withInput($sourceModel)
     ->withResponseClass(get_class($sourceModel))
diff --git a/examples/B02_LLMAdvanced/ReasoningContent/run.php b/examples/B02_LLMAdvanced/ReasoningContent/run.php
@@ -20,8 +20,8 @@
 
 // EXAMPLE 1: regular API, allows to customize inference options
 $response = Inference::using('deepseek-r')
-    ->withMessages(Messages::fromString('What is the capital of France. Answer with just a name.'))
-    ->withMaxTokens(256)
+    ->withMessages(Messages::fromString('What is the capital of France? Answer with just the city name, nothing else.'))
+    ->withMaxTokens(1024)
     ->response();
 
 echo "\nCASE #1: Sync response\n";
@@ -34,8 +34,8 @@
 // EXAMPLE 2: streaming response
 $stream = Inference::using('deepseek-r')
     ->with(
-        messages: Messages::fromString('What is capital of Brasil. Answer with just a name.'),
-        options: ['max_tokens' => 256]
+        messages: Messages::fromString('What is the capital of Brasil? Answer with just the city name, nothing else.'),
+        options: ['max_tokens' => 1024]
     )
     ->withStreaming()
     ->stream();
diff --git a/examples/B04_LLMApiSupport/A21/run.php b/examples/B04_LLMApiSupport/A21/run.php
@@ -2,6 +2,7 @@
 title: 'A21'
 docname: 'llm_a21'
 id: '1dd0'
+skip: true
 ---
 ## Overview
 
diff --git a/examples/B04_LLMApiSupport/Meta/run.php b/examples/B04_LLMApiSupport/Meta/run.php
@@ -2,6 +2,7 @@
 title: 'Meta'
 docname: 'llm_meta'
 id: '2dca'
+skip: true
 ---
 ## Overview
 
diff --git a/examples/B04_LLMApiSupport/Minimaxi/run.php b/examples/B04_LLMApiSupport/Minimaxi/run.php
@@ -2,6 +2,7 @@
 title: 'Minimaxi'
 docname: 'llm_minimaxi'
 id: '89a2'
+skip: true
 ---
 ## Overview
 
diff --git a/packages/hub/src/Commands/EnhancedRunAllExamples.php b/packages/hub/src/Commands/EnhancedRunAllExamples.php
@@ -169,6 +169,18 @@ private function executeExamples(array $examples, bool $stopOnError): int
             if ($result->isSuccessful()) {
                 Cli::out(str_pad("OK", 8, ' ', STR_PAD_BOTH), [Color::GREEN, Color::BOLD]);
                 $success++;
+            } elseif ($result->error?->isAssertion()) {
+                Cli::out(str_pad("ASSERT", 8, ' ', STR_PAD_BOTH), [Color::YELLOW, Color::BOLD]);
+                $errors++;
+
+                Cli::outln('');
+                Cli::outln("    " . Cli::limit($result->error->message, 70), [Color::YELLOW]);
+
+                if ($stopOnError) {
+                    Cli::outln('');
+                    Cli::outln('Stopping on first error as requested.', [Color::YELLOW]);
+                    break;
+                }
             } else {
                 Cli::out(str_pad("ERROR", 8, ' ', STR_PAD_BOTH), [Color::RED, Color::BOLD]);
                 $errors++;
diff --git a/packages/hub/src/Commands/ErrorsCommand.php b/packages/hub/src/Commands/ErrorsCommand.php
@@ -7,6 +7,7 @@
 use Cognesy\InstructorHub\Core\Cli;
 use Cognesy\InstructorHub\Data\ExecutionStatus;
 use Cognesy\InstructorHub\Services\ExampleRepository;
+use Cognesy\InstructorHub\Services\ExecutionTracker;
 use Cognesy\Utils\Cli\Color;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputOption;
@@ -18,7 +19,7 @@ class ErrorsCommand extends Command
     public function __construct(
         private CanExecuteExample $runner,
         private ExampleRepository $examples,
-        private CanTrackExecution $tracker,
+        private ExecutionTracker $tracker,
     ) {
         parent::__construct();
     }
@@ -28,32 +29,96 @@ protected function configure(): void
     {
         $this
             ->setName('errors')
-            ->setDescription('Re-run examples that failed in previous executions')
+            ->setDescription('Manage examples that failed in previous executions')
+            ->addOption('list', null, InputOption::VALUE_NONE,
+                'List failed examples with error details')
+            ->addOption('run', null, InputOption::VALUE_NONE,
+                'Re-run failed examples')
+            ->addOption('clear', null, InputOption::VALUE_NONE,
+                'Clear all error statuses')
             ->addOption('since', 's', InputOption::VALUE_REQUIRED,
                 'Only errors since date (e.g. "yesterday", "1 hour ago")')
             ->addOption('limit', 'l', InputOption::VALUE_REQUIRED,
-                'Limit number of examples to run', 0)
+                'Limit number of examples', 0)
             ->addOption('dry-run', null, InputOption::VALUE_NONE,
-                'Show what would be executed without running')
-            ->addOption('show-errors', null, InputOption::VALUE_NONE,
-                'Show error details without re-running');
+                'Show what would be executed without running');
     }
 
     #[\Override]
     protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $list = $input->getOption('list');
+        $run = $input->getOption('run');
+        $clear = $input->getOption('clear');
+
+        if (!$list && !$run && !$clear) {
+            return $this->showHelp();
+        }
+
+        if ($clear) {
+            return $this->clearErrors();
+        }
+
+        $errorStatuses = $this->getErrorStatuses($input);
+
+        if ($errorStatuses === null) {
+            return Command::FAILURE;
+        }
+
+        if (empty($errorStatuses)) {
+            Cli::outln('');
+            Cli::outln('No failed examples found.', [Color::GREEN]);
+            Cli::outln('');
+            return Command::SUCCESS;
+        }
+
+        if ($list) {
+            Cli::outln('');
+            Cli::outln('Found ' . count($errorStatuses) . ' failed example(s)', [Color::YELLOW, Color::BOLD]);
+            Cli::outln('');
+            $this->showErrorDetails($errorStatuses);
+            return Command::SUCCESS;
+        }
+
+        // --run
+        if ($input->getOption('dry-run')) {
+            Cli::outln('');
+            Cli::outln('Found ' . count($errorStatuses) . ' failed example(s)', [Color::YELLOW, Color::BOLD]);
+            Cli::outln('');
+            $this->showDryRun($errorStatuses);
+            return Command::SUCCESS;
+        }
+
+        return $this->reRunErrors($errorStatuses);
+    }
+
+    private function showHelp(): int
+    {
+        Cli::outln('');
+        Cli::outln('Manage failed examples:', [Color::YELLOW, Color::BOLD]);
+        Cli::outln('');
+        Cli::outln('  composer hub errors --list              List failed examples with error details', [Color::WHITE]);
+        Cli::outln('  composer hub errors --run               Re-run failed examples', [Color::WHITE]);
+        Cli::outln('  composer hub errors --run --dry-run     Show what would be re-run', [Color::WHITE]);
+        Cli::outln('  composer hub errors --clear             Clear all error statuses', [Color::WHITE]);
+        Cli::outln('');
+        Cli::outln('Options:', [Color::YELLOW]);
+        Cli::outln('  --since, -s    Filter by date (e.g. "yesterday", "1 hour ago")', [Color::DARK_GRAY]);
+        Cli::outln('  --limit, -l    Limit number of examples', [Color::DARK_GRAY]);
+        Cli::outln('');
+        return Command::SUCCESS;
+    }
+
+    private function getErrorStatuses(InputInterface $input): ?array
     {
         $since = $input->getOption('since');
         $limit = (int) $input->getOption('limit');
-        $dryRun = $input->getOption('dry-run');
-        $showErrors = $input->getOption('show-errors');
 
-        // Get all statuses with errors
         $errorStatuses = array_filter(
             $this->tracker->getAllStatuses(),
             fn($s) => $s->status === ExecutionStatus::ERROR
         );
 
-        // Filter by time if specified
         if ($since) {
             try {
                 $sinceTime = new \DateTimeImmutable($since);
@@ -63,43 +128,36 @@ protected function execute(InputInterface $input, OutputInterface $output): int
                 );
             } catch (\Exception $e) {
                 Cli::outln("Invalid date format: {$since}", [Color::RED]);
-                return Command::FAILURE;
+                return null;
             }
         }
 
-        if (empty($errorStatuses)) {
-            Cli::outln('');
-            Cli::outln('No failed examples found.', [Color::GREEN]);
-            Cli::outln('');
-            return Command::SUCCESS;
-        }
-
-        // Apply limit
         if ($limit > 0) {
             $errorStatuses = array_slice($errorStatuses, 0, $limit);
         }
 
-        Cli::outln('');
-        Cli::outln('Found ' . count($errorStatuses) . ' failed example(s)', [Color::YELLOW, Color::BOLD]);
-        Cli::outln('');
+        return $errorStatuses;
+    }
 
-        if ($showErrors) {
-            $this->showErrorDetails($errorStatuses);
-            return Command::SUCCESS;
-        }
+    private function clearErrors(): int
+    {
+        $removed = $this->tracker->removeErrorStatuses();
 
-        if ($dryRun) {
-            $this->showDryRun($errorStatuses);
-            return Command::SUCCESS;
+        Cli::outln('');
+        if ($removed > 0) {
+            Cli::outln("Cleared {$removed} error status(es).", [Color::GREEN]);
+        } else {
+            Cli::outln('No error statuses to clear.', [Color::YELLOW]);
         }
+        Cli::outln('');
 
-        return $this->reRunErrors($errorStatuses);
+        return Command::SUCCESS;
     }
 
     private function showErrorDetails(array $errorStatuses): void
     {
         foreach ($errorStatuses as $status) {
-            Cli::outln("[" . ($status->index + 1) . "] {$status->group}/{$status->name}", [Color::WHITE, Color::BOLD]);
+            Cli::outln("[{$status->index}] {$status->group}/{$status->name}", [Color::WHITE, Color::BOLD]);
 
             $lastError = $status->lastError();
             if ($lastError) {
@@ -136,10 +194,10 @@ private function reRunErrors(array $errorStatuses): int
         $fixed = 0;
         $stillFailing = 0;
 
-        Cli::outln('Re-running failed examples...', [Color::YELLOW]);
+        Cli::outln('');
+        Cli::outln("Re-running {$total} failed example(s)...", [Color::YELLOW, Color::BOLD]);
         Cli::outln('');
 
-        // Map status indices to examples
         $exampleMap = [];
         $this->examples->forEachExample(function($example) use (&$exampleMap) {
             $exampleMap[$example->index] = $example;
diff --git a/packages/hub/src/Commands/ExcludeCommand.php b/packages/hub/src/Commands/ExcludeCommand.php
diff --git a/packages/hub/src/Data/ExecutionError.php b/packages/hub/src/Data/ExecutionError.php
diff --git a/packages/hub/src/Hub.php b/packages/hub/src/Hub.php
diff --git a/packages/hub/src/Services/ExecutionTracker.php b/packages/hub/src/Services/ExecutionTracker.php
diff --git a/packages/polyglot/resources/config/llm/presets/meta.yaml b/packages/polyglot/resources/config/llm/presets/meta.yaml
diff --git a/packages/polyglot/src/Inference/Drivers/CohereV2/CohereV2BodyFormat.php b/packages/polyglot/src/Inference/Drivers/CohereV2/CohereV2BodyFormat.php
