feature #481 [Store] Refactor VectorizerInterface to separate string and document vectorization (OskarStark)

This PR was squashed before being merged into the main branch.

Discussion
----------

[Store] Refactor `VectorizerInterface` to separate string and document vectorization

| Q             | A
| ------------- | ---
| Bug fix?      | no
| New feature?  | yes
| Docs?         | no
| Issues        | --
| License       | MIT

Commits
-------

1bd8cc4 [Store] Refactor `VectorizerInterface` to separate string and document vectorization

Author: OskarStark

demo/config/packages/ai.yaml

@@ -74,4 +74,4 @@ services:
     #     $apiKey: '%env(SERP_API_KEY)%'
     Symfony\AI\Agent\Toolbox\Tool\Wikipedia: ~
     Symfony\AI\Agent\Toolbox\Tool\SimilaritySearch:
-        $model: '@ai.indexer.default.model'
+        $vectorizer: '@ai.vectorizer.openai_embeddings'

examples/document/vectorizing-text-documents.php

@@ -0,0 +1,33 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Platform\Bridge\OpenAi\Embeddings;
+use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\VectorDocument;
+use Symfony\AI\Store\Document\Vectorizer;
+use Symfony\Component\Uid\Uuid;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+$platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
+$embeddings = new Embeddings(Embeddings::TEXT_3_LARGE);
+
+$textDocuments = [
+    new TextDocument(Uuid::v4(), 'Hello World'),
+    new TextDocument(Uuid::v4(), 'Lorem ipsum dolor sit amet'),
+    new TextDocument(Uuid::v4(), 'PHP Hypertext Preprocessor'),
+];
+
+$vectorizer = new Vectorizer($platform, $embeddings);
+$vectorDocuments = $vectorizer->vectorizeTextDocuments($textDocuments);
+
+dump(array_map(fn (VectorDocument $document) => $document->vector->getDimensions(), $vectorDocuments));

examples/document/vectorizing.php

@@ -11,23 +11,21 @@
 
 use Symfony\AI\Platform\Bridge\OpenAi\Embeddings;
 use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
-use Symfony\AI\Store\Document\TextDocument;
-use Symfony\AI\Store\Document\VectorDocument;
 use Symfony\AI\Store\Document\Vectorizer;
-use Symfony\Component\Uid\Uuid;
 
 require_once dirname(__DIR__).'/bootstrap.php';
 
 $platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
 $embeddings = new Embeddings(Embeddings::TEXT_3_LARGE);
 
-$textDocuments = [
-    new TextDocument(Uuid::v4(), 'Hello World'),
-    new TextDocument(Uuid::v4(), 'Lorem ipsum dolor sit amet'),
-    new TextDocument(Uuid::v4(), 'PHP Hypertext Preprocessor'),
-];
-
 $vectorizer = new Vectorizer($platform, $embeddings);
-$vectorDocuments = $vectorizer->vectorizeDocuments($textDocuments);
 
-dump(array_map(fn (VectorDocument $document) => $document->vector->getDimensions(), $vectorDocuments));
+$string = 'Hello World';
+$vector = $vectorizer->vectorize($string);
+
+printf(
+    "String: %s\nVector dimensions: %d\nFirst 5 values: [%s]\n",
+    $string,
+    $vector->getDimensions(),
+    implode(', ', array_map(fn ($val) => number_format($val, 6), array_slice($vector->getData(), 0, 5)))
+);

src/agent/src/Toolbox/Tool/SimilaritySearch.php

@@ -12,9 +12,8 @@
 namespace Symfony\AI\Agent\Toolbox\Tool;
 
 use Symfony\AI\Agent\Toolbox\Attribute\AsTool;
-use Symfony\AI\Platform\Model;
-use Symfony\AI\Platform\PlatformInterface;
 use Symfony\AI\Store\Document\VectorDocument;
+use Symfony\AI\Store\Document\VectorizerInterface;
 use Symfony\AI\Store\StoreInterface;
 
 /**
@@ -29,8 +28,7 @@ final class SimilaritySearch
     public array $usedDocuments = [];
 
     public function __construct(
-        private readonly PlatformInterface $platform,
-        private readonly Model $model,
+        private readonly VectorizerInterface $vectorizer,
         private readonly StoreInterface $store,
     ) {
     }
@@ -40,8 +38,8 @@ public function __construct(
      */
     public function __invoke(string $searchTerm): string
     {
-        $vectors = $this->platform->invoke($this->model, $searchTerm)->asVectors();
-        $this->usedDocuments = $this->store->query($vectors[0]);
+        $vector = $this->vectorizer->vectorize($searchTerm);
+        $this->usedDocuments = $this->store->query($vector);
 
         if ([] === $this->usedDocuments) {
             return 'No results found';

src/agent/tests/Toolbox/Tool/SimilaritySearchTest.php

@@ -14,14 +14,10 @@
 use PHPUnit\Framework\Attributes\CoversClass;
 use PHPUnit\Framework\TestCase;
 use Symfony\AI\Agent\Toolbox\Tool\SimilaritySearch;
-use Symfony\AI\Platform\Model;
-use Symfony\AI\Platform\PlatformInterface;
-use Symfony\AI\Platform\Result\RawResultInterface;
-use Symfony\AI\Platform\Result\ResultPromise;
-use Symfony\AI\Platform\Result\VectorResult;
 use Symfony\AI\Platform\Vector\Vector;
 use Symfony\AI\Store\Document\Metadata;
 use Symfony\AI\Store\Document\VectorDocument;
+use Symfony\AI\Store\Document\VectorizerInterface;
 use Symfony\AI\Store\StoreInterface;
 use Symfony\Component\Uid\Uuid;
 
@@ -44,27 +40,19 @@ public function testSearchWithResults()
             new Metadata(['title' => 'Document 2', 'content' => 'Second document content']),
         );
 
-        $rawResult = $this->createMock(RawResultInterface::class);
-        $vectorResult = new VectorResult($vector);
-        $resultPromise = new ResultPromise(
-            fn () => $vectorResult,
-            $rawResult
-        );
-
-        $platform = $this->createMock(PlatformInterface::class);
-        $platform->expects($this->once())
-            ->method('invoke')
-            ->with($this->isInstanceOf(Model::class), $searchTerm)
-            ->willReturn($resultPromise);
+        $vectorizer = $this->createMock(VectorizerInterface::class);
+        $vectorizer->expects($this->once())
+            ->method('vectorize')
+            ->with($searchTerm)
+            ->willReturn($vector);
 
         $store = $this->createMock(StoreInterface::class);
         $store->expects($this->once())
             ->method('query')
             ->with($vector)
             ->willReturn([$document1, $document2]);
 
-        $model = new Model('test-model');
-        $similaritySearch = new SimilaritySearch($platform, $model, $store);
+        $similaritySearch = new SimilaritySearch($vectorizer, $store);
 
         $result = $similaritySearch($searchTerm);
 
@@ -77,27 +65,19 @@ public function testSearchWithoutResults()
         $searchTerm = 'find nothing';
         $vector = new Vector([0.1, 0.2, 0.3]);
 
-        $rawResult = $this->createMock(RawResultInterface::class);
-        $vectorResult = new VectorResult($vector);
-        $resultPromise = new ResultPromise(
-            fn () => $vectorResult,
-            $rawResult
-        );
-
-        $platform = $this->createMock(PlatformInterface::class);
-        $platform->expects($this->once())
-            ->method('invoke')
-            ->with($this->isInstanceOf(Model::class), $searchTerm)
-            ->willReturn($resultPromise);
+        $vectorizer = $this->createMock(VectorizerInterface::class);
+        $vectorizer->expects($this->once())
+            ->method('vectorize')
+            ->with($searchTerm)
+            ->willReturn($vector);
 
         $store = $this->createMock(StoreInterface::class);
         $store->expects($this->once())
             ->method('query')
             ->with($vector)
             ->willReturn([]);
 
-        $model = new Model('test-model');
-        $similaritySearch = new SimilaritySearch($platform, $model, $store);
+        $similaritySearch = new SimilaritySearch($vectorizer, $store);
 
         $result = $similaritySearch($searchTerm);
 
@@ -116,27 +96,19 @@ public function testSearchWithSingleResult()
             new Metadata(['title' => 'Single Document', 'description' => 'Only one match']),
         );
 
-        $rawResult = $this->createMock(RawResultInterface::class);
-        $vectorResult = new VectorResult($vector);
-        $resultPromise = new ResultPromise(
-            fn () => $vectorResult,
-            $rawResult
-        );
-
-        $platform = $this->createMock(PlatformInterface::class);
-        $platform->expects($this->once())
-            ->method('invoke')
-            ->with($this->isInstanceOf(Model::class), $searchTerm)
-            ->willReturn($resultPromise);
+        $vectorizer = $this->createMock(VectorizerInterface::class);
+        $vectorizer->expects($this->once())
+            ->method('vectorize')
+            ->with($searchTerm)
+            ->willReturn($vector);
 
         $store = $this->createMock(StoreInterface::class);
         $store->expects($this->once())
             ->method('query')
             ->with($vector)
             ->willReturn([$document]);
 
-        $model = new Model('test-model');
-        $similaritySearch = new SimilaritySearch($platform, $model, $store);
+        $similaritySearch = new SimilaritySearch($vectorizer, $store);
 
         $result = $similaritySearch($searchTerm);
 

src/ai-bundle/config/options.php

@@ -373,6 +373,7 @@
                 ->end()
             ->end()
             ->arrayNode('vectorizer')
+                ->info('Vectorizers for converting strings to Vector objects and transforming TextDocument arrays to VectorDocument arrays')
                 ->useAttributeAsKey('name')
                 ->arrayPrototype()
                     ->children()

src/store/src/Document/Vectorizer.php

@@ -16,6 +16,8 @@
 use Symfony\AI\Platform\Capability;
 use Symfony\AI\Platform\Model;
 use Symfony\AI\Platform\PlatformInterface;
+use Symfony\AI\Platform\Vector\Vector;
+use Symfony\AI\Store\Exception\RuntimeException;
 
 final readonly class Vectorizer implements VectorizerInterface
 {
@@ -26,7 +28,7 @@ public function __construct(
     ) {
     }
 
-    public function vectorize(array $documents): array
+    public function vectorizeTextDocuments(array $documents): array
     {
         $documentCount = \count($documents);
         $this->logger->info('Starting vectorization process', ['document_count' => $documentCount]);
@@ -64,4 +66,18 @@ public function vectorize(array $documents): array
 
         return $vectorDocuments;
     }
+
+    public function vectorize(string $string): Vector
+    {
+        $this->logger->debug('Vectorizing string', ['string' => $string]);
+
+        $result = $this->platform->invoke($this->model, $string);
+        $vectors = $result->asVectors();
+
+        if (!isset($vectors[0])) {
+            throw new RuntimeException('No vector returned for string vectorization.');
+        }
+
+        return $vectors[0];
+    }
 }

src/store/src/Document/VectorizerInterface.php

@@ -11,8 +11,11 @@
 
 namespace Symfony\AI\Store\Document;
 
+use Symfony\AI\Platform\Vector\Vector;
+
 /**
- * Interface for converting a collection of TextDocuments into VectorDocuments.
+ * Interface for converting a collection of TextDocuments into VectorDocuments
+ * and for vectorizing individual strings.
  *
  * @author Oskar Stark <[email protected]>
  */
@@ -23,5 +26,10 @@ interface VectorizerInterface
      *
      * @return VectorDocument[]
      */
-    public function vectorize(array $documents): array;
+    public function vectorizeTextDocuments(array $documents): array;
+
+    /**
+     * Vectorizes a single string into a Vector.
+     */
+    public function vectorize(string $string): Vector;
 }

src/store/src/Indexer.php

@@ -45,13 +45,13 @@ public function index(TextDocument|iterable $documents, int $chunkSize = 50): vo
             ++$counter;
 
             if ($chunkSize === \count($chunk)) {
-                $this->store->add(...$this->vectorizer->vectorize($chunk));
+                $this->store->add(...$this->vectorizer->vectorizeTextDocuments($chunk));
                 $chunk = [];
             }
         }
 
         if (\count($chunk) > 0) {
-            $this->store->add(...$this->vectorizer->vectorize($chunk));
+            $this->store->add(...$this->vectorizer->vectorizeTextDocuments($chunk));
         }
 
         $this->logger->debug(0 === $counter ? 'No documents to index' : \sprintf('Indexed %d documents', $counter));