bug #137 [Store][Meilisearch] Remove id, score and vectors from metadata (chr-hertel)

This PR was merged into the main branch.

Discussion
----------

[Store][Meilisearch] Remove id, score and vectors from metadata

| Q             | A
| ------------- | ---
| Bug fix?      | yes
| New feature?  | no
| Docs?         | no
| Issues        |
| License       | MIT

Metadata should only include additional but relevant information. It is handed over by the SimilaritySearch to the LLM and too much unnecessary information hurt the context size and token consumption.

On top, score was not set.

Commits
-------

0ace629 Remove id, score and vectors from metadata

Author: chr-hertel

examples/store/memory-similarity-search.php

@@ -13,6 +13,7 @@
 use Symfony\AI\Agent\Toolbox\AgentProcessor;
 use Symfony\AI\Agent\Toolbox\Tool\SimilaritySearch;
 use Symfony\AI\Agent\Toolbox\Toolbox;
+use Symfony\AI\Fixtures\Movies;
 use Symfony\AI\Platform\Bridge\OpenAI\Embeddings;
 use Symfony\AI\Platform\Bridge\OpenAI\GPT;
 use Symfony\AI\Platform\Bridge\OpenAI\PlatformFactory;
@@ -37,15 +38,8 @@
 // initialize the store
 $store = new InMemoryStore();
 
-// our data
-$movies = [
-    ['title' => 'Inception', 'description' => 'A skilled thief is given a chance at redemption if he can successfully perform inception, the act of planting an idea in someone\'s subconscious.', 'director' => 'Christopher Nolan'],
-    ['title' => 'The Matrix', 'description' => 'A hacker discovers the world he lives in is a simulated reality and joins a rebellion to overthrow its controllers.', 'director' => 'The Wachowskis'],
-    ['title' => 'The Godfather', 'description' => 'The aging patriarch of an organized crime dynasty transfers control of his empire to his reluctant son.', 'director' => 'Francis Ford Coppola'],
-];
-
 // create embeddings and documents
-foreach ($movies as $i => $movie) {
+foreach (Movies::all() as $i => $movie) {
     $documents[] = new TextDocument(
         id: Uuid::v4(),
         content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'],

src/store/src/Bridge/Meilisearch/Store.php

@@ -123,12 +123,13 @@ private function convertToIndexableArray(VectorDocument $document): array
      */
     private function convertToVectorDocument(array $data): VectorDocument
     {
-        return new VectorDocument(
-            id: Uuid::fromString($data['id']),
-            vector: !\array_key_exists($this->vectorFieldName, $data) || null === $data[$this->vectorFieldName]
-                ? new NullVector()
-                : new Vector($data[$this->vectorFieldName][$this->embedder]['embeddings']),
-            metadata: new Metadata($data),
-        );
+        $id = $data['id'] ?? throw new InvalidArgumentException('Missing "id" field in the document data');
+        $vector = !\array_key_exists($this->vectorFieldName, $data) || null === $data[$this->vectorFieldName]
+            ? new NullVector() : new Vector($data[$this->vectorFieldName][$this->embedder]['embeddings']);
+        $score = $data['_rankingScore'] ?? null;
+
+        unset($data['id'], $data[$this->vectorFieldName], $data['_rankingScore']);
+
+        return new VectorDocument(Uuid::fromString($id), $vector, new Metadata($data), $score);
     }
 }

src/store/tests/Bridge/Meilisearch/StoreTest.php

@@ -18,7 +18,7 @@
 use Symfony\AI\Store\Document\VectorDocument;
 use Symfony\Component\HttpClient\Exception\ClientException;
 use Symfony\Component\HttpClient\MockHttpClient;
-use Symfony\Component\HttpClient\Response\MockResponse;
+use Symfony\Component\HttpClient\Response\JsonMockResponse;
 use Symfony\Component\Uid\Uuid;
 
 #[CoversClass(Store::class)]
@@ -27,12 +27,12 @@ final class StoreTest extends TestCase
     public function testStoreCannotInitializeOnInvalidResponse(): void
     {
         $httpClient = new MockHttpClient([
-            new MockResponse(json_encode([
+            new JsonMockResponse([
                 'message' => 'error',
                 'code' => 'index_creation_failed',
                 'type' => 'invalid_request',
                 'link' => 'https://docs.meilisearch.com/errors#index_creation_failed',
-            ]), [
+            ], [
                 'http_code' => 400,
             ]),
         ], 'http://localhost:7700');
@@ -53,22 +53,22 @@ public function testStoreCannotInitializeOnInvalidResponse(): void
     public function testStoreCanInitialize(): void
     {
         $httpClient = new MockHttpClient([
-            new MockResponse(json_encode([
+            new JsonMockResponse([
                 'taskUid' => 1,
                 'indexUid' => 'test',
                 'status' => 'enqueued',
                 'type' => 'indexCreation',
                 'enqueuedAt' => '2025-01-01T00:00:00Z',
-            ]), [
+            ], [
                 'http_code' => 202,
             ]),
-            new MockResponse(json_encode([
+            new JsonMockResponse([
                 'taskUid' => 2,
                 'indexUid' => 'test',
                 'status' => 'enqueued',
                 'type' => 'indexUpdate',
                 'enqueuedAt' => '2025-01-01T01:00:00Z',
-            ]), [
+            ], [
                 'http_code' => 202,
             ]),
         ], 'http://localhost:7700');
@@ -88,12 +88,12 @@ public function testStoreCanInitialize(): void
     public function testStoreCannotAddOnInvalidResponse(): void
     {
         $httpClient = new MockHttpClient([
-            new MockResponse(json_encode([
+            new JsonMockResponse([
                 'message' => 'error',
                 'code' => 'invalid_document_fields',
                 'type' => 'invalid_request',
                 'link' => 'https://docs.meilisearch.com/errors#invalid_document_fields',
-            ]), [
+            ], [
                 'http_code' => 400,
             ]),
         ], 'http://localhost:7700');
@@ -114,13 +114,13 @@ public function testStoreCannotAddOnInvalidResponse(): void
     public function testStoreCanAdd(): void
     {
         $httpClient = new MockHttpClient([
-            new MockResponse(json_encode([
+            new JsonMockResponse([
                 'taskUid' => 1,
                 'indexUid' => 'test',
                 'status' => 'enqueued',
                 'type' => 'documentAdditionOrUpdate',
                 'enqueuedAt' => '2025-01-01T00:00:00Z',
-            ]), [
+            ], [
                 'http_code' => 202,
             ]),
         ], 'http://localhost:7700');
@@ -140,12 +140,12 @@ public function testStoreCanAdd(): void
     public function testStoreCannotQueryOnInvalidResponse(): void
     {
         $httpClient = new MockHttpClient([
-            new MockResponse(json_encode([
+            new JsonMockResponse([
                 'message' => 'error',
                 'code' => 'invalid_search_hybrid_query',
                 'type' => 'invalid_request',
                 'link' => 'https://docs.meilisearch.com/errors#invalid_search_hybrid_query',
-            ]), [
+            ], [
                 'http_code' => 400,
             ]),
         ], 'http://localhost:7700');
@@ -166,7 +166,7 @@ public function testStoreCannotQueryOnInvalidResponse(): void
     public function testStoreCanQuery(): void
     {
         $httpClient = new MockHttpClient([
-            new MockResponse(json_encode([
+            new JsonMockResponse([
                 'hits' => [
                     [
                         'id' => Uuid::v4()->toRfc4122(),
@@ -176,6 +176,7 @@ public function testStoreCanQuery(): void
                                 'regenerate' => false,
                             ],
                         ],
+                        '_rankingScore' => 0.95,
                     ],
                     [
                         'id' => Uuid::v4()->toRfc4122(),
@@ -185,9 +186,10 @@ public function testStoreCanQuery(): void
                                 'regenerate' => false,
                             ],
                         ],
+                        '_rankingScore' => 0.85,
                     ],
                 ],
-            ]), [
+            ], [
                 'http_code' => 200,
             ]),
         ], 'http://localhost:7700');
@@ -204,5 +206,49 @@ public function testStoreCanQuery(): void
 
         self::assertSame(1, $httpClient->getRequestsCount());
         self::assertCount(2, $vectors);
+        self::assertInstanceOf(VectorDocument::class, $vectors[0]);
+        self::assertInstanceOf(VectorDocument::class, $vectors[1]);
+        self::assertSame(0.95, $vectors[0]->score);
+        self::assertSame(0.85, $vectors[1]->score);
+    }
+
+    public function testMetadataWithoutIDRankingandVector(): void
+    {
+        $httpClient = new MockHttpClient([
+            new JsonMockResponse([
+                'hits' => [
+                    [
+                        'id' => Uuid::v4()->toRfc4122(),
+                        'title' => 'The Matrix',
+                        'description' => 'A science fiction action film.',
+                        '_vectors' => [
+                            'default' => [
+                                'embeddings' => [0.1, 0.2, 0.3],
+                                'regenerate' => false,
+                            ],
+                        ],
+                        '_rankingScore' => 0.95,
+                    ],
+                ],
+            ], [
+                'http_code' => 200,
+            ]),
+        ], 'http://localhost:7700');
+
+        $store = new Store(
+            $httpClient,
+            'http://localhost:7700',
+            'test',
+            'test',
+            embeddingsDimension: 3,
+        );
+
+        $vectors = $store->query(new Vector([0.1, 0.2, 0.3]));
+        $expected = [
+            'title' => 'The Matrix',
+            'description' => 'A science fiction action film.',
+        ];
+
+        self::assertSame($expected, $vectors[0]->metadata->getArrayCopy());
     }
 }