feature #259 [Store] Add Typesense (Guikingone)

This PR was squashed before being merged into the main branch.

Discussion
----------

[Store] Add Typesense

| Q             | A
| ------------- | ---
| Bug fix?      | no
| New feature?  | yes
| Docs?         | yes
| Issues        | None
| License       | MIT

Hi 👋🏻

This PR aim to introduce the `Typesense` store for vectors, entirely based on the HttpClient for convenience, regarding the technical implementation, the only "tricky" part is the metadata that are stored as string (easier for storing / retrieving).

The `add` part is not tested against errors as like Neo4j, Typesense doesn't seem to care about the insertion of an existing document 😅

Commits
-------

5c1bae7 [Store] Add Typesense

Author: OskarStark

examples/.env

@@ -96,3 +96,7 @@ NEO4J_HOST=http://127.0.0.1:7474
 NEO4J_DATABASE=neo4j
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=symfonyai
+
+# Typesense
+TYPESENSE_HOST=http://127.0.0.1:8108
+TYPESENSE_API_KEY=changeMe

examples/compose.yaml

@@ -45,3 +45,16 @@ services:
       ports:
           - '7474:7474'
           - '7687:7687'
+
+  typesense:
+      image: typesense/typesense:29.0
+      environment:
+          TYPESENSE_API_KEY: '${TYPESENSE_API_KEY:-changeMe}'
+          TYPESENSE_DATA_DIR: '/data'
+      volumes:
+          - typesense_data:/data
+      ports:
+          - '8108:8108'
+
+volumes:
+    typesense_data:

examples/rag/typesense.php

@@ -0,0 +1,71 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Agent\Agent;
+use Symfony\AI\Agent\Toolbox\AgentProcessor;
+use Symfony\AI\Agent\Toolbox\Tool\SimilaritySearch;
+use Symfony\AI\Agent\Toolbox\Toolbox;
+use Symfony\AI\Fixtures\Movies;
+use Symfony\AI\Platform\Bridge\OpenAi\Embeddings;
+use Symfony\AI\Platform\Bridge\OpenAi\Gpt;
+use Symfony\AI\Platform\Bridge\OpenAi\PlatformFactory;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+use Symfony\AI\Store\Bridge\Typesense\Store;
+use Symfony\AI\Store\Document\Metadata;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\Vectorizer;
+use Symfony\AI\Store\Indexer;
+use Symfony\Component\Uid\Uuid;
+
+require_once dirname(__DIR__).'/bootstrap.php';
+
+// initialize the store
+$store = new Store(
+    httpClient: http_client(),
+    endpointUrl: env('TYPESENSE_HOST'),
+    apiKey: env('TYPESENSE_API_KEY'),
+    collection: 'movies',
+);
+
+// initialize the index
+$store->initialize();
+
+// create embeddings and documents
+$documents = [];
+foreach (Movies::all() as $i => $movie) {
+    $documents[] = new TextDocument(
+        id: Uuid::v4(),
+        content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'],
+        metadata: new Metadata($movie),
+    );
+}
+
+// create embeddings for documents
+$platform = PlatformFactory::create(env('OPENAI_API_KEY'), http_client());
+$vectorizer = new Vectorizer($platform, $embeddings = new Embeddings());
+$indexer = new Indexer($vectorizer, $store, logger());
+$indexer->index($documents);
+
+$model = new Gpt(Gpt::GPT_4O_MINI);
+
+$similaritySearch = new SimilaritySearch($platform, $embeddings, $store);
+$toolbox = new Toolbox([$similaritySearch], logger: logger());
+$processor = new AgentProcessor($toolbox);
+$agent = new Agent($platform, $model, [$processor], [$processor], logger());
+
+$messages = new MessageBag(
+    Message::forSystem('Please answer all user questions only using SimilaritySearch function.'),
+    Message::ofUser('Which movie fits the theme of technology?')
+);
+$result = $agent->call($messages);
+
+echo $result->getContent().\PHP_EOL;

src/ai-bundle/config/options.php

@@ -271,6 +271,19 @@
                             ->end()
                         ->end()
                     ->end()
+                    ->arrayNode('typesense')
+                        ->normalizeKeys(false)
+                        ->useAttributeAsKey('name')
+                        ->arrayPrototype()
+                            ->children()
+                                ->scalarNode('endpoint')->cannotBeEmpty()->end()
+                                ->scalarNode('api_key')->isRequired()->end()
+                                ->scalarNode('collection')->isRequired()->end()
+                                ->scalarNode('vector_field')->end()
+                                ->scalarNode('dimensions')->end()
+                            ->end()
+                        ->end()
+                    ->end()
                 ->end()
             ->end()
             ->arrayNode('indexer')

src/ai-bundle/src/AiBundle.php

@@ -46,6 +46,7 @@
 use Symfony\AI\Store\Bridge\Pinecone\Store as PineconeStore;
 use Symfony\AI\Store\Bridge\Qdrant\Store as QdrantStore;
 use Symfony\AI\Store\Bridge\SurrealDb\Store as SurrealDbStore;
+use Symfony\AI\Store\Bridge\Typesense\Store as TypesenseStore;
 use Symfony\AI\Store\Document\Vectorizer;
 use Symfony\AI\Store\Indexer;
 use Symfony\AI\Store\InMemoryStore;
@@ -678,6 +679,32 @@ private function processStoreConfig(string $type, array $stores, ContainerBuilde
                 $container->setDefinition('ai.store.'.$type.'.'.$name, $definition);
             }
         }
+
+        if ('typesense' === $type) {
+            foreach ($stores as $name => $store) {
+                $arguments = [
+                    new Reference('http_client'),
+                    $store['endpoint'],
+                    $store['api_key'],
+                    $store['collection'],
+                ];
+
+                if (\array_key_exists('vector_field', $store)) {
+                    $arguments[4] = $store['vector_field'];
+                }
+
+                if (\array_key_exists('dimensions', $store)) {
+                    $arguments[5] = $store['dimensions'];
+                }
+
+                $definition = new Definition(TypesenseStore::class);
+                $definition
+                    ->addTag('ai.store')
+                    ->setArguments($arguments);
+
+                $container->setDefinition('ai.store.'.$type.'.'.$name, $definition);
+            }
+        }
     }
 
     /**

src/ai-bundle/tests/DependencyInjection/AiBundleTest.php

@@ -274,6 +274,15 @@ private function getFullConfig(): array
                             'namespaced_user' => true,
                         ],
                     ],
+                    'typesense' => [
+                        'my_typesense_store' => [
+                            'endpoint' => 'http://localhost:8108',
+                            'api_key' => 'foo',
+                            'collection' => 'my_collection',
+                            'vector_field' => 'vector',
+                            'dimensions' => 768,
+                        ],
+                    ],
                 ],
                 'indexer' => [
                     'my_text_indexer' => [

src/store/CHANGELOG.md

@@ -45,6 +45,7 @@ CHANGELOG
    - Qdrant
    - SurrealDB
    - Neo4j
+   - Typesense
  * Add Retrieval Augmented Generation (RAG) support:
    - Document embedding storage
    - Similarity search for relevant documents

src/store/doc/index.rst

@@ -39,12 +39,13 @@ You can find more advanced usage in combination with an Agent using the store fo
 
 * `Similarity Search with MariaDB (RAG)`_
 * `Similarity Search with Meilisearch (RAG)`_
+* `Similarity Search with memory storage (RAG)`_
 * `Similarity Search with MongoDB (RAG)`_
 * `Similarity Search with Neo4j (RAG)`_
 * `Similarity Search with Pinecone (RAG)`_
 * `Similarity Search with Qdrant (RAG)`_
 * `Similarity Search with SurrealDB (RAG)`_
-* `Similarity Search with memory storage (RAG)`_
+* `Similarity Search with Typesense (RAG)`_
 
 Supported Stores
 ----------------
@@ -60,6 +61,7 @@ Supported Stores
 * `Postgres`_ (requires `ext-pdo`)
 * `Qdrant`_
 * `SurrealDB`_
+* `Typesense`_
 
 .. note::
 
@@ -101,6 +103,7 @@ This leads to a store implementing two methods::
 .. _`Similarity Search with memory storage (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/in-memory.php
 .. _`Similarity Search with Qdrant (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/qdrant.php
 .. _`Similarity Search with Neo4j (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/neo4j.php
+.. _`Similarity Search with Typesense (RAG)`: https://github.com/symfony/ai/blob/main/examples/rag/typesense.php
 .. _`Azure AI Search`: https://azure.microsoft.com/products/ai-services/ai-search
 .. _`Chroma`: https://www.trychroma.com/
 .. _`MariaDB`: https://mariadb.org/projects/mariadb-vector/
@@ -112,4 +115,5 @@ This leads to a store implementing two methods::
 .. _`InMemory`: https://www.php.net/manual/en/language.types.array.php
 .. _`Qdrant`: https://qdrant.tech/
 .. _`Neo4j`: https://neo4j.com/
+.. _`Typesense`: https://typesense.org/
 .. _`GitHub`: https://github.com/symfony/ai/issues/16

src/store/src/Bridge/Typesense/Store.php

@@ -0,0 +1,133 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Store\Bridge\Typesense;
+
+use Symfony\AI\Platform\Vector\NullVector;
+use Symfony\AI\Platform\Vector\Vector;
+use Symfony\AI\Store\Document\Metadata;
+use Symfony\AI\Store\Document\VectorDocument;
+use Symfony\AI\Store\Exception\InvalidArgumentException;
+use Symfony\AI\Store\InitializableStoreInterface;
+use Symfony\AI\Store\VectorStoreInterface;
+use Symfony\Component\Uid\Uuid;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+
+/**
+ * @author Guillaume Loulier <[email protected]>
+ */
+final readonly class Store implements InitializableStoreInterface, VectorStoreInterface
+{
+    public function __construct(
+        private HttpClientInterface $httpClient,
+        private string $endpointUrl,
+        #[\SensitiveParameter] private string $apiKey,
+        #[\SensitiveParameter] private string $collection,
+        private string $vectorFieldName = '_vectors',
+        private int $embeddingsDimension = 1536,
+    ) {
+    }
+
+    public function add(VectorDocument ...$documents): void
+    {
+        foreach ($documents as $document) {
+            $this->request('POST', \sprintf('collections/%s/documents', $this->collection), $this->convertToIndexableArray($document));
+        }
+    }
+
+    public function query(Vector $vector, array $options = []): array
+    {
+        $documents = $this->request('POST', 'multi_search', [
+            'searches' => [
+                [
+                    'collection' => $this->collection,
+                    'q' => '*',
+                    'vector_query' => \sprintf('%s:([%s], k:%d)', $this->vectorFieldName, implode(', ', $vector->getData()), $options['k'] ?? 10),
+                ],
+            ],
+        ]);
+
+        return array_map($this->convertToVectorDocument(...), $documents['results'][0]['hits']);
+    }
+
+    public function initialize(array $options = []): void
+    {
+        if ([] !== $options) {
+            throw new InvalidArgumentException('No supported options.');
+        }
+
+        $this->request('POST', 'collections', [
+            'name' => $this->collection,
+            'fields' => [
+                [
+                    'name' => 'id',
+                    'type' => 'string',
+                ],
+                [
+                    'name' => $this->vectorFieldName,
+                    'type' => 'float[]',
+                    'num_dim' => $this->embeddingsDimension,
+                ],
+                [
+                    'name' => 'metadata',
+                    'type' => 'string',
+                ],
+            ],
+        ]);
+    }
+
+    /**
+     * @param array<string, mixed> $payload
+     *
+     * @return array<string, mixed>
+     */
+    private function request(string $method, string $endpoint, array $payload): array
+    {
+        $url = \sprintf('%s/%s', $this->endpointUrl, $endpoint);
+        $result = $this->httpClient->request($method, $url, [
+            'headers' => [
+                'X-TYPESENSE-API-KEY' => $this->apiKey,
+            ],
+            'json' => $payload,
+        ]);
+
+        return $result->toArray();
+    }
+
+    /**
+     * @return array<string, mixed>
+     */
+    private function convertToIndexableArray(VectorDocument $document): array
+    {
+        return [
+            'id' => $document->id->toRfc4122(),
+            $this->vectorFieldName => $document->vector->getData(),
+            'metadata' => json_encode($document->metadata->getArrayCopy()),
+        ];
+    }
+
+    /**
+     * @param array<string, mixed> $data
+     */
+    private function convertToVectorDocument(array $data): VectorDocument
+    {
+        $document = $data['document'] ?? throw new InvalidArgumentException('Missing "document" field in the document data.');
+
+        $id = $document['id'] ?? throw new InvalidArgumentException('Missing "id" field in the document data.');
+
+        $vector = !\array_key_exists($this->vectorFieldName, $document) || null === $document[$this->vectorFieldName]
+            ? new NullVector() : new Vector($document[$this->vectorFieldName]);
+
+        $score = $data['vector_distance'] ?? null;
+
+        return new VectorDocument(Uuid::fromString($id), $vector, new Metadata(json_decode($document['metadata'], true)), $score);
+    }
+}