feature #111 [Store] Add Qdrant (Guikingone)

This PR was merged into the main branch.

Discussion
----------

[Store] Add Qdrant

| Q             | A
| ------------- | ---
| Bug fix?      | no
| New feature?  | yes
| Docs?         | yes
| Issues        | None
| License       | MIT

Hi 👋🏻

Alright, last one for today, here's the implementation for `Qdrant`, this one's tricky as the default search route is deprecated, the implementation is based on https://qdrant.tech/documentation/quickstart/#run-a-query, the whole store use the HTTP API along with the API key.

Some options are supported (not all as some depends on the query you're building), more can be added if needed.

Commits
-------

1a94a19 feat(store): qdrant store started

Author: chr-hertel

examples/.env

@@ -81,3 +81,7 @@ MEILISEARCH_API_KEY=changeMe
 
 # For using LMStudio
 LMSTUDIO_HOST_URL=http://127.0.0.1:1234
+
+# Qdrant
+QDRANT_HOST='http://127.0.0.1:6333'
+QDRANT_SERVICE_API_KEY=changeMe

examples/compose.yaml

@@ -5,7 +5,7 @@ services:
       MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1
       MARIADB_DATABASE: my_database
     ports:
-      - "3309:3306"
+      - '3309:3306'
 
   postgres:
     image: pgvector/pgvector:0.8.0-pg17
@@ -14,11 +14,18 @@ services:
         POSTGRES_USER: postgres
         POSTGRES_PASSWORD: postgres
     ports:
-        - "5432:5432"
+        - '5432:5432'
 
   meilisearch:
       image: getmeili/meilisearch:v1.15
       environment:
-          MEILI_MASTER_KEY: "${MEILISEARCH_MASTER_KEY:-changeMe}"
+          MEILI_MASTER_KEY: '${MEILISEARCH_MASTER_KEY:-changeMe}'
       ports:
-          - "7700:7700"
+          - '7700:7700'
+
+  qdrant:
+      image: qdrant/qdrant
+      environment:
+          QDRANT__SERVICE__API_KEY: '${QDRAT_SERVICE_API_KEY:-changeMe}'
+      ports:
+          - '6333:6333'

examples/store/qdrant-similarity-search.php

@@ -0,0 +1,78 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Symfony\AI\Agent\Agent;
+use Symfony\AI\Agent\Toolbox\AgentProcessor;
+use Symfony\AI\Agent\Toolbox\Tool\SimilaritySearch;
+use Symfony\AI\Agent\Toolbox\Toolbox;
+use Symfony\AI\Fixtures\Movies;
+use Symfony\AI\Platform\Bridge\OpenAI\Embeddings;
+use Symfony\AI\Platform\Bridge\OpenAI\GPT;
+use Symfony\AI\Platform\Bridge\OpenAI\PlatformFactory;
+use Symfony\AI\Platform\Message\Message;
+use Symfony\AI\Platform\Message\MessageBag;
+use Symfony\AI\Store\Bridge\Qdrant\Store;
+use Symfony\AI\Store\Document\Metadata;
+use Symfony\AI\Store\Document\TextDocument;
+use Symfony\AI\Store\Document\Vectorizer;
+use Symfony\AI\Store\Indexer;
+use Symfony\Component\Dotenv\Dotenv;
+use Symfony\Component\HttpClient\HttpClient;
+use Symfony\Component\Uid\Uuid;
+
+require_once dirname(__DIR__).'/vendor/autoload.php';
+(new Dotenv())->loadEnv(dirname(__DIR__).'/.env');
+
+if (!isset($_SERVER['OPENAI_API_KEY'], $_SERVER['QDRANT_HOST'], $_SERVER['QDRANT_SERVICE_API_KEY'])) {
+    echo 'Please set OPENAI_API_KEY, QDRANT_HOST and QDRANT_SERVICE_API_KEY environment variables.'.\PHP_EOL;
+    exit(1);
+}
+
+// initialize the store
+$store = new Store(
+    HttpClient::create(),
+    'http://127.0.0.1:6333',
+    $_SERVER['QDRANT_SERVICE_API_KEY'],
+    'movies',
+);
+
+// initialize the collection (needs to be called before the indexer)
+$store->initialize();
+
+// create embeddings and documents
+foreach (Movies::all() as $movie) {
+    $documents[] = new TextDocument(
+        id: Uuid::v4(),
+        content: 'Title: '.$movie['title'].\PHP_EOL.'Director: '.$movie['director'].\PHP_EOL.'Description: '.$movie['description'],
+        metadata: new Metadata($movie),
+    );
+}
+
+// create embeddings for documents
+$platform = PlatformFactory::create($_SERVER['OPENAI_API_KEY']);
+$vectorizer = new Vectorizer($platform, $embeddings = new Embeddings());
+$indexer = new Indexer($vectorizer, $store);
+$indexer->index($documents);
+
+$model = new GPT(GPT::GPT_4O_MINI);
+
+$similaritySearch = new SimilaritySearch($platform, $embeddings, $store);
+$toolbox = Toolbox::create($similaritySearch);
+$processor = new AgentProcessor($toolbox);
+$agent = new Agent($platform, $model, [$processor], [$processor]);
+
+$messages = new MessageBag(
+    Message::forSystem('Please answer all user questions only using SimilaritySearch function.'),
+    Message::ofUser('Which movie fits the theme of the mafia?')
+);
+$response = $agent->call($messages);
+
+echo $response->getContent().\PHP_EOL;

src/store/CHANGELOG.md

@@ -42,6 +42,7 @@ CHANGELOG
    - Meilisearch
    - ChromaDB
    - Pinecone
+   - Qdrant
  * Add Retrieval Augmented Generation (RAG) support:
    - Document embedding storage
    - Similarity search for relevant documents
@@ -52,4 +53,4 @@ CHANGELOG
    - Result limiting
    - Distance/similarity scoring
  * Add custom exception hierarchy with `ExceptionInterface`
- * Add support for specific exceptions for invalid arguments and runtime errors
+ * Add support for specific exceptions for invalid arguments and runtime errors

src/store/doc/index.rst

@@ -42,6 +42,7 @@ You can find more advanced usage in combination with an Agent using the store fo
 * `Similarity Search with Pinecone (RAG)`_
 * `Similarity Search with Meilisearch (RAG)`_
 * `Similarity Search with memory storage (RAG)`_
+* `Similarity Search with Qdrant (RAG)`_
 
 Supported Stores
 ----------------
@@ -54,6 +55,7 @@ Supported Stores
 * `Postgres`_ (requires `ext-pdo`)
 * `Meilisearch`_
 * `InMemory`_
+* `Qdrant`_
 
 .. note::
 
@@ -92,6 +94,7 @@ This leads to a store implementing two methods::
 .. _`Similarity Search with Pinecone (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/pinecone-similarity-search.php
 .. _`Similarity Search with Meilisearch (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/meilisearch-similarity-search.php
 .. _`Similarity Search with memory storage (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/memory-similarity-search.php
+.. _`Similarity Search with Qdrant (RAG)`: https://github.com/symfony/ai/blob/main/examples/store/qdrant-similarity-search.php
 .. _`Azure AI Search`: https://azure.microsoft.com/products/ai-services/ai-search
 .. _`Chroma`: https://www.trychroma.com/
 .. _`MariaDB`: https://mariadb.org/projects/mariadb-vector/
@@ -100,4 +103,5 @@ This leads to a store implementing two methods::
 .. _`Postgres`: https://www.postgresql.org/about/news/pgvector-070-released-2852/
 .. _`Meilisearch`: https://www.meilisearch.com/
 .. _`InMemory`: https://www.php.net/manual/en/language.types.array.php
+.. _`Qdrant`: https://qdrant.tech/
 .. _`GitHub`: https://github.com/symfony/ai/issues/16

src/store/src/Bridge/Qdrant/Store.php

@@ -0,0 +1,142 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <[email protected]>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Store\Bridge\Qdrant;
+
+use Symfony\AI\Platform\Vector\NullVector;
+use Symfony\AI\Platform\Vector\Vector;
+use Symfony\AI\Store\Document\Metadata;
+use Symfony\AI\Store\Document\VectorDocument;
+use Symfony\AI\Store\Exception\InvalidArgumentException;
+use Symfony\AI\Store\InitializableStoreInterface;
+use Symfony\AI\Store\VectorStoreInterface;
+use Symfony\Component\Uid\Uuid;
+use Symfony\Contracts\HttpClient\HttpClientInterface;
+
+/**
+ * @author Guillaume Loulier <[email protected]>
+ */
+final readonly class Store implements InitializableStoreInterface, VectorStoreInterface
+{
+    public function __construct(
+        private HttpClientInterface $httpClient,
+        private string $endpointUrl,
+        #[\SensitiveParameter] private string $apiKey,
+        private string $collectionName,
+        private int $embeddingsDimension = 1536,
+        private string $embeddingsDistance = 'Cosine',
+    ) {
+    }
+
+    public function add(VectorDocument ...$documents): void
+    {
+        $this->request('PUT', \sprintf('collections/%s/points', $this->collectionName), [
+            'points' => array_map($this->convertToIndexableArray(...), $documents),
+        ]);
+    }
+
+    /**
+     * @param array{
+     *     limit?: positive-int,
+     *     offset?: positive-int
+     * } $options
+     */
+    public function query(Vector $vector, array $options = [], ?float $minScore = null): array
+    {
+        $payload = [
+            'vector' => $vector->getData(),
+            'with_payload' => true,
+            'with_vector' => true,
+        ];
+
+        if (\array_key_exists('limit', $options)) {
+            $payload['limit'] = $options['limit'];
+        }
+
+        if (\array_key_exists('offset', $options)) {
+            $payload['offset'] = $options['offset'];
+        }
+
+        $response = $this->request('POST', \sprintf('collections/%s/points/query', $this->collectionName), $payload);
+
+        return array_map($this->convertToVectorDocument(...), $response['result']['points']);
+    }
+
+    public function initialize(array $options = []): void
+    {
+        if ([] !== $options) {
+            throw new InvalidArgumentException('No supported options');
+        }
+
+        $collectionExistResponse = $this->request('GET', \sprintf('collections/%s/exists', $this->collectionName));
+
+        if ($collectionExistResponse['result']['exists']) {
+            return;
+        }
+
+        $this->request('PUT', \sprintf('collections/%s', $this->collectionName), [
+            'vectors' => [
+                'size' => $this->embeddingsDimension,
+                'distance' => $this->embeddingsDistance,
+            ],
+        ]);
+    }
+
+    /**
+     * @param array<string, mixed> $payload
+     *
+     * @return array<string, mixed>
+     */
+    private function request(string $method, string $endpoint, array $payload = []): array
+    {
+        $url = \sprintf('%s/%s', $this->endpointUrl, $endpoint);
+
+        $response = $this->httpClient->request($method, $url, [
+            'headers' => [
+                'api-key' => $this->apiKey,
+            ],
+            'json' => $payload,
+        ]);
+
+        return $response->toArray();
+    }
+
+    /**
+     * @return array<string, mixed>
+     */
+    private function convertToIndexableArray(VectorDocument $document): array
+    {
+        return [
+            'id' => $document->id->toRfc4122(),
+            'vector' => $document->vector->getData(),
+            'payload' => $document->metadata->getArrayCopy(),
+        ];
+    }
+
+    /**
+     * @param array<string, mixed> $data
+     */
+    private function convertToVectorDocument(array $data): VectorDocument
+    {
+        $id = $data['id'] ?? throw new InvalidArgumentException('Missing "id" field in the document data');
+
+        $vector = !\array_key_exists('vector', $data) || null === $data['vector']
+            ? new NullVector()
+            : new Vector($data['vector']);
+
+        return new VectorDocument(
+            id: Uuid::fromString($id),
+            vector: $vector,
+            metadata: new Metadata($data['payload']),
+            score: $data['score'] ?? null
+        );
+    }
+}