doc: End-to-end example

Signed-off-by: Anush008 <[email protected]>

Author: Anush008

docs/docs/ops/storages.md

@@ -21,6 +21,12 @@ Exports data to a [Qdrant](https://qdrant.tech/) collection.
 
 The spec takes the following fields:
 
-*   `qdrant_url` (type: `str`, required): The [gRPC URL](https://qdrant.tech/documentation/interfaces/#grpc-interface) of the Qdrant instance. Defaults to `http://localhost:6334/`.
+*   `grpc_url` (type: `str`, required): The [gRPC URL](https://qdrant.tech/documentation/interfaces/#grpc-interface) of the Qdrant instance. Defaults to `http://localhost:6334/`.
 
-*   `collection` (type: `str`, required): The name of the collection to export the data to.
+*   `collection_name` (type: `str`, required): The name of the collection to export the data to.
+
+The field name for the vector embeddings must match the [vector name](https://qdrant.tech/documentation/concepts/vectors/#named-vectors) used when the collection was created.
+
+If no primary key is set during export, a random UUID is used as the Qdrant point ID.
+
+You can find an end-to-end example [here](https://github.com/cocoindex-io/cocoindex/tree/main/examples/text_embedding).

examples/text_embedding/README.md

@@ -1,7 +1,32 @@
-Simple example for cocoindex: build embedding index based on local files.
+## Description
 
-## Prerequisite
-[Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one.
+Example to build a vector index in Qdrant based on local files.
+
+## Pre-requisites
+
+- [Install Postgres](https://cocoindex.io/docs/getting_started/installation#-install-postgres) if you don't have one.
+
+- Run Qdrant.
+
+```bash
+docker run -d -p 6334:6334 -p 6333:6333 qdrant/qdrant
+```
+
+- [Create a collection](https://qdrant.tech/documentation/concepts/vectors/#named-vectors) to export the embeddings to.
+
+```bash
+curl  -X PUT \
+  'http://localhost:6333/collections/cocoindex' \
+  --header 'Content-Type: application/json' \
+  --data-raw '{
+  "vectors": {
+    "text_embedding": {
+      "size": 384,
+      "distance": "Cosine"
+    }
+  }
+}'
+```
 
 ## Run
 
@@ -23,19 +48,22 @@ Update index:
 python main.py cocoindex update
 ```
 
+You can now view the data in the Qdrant dashboard at <http://localhost:6333/dashboard>.
+
 Run:
 
 ```bash
 python main.py
 ```
 
-## CocoInsight 
+## CocoInsight
+
 CocoInsight is in Early Access now (Free) 😊 You found us! A quick 3 minute video tutorial about CocoInsight: [Watch on YouTube](https://youtu.be/ZnmyoHslBSc?si=pPLXWALztkA710r9).
 
 Run CocoInsight to understand your RAG data pipeline:
 
-```
+```bash
 python main.py cocoindex server -c https://cocoindex.io
 ```
 
-Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).
+Then open the CocoInsight UI at [https://cocoindex.io/cocoinsight](https://cocoindex.io/cocoinsight).

examples/text_embedding/main.py

@@ -2,55 +2,77 @@
 
 import cocoindex
 
+
 def text_to_embedding(text: cocoindex.DataSlice) -> cocoindex.DataSlice:
     """
     Embed the text using a SentenceTransformer model.
     This is a shared logic between indexing and querying, so extract it as a function.
     """
     return text.transform(
         cocoindex.functions.SentenceTransformerEmbed(
-            model="sentence-transformers/all-MiniLM-L6-v2"))
+            model="sentence-transformers/all-MiniLM-L6-v2"
+        )
+    )
+
 
 @cocoindex.flow_def(name="TextEmbedding")
-def text_embedding_flow(flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope):
+def text_embedding_flow(
+    flow_builder: cocoindex.FlowBuilder, data_scope: cocoindex.DataScope
+):
     """
     Define an example flow that embeds text into a vector database.
     """
     data_scope["documents"] = flow_builder.add_source(
-        cocoindex.sources.LocalFile(path="markdown_files"))
+        cocoindex.sources.LocalFile(path="markdown_files")
+    )
 
     doc_embeddings = data_scope.add_collector()
 
     with data_scope["documents"].row() as doc:
         doc["chunks"] = doc["content"].transform(
             cocoindex.functions.SplitRecursively(),
-            language="markdown", chunk_size=2000, chunk_overlap=500)
+            language="markdown",
+            chunk_size=2000,
+            chunk_overlap=500,
+        )
 
         with doc["chunks"].row() as chunk:
             chunk["embedding"] = text_to_embedding(chunk["text"])
-            doc_embeddings.collect(filename=doc["filename"], location=chunk["location"],
-                                   text=chunk["text"], embedding=chunk["embedding"])
+            doc_embeddings.collect(
+                id=cocoindex.GeneratedField.UUID,
+                filename=doc["filename"],
+                location=chunk["location"],
+                text=chunk["text"],
+                # 'text_embedding' is the name of the vector we've created the Qdrant collection with.
+                text_embedding=chunk["embedding"],
+            )
 
     doc_embeddings.export(
         "doc_embeddings",
-        cocoindex.storages.Postgres(),
-        primary_key_fields=["filename", "location"],
-        vector_index=[("embedding", cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)])
+        cocoindex.storages.Qdrant(
+            collection_name="cocoindex", grpc_url="http://localhost:6334/"
+        ),
+        primary_key_fields=["id"],
+        setup_by_user=True,
+    )
+
 
 query_handler = cocoindex.query.SimpleSemanticsQueryHandler(
     name="SemanticsSearch",
     flow=text_embedding_flow,
     target_name="doc_embeddings",
     query_transform_flow=text_to_embedding,
-    default_similarity_metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY)
+    default_similarity_metric=cocoindex.VectorSimilarityMetric.COSINE_SIMILARITY,
+)
+
 
 @cocoindex.main_fn()
 def _run():
     # Run queries in a loop to demonstrate the query capabilities.
     while True:
         try:
             query = input("Enter search query (or Enter to quit): ")
-            if query == '':
+            if query == "":
                 break
             results, _ = query_handler.search(query, 10)
             print("\nSearch results:")
@@ -62,6 +84,7 @@ def _run():
         except KeyboardInterrupt:
             break
 
+
 if __name__ == "__main__":
     load_dotenv(override=True)
     _run()