File tree Expand file tree Collapse file tree 1 file changed +14
-6
lines changed Expand file tree Collapse file tree 1 file changed +14
-6
lines changed Original file line number Diff line number Diff line change 1- from dotenv import load_dotenv
2- from psycopg_pool import ConnectionPool
3- from pgvector .psycopg import register_vector
1+ import os
42from typing import Any
3+
54import cocoindex
6- import os
7- from numpy .typing import NDArray
85import numpy as np
6+ from dotenv import load_dotenv
7+ from numpy .typing import NDArray
8+ from pgvector .psycopg import register_vector
9+ from psycopg_pool import ConnectionPool
910
1011
1112@cocoindex .transform_flow ()
@@ -50,8 +51,15 @@ def text_embedding_flow(
5051 chunk_size = 2000 ,
5152 chunk_overlap = 500 ,
5253 )
54+ func = cocoindex .functions .SplitRecursively ()
55+ chunks = func (
56+ text = doc ["content" ],
57+ language = "markdown" ,
58+ chunk_size = 2000 ,
59+ chunk_overlap = 500 ,
60+ )
5361
54- with doc [ " chunks" ] .row () as chunk :
62+ with chunks .row () as chunk :
5563 chunk ["embedding" ] = text_to_embedding (chunk ["text" ])
5664 doc_embeddings .collect (
5765 filename = doc ["filename" ],
You can’t perform that action at this time.
0 commit comments