66 @date:2024/10/16 16:34
77 @desc:
88"""
9+ from functools import reduce
910from typing import Dict , List
1011
1112from langchain_community .embeddings import DashScopeEmbeddings
1415from setting .models_provider .base_model_provider import MaxKBBaseModel
1516
1617
18+ def proxy_embed_documents (texts : List [str ], step_size , embed_documents ):
19+ value = [embed_documents (texts [start_index :start_index + step_size ]) for start_index in
20+ range (0 , len (texts ), step_size )]
21+ return reduce (lambda x , y : [* x , * y ], value , [])
22+
23+
1724class AliyunBaiLianEmbedding (MaxKBBaseModel , DashScopeEmbeddings ):
1825 @staticmethod
1926 def new_instance (model_type , model_name , model_credential : Dict [str , object ], ** model_kwargs ):
@@ -23,6 +30,11 @@ def new_instance(model_type, model_name, model_credential: Dict[str, object], **
2330 )
2431
2532 def embed_documents (self , texts : List [str ]) -> List [List [float ]]:
33+ if self .model == 'text-embedding-v3' :
34+ return proxy_embed_documents (texts , 6 , self ._embed_documents )
35+ return self ._embed_documents (texts )
36+
37+ def _embed_documents (self , texts : List [str ]) -> List [List [float ]]:
2638 """Call out to DashScope's embedding endpoint for embedding search docs.
2739
2840 Args:
0 commit comments