File tree Expand file tree Collapse file tree 2 files changed +4
-0
lines changed
Expand file tree Collapse file tree 2 files changed +4
-0
lines changed Original file line number Diff line number Diff line change @@ -110,6 +110,7 @@ async def semantic_text_chunker(req: func.HttpRequest) -> func.HttpResponse:
110110 "similarity_threshold" , 0.8
111111 )
112112 max_chunk_tokens = semantic_text_chunker_config .get ("max_chunk_tokens" , 500 )
113+ min_chunk_tokens = semantic_text_chunker_config .get ("min_chunk_tokens" , 50 )
113114
114115 except ValueError :
115116 return func .HttpResponse (
@@ -124,6 +125,7 @@ async def semantic_text_chunker(req: func.HttpRequest) -> func.HttpResponse:
124125 num_surrounding_sentences = num_surrounding_sentences ,
125126 similarity_threshold = similarity_threshold ,
126127 max_chunk_tokens = max_chunk_tokens ,
128+ min_chunk_tokens = min_chunk_tokens ,
127129 )
128130
129131 for value in values :
Original file line number Diff line number Diff line change @@ -260,6 +260,7 @@ def get_text_split_skill(
260260 num_surrounding_sentences : int = 1 ,
261261 similarity_threshold : float = 0.8 ,
262262 max_chunk_tokens : int = 200 ,
263+ min_chunk_tokens : int = 50 ,
263264 ) -> SplitSkill :
264265 """Get the skill for text split.
265266
@@ -303,6 +304,7 @@ def get_text_split_skill(
303304 "num_surrounding_sentences" : num_surrounding_sentences ,
304305 "similarity_threshold" : similarity_threshold ,
305306 "max_chunk_tokens" : max_chunk_tokens ,
307+ "min_chunk_tokens" : min_chunk_tokens ,
306308 },
307309 inputs = semantic_text_chunker_skill_inputs ,
308310 outputs = semantic_text_chunker_skill_outputs ,
You can’t perform that action at this time.
0 commit comments