File tree Expand file tree Collapse file tree 1 file changed +2
-7
lines changed Expand file tree Collapse file tree 1 file changed +2
-7
lines changed Original file line number Diff line number Diff line change @@ -120,11 +120,7 @@ def chunk(
120120
121121 if overlap :
122122 # Make relative overlaps absolute and floor both relative and absolute overlaps to prevent ever having an overlap >= chunk_size.
123- if overlap < 1 :
124- overlap = math .floor (chunk_size * overlap )
125-
126- else :
127- overlap = min (overlap , chunk_size - 1 )
123+ overlap = math .floor (chunk_size * overlap ) if overlap < 1 else min (overlap , chunk_size - 1 )
128124
129125 # If the overlap has not been zeroed, compute the effective chunk size as the minimum of the chunk size and the chunk size minus the overlap.
130126 if overlap :
@@ -387,8 +383,7 @@ def token_counter(text: str) -> int:
387383 def faster_token_counter (text : str ) -> int :
388384 heuristic = chunk_size * 6
389385
390- if len (text ) > heuristic and original_token_counter (text [:heuristic + max_token_chars ]) > chunk_size :
391- return chunk_size + 1
386+ if len (text ) > heuristic and original_token_counter (text [:heuristic + max_token_chars ]) > chunk_size : return chunk_size + 1
392387
393388 return original_token_counter (text )
394389
You can’t perform that action at this time.
0 commit comments