Skip to content

Commit dd6c44e

Browse files
committed
Condensed line
1 parent f7b5b7b commit dd6c44e

File tree

1 file changed

+2
-7
lines changed

1 file changed

+2
-7
lines changed

src/semchunk/semchunk.py

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -120,11 +120,7 @@ def chunk(
120120

121121
if overlap:
122122
# Make relative overlaps absolute and floor both relative and absolute overlaps to prevent ever having an overlap >= chunk_size.
123-
if overlap < 1:
124-
overlap = math.floor(chunk_size * overlap)
125-
126-
else:
127-
overlap = min(overlap, chunk_size - 1)
123+
overlap = math.floor(chunk_size * overlap) if overlap < 1 else min(overlap, chunk_size - 1)
128124

129125
# If the overlap has not been zeroed, compute the effective chunk size as the minimum of the chunk size and the chunk size minus the overlap.
130126
if overlap:
@@ -387,8 +383,7 @@ def token_counter(text: str) -> int:
387383
def faster_token_counter(text: str) -> int:
388384
heuristic = chunk_size * 6
389385

390-
if len(text) > heuristic and original_token_counter(text[:heuristic + max_token_chars]) > chunk_size:
391-
return chunk_size + 1
386+
if len(text) > heuristic and original_token_counter(text[:heuristic + max_token_chars]) > chunk_size: return chunk_size + 1
392387

393388
return original_token_counter(text)
394389

0 commit comments

Comments
 (0)