Skip to content

Commit d6f4560

Browse files
committed
Skip chunks without alphanumeric characters.
1 parent 5bf63f3 commit d6f4560

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

src/ops/functions/split_recursively.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -449,8 +449,8 @@ impl<'t, 's: 't> RecursiveChunker<'s> {
449449
// Trim trailing whitespaces
450450
let trimmed_text = trimmed_text.trim_end();
451451

452-
// Only record non-empty chunks.
453-
if !trimmed_text.is_empty() {
452+
// Only record chunks with alphanumeric characters.
453+
if trimmed_text.chars().any(|ch| ch.is_alphanumeric()) {
454454
output.push((
455455
RangeValue::new(adjusted_start, adjusted_start + trimmed_text.len()),
456456
trimmed_text,

0 commit comments

Comments
 (0)