Skip to content

Commit 71a1ca2

Browse files
committed
exclude empty sentences from sentenceit
1 parent bc1a4a1 commit 71a1ca2

File tree

1 file changed

+3
-1
lines changed

1 file changed

+3
-1
lines changed

chunkit.js

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,9 @@ export async function sentenceit(
333333
// Split the text into sentences
334334
const chunks = [];
335335
for (const { segment } of splitBySentence(doc.document_text)) {
336-
chunks.push(segment.trim());
336+
if (segment.trim().length > 0) {
337+
chunks.push(segment.trim());
338+
}
337339
}
338340

339341
if (logging) {

0 commit comments

Comments
 (0)