File tree Expand file tree Collapse file tree 2 files changed +2
-3
lines changed
Expand file tree Collapse file tree 2 files changed +2
-3
lines changed Original file line number Diff line number Diff line change @@ -42,9 +42,8 @@ pub trait Language {
4242 let mut boundaries = Vec :: with_capacity ( estimated_sentences) ;
4343
4444 // Split by paragraph breaks (one or more newlines with optional whitespace)
45- let para_split_re = Regex :: new ( r"\n[\r\s ]*\n" ) . unwrap ( ) ;
45+ let para_split_re = Regex :: new ( r"\n[\r]*\n" ) . unwrap ( ) ;
4646 let paragraphs: Vec < & str > = para_split_re. split ( text) . collect ( ) ;
47-
4847 // Pre-calculate all paragraph offsets in one pass
4948 let mut paragraph_offsets = Vec :: with_capacity ( paragraphs. len ( ) ) ;
5049 let mut current_offset = 0 ;
Original file line number Diff line number Diff line change @@ -107,7 +107,7 @@ fn chunk_text(text: &str, chunk_size: usize) -> Vec<&str> {
107107 let mut chunks = Vec :: new ( ) ;
108108
109109 // Split by paragraph breaks (one or more newlines with optional whitespace)
110- let re = Regex :: new ( r"\n[\r\s ]*\n" ) . unwrap ( ) ;
110+ let re = Regex :: new ( r"\n[\r]*\n" ) . unwrap ( ) ;
111111
112112 // Get paragraph parts and their positions
113113 let mut paragraphs = Vec :: new ( ) ;
You can’t perform that action at this time.
0 commit comments