@@ -169,9 +169,9 @@ impl<'a> Tokenizer<'a> {
169
169
self . cur_pos ( )
170
170
}
171
171
172
- pub fn take_line_until_punc ( & mut self ) -> Result < & ' a str , Error < ' a > > {
172
+ pub fn take_line ( & mut self ) -> Result < & ' a str , Error < ' a > > {
173
173
let start = self . cur_pos ( ) ;
174
- while self . cur_punct ( ) . is_none ( ) && !self . at_end ( ) {
174
+ while ! matches ! ( self . cur_punct( ) , Some ( Token :: EndOfLine ) ) && !self . at_end ( ) {
175
175
self . advance ( ) ;
176
176
}
177
177
Ok ( self . str_from ( start) )
@@ -359,22 +359,30 @@ fn tokenize_raw_string_prohibit_1() {
359
359
}
360
360
361
361
#[ test]
362
- fn take_line_until_punc ( ) {
362
+ fn tokennize_take_line ( ) {
363
363
assert_eq ! (
364
- Tokenizer :: new( "this is a text. this another one." ) . take_line_until_punc ( ) ,
365
- Ok ( "this is a text" )
364
+ Tokenizer :: new( "this is a text. this another one." ) . take_line ( ) ,
365
+ Ok ( "this is a text. this another one. " )
366
366
) ;
367
367
}
368
368
369
369
#[ test]
370
- fn take_line_until_punc_2 ( ) {
370
+ fn tokennize_take_line_2 ( ) {
371
371
assert_eq ! (
372
- Tokenizer :: new( "punc is \n newline" ) . take_line_until_punc ( ) ,
372
+ Tokenizer :: new( "punc is \n newline" ) . take_line ( ) ,
373
373
Ok ( "punc is " )
374
374
) ;
375
375
}
376
376
377
377
#[ test]
378
- fn take_line_until_punc_3 ( ) {
379
- assert_eq ! ( Tokenizer :: new( "" ) . take_line_until_punc( ) , Ok ( "" ) ) ;
378
+ fn tokennize_take_line_3 ( ) {
379
+ assert_eq ! ( Tokenizer :: new( "" ) . take_line( ) , Ok ( "" ) ) ;
380
+ }
381
+
382
+ #[ test]
383
+ fn tokennize_take_line_4 ( ) {
384
+ assert_eq ! (
385
+ Tokenizer :: new( "To be used in 1.84. Another string." ) . take_line( ) ,
386
+ Ok ( "To be used in 1.84. Another string." )
387
+ ) ;
380
388
}
0 commit comments