1- use crate :: JSError ;
1+ use crate :: { JSError , raise_tokenize_error } ;
22
33#[ derive( Debug , Clone ) ]
44pub enum Token {
@@ -218,7 +218,7 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
218218 i += 1 ;
219219 }
220220 if i >= chars. len ( ) {
221- return Err ( JSError :: TokenizationError ) ; // Unterminated comment
221+ return Err ( raise_tokenize_error ! ( ) ) ; // Unterminated comment
222222 }
223223 } else {
224224 // Heuristic: when '/' occurs in a position that cannot end an
@@ -268,7 +268,7 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
268268 j += 1 ;
269269 }
270270 if j >= chars. len ( ) || chars[ j] != '/' {
271- return Err ( JSError :: TokenizationError ) ; // unterminated regex
271+ return Err ( raise_tokenize_error ! ( ) ) ; // unterminated regex
272272 }
273273 // pattern is between i+1 and j-1
274274 let pattern: String = chars[ i + 1 ..j] . iter ( ) . collect ( ) ;
@@ -418,7 +418,7 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
418418 tokens. push ( Token :: LogicalAnd ) ;
419419 i += 2 ;
420420 } else {
421- return Err ( JSError :: TokenizationError ) ;
421+ return Err ( raise_tokenize_error ! ( ) ) ;
422422 }
423423 }
424424 '|' => {
@@ -430,7 +430,7 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
430430 tokens. push ( Token :: LogicalOr ) ;
431431 i += 2 ;
432432 } else {
433- return Err ( JSError :: TokenizationError ) ;
433+ return Err ( raise_tokenize_error ! ( ) ) ;
434434 }
435435 }
436436 '0' ..='9' => {
@@ -445,7 +445,7 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
445445 let mut num_str: String = chars[ start..i] . iter ( ) . collect ( ) ;
446446 num_str. retain ( |c| c != '_' ) ;
447447 if num_str. is_empty ( ) || !num_str. chars ( ) . all ( |c| c. is_ascii_digit ( ) ) {
448- return Err ( JSError :: TokenizationError ) ;
448+ return Err ( raise_tokenize_error ! ( ) ) ;
449449 }
450450 tokens. push ( Token :: BigInt ( num_str) ) ;
451451 i += 1 ; // consume trailing 'n'
@@ -469,7 +469,7 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
469469 }
470470 // require at least one digit in exponent (underscores allowed inside digits)
471471 if j >= chars. len ( ) || !( chars[ j] . is_ascii_digit ( ) ) {
472- return Err ( JSError :: TokenizationError ) ;
472+ return Err ( raise_tokenize_error ! ( ) ) ;
473473 }
474474 while j < chars. len ( ) && ( chars[ j] . is_ascii_digit ( ) || chars[ j] == '_' ) {
475475 j += 1 ;
@@ -483,7 +483,7 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
483483 // Convert to f64
484484 match num_str. parse :: < f64 > ( ) {
485485 Ok ( n) => tokens. push ( Token :: Number ( n) ) ,
486- Err ( _) => return Err ( JSError :: TokenizationError ) ,
486+ Err ( _) => return Err ( raise_tokenize_error ! ( ) ) ,
487487 }
488488 }
489489 '"' => {
@@ -525,7 +525,7 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
525525 i += 1 ;
526526 }
527527 if brace_count != 0 {
528- return Err ( JSError :: TokenizationError ) ;
528+ return Err ( raise_tokenize_error ! ( ) ) ;
529529 }
530530 let expr_str: String = chars[ expr_start..i - 1 ] . iter ( ) . collect ( ) ;
531531 // Tokenize the expression inside ${}
@@ -537,7 +537,7 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
537537 }
538538 }
539539 if i >= chars. len ( ) {
540- return Err ( JSError :: TokenizationError ) ;
540+ return Err ( raise_tokenize_error ! ( ) ) ;
541541 }
542542 // Add remaining string part
543543 if current_start < i {
@@ -600,7 +600,7 @@ pub fn tokenize(expr: &str) -> Result<Vec<Token>, JSError> {
600600 tokens. push ( Token :: Semicolon ) ;
601601 i += 1 ;
602602 }
603- _ => return Err ( JSError :: TokenizationError ) ,
603+ _ => return Err ( raise_tokenize_error ! ( ) ) ,
604604 }
605605 }
606606 Ok ( tokens)
@@ -612,7 +612,7 @@ fn parse_string_literal(chars: &[char], start: &mut usize, end_char: char) -> Re
612612 if chars[ * start] == '\\' {
613613 * start += 1 ;
614614 if * start >= chars. len ( ) {
615- return Err ( JSError :: TokenizationError ) ;
615+ return Err ( raise_tokenize_error ! ( ) ) ;
616616 }
617617 match chars[ * start] {
618618 'n' => result. push ( '\n' as u16 ) ,
@@ -626,7 +626,7 @@ fn parse_string_literal(chars: &[char], start: &mut usize, end_char: char) -> Re
626626 // Unicode escape sequences: either \uXXXX or \u{HEX...}
627627 * start += 1 ;
628628 if * start >= chars. len ( ) {
629- return Err ( JSError :: TokenizationError ) ;
629+ return Err ( raise_tokenize_error ! ( ) ) ;
630630 }
631631 if chars[ * start] == '{' {
632632 // \u{HEX...}
@@ -637,7 +637,7 @@ fn parse_string_literal(chars: &[char], start: &mut usize, end_char: char) -> Re
637637 * start += 1 ;
638638 }
639639 if * start >= chars. len ( ) || chars[ * start] != '}' {
640- return Err ( JSError :: TokenizationError ) ; // no closing brace
640+ return Err ( raise_tokenize_error ! ( ) ) ; // no closing brace
641641 }
642642 // parse hex as codepoint
643643 match u32:: from_str_radix ( & hex_str, 16 ) {
@@ -653,37 +653,37 @@ fn parse_string_literal(chars: &[char], start: &mut usize, end_char: char) -> Re
653653 result. push ( low) ;
654654 }
655655 }
656- _ => return Err ( JSError :: TokenizationError ) ,
656+ _ => return Err ( raise_tokenize_error ! ( ) ) ,
657657 }
658658 // `start` currently at closing '}', the outer loop will increment it further
659659 } else {
660660 // Unicode escape sequence \uXXXX
661661 if * start + 4 > chars. len ( ) {
662- return Err ( JSError :: TokenizationError ) ;
662+ return Err ( raise_tokenize_error ! ( ) ) ;
663663 }
664664 let hex_str: String = chars[ * start..* start + 4 ] . iter ( ) . collect ( ) ;
665665 * start += 3 ; // will be incremented by 1 at the end
666666 match u16:: from_str_radix ( & hex_str, 16 ) {
667667 Ok ( code) => {
668668 result. push ( code) ;
669669 }
670- Err ( _) => return Err ( JSError :: TokenizationError ) , // Invalid hex
670+ Err ( _) => return Err ( raise_tokenize_error ! ( ) ) , // Invalid hex
671671 }
672672 }
673673 }
674674 'x' => {
675675 // Hex escape sequence \xHH
676676 * start += 1 ;
677677 if * start + 2 > chars. len ( ) {
678- return Err ( JSError :: TokenizationError ) ;
678+ return Err ( raise_tokenize_error ! ( ) ) ;
679679 }
680680 let hex_str: String = chars[ * start..* start + 2 ] . iter ( ) . collect ( ) ;
681681 * start += 1 ; // will be incremented by 1 at the end
682682 match u8:: from_str_radix ( & hex_str, 16 ) {
683683 Ok ( code) => {
684684 result. push ( code as u16 ) ;
685685 }
686- Err ( _) => return Err ( JSError :: TokenizationError ) ,
686+ Err ( _) => return Err ( raise_tokenize_error ! ( ) ) ,
687687 }
688688 }
689689 // For other escapes (regex escapes like \., \s, \], etc.) keep the backslash
@@ -699,7 +699,7 @@ fn parse_string_literal(chars: &[char], start: &mut usize, end_char: char) -> Re
699699 * start += 1 ;
700700 }
701701 if * start >= chars. len ( ) {
702- return Err ( JSError :: TokenizationError ) ;
702+ return Err ( raise_tokenize_error ! ( ) ) ; // Unterminated string literal
703703 }
704704 Ok ( result)
705705}
0 commit comments