Simplify and refactor Reader

Simplify reader's implementation, rename methods to be more consistent,
add 1-character lookbehind support, expose unconsumed (tail) range and text.

Author: taminomara

crates/emmylua_parser/src/lexer/lua_doc_lexer.rs

@@ -47,7 +47,8 @@ impl LuaDocLexer<'_> {
     }
 
     pub fn reset(&mut self, kind: LuaTokenKind, range: SourceRange) {
-        self.reader = Some(Reader::new_with_range(self.origin_text, range));
+        let text = &self.origin_text[range.start_offset..range.end_offset()];
+        self.reader = Some(Reader::new_with_range(text, range));
         self.origin_token_kind = kind;
     }
 
@@ -76,7 +77,7 @@ impl LuaDocLexer<'_> {
     }
 
     pub fn current_token_range(&self) -> SourceRange {
-        self.reader.as_ref().unwrap().saved_range()
+        self.reader.as_ref().unwrap().current_range()
     }
 
     fn lex_init(&mut self) -> LuaTokenKind {
@@ -135,7 +136,7 @@ impl LuaDocLexer<'_> {
             ch if is_name_start(ch) => {
                 reader.bump();
                 reader.eat_while(is_name_continue);
-                let text = reader.current_saved_text();
+                let text = reader.current_text();
                 to_tag(text)
             }
             _ => {
@@ -490,7 +491,7 @@ impl LuaDocLexer<'_> {
                 }
 
                 reader.eat_while(|c| c.is_ascii_alphabetic());
-                let text = reader.current_saved_text();
+                let text = reader.current_text();
                 match text {
                     "region" | "#region" => LuaTokenKind::TkDocRegion,
                     "endregion" | "#endregion" => LuaTokenKind::TkDocEndRegion,
@@ -651,7 +652,7 @@ fn read_doc_name<'a>(reader: &'a mut Reader) -> (&'a str, bool /* str tpl */) {
         }
     }
 
-    (reader.current_saved_text(), str_tpl)
+    (reader.current_text(), str_tpl)
 }
 
 fn is_source_continue(ch: char) -> bool {

crates/emmylua_parser/src/lexer/lua_lexer.rs

@@ -30,7 +30,7 @@ impl LuaLexer<'_> {
                 break;
             }
 
-            tokens.push(LuaTokenData::new(kind, self.reader.saved_range()));
+            tokens.push(LuaTokenData::new(kind, self.reader.current_range()));
         }
 
         tokens
@@ -122,7 +122,7 @@ impl LuaLexer<'_> {
                 if self.reader.current_char() != '[' {
                     self.errors.push(LuaParseError::syntax_error_from(
                         &t!("invalid long string delimiter"),
-                        self.reader.saved_range(),
+                        self.reader.current_range(),
                     ));
                     return LuaTokenKind::TkLongString;
                 }
@@ -149,7 +149,7 @@ impl LuaLexer<'_> {
                         if !self.lexer_config.support_integer_operation() {
                             self.errors.push(LuaParseError::syntax_error_from(
                                 &t!("bitwise operation is not supported"),
-                                self.reader.saved_range(),
+                                self.reader.current_range(),
                             ));
                         }
 
@@ -176,7 +176,7 @@ impl LuaLexer<'_> {
                         if !self.lexer_config.support_integer_operation() {
                             self.errors.push(LuaParseError::syntax_error_from(
                                 &t!("bitwise operation is not supported"),
-                                self.reader.saved_range(),
+                                self.reader.current_range(),
                             ));
                         }
 
@@ -198,7 +198,7 @@ impl LuaLexer<'_> {
                     if !self.lexer_config.support_integer_operation() {
                         self.errors.push(LuaParseError::syntax_error_from(
                             &t!("bitwise operation is not supported"),
-                            self.reader.saved_range(),
+                            self.reader.current_range(),
                         ));
                     }
                     return LuaTokenKind::TkBitXor;
@@ -252,7 +252,7 @@ impl LuaLexer<'_> {
                 if self.reader.current_char() != quote {
                     self.errors.push(LuaParseError::syntax_error_from(
                         &t!("unfinished string"),
-                        self.reader.saved_range(),
+                        self.reader.current_range(),
                     ));
                     return LuaTokenKind::TkString;
                 }
@@ -297,7 +297,7 @@ impl LuaLexer<'_> {
                                 _ if self.reader.is_eof() => {
                                     self.errors.push(LuaParseError::syntax_error_from(
                                         &t!("unfinished long comment"),
-                                        self.reader.saved_range(),
+                                        self.reader.current_range(),
                                     ));
                                     return LuaTokenKind::TkLongComment;
                                 }
@@ -323,7 +323,7 @@ impl LuaLexer<'_> {
                         if !self.lexer_config.support_integer_operation() {
                             self.errors.push(LuaParseError::syntax_error_from(
                                 &t!("integer division is not supported"),
-                                self.reader.saved_range(),
+                                self.reader.current_range(),
                             ));
                         }
 
@@ -405,7 +405,7 @@ impl LuaLexer<'_> {
                 if !self.lexer_config.support_integer_operation() {
                     self.errors.push(LuaParseError::syntax_error_from(
                         &t!("bitwise operation is not supported"),
-                        self.reader.saved_range(),
+                        self.reader.current_range(),
                     ));
                 }
 
@@ -428,7 +428,7 @@ impl LuaLexer<'_> {
                 if !self.lexer_config.support_integer_operation() {
                     self.errors.push(LuaParseError::syntax_error_from(
                         &t!("bitwise operation is not supported"),
-                        self.reader.saved_range(),
+                        self.reader.current_range(),
                     ));
                 }
 
@@ -483,7 +483,7 @@ impl LuaLexer<'_> {
             ch if is_name_start(ch) => {
                 self.reader.bump();
                 self.reader.eat_while(is_name_continue);
-                let name = self.reader.current_saved_text();
+                let name = self.reader.current_text();
                 self.name_to_kind(name)
             }
             _ => {
@@ -546,7 +546,7 @@ impl LuaLexer<'_> {
         if !end {
             self.errors.push(LuaParseError::syntax_error_from(
                 &t!("unfinished long string or comment"),
-                self.reader.saved_range(),
+                self.reader.current_range(),
             ));
         }
 
@@ -671,7 +671,7 @@ impl LuaLexer<'_> {
                     "unexpected character '{}' after number literal",
                     self.reader.current_char()
                 ),
-                self.reader.saved_range(),
+                self.reader.current_range(),
             ));
         }