Reimplemented lexer with vectors instead of iterators

Author: Veetaha

crates/ra_ide/src/references/rename.rs

@@ -2,7 +2,7 @@
 
 use hir::ModuleSource;
 use ra_db::{RelativePath, RelativePathBuf, SourceDatabase, SourceDatabaseExt};
-use ra_syntax::{algo::find_node_at_offset, ast, tokenize, AstNode, SyntaxKind, SyntaxNode};
+use ra_syntax::{algo::find_node_at_offset, ast, single_token, AstNode, SyntaxKind, SyntaxNode};
 use ra_text_edit::TextEdit;
 
 use crate::{
@@ -17,11 +17,9 @@ pub(crate) fn rename(
     position: FilePosition,
     new_name: &str,
 ) -> Option<RangeInfo<SourceChange>> {
-    let tokens = tokenize(new_name);
-    if tokens.len() != 1
-        || (tokens[0].kind != SyntaxKind::IDENT && tokens[0].kind != SyntaxKind::UNDERSCORE)
-    {
-        return None;
+    match single_token(new_name)?.token.kind {
+        SyntaxKind::IDENT | SyntaxKind::UNDERSCORE => (),
+        _ => return None,
     }
 
     let parse = db.parse(position.file_id);

crates/ra_mbe/src/subtree_source.rs

@@ -1,7 +1,7 @@
 //! FIXME: write short doc here
 
 use ra_parser::{Token, TokenSource};
-use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T};
+use ra_syntax::{single_token, SmolStr, SyntaxKind, SyntaxKind::*, T};
 use std::cell::{Cell, Ref, RefCell};
 use tt::buffer::{Cursor, TokenBuffer};
 
@@ -129,8 +129,10 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
 }
 
 fn convert_literal(l: &tt::Literal) -> TtToken {
-    let kind =
-        classify_literal(&l.text).map(|tkn| tkn.kind).unwrap_or_else(|| match l.text.as_ref() {
+    let kind = single_token(&l.text)
+        .map(|parsed| parsed.token.kind)
+        .filter(|kind| kind.is_literal())
+        .unwrap_or_else(|| match l.text.as_ref() {
             "true" => T![true],
             "false" => T![false],
             _ => panic!("Fail to convert given literal {:#?}", &l),

crates/ra_syntax/src/lib.rs

@@ -41,7 +41,7 @@ use crate::syntax_node::GreenNode;
 pub use crate::{
     algo::InsertPosition,
     ast::{AstNode, AstToken},
-    parsing::{classify_literal, tokenize, Token},
+    parsing::{first_token, single_token, tokenize, tokenize_append, Token, TokenizeError},
     ptr::{AstPtr, SyntaxNodePtr},
     syntax_error::{Location, SyntaxError, SyntaxErrorKind},
     syntax_node::{

crates/ra_syntax/src/parsing.rs

@@ -7,15 +7,17 @@ mod text_tree_sink;
 mod reparsing;
 
 use crate::{syntax_node::GreenNode, SyntaxError};
+use text_token_source::TextTokenSource;
+use text_tree_sink::TextTreeSink;
 
-pub use self::lexer::{classify_literal, tokenize, Token};
+pub use lexer::*;
 
 pub(crate) use self::reparsing::incremental_reparse;
 
 pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
-    let tokens = tokenize(&text);
-    let mut token_source = text_token_source::TextTokenSource::new(text, &tokens);
-    let mut tree_sink = text_tree_sink::TextTreeSink::new(text, &tokens);
+    let ParsedTokens { tokens, errors } = tokenize(&text);
+    let mut token_source = TextTokenSource::new(text, &tokens);
+    let mut tree_sink = TextTreeSink::new(text, &tokens, errors);
     ra_parser::parse(&mut token_source, &mut tree_sink);
     tree_sink.finish()
 }