ra_syntax: refactored the lexer design as per @matklad and @kiljacken PR review

Author: Veetaha

crates/ra_ide/src/references/rename.rs

@@ -2,7 +2,9 @@
 
 use hir::ModuleSource;
 use ra_db::{RelativePath, RelativePathBuf, SourceDatabase, SourceDatabaseExt};
-use ra_syntax::{algo::find_node_at_offset, ast, single_token, AstNode, SyntaxKind, SyntaxNode};
+use ra_syntax::{
+    algo::find_node_at_offset, ast, lex_single_valid_syntax_kind, AstNode, SyntaxKind, SyntaxNode,
+};
 use ra_text_edit::TextEdit;
 
 use crate::{
@@ -17,7 +19,7 @@ pub(crate) fn rename(
     position: FilePosition,
     new_name: &str,
 ) -> Option<RangeInfo<SourceChange>> {
-    match single_token(new_name)?.token.kind {
+    match lex_single_valid_syntax_kind(new_name)? {
         SyntaxKind::IDENT | SyntaxKind::UNDERSCORE => (),
         _ => return None,
     }

crates/ra_mbe/src/subtree_source.rs

@@ -1,7 +1,7 @@
 //! FIXME: write short doc here
 
 use ra_parser::{Token, TokenSource};
-use ra_syntax::{single_token, SmolStr, SyntaxKind, SyntaxKind::*, T};
+use ra_syntax::{lex_single_valid_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
 use std::cell::{Cell, Ref, RefCell};
 use tt::buffer::{Cursor, TokenBuffer};
 
@@ -129,8 +129,7 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
 }
 
 fn convert_literal(l: &tt::Literal) -> TtToken {
-    let kind = single_token(&l.text)
-        .map(|parsed| parsed.token.kind)
+    let kind = lex_single_valid_syntax_kind(&l.text)
         .filter(|kind| kind.is_literal())
         .unwrap_or_else(|| match l.text.as_ref() {
             "true" => T![true],

crates/ra_syntax/src/lib.rs

@@ -41,7 +41,9 @@ use crate::syntax_node::GreenNode;
 pub use crate::{
     algo::InsertPosition,
     ast::{AstNode, AstToken},
-    parsing::{first_token, single_token, tokenize, tokenize_append, Token, TokenizeError},
+    parsing::{
+        lex_single_syntax_kind, lex_single_valid_syntax_kind, tokenize, Token, TokenizeError,
+    },
     ptr::{AstPtr, SyntaxNodePtr},
     syntax_error::{Location, SyntaxError, SyntaxErrorKind},
     syntax_node::{

crates/ra_syntax/src/parsing.rs

@@ -15,9 +15,15 @@ pub use lexer::*;
 pub(crate) use self::reparsing::incremental_reparse;
 
 pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
-    let ParsedTokens { tokens, errors } = tokenize(&text);
+    let (tokens, lexer_errors) = tokenize(&text);
+
     let mut token_source = TextTokenSource::new(text, &tokens);
-    let mut tree_sink = TextTreeSink::new(text, &tokens, errors);
+    let mut tree_sink = TextTreeSink::new(text, &tokens);
+
     ra_parser::parse(&mut token_source, &mut tree_sink);
-    tree_sink.finish()
+
+    let (tree, mut parser_errors) = tree_sink.finish();
+    parser_errors.extend(lexer_errors);
+
+    (tree, parser_errors)
 }