Merge #11117

11117: internal: replace TreeSink with a data structure  r=matklad a=matklad

The general theme of this is to make parser a better independent
library.

The specific thing we do here is replacing callback based TreeSink with
a data structure. That is, rather than calling user-provided tree
construction methods, the parser now spits out a very bare-bones tree,
effectively a log of a DFS traversal.

This makes the parser usable without any *specifc* tree sink, and allows
us to, eg, move tests into this crate.

Now, it's also true that this is a distinction without a difference, as
the old and the new interface are equivalent in expressiveness. Still,
this new thing seems somewhat simpler. But yeah, I admit I don't have a
suuper strong motivation here, just a hunch that this is better.

cc #10765 

Co-authored-by: Aleksey Kladov <[email protected]>

Author: bors[bot]

crates/mbe/src/lib.rs

@@ -10,7 +10,7 @@ mod parser;
 mod expander;
 mod syntax_bridge;
 mod tt_iter;
-mod to_parser_tokens;
+mod to_parser_input;
 
 #[cfg(test)]
 mod benchmark;

crates/mbe/src/syntax_bridge.rs

@@ -1,6 +1,5 @@
 //! Conversions between [`SyntaxNode`] and [`tt::TokenTree`].
 
-use parser::{ParseError, TreeSink};
 use rustc_hash::{FxHashMap, FxHashSet};
 use syntax::{
     ast::{self, make::tokens::doc_comment},
@@ -11,7 +10,7 @@ use syntax::{
 use tt::buffer::{Cursor, TokenBuffer};
 
 use crate::{
-    to_parser_tokens::to_parser_tokens, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
+    to_parser_input::to_parser_input, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
 };
 
 /// Convert the syntax node to a `TokenTree` (what macro
@@ -55,9 +54,19 @@ pub fn token_tree_to_syntax_node(
         }
         _ => TokenBuffer::from_subtree(tt),
     };
-    let parser_tokens = to_parser_tokens(&buffer);
+    let parser_input = to_parser_input(&buffer);
+    let parser_output = parser::parse(&parser_input, entry_point);
     let mut tree_sink = TtTreeSink::new(buffer.begin());
-    parser::parse(&parser_tokens, &mut tree_sink, entry_point);
+    for event in parser_output.iter() {
+        match event {
+            parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
+                tree_sink.token(kind, n_raw_tokens)
+            }
+            parser::Step::Enter { kind } => tree_sink.start_node(kind),
+            parser::Step::Exit => tree_sink.finish_node(),
+            parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
+        }
+    }
     if tree_sink.roots.len() != 1 {
         return Err(ExpandError::ConversionError);
     }
@@ -643,7 +652,7 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str {
     &texts[idx..texts.len() - (1 - idx)]
 }
 
-impl<'a> TreeSink for TtTreeSink<'a> {
+impl<'a> TtTreeSink<'a> {
     fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
         if kind == LIFETIME_IDENT {
             n_tokens = 2;
@@ -741,7 +750,7 @@ impl<'a> TreeSink for TtTreeSink<'a> {
         *self.roots.last_mut().unwrap() -= 1;
     }
 
-    fn error(&mut self, error: ParseError) {
+    fn error(&mut self, error: String) {
         self.inner.error(error, self.text_pos)
     }
 }

crates/mbe/src/to_parser_tokens.rs renamed to crates/mbe/src/to_parser_input.rs

@@ -4,8 +4,8 @@
 use syntax::{SyntaxKind, SyntaxKind::*, T};
 use tt::buffer::TokenBuffer;
 
-pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens {
-    let mut res = parser::Tokens::default();
+pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input {
+    let mut res = parser::Input::default();
 
     let mut current = buffer.begin();
 

crates/mbe/src/tt_iter.rs

@@ -1,11 +1,10 @@
 //! A "Parser" structure for token trees. We use this when parsing a declarative
 //! macro definition into a list of patterns and templates.
 
-use crate::{to_parser_tokens::to_parser_tokens, ExpandError, ExpandResult, ParserEntryPoint};
+use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult, ParserEntryPoint};
 
-use parser::TreeSink;
 use syntax::SyntaxKind;
-use tt::buffer::{Cursor, TokenBuffer};
+use tt::buffer::TokenBuffer;
 
 macro_rules! err {
     () => {
@@ -94,34 +93,28 @@ impl<'a> TtIter<'a> {
         &mut self,
         entry_point: ParserEntryPoint,
     ) -> ExpandResult<Option<tt::TokenTree>> {
-        struct OffsetTokenSink<'a> {
-            cursor: Cursor<'a>,
-            error: bool,
-        }
-
-        impl<'a> TreeSink for OffsetTokenSink<'a> {
-            fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
-                if kind == SyntaxKind::LIFETIME_IDENT {
-                    n_tokens = 2;
-                }
-                for _ in 0..n_tokens {
-                    self.cursor = self.cursor.bump_subtree();
+        let buffer = TokenBuffer::from_tokens(self.inner.as_slice());
+        let parser_input = to_parser_input(&buffer);
+        let tree_traversal = parser::parse(&parser_input, entry_point);
+
+        let mut cursor = buffer.begin();
+        let mut error = false;
+        for step in tree_traversal.iter() {
+            match step {
+                parser::Step::Token { kind, mut n_input_tokens } => {
+                    if kind == SyntaxKind::LIFETIME_IDENT {
+                        n_input_tokens = 2;
+                    }
+                    for _ in 0..n_input_tokens {
+                        cursor = cursor.bump_subtree();
+                    }
                 }
-            }
-            fn start_node(&mut self, _kind: SyntaxKind) {}
-            fn finish_node(&mut self) {}
-            fn error(&mut self, _error: parser::ParseError) {
-                self.error = true;
+                parser::Step::Enter { .. } | parser::Step::Exit => (),
+                parser::Step::Error { .. } => error = true,
             }
         }
 
-        let buffer = TokenBuffer::from_tokens(self.inner.as_slice());
-        let parser_tokens = to_parser_tokens(&buffer);
-        let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
-
-        parser::parse(&parser_tokens, &mut sink, entry_point);
-
-        let mut err = if !sink.cursor.is_root() || sink.error {
+        let mut err = if !cursor.is_root() || error {
             Some(err!("expected {:?}", entry_point))
         } else {
             None
@@ -130,8 +123,8 @@ impl<'a> TtIter<'a> {
         let mut curr = buffer.begin();
         let mut res = vec![];
 
-        if sink.cursor.is_root() {
-            while curr != sink.cursor {
+        if cursor.is_root() {
+            while curr != cursor {
                 if let Some(token) = curr.token_tree() {
                     res.push(token);
                 }

crates/parser/src/event.rs

@@ -10,9 +10,8 @@
 use std::mem;
 
 use crate::{
-    ParseError,
+    output::Output,
     SyntaxKind::{self, *},
-    TreeSink,
 };
 
 /// `Parser` produces a flat list of `Event`s.
@@ -77,7 +76,7 @@ pub(crate) enum Event {
     },
 
     Error {
-        msg: ParseError,
+        msg: String,
     },
 }
 
@@ -88,7 +87,8 @@ impl Event {
 }
 
 /// Generate the syntax tree with the control of events.
-pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec<Event>) {
+pub(super) fn process(mut events: Vec<Event>) -> Output {
+    let mut res = Output::default();
     let mut forward_parents = Vec::new();
 
     for i in 0..events.len() {
@@ -117,15 +117,17 @@ pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec<Event>) {
 
                 for kind in forward_parents.drain(..).rev() {
                     if kind != TOMBSTONE {
-                        sink.start_node(kind);
+                        res.enter_node(kind);
                     }
                 }
             }
-            Event::Finish => sink.finish_node(),
+            Event::Finish => res.leave_node(),
             Event::Token { kind, n_raw_tokens } => {
-                sink.token(kind, n_raw_tokens);
+                res.token(kind, n_raw_tokens);
             }
-            Event::Error { msg } => sink.error(msg),
+            Event::Error { msg } => res.error(msg),
         }
     }
+
+    res
 }

crates/parser/src/tokens.rs renamed to crates/parser/src/input.rs

@@ -1,26 +1,26 @@
-//! Input for the parser -- a sequence of tokens.
-//!
-//! As of now, parser doesn't have access to the *text* of the tokens, and makes
-//! decisions based solely on their classification. Unlike `LexerToken`, the
-//! `Tokens` doesn't include whitespace and comments.
+//! See [`Input`].
 
 use crate::SyntaxKind;
 
 #[allow(non_camel_case_types)]
 type bits = u64;
 
-/// Main input to the parser.
+/// Input for the parser -- a sequence of tokens.
 ///
-/// A sequence of tokens represented internally as a struct of arrays.
+/// As of now, parser doesn't have access to the *text* of the tokens, and makes
+/// decisions based solely on their classification. Unlike `LexerToken`, the
+/// `Tokens` doesn't include whitespace and comments. Main input to the parser.
+///
+/// Struct of arrays internally, but this shouldn't really matter.
 #[derive(Default)]
-pub struct Tokens {
+pub struct Input {
     kind: Vec<SyntaxKind>,
     joint: Vec<bits>,
     contextual_kind: Vec<SyntaxKind>,
 }
 
 /// `pub` impl used by callers to create `Tokens`.
-impl Tokens {
+impl Input {
     #[inline]
     pub fn push(&mut self, kind: SyntaxKind) {
         self.push_impl(kind, SyntaxKind::EOF)
@@ -63,7 +63,7 @@ impl Tokens {
 }
 
 /// pub(crate) impl used by the parser to consume `Tokens`.
-impl Tokens {
+impl Input {
     pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {
         self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
     }
@@ -76,7 +76,7 @@ impl Tokens {
     }
 }
 
-impl Tokens {
+impl Input {
     fn bit_index(&self, n: usize) -> (usize, usize) {
         let idx = n / (bits::BITS as usize);
         let b_idx = n % (bits::BITS as usize);

crates/parser/src/lexed_str.rs

@@ -122,8 +122,8 @@ impl<'a> LexedStr<'a> {
         self.error.iter().map(|it| (it.token as usize, it.msg.as_str()))
     }
 
-    pub fn to_tokens(&self) -> crate::Tokens {
-        let mut res = crate::Tokens::default();
+    pub fn to_input(&self) -> crate::Input {
+        let mut res = crate::Input::default();
         let mut was_joint = false;
         for i in 0..self.len() {
             let kind = self.kind(i);

crates/parser/src/lib.rs

@@ -24,32 +24,20 @@ mod syntax_kind;
 mod event;
 mod parser;
 mod grammar;
-mod tokens;
+mod input;
+mod output;
 
 #[cfg(test)]
 mod tests;
 
 pub(crate) use token_set::TokenSet;
 
-pub use crate::{lexed_str::LexedStr, syntax_kind::SyntaxKind, tokens::Tokens};
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct ParseError(pub Box<String>);
-
-/// `TreeSink` abstracts details of a particular syntax tree implementation.
-pub trait TreeSink {
-    /// Adds new token to the current branch.
-    fn token(&mut self, kind: SyntaxKind, n_tokens: u8);
-
-    /// Start new branch and make it current.
-    fn start_node(&mut self, kind: SyntaxKind);
-
-    /// Finish current branch and restore previous
-    /// branch as current.
-    fn finish_node(&mut self);
-
-    fn error(&mut self, error: ParseError);
-}
+pub use crate::{
+    input::Input,
+    lexed_str::LexedStr,
+    output::{Output, Step},
+    syntax_kind::SyntaxKind,
+};
 
 /// rust-analyzer parser allows you to choose one of the possible entry points.
 ///
@@ -74,11 +62,19 @@ pub enum ParserEntryPoint {
 }
 
 /// Parse given tokens into the given sink as a rust file.
-pub fn parse_source_file(tokens: &Tokens, tree_sink: &mut dyn TreeSink) {
-    parse(tokens, tree_sink, ParserEntryPoint::SourceFile);
+pub fn parse_source_file(inp: &Input) -> Output {
+    parse(inp, ParserEntryPoint::SourceFile)
 }
 
-pub fn parse(tokens: &Tokens, tree_sink: &mut dyn TreeSink, entry_point: ParserEntryPoint) {
+/// Parses the given [`Input`] into [`Output`] assuming that the top-level
+/// syntactic construct is the given [`ParserEntryPoint`].
+///
+/// Both input and output here are fairly abstract. The overall flow is that the
+/// caller has some "real" tokens, converts them to [`Input`], parses them to
+/// [`Output`], and then converts that into a "real" tree. The "real" tree is
+/// made of "real" tokens, so this all hinges on rather tight coordination of
+/// indices between the four stages.
+pub fn parse(inp: &Input, entry_point: ParserEntryPoint) -> Output {
     let entry_point: fn(&'_ mut parser::Parser) = match entry_point {
         ParserEntryPoint::SourceFile => grammar::entry_points::source_file,
         ParserEntryPoint::Path => grammar::entry_points::path,
@@ -96,10 +92,10 @@ pub fn parse(tokens: &Tokens, tree_sink: &mut dyn TreeSink, entry_point: ParserE
         ParserEntryPoint::Attr => grammar::entry_points::attr,
     };
 
-    let mut p = parser::Parser::new(tokens);
+    let mut p = parser::Parser::new(inp);
     entry_point(&mut p);
     let events = p.finish();
-    event::process(tree_sink, events);
+    event::process(events)
 }
 
 /// A parsing function for a specific braced-block.
@@ -119,11 +115,11 @@ impl Reparser {
     ///
     /// Tokens must start with `{`, end with `}` and form a valid brace
     /// sequence.
-    pub fn parse(self, tokens: &Tokens, tree_sink: &mut dyn TreeSink) {
+    pub fn parse(self, tokens: &Input) -> Output {
         let Reparser(r) = self;
         let mut p = parser::Parser::new(tokens);
         r(&mut p);
         let events = p.finish();
-        event::process(tree_sink, events);
+        event::process(events)
     }
 }