Merge #11134

11134: internal: tighten up parser API r=matklad a=matklad

It's tempting to expose things like `Expr::parse`, 
but they'll necessary have somewhat ad-hoc semantics. 

Instead, we narrow down the parser's interface strictly 
to what's needed for MBE. For everything else (eg, parsing
imports), the proper way is enclose the input string into 
some context, parse the whole as a file, and then verify 
that the input was parsed as intended.

bors r+
🤖

Co-authored-by: Aleksey Kladov <[email protected]>

Author: bors[bot]

crates/hir/src/attrs.rs

@@ -11,7 +11,7 @@ use hir_def::{
 };
 use hir_expand::{hygiene::Hygiene, MacroDefId};
 use hir_ty::db::HirDatabase;
-use syntax::ast;
+use syntax::{ast, AstNode};
 
 use crate::{
     Adt, AssocItem, Const, ConstParam, Enum, Field, Function, GenericParam, Impl, LifetimeParam,
@@ -147,8 +147,18 @@ fn resolve_doc_path(
         // FIXME
         AttrDefId::MacroDefId(_) => return None,
     };
-    let path = ast::Path::parse(link).ok()?;
-    let modpath = ModPath::from_src(db.upcast(), path, &Hygiene::new_unhygienic())?;
+
+    let modpath = {
+        let ast_path = ast::SourceFile::parse(&format!("type T = {};", link))
+            .syntax_node()
+            .descendants()
+            .find_map(ast::Path::cast)?;
+        if ast_path.to_string() != link {
+            return None;
+        }
+        ModPath::from_src(db.upcast(), ast_path, &Hygiene::new_unhygienic())?
+    };
+
     let resolved = resolver.resolve_module_path_in_items(db.upcast(), &modpath);
     let resolved = if resolved == PerNs::none() {
         resolver.resolve_module_path_in_trait_assoc_items(db.upcast(), &modpath)?

crates/hir_def/src/attr.rs

@@ -714,8 +714,7 @@ impl Attr {
         hygiene: &Hygiene,
         id: AttrId,
     ) -> Option<Attr> {
-        let (parse, _) =
-            mbe::token_tree_to_syntax_node(tt, mbe::ParserEntryPoint::MetaItem).ok()?;
+        let (parse, _) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem).ok()?;
         let ast = ast::Meta::cast(parse.syntax_node())?;
 
         Self::from_src(db, ast, hygiene, id)

crates/hir_expand/src/builtin_derive_macro.rs

@@ -72,7 +72,7 @@ struct BasicAdtInfo {
 }
 
 fn parse_adt(tt: &tt::Subtree) -> Result<BasicAdtInfo, mbe::ExpandError> {
-    let (parsed, token_map) = mbe::token_tree_to_syntax_node(tt, mbe::ParserEntryPoint::Items)?; // FragmentKind::Items doesn't parse attrs?
+    let (parsed, token_map) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MacroItems)?; // FragmentKind::Items doesn't parse attrs?
     let macro_items = ast::MacroItems::cast(parsed.syntax_node()).ok_or_else(|| {
         debug!("derive node didn't parse");
         mbe::ExpandError::UnexpectedToken

crates/hir_expand/src/db.rs

@@ -497,11 +497,11 @@ fn token_tree_to_syntax_node(
     expand_to: ExpandTo,
 ) -> Result<(Parse<SyntaxNode>, mbe::TokenMap), ExpandError> {
     let entry_point = match expand_to {
-        ExpandTo::Statements => mbe::ParserEntryPoint::Statements,
-        ExpandTo::Items => mbe::ParserEntryPoint::Items,
-        ExpandTo::Pattern => mbe::ParserEntryPoint::Pattern,
-        ExpandTo::Type => mbe::ParserEntryPoint::Type,
-        ExpandTo::Expr => mbe::ParserEntryPoint::Expr,
+        ExpandTo::Statements => mbe::TopEntryPoint::MacroStmts,
+        ExpandTo::Items => mbe::TopEntryPoint::MacroItems,
+        ExpandTo::Pattern => mbe::TopEntryPoint::Pattern,
+        ExpandTo::Type => mbe::TopEntryPoint::Type,
+        ExpandTo::Expr => mbe::TopEntryPoint::Expr,
     };
     mbe::token_tree_to_syntax_node(tt, entry_point)
 }

crates/hir_expand/src/eager.rs

@@ -131,7 +131,7 @@ pub fn expand_eager_macro(
     let arg_file_id = arg_id;
 
     let parsed_args = diagnostic_sink
-        .result(mbe::token_tree_to_syntax_node(&parsed_args, mbe::ParserEntryPoint::Expr))?
+        .result(mbe::token_tree_to_syntax_node(&parsed_args, mbe::TopEntryPoint::Expr))?
         .0;
     let result = eager_macro_recur(
         db,

crates/ide_assists/src/handlers/remove_dbg.rs

@@ -36,9 +36,8 @@ pub(crate) fn remove_dbg(acc: &mut Assists, ctx: &AssistContext) -> Option<()> {
     let input_expressions = input_expressions
         .into_iter()
         .filter_map(|(is_sep, group)| (!is_sep).then(|| group))
-        .map(|mut tokens| ast::Expr::parse(&tokens.join("")))
-        .collect::<Result<Vec<ast::Expr>, _>>()
-        .ok()?;
+        .map(|mut tokens| syntax::hacks::parse_expr_from_str(&tokens.join("")))
+        .collect::<Option<Vec<ast::Expr>>>()?;
 
     let parent = macro_call.syntax().parent()?;
     let (range, text) = match &*input_expressions {

crates/ide_completion/src/completions/attribute.rs

@@ -309,7 +309,7 @@ fn parse_comma_sep_expr(input: ast::TokenTree) -> Option<Vec<ast::Expr>> {
         input_expressions
             .into_iter()
             .filter_map(|(is_sep, group)| (!is_sep).then(|| group))
-            .filter_map(|mut tokens| ast::Expr::parse(&tokens.join("")).ok())
+            .filter_map(|mut tokens| syntax::hacks::parse_expr_from_str(&tokens.join("")))
             .collect::<Vec<ast::Expr>>(),
     )
 }

crates/ide_completion/src/snippet.rs

@@ -212,15 +212,14 @@ fn validate_snippet(
 ) -> Option<(Box<[GreenNode]>, String, Option<Box<str>>)> {
     let mut imports = Vec::with_capacity(requires.len());
     for path in requires.iter() {
-        let path = ast::Path::parse(path).ok()?;
-        let valid_use_path = path.segments().all(|seg| {
-            matches!(seg.kind(), Some(ast::PathSegmentKind::Name(_)))
-                || seg.generic_arg_list().is_none()
-        });
-        if !valid_use_path {
+        let use_path = ast::SourceFile::parse(&format!("use {};", path))
+            .syntax_node()
+            .descendants()
+            .find_map(ast::Path::cast)?;
+        if use_path.syntax().text() != path.as_str() {
             return None;
         }
-        let green = path.syntax().green().into_owned();
+        let green = use_path.syntax().green().into_owned();
         imports.push(green);
     }
     let snippet = snippet.iter().join("\n");

crates/ide_db/src/helpers.rs

@@ -67,7 +67,11 @@ pub fn get_path_at_cursor_in_tt(cursor: &ast::Ident) -> Option<ast::Path> {
         .filter_map(SyntaxElement::into_token)
         .take_while(|tok| tok != cursor);
 
-    ast::Path::parse(&path_tokens.chain(iter::once(cursor.clone())).join("")).ok()
+    syntax::hacks::parse_expr_from_str(&path_tokens.chain(iter::once(cursor.clone())).join(""))
+        .and_then(|expr| match expr {
+            ast::Expr::PathExpr(it) => it.path(),
+            _ => None,
+        })
 }
 
 /// Parses and resolves the path at the cursor position in the given attribute, if it is a derive.
@@ -323,7 +327,12 @@ pub fn parse_tt_as_comma_sep_paths(input: ast::TokenTree) -> Option<Vec<ast::Pat
     let paths = input_expressions
         .into_iter()
         .filter_map(|(is_sep, group)| (!is_sep).then(|| group))
-        .filter_map(|mut tokens| ast::Path::parse(&tokens.join("")).ok())
+        .filter_map(|mut tokens| {
+            syntax::hacks::parse_expr_from_str(&tokens.join("")).and_then(|expr| match expr {
+                ast::Expr::PathExpr(it) => it.path(),
+                _ => None,
+            })
+        })
         .collect();
     Some(paths)
 }

crates/ide_ssr/src/fragments.rs

@@ -0,0 +1,58 @@
+//! When specifying SSR rule, you generally want to map one *kind* of thing to
+//! the same kind of thing: path to path, expression to expression, type to
+//! type.
+//!
+//! The problem is, while this *kind* is generally obvious to the human, the ide
+//! needs to determine it somehow. We do this in a stupid way -- by pasting SSR
+//! rule into different contexts and checking what works.
+
+use syntax::{ast, AstNode, SyntaxNode};
+
+pub(crate) fn ty(s: &str) -> Result<SyntaxNode, ()> {
+    fragment::<ast::Type>("type T = {};", s)
+}
+
+pub(crate) fn item(s: &str) -> Result<SyntaxNode, ()> {
+    fragment::<ast::Item>("{}", s)
+}
+
+pub(crate) fn pat(s: &str) -> Result<SyntaxNode, ()> {
+    fragment::<ast::Pat>("const _: () = {let {} = ();};", s)
+}
+
+pub(crate) fn expr(s: &str) -> Result<SyntaxNode, ()> {
+    fragment::<ast::Expr>("const _: () = {};", s)
+}
+
+pub(crate) fn stmt(s: &str) -> Result<SyntaxNode, ()> {
+    let template = "const _: () = { {}; };";
+    let input = template.replace("{}", s);
+    let parse = syntax::SourceFile::parse(&input);
+    if !parse.errors().is_empty() {
+        return Err(());
+    }
+    let mut node =
+        parse.tree().syntax().descendants().skip(2).find_map(ast::Stmt::cast).ok_or(())?;
+    if !s.ends_with(';') && node.to_string().ends_with(';') {
+        node = node.clone_for_update();
+        node.syntax().last_token().map(|it| it.detach());
+    }
+    if node.to_string() != s {
+        return Err(());
+    }
+    Ok(node.syntax().clone_subtree())
+}
+
+fn fragment<T: AstNode>(template: &str, s: &str) -> Result<SyntaxNode, ()> {
+    let s = s.trim();
+    let input = template.replace("{}", s);
+    let parse = syntax::SourceFile::parse(&input);
+    if !parse.errors().is_empty() {
+        return Err(());
+    }
+    let node = parse.tree().syntax().descendants().find_map(T::cast).ok_or(())?;
+    if node.syntax().text() != s {
+        return Err(());
+    }
+    Ok(node.syntax().clone_subtree())
+}