Attach TokenStream to ast::Path

Aaron1011 · Aaron1011 · commit 55082ce41334 · 2020-09-10T17:33:06.000-04:00
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
@@ -96,6 +96,7 @@ pub struct Path {
     /// The segments in the path: the things separated by `::`.
     /// Global paths begin with `kw::PathRoot`.
     pub segments: Vec<PathSegment>,
+    pub tokens: Option<TokenStream>,
 }
 
 impl PartialEq<Symbol> for Path {
@@ -117,7 +118,7 @@ impl Path {
     // Convert a span and an identifier to the corresponding
     // one-segment path.
     pub fn from_ident(ident: Ident) -> Path {
-        Path { segments: vec![PathSegment::from_ident(ident)], span: ident.span }
+        Path { segments: vec![PathSegment::from_ident(ident)], span: ident.span, tokens: None }
     }
 
     pub fn is_global(&self) -> bool {
@@ -1069,7 +1070,7 @@ pub struct Expr {
 
 // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
-rustc_data_structures::static_assert_size!(Expr, 104);
+rustc_data_structures::static_assert_size!(Expr, 112);
 
 impl Expr {
     /// Returns `true` if this expression would be valid somewhere that expects a value;
diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs
@@ -415,7 +415,7 @@ impl MetaItem {
                     }
                 }
                 let span = span.with_hi(segments.last().unwrap().ident.span.hi());
-                Path { span, segments }
+                Path { span, segments, tokens: None }
             }
             Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
                 token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
@@ -513,7 +513,7 @@ pub fn noop_visit_ident<T: MutVisitor>(Ident { name: _, span }: &mut Ident, vis:
     vis.visit_span(span);
 }
 
-pub fn noop_visit_path<T: MutVisitor>(Path { segments, span }: &mut Path, vis: &mut T) {
+pub fn noop_visit_path<T: MutVisitor>(Path { segments, span, tokens: _ }: &mut Path, vis: &mut T) {
     vis.visit_span(span);
     for PathSegment { ident, id, args } in segments {
         vis.visit_ident(ident);
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
@@ -251,7 +251,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             ItemKind::ExternCrate(orig_name) => hir::ItemKind::ExternCrate(orig_name),
             ItemKind::Use(ref use_tree) => {
                 // Start with an empty prefix.
-                let prefix = Path { segments: vec![], span: use_tree.span };
+                let prefix = Path { segments: vec![], span: use_tree.span, tokens: None };
 
                 self.lower_use_tree(use_tree, &prefix, id, vis, ident, attrs)
             }
@@ -488,7 +488,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 *ident = tree.ident();
 
                 // First, apply the prefix to the path.
-                let mut path = Path { segments, span: path.span };
+                let mut path = Path { segments, span: path.span, tokens: None };
 
                 // Correctly resolve `self` imports.
                 if path.segments.len() > 1
@@ -540,8 +540,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 hir::ItemKind::Use(path, hir::UseKind::Single)
             }
             UseTreeKind::Glob => {
-                let path =
-                    self.lower_path(id, &Path { segments, span: path.span }, ParamMode::Explicit);
+                let path = self.lower_path(
+                    id,
+                    &Path { segments, span: path.span, tokens: None },
+                    ParamMode::Explicit,
+                );
                 hir::ItemKind::Use(path, hir::UseKind::Glob)
             }
             UseTreeKind::Nested(ref trees) => {
@@ -569,7 +572,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 // for that we return the `{}` import (called the
                 // `ListStem`).
 
-                let prefix = Path { segments, span: prefix.span.to(path.span) };
+                let prefix = Path { segments, span: prefix.span.to(path.span), tokens: None };
 
                 // Add all the nested `PathListItem`s to the HIR.
                 for &(ref use_tree, id) in trees {
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
@@ -46,7 +46,7 @@ impl<'a> ExtCtxt<'a> {
             id: ast::DUMMY_NODE_ID,
             args,
         });
-        ast::Path { span, segments }
+        ast::Path { span, segments, tokens: None }
     }
 
     pub fn ty_mt(&self, ty: P<ast::Ty>, mutbl: ast::Mutability) -> ast::MutTy {
diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs
@@ -18,7 +18,7 @@ pub fn placeholder(
 ) -> AstFragment {
     fn mac_placeholder() -> ast::MacCall {
         ast::MacCall {
-            path: ast::Path { span: DUMMY_SP, segments: Vec::new() },
+            path: ast::Path { span: DUMMY_SP, segments: Vec::new(), tokens: None },
             args: P(ast::MacArgs::Empty),
             prior_type_ascription: None,
         }
diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs
@@ -278,6 +278,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke
             Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into())
         }
         Nonterminal::NtMeta(ref attr) => attr.tokens.clone(),
+        Nonterminal::NtPath(ref path) => path.tokens.clone(),
         Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),
         Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => {
             if expr.tokens.is_none() {
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -901,7 +901,7 @@ impl<'a> Parser<'a> {
     ) -> PResult<'a, P<T>> {
         self.expect(&token::ModSep)?;
 
-        let mut path = ast::Path { segments: Vec::new(), span: DUMMY_SP };
+        let mut path = ast::Path { segments: Vec::new(), span: DUMMY_SP, tokens: None };
         self.parse_path_segments(&mut path.segments, T::PATH_STYLE)?;
         path.span = ty_span.to(self.prev_token.span);
 
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
@@ -787,7 +787,7 @@ impl<'a> Parser<'a> {
     fn parse_use_tree(&mut self) -> PResult<'a, UseTree> {
         let lo = self.token.span;
 
-        let mut prefix = ast::Path { segments: Vec::new(), span: lo.shrink_to_lo() };
+        let mut prefix = ast::Path { segments: Vec::new(), span: lo.shrink_to_lo(), tokens: None };
         let kind = if self.check(&token::OpenDelim(token::Brace))
             || self.check(&token::BinOp(token::Star))
             || self.is_import_coupler()
diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs
@@ -168,7 +168,15 @@ impl<'a> Parser<'a> {
                     return Err(self.struct_span_err(self.token.span, msg));
                 }
             }
-            NonterminalKind::Path => token::NtPath(self.parse_path(PathStyle::Type)?),
+            NonterminalKind::Path => {
+                let (mut path, tokens) =
+                    self.collect_tokens(|this| this.parse_path(PathStyle::Type))?;
+                // We have have eaten an NtPath, which could already have tokens
+                if path.tokens.is_none() {
+                    path.tokens = Some(tokens);
+                }
+                token::NtPath(path)
+            }
             NonterminalKind::Meta => {
                 let (mut attr, tokens) = self.collect_tokens(|this| this.parse_attr_item())?;
                 // We may have eaten a nonterminal, which could already have tokens
diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs
@@ -64,7 +64,7 @@ impl<'a> Parser<'a> {
             path_span = path_lo.to(self.prev_token.span);
         } else {
             path_span = self.token.span.to(self.token.span);
-            path = ast::Path { segments: Vec::new(), span: path_span };
+            path = ast::Path { segments: Vec::new(), span: path_span, tokens: None };
         }
 
         // See doc comment for `unmatched_angle_bracket_count`.
@@ -81,7 +81,10 @@ impl<'a> Parser<'a> {
         let qself = QSelf { ty, path_span, position: path.segments.len() };
         self.parse_path_segments(&mut path.segments, style)?;
 
-        Ok((qself, Path { segments: path.segments, span: lo.to(self.prev_token.span) }))
+        Ok((
+            qself,
+            Path { segments: path.segments, span: lo.to(self.prev_token.span), tokens: None },
+        ))
     }
 
     /// Recover from an invalid single colon, when the user likely meant a qualified path.
@@ -144,7 +147,7 @@ impl<'a> Parser<'a> {
         }
         self.parse_path_segments(&mut segments, style)?;
 
-        Ok(Path { segments, span: lo.to(self.prev_token.span) })
+        Ok(Path { segments, span: lo.to(self.prev_token.span), tokens: None })
     }
 
     pub(super) fn parse_path_segments(
diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs
@@ -794,7 +794,7 @@ impl<'a> Resolver<'a> {
                         }
 
                         segms.push(ast::PathSegment::from_ident(ident));
-                        let path = Path { span: name_binding.span, segments: segms };
+                        let path = Path { span: name_binding.span, segments: segms, tokens: None };
                         let did = match res {
                             Res::Def(DefKind::Ctor(..), did) => this.parent(did),
                             _ => res.opt_def_id(),
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
@@ -1967,7 +1967,7 @@ impl<'a, 'b, 'ast> LateResolutionVisitor<'a, 'b, 'ast> {
 
         if qself.is_none() {
             let path_seg = |seg: &Segment| PathSegment::from_ident(seg.ident);
-            let path = Path { segments: path.iter().map(path_seg).collect(), span };
+            let path = Path { segments: path.iter().map(path_seg).collect(), span, tokens: None };
             if let Ok((_, res)) =
                 self.r.resolve_macro_path(&path, None, &self.parent_scope, false, false)
             {
diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs
@@ -83,6 +83,7 @@ fn import_candidate_to_enum_paths(suggestion: &ImportSuggestion) -> (String, Str
     let enum_path = ast::Path {
         span: suggestion.path.span,
         segments: suggestion.path.segments[0..path_len - 1].to_vec(),
+        tokens: None,
     };
     let enum_path_string = path_names_to_string(&enum_path);
 
@@ -1065,7 +1066,8 @@ impl<'a> LateResolutionVisitor<'a, '_, '_> {
                     path_segments.push(ast::PathSegment::from_ident(ident));
                     let module_def_id = module.def_id().unwrap();
                     if module_def_id == def_id {
-                        let path = Path { span: name_binding.span, segments: path_segments };
+                        let path =
+                            Path { span: name_binding.span, segments: path_segments, tokens: None };
                         result = Some((
                             module,
                             ImportSuggestion {
@@ -1095,7 +1097,7 @@ impl<'a> LateResolutionVisitor<'a, '_, '_> {
                 if let Res::Def(DefKind::Variant, _) = name_binding.res() {
                     let mut segms = enum_import_suggestion.path.segments.clone();
                     segms.push(ast::PathSegment::from_ident(ident));
-                    variants.push(Path { span: name_binding.span, segments: segms });
+                    variants.push(Path { span: name_binding.span, segments: segms, tokens: None });
                 }
             });
             variants
diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs
@@ -3189,6 +3189,7 @@ impl<'a> Resolver<'a> {
                     .chain(path_str.split("::").skip(1).map(Ident::from_str))
                     .map(|i| self.new_ast_path_segment(i))
                     .collect(),
+                tokens: None,
             }
         } else {
             ast::Path {
@@ -3198,6 +3199,7 @@ impl<'a> Resolver<'a> {
                     .map(Ident::from_str)
                     .map(|i| self.new_ast_path_segment(i))
                     .collect(),
+                tokens: None,
             }
         };
         let module = self.get_module(module_id);

Original file line number	Diff line number	Diff line change
`@@ -415,7 +415,7 @@ impl MetaItem {`
`415`	`415`	`}`
`416`	`416`	`}`
`417`	`417`	`let span = span.with_hi(segments.last().unwrap().ident.span.hi());`
`418`		`- Path { span, segments }`
	`418`	`+ Path { span, segments, tokens: None }`
`419`	`419`	`}`
`420`	`420`	`Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {`
`421`	`421`	`token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),`
Original file line number	Diff line number	Diff line change
`@@ -513,7 +513,7 @@ pub fn noop_visit_ident<T: MutVisitor>(Ident { name: _, span }: &mut Ident, vis:`
`513`	`513`	`vis.visit_span(span);`
`514`	`514`	`}`
`515`	`515`
`516`		`-pub fn noop_visit_path<T: MutVisitor>(Path { segments, span }: &mut Path, vis: &mut T) {`
	`516`	`+pub fn noop_visit_path<T: MutVisitor>(Path { segments, span, tokens: _ }: &mut Path, vis: &mut T) {`
`517`	`517`	`vis.visit_span(span);`
`518`	`518`	`for PathSegment { ident, id, args } in segments {`
`519`	`519`	`vis.visit_ident(ident);`
Original file line number	Diff line number	Diff line change
`@@ -46,7 +46,7 @@ impl<'a> ExtCtxt<'a> {`
`46`	`46`	`id: ast::DUMMY_NODE_ID,`
`47`	`47`	`args,`
`48`	`48`	`});`
`49`		`- ast::Path { span, segments }`
	`49`	`+ ast::Path { span, segments, tokens: None }`
`50`	`50`	`}`
`51`	`51`
`52`	`52`	`pub fn ty_mt(&self, ty: P<ast::Ty>, mutbl: ast::Mutability) -> ast::MutTy {`
Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@ pub fn placeholder(`
`18`	`18`	`) -> AstFragment {`
`19`	`19`	`fn mac_placeholder() -> ast::MacCall {`
`20`	`20`	`ast::MacCall {`
`21`		`- path: ast::Path { span: DUMMY_SP, segments: Vec::new() },`
	`21`	`+ path: ast::Path { span: DUMMY_SP, segments: Vec::new(), tokens: None },`
`22`	`22`	`args: P(ast::MacArgs::Empty),`
`23`	`23`	`prior_type_ascription: None,`
`24`	`24`	`}`
Original file line number	Diff line number	Diff line change
`@@ -278,6 +278,7 @@ pub fn nt_to_tokenstream(nt: &Nonterminal, sess: &ParseSess, span: Span) -> Toke`
`278`	`278`	`Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into())`
`279`	`279`	`}`
`280`	`280`	`Nonterminal::NtMeta(ref attr) => attr.tokens.clone(),`
	`281`	`+ Nonterminal::NtPath(ref path) => path.tokens.clone(),`
`281`	`282`	`Nonterminal::NtTT(ref tt) => Some(tt.clone().into()),`
`282`	`283`	`Nonterminal::NtExpr(ref expr) \| Nonterminal::NtLiteral(ref expr) => {`
`283`	`284`	`if expr.tokens.is_none() {`