Address review comments.

Author: jseyfried

src/libproc_macro/lib.rs

@@ -9,13 +9,17 @@
 // except according to those terms.
 
 //! # Quasiquoter
-//! This file contains the implementation internals of the quasiquoter provided by `qquote!`.
+//! This file contains the implementation internals of the quasiquoter provided by `quote!`.
+
+//! This quasiquoter uses macros 2.0 hygiene to reliably use items from `__rt`,
+//! including re-exported API `libsyntax`, to build a `syntax::tokenstream::TokenStream`
+//! and wrap it into a `proc_macro::TokenStream`.
 
 use syntax::ast::Ident;
 use syntax::ext::base::{ExtCtxt, ProcMacro};
 use syntax::parse::token::{self, Token, Lit};
 use syntax::symbol::Symbol;
-use syntax::tokenstream::{Delimited, TokenTree, TokenStream};
+use syntax::tokenstream::{Delimited, TokenTree, TokenStream, TokenStreamBuilder};
 use syntax_pos::{DUMMY_SP, Span};
 use syntax_pos::hygiene::SyntaxContext;
 
@@ -25,7 +29,7 @@ pub mod __rt {
     pub use syntax::ast::Ident;
     pub use syntax::parse::token;
     pub use syntax::symbol::Symbol;
-    pub use syntax::tokenstream::{TokenStream, TokenTree, Delimited};
+    pub use syntax::tokenstream::{TokenStream, TokenStreamBuilder, TokenTree, Delimited};
     pub use super::{ctxt, span};
 
     pub fn unquote<T: Into<::TokenStream> + Clone>(tokens: &T) -> TokenStream {
@@ -41,7 +45,7 @@ pub fn span() -> Span {
     ::Span::default().0
 }
 
-trait Quote {
+pub trait Quote {
     fn quote(&self) -> TokenStream;
 }
 
@@ -98,8 +102,8 @@ impl<T: Quote> Quote for Option<T> {
 
 impl Quote for TokenStream {
     fn quote(&self) -> TokenStream {
-        let mut builder = TokenStream::builder();
-        builder.push(quote!(rt::TokenStream::builder()));
+        let mut builder = TokenStreamBuilder::new();
+        builder.push(quote!(rt::TokenStreamBuilder::new()));
 
         let mut trees = self.trees();
         loop {

src/libproc_macro/quote.rs

@@ -14,7 +14,7 @@ use cstore::{self, CStore, CrateSource, MetadataBlob};
 use locator::{self, CratePaths};
 use schema::{CrateRoot, Tracked};
 
-use rustc::hir::def_id::{CrateNum, DefIndex};
+use rustc::hir::def_id::{CrateNum, DefIndex, CRATE_DEF_INDEX};
 use rustc::hir::svh::Svh;
 use rustc::middle::cstore::DepKind;
 use rustc::session::Session;
@@ -35,7 +35,7 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::{cmp, fs};
 
-use syntax::ast::{self, Ident};
+use syntax::ast;
 use syntax::abi::Abi;
 use syntax::attr;
 use syntax::ext::base::SyntaxExtension;
@@ -1238,7 +1238,7 @@ fn proc_macro_def_path_table(proc_macros: &[(ast::Name, Rc<SyntaxExtension>)]) -
         let key = DefKey {
             parent: Some(CRATE_DEF_INDEX),
             disambiguated_data: DisambiguatedDefPathData {
-                data: DefPathData::MacroDef(Ident::with_empty_ctxt(proc_macro.0)),
+                data: DefPathData::MacroDef(proc_macro.0),
                 disambiguator: 0,
             },
         };

src/librustc_metadata/creader.rs

@@ -483,7 +483,7 @@ impl<'a> StringReader<'a> {
         self.with_str_from(start, |string| {
             if string == "_" {
                 self.sess.span_diagnostic
-                    .struct_span_warn(mk_sp(start, self.pos),
+                    .struct_span_warn(self.mk_sp(start, self.pos),
                                       "underscore literal suffix is not allowed")
                     .warn("this was previously accepted by the compiler but is \
                           being phased out; it will become a hard error in \

src/libsyntax/parse/lexer/mod.rs

@@ -20,8 +20,8 @@ use serialize::{Decodable, Decoder, Encodable, Encoder};
 use symbol::keywords;
 use tokenstream::{TokenStream, TokenTree};
 
-use std::cell::RefCell;
-use std::fmt;
+use std::cell::Cell;
+use std::{cmp, fmt};
 use std::rc::Rc;
 
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
@@ -169,7 +169,8 @@ pub enum Token {
     Underscore,
     Lifetime(ast::Ident),
 
-    /* For interpolation */
+    // The `LazyTokenStream` is a pure function of the `Nonterminal`,
+    // and so the `LazyTokenStream` can be ignored by Eq, Hash, etc.
     Interpolated(Rc<(Nonterminal, LazyTokenStream)>),
     // Can be expanded into several tokens.
     /// Doc comment
@@ -468,19 +469,40 @@ pub fn is_op(tok: &Token) -> bool {
     }
 }
 
-#[derive(Clone, Eq, PartialEq, Debug)]
-pub struct LazyTokenStream(RefCell<Option<TokenStream>>);
+pub struct LazyTokenStream(Cell<Option<TokenStream>>);
+
+impl Clone for LazyTokenStream {
+    fn clone(&self) -> Self {
+        let opt_stream = self.0.take();
+        self.0.set(opt_stream.clone());
+        LazyTokenStream(Cell::new(opt_stream))
+    }
+}
+
+impl cmp::Eq for LazyTokenStream {}
+impl PartialEq for LazyTokenStream {
+    fn eq(&self, _other: &LazyTokenStream) -> bool {
+        true
+    }
+}
+
+impl fmt::Debug for LazyTokenStream {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        fmt::Debug::fmt(&self.clone().0.into_inner(), f)
+    }
+}
 
 impl LazyTokenStream {
     pub fn new() -> Self {
-        LazyTokenStream(RefCell::new(None))
+        LazyTokenStream(Cell::new(None))
     }
 
     pub fn force<F: FnOnce() -> TokenStream>(&self, f: F) -> TokenStream {
-        let mut opt_stream = self.0.borrow_mut();
+        let mut opt_stream = self.0.take();
         if opt_stream.is_none() {
-            *opt_stream = Some(f());
-        };
+            opt_stream = Some(f());
+        }
+        self.0.set(opt_stream.clone());
         opt_stream.clone().unwrap()
     }
 }
@@ -498,7 +520,5 @@ impl Decodable for LazyTokenStream {
 }
 
 impl ::std::hash::Hash for LazyTokenStream {
-    fn hash<H: ::std::hash::Hasher>(&self, hasher: &mut H) {
-        self.0.borrow().hash(hasher);
-    }
+    fn hash<H: ::std::hash::Hasher>(&self, _hasher: &mut H) {}
 }

src/libsyntax/parse/token.rs

@@ -201,10 +201,6 @@ impl TokenStream {
         }
     }
 
-    pub fn builder() -> TokenStreamBuilder {
-        TokenStreamBuilder(Vec::new())
-    }
-
     pub fn concat(mut streams: Vec<TokenStream>) -> TokenStream {
         match streams.len() {
             0 => TokenStream::empty(),
@@ -235,6 +231,8 @@ impl TokenStream {
         true
     }
 
+    /// Precondition: `self` consists of a single token tree.
+    /// Returns true if the token tree is a joint operation w.r.t. `proc_macro::TokenNode`.
     pub fn as_tree(self) -> (TokenTree, bool /* joint? */) {
         match self.kind {
             TokenStreamKind::Tree(tree) => (tree, false),
@@ -277,6 +275,10 @@ impl TokenStream {
 pub struct TokenStreamBuilder(Vec<TokenStream>);
 
 impl TokenStreamBuilder {
+    pub fn new() -> TokenStreamBuilder {
+        TokenStreamBuilder(Vec::new())
+    }
+
     pub fn push<T: Into<TokenStream>>(&mut self, stream: T) {
         let stream = stream.into();
         let last_tree_if_joint = self.0.last().and_then(TokenStream::last_tree_if_joint);

src/libsyntax/tokenstream.rs

@@ -15,15 +15,15 @@
 
 extern crate proc_macro;
 
-use proc_macro::{TokenStream, TokenKind, quote};
+use proc_macro::{TokenStream, TokenNode, quote};
 
 #[proc_macro]
 pub fn cond(input: TokenStream) -> TokenStream {
     let mut conds = Vec::new();
     let mut input = input.into_iter().peekable();
     while let Some(tree) = input.next() {
         let cond = match tree.kind {
-            TokenKind::Sequence(_, cond) => cond,
+            TokenNode::Sequence(_, cond) => cond,
             _ => panic!("Invalid input"),
         };
         let mut cond_trees = cond.clone().into_iter();
@@ -33,7 +33,7 @@ pub fn cond(input: TokenStream) -> TokenStream {
             panic!("Invalid macro usage in cond: {}", cond);
         }
         let is_else = match test.kind {
-            TokenKind::Word(word) => *word == *"else",
+            TokenNode::Word(word) => word.as_str() == "else",
             _ => false,
         };
         conds.push(if is_else || input.peek().is_none() {

src/test/run-pass-fulldeps/auxiliary/cond_plugin.rs

@@ -15,20 +15,20 @@
 
 extern crate proc_macro;
 
-use proc_macro::{TokenStream, TokenKind, OpKind, Literal, quote};
+use proc_macro::{TokenStream, TokenNode, OpKind, Literal, quote};
 
 #[proc_macro]
 pub fn count_compound_ops(input: TokenStream) -> TokenStream {
     assert_eq!(count_compound_ops_helper(quote!(++ (&&) 4@a)), 3);
-    TokenKind::Literal(Literal::u32(count_compound_ops_helper(input))).into()
+    TokenNode::Literal(Literal::u32(count_compound_ops_helper(input))).into()
 }
 
 fn count_compound_ops_helper(input: TokenStream) -> u32 {
     let mut count = 0;
     for token in input {
         match token.kind {
-            TokenKind::Op(c, OpKind::Alone) => count += 1,
-            TokenKind::Sequence(_, tokens) => count += count_compound_ops_helper(tokens),
+            TokenNode::Op(c, OpKind::Alone) => count += 1,
+            TokenNode::Sequence(_, tokens) => count += count_compound_ops_helper(tokens),
             _ => {}
         }
     }