Skip to content

Commit 29c19f4

Browse files
alexcrichtonalexcrichton
authored
Merge pull request #199 from dtolnay/extend
Defer calls to proc_macro::TokenStream::extend
2 parents 74f2200 + b07092c commit 29c19f4

File tree

1 file changed

+61
-18
lines changed

1 file changed

+61
-18
lines changed

‎src/wrapper.rs‎

Lines changed: 61 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,20 @@ use crate::{fallback, Delimiter, Punct, Spacing, TokenTree};
1010

1111
#[derive(Clone)]
1212
pub enum TokenStream {
13-
Compiler(proc_macro::TokenStream),
13+
Compiler(DeferredTokenStream),
1414
Fallback(fallback::TokenStream),
1515
}
1616

17+
// Work around https://github.com/rust-lang/rust/issues/65080.
18+
// In `impl Extend<TokenTree> for TokenStream` which is used heavily by quote,
19+
// we hold on to the appended tokens and do proc_macro::TokenStream::extend as
20+
// late as possible to batch together consecutive uses of the Extend impl.
21+
#[derive(Clone)]
22+
pub struct DeferredTokenStream {
23+
stream: proc_macro::TokenStream,
24+
extra: Vec<proc_macro::TokenTree>,
25+
}
26+
1727
pub enum LexError {
1828
Compiler(proc_macro::LexError),
1929
Fallback(fallback::LexError),
@@ -80,10 +90,32 @@ fn mismatch() -> ! {
8090
panic!("stable/nightly mismatch")
8191
}
8292

93+
impl DeferredTokenStream {
94+
fn new(stream: proc_macro::TokenStream) -> Self {
95+
DeferredTokenStream {
96+
stream,
97+
extra: Vec::new(),
98+
}
99+
}
100+
101+
fn is_empty(&self) -> bool {
102+
self.stream.is_empty() && self.extra.is_empty()
103+
}
104+
105+
fn evaluate_now(&mut self) {
106+
self.stream.extend(self.extra.drain(..));
107+
}
108+
109+
fn into_token_stream(mut self) -> proc_macro::TokenStream {
110+
self.evaluate_now();
111+
self.stream
112+
}
113+
}
114+
83115
impl TokenStream {
84116
pub fn new() -> TokenStream {
85117
if nightly_works() {
86-
TokenStream::Compiler(proc_macro::TokenStream::new())
118+
TokenStream::Compiler(DeferredTokenStream::new(proc_macro::TokenStream::new()))
87119
} else {
88120
TokenStream::Fallback(fallback::TokenStream::new())
89121
}
@@ -98,7 +130,7 @@ impl TokenStream {
98130

99131
fn unwrap_nightly(self) -> proc_macro::TokenStream {
100132
match self {
101-
TokenStream::Compiler(s) => s,
133+
TokenStream::Compiler(s) => s.into_token_stream(),
102134
TokenStream::Fallback(_) => mismatch(),
103135
}
104136
}
@@ -116,7 +148,9 @@ impl FromStr for TokenStream {
116148

117149
fn from_str(src: &str) -> Result<TokenStream, LexError> {
118150
if nightly_works() {
119-
Ok(TokenStream::Compiler(src.parse()?))
151+
Ok(TokenStream::Compiler(DeferredTokenStream::new(
152+
src.parse()?,
153+
)))
120154
} else {
121155
Ok(TokenStream::Fallback(src.parse()?))
122156
}
@@ -126,22 +160,22 @@ impl FromStr for TokenStream {
126160
impl fmt::Display for TokenStream {
127161
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
128162
match self {
129-
TokenStream::Compiler(tts) => tts.fmt(f),
163+
TokenStream::Compiler(tts) => tts.clone().into_token_stream().fmt(f),
130164
TokenStream::Fallback(tts) => tts.fmt(f),
131165
}
132166
}
133167
}
134168

135169
impl From<proc_macro::TokenStream> for TokenStream {
136170
fn from(inner: proc_macro::TokenStream) -> TokenStream {
137-
TokenStream::Compiler(inner)
171+
TokenStream::Compiler(DeferredTokenStream::new(inner))
138172
}
139173
}
140174

141175
impl From<TokenStream> for proc_macro::TokenStream {
142176
fn from(inner: TokenStream) -> proc_macro::TokenStream {
143177
match inner {
144-
TokenStream::Compiler(inner) => inner,
178+
TokenStream::Compiler(inner) => inner.into_token_stream(),
145179
TokenStream::Fallback(inner) => inner.to_string().parse().unwrap(),
146180
}
147181
}
@@ -174,7 +208,7 @@ fn into_compiler_token(token: TokenTree) -> proc_macro::TokenTree {
174208
impl From<TokenTree> for TokenStream {
175209
fn from(token: TokenTree) -> TokenStream {
176210
if nightly_works() {
177-
TokenStream::Compiler(into_compiler_token(token).into())
211+
TokenStream::Compiler(DeferredTokenStream::new(into_compiler_token(token).into()))
178212
} else {
179213
TokenStream::Fallback(token.into())
180214
}
@@ -184,7 +218,9 @@ impl From<TokenTree> for TokenStream {
184218
impl iter::FromIterator<TokenTree> for TokenStream {
185219
fn from_iter<I: IntoIterator<Item = TokenTree>>(trees: I) -> Self {
186220
if nightly_works() {
187-
TokenStream::Compiler(trees.into_iter().map(into_compiler_token).collect())
221+
TokenStream::Compiler(DeferredTokenStream::new(
222+
trees.into_iter().map(into_compiler_token).collect(),
223+
))
188224
} else {
189225
TokenStream::Fallback(trees.into_iter().collect())
190226
}
@@ -196,8 +232,9 @@ impl iter::FromIterator<TokenStream> for TokenStream {
196232
let mut streams = streams.into_iter();
197233
match streams.next() {
198234
Some(TokenStream::Compiler(mut first)) => {
199-
first.extend(streams.map(|s| match s {
200-
TokenStream::Compiler(s) => s,
235+
first.evaluate_now();
236+
first.stream.extend(streams.map(|s| match s {
237+
TokenStream::Compiler(s) => s.into_token_stream(),
201238
TokenStream::Fallback(_) => mismatch(),
202239
}));
203240
TokenStream::Compiler(first)
@@ -218,7 +255,9 @@ impl Extend<TokenTree> for TokenStream {
218255
fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
219256
match self {
220257
TokenStream::Compiler(tts) => {
221-
tts.extend(streams.into_iter().map(into_compiler_token));
258+
// Here is the reason for DeferredTokenStream.
259+
tts.extra
260+
.extend(streams.into_iter().map(into_compiler_token));
222261
}
223262
TokenStream::Fallback(tts) => tts.extend(streams),
224263
}
@@ -229,7 +268,9 @@ impl Extend<TokenStream> for TokenStream {
229268
fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
230269
match self {
231270
TokenStream::Compiler(tts) => {
232-
tts.extend(streams.into_iter().map(|stream| stream.unwrap_nightly()));
271+
tts.evaluate_now();
272+
tts.stream
273+
.extend(streams.into_iter().map(|stream| stream.unwrap_nightly()));
233274
}
234275
TokenStream::Fallback(tts) => {
235276
tts.extend(streams.into_iter().map(|stream| stream.unwrap_stable()));
@@ -241,7 +282,7 @@ impl Extend<TokenStream> for TokenStream {
241282
impl fmt::Debug for TokenStream {
242283
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
243284
match self {
244-
TokenStream::Compiler(tts) => tts.fmt(f),
285+
TokenStream::Compiler(tts) => tts.clone().into_token_stream().fmt(f),
245286
TokenStream::Fallback(tts) => tts.fmt(f),
246287
}
247288
}
@@ -280,7 +321,9 @@ impl IntoIterator for TokenStream {
280321

281322
fn into_iter(self) -> TokenTreeIter {
282323
match self {
283-
TokenStream::Compiler(tts) => TokenTreeIter::Compiler(tts.into_iter()),
324+
TokenStream::Compiler(tts) => {
325+
TokenTreeIter::Compiler(tts.into_token_stream().into_iter())
326+
}
284327
TokenStream::Fallback(tts) => TokenTreeIter::Fallback(tts.into_iter()),
285328
}
286329
}
@@ -526,14 +569,14 @@ pub enum Group {
526569
impl Group {
527570
pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
528571
match stream {
529-
TokenStream::Compiler(stream) => {
572+
TokenStream::Compiler(tts) => {
530573
let delimiter = match delimiter {
531574
Delimiter::Parenthesis => proc_macro::Delimiter::Parenthesis,
532575
Delimiter::Bracket => proc_macro::Delimiter::Bracket,
533576
Delimiter::Brace => proc_macro::Delimiter::Brace,
534577
Delimiter::None => proc_macro::Delimiter::None,
535578
};
536-
Group::Compiler(proc_macro::Group::new(delimiter, stream))
579+
Group::Compiler(proc_macro::Group::new(delimiter, tts.into_token_stream()))
537580
}
538581
TokenStream::Fallback(stream) => {
539582
Group::Fallback(fallback::Group::new(delimiter, stream))
@@ -555,7 +598,7 @@ impl Group {
555598

556599
pub fn stream(&self) -> TokenStream {
557600
match self {
558-
Group::Compiler(g) => TokenStream::Compiler(g.stream()),
601+
Group::Compiler(g) => TokenStream::Compiler(DeferredTokenStream::new(g.stream())),
559602
Group::Fallback(g) => TokenStream::Fallback(g.stream()),
560603
}
561604
}

0 commit comments

Comments
 (0)