From 71969a54a15acf5c83b5a834fd14a2ec28a8ff16 Mon Sep 17 00:00:00 2001 From: Felix Rath Date: Wed, 13 Aug 2025 13:05:07 +0200 Subject: [PATCH 1/6] Implement incremental caching for derive macro expansions --- Cargo.lock | 1 + compiler/rustc_ast/src/tokenstream.rs | 100 +++++++++- compiler/rustc_expand/Cargo.toml | 1 + compiler/rustc_expand/src/lib.rs | 4 + compiler/rustc_expand/src/proc_macro.rs | 177 +++++++++++++++--- compiler/rustc_interface/src/passes.rs | 1 + compiler/rustc_middle/src/arena.rs | 1 + compiler/rustc_middle/src/query/erase.rs | 5 + compiler/rustc_middle/src/query/keys.rs | 17 +- compiler/rustc_middle/src/query/mod.rs | 10 +- .../rustc_middle/src/query/on_disk_cache.rs | 7 + compiler/rustc_session/src/options.rs | 2 + compiler/rustc_span/src/hygiene.rs | 6 + .../auxiliary/derive_nothing.rs | 20 ++ .../proc_macro_unchanged.rs | 39 ++++ 15 files changed, 356 insertions(+), 35 deletions(-) create mode 100644 tests/incremental/derive_macro_expansion/auxiliary/derive_nothing.rs create mode 100644 tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs diff --git a/Cargo.lock b/Cargo.lock index 4eb246995b1ca..3bcdb034126e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3833,6 +3833,7 @@ dependencies = [ "rustc_lexer", "rustc_lint_defs", "rustc_macros", + "rustc_middle", "rustc_parse", "rustc_proc_macro", "rustc_serialize", diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index e55399adfb85a..1d4e7cddc8f6a 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -14,6 +14,7 @@ //! ownership of the original. use std::borrow::Cow; +use std::hash::Hash; use std::ops::Range; use std::sync::Arc; use std::{cmp, fmt, iter, mem}; @@ -21,8 +22,9 @@ use std::{cmp, fmt, iter, mem}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync; use rustc_macros::{Decodable, Encodable, HashStable_Generic, Walkable}; -use rustc_serialize::{Decodable, Encodable}; -use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym}; +use rustc_serialize::{Decodable, Encodable, Encoder}; +use rustc_span::def_id::{CrateNum, DefIndex}; +use rustc_span::{ByteSymbol, DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym}; use thin_vec::ThinVec; use crate::ast::AttrStyle; @@ -560,6 +562,100 @@ pub struct AttrsTarget { #[derive(Clone, Debug, Default, Encodable, Decodable)] pub struct TokenStream(pub(crate) Arc>); +struct HashEncoder { + hasher: H, +} + +impl Encoder for HashEncoder { + fn emit_usize(&mut self, v: usize) { + self.hasher.write_usize(v) + } + + fn emit_u128(&mut self, v: u128) { + self.hasher.write_u128(v) + } + + fn emit_u64(&mut self, v: u64) { + self.hasher.write_u64(v) + } + + fn emit_u32(&mut self, v: u32) { + self.hasher.write_u32(v) + } + + fn emit_u16(&mut self, v: u16) { + self.hasher.write_u16(v) + } + + fn emit_u8(&mut self, v: u8) { + self.hasher.write_u8(v) + } + + fn emit_isize(&mut self, v: isize) { + self.hasher.write_isize(v) + } + + fn emit_i128(&mut self, v: i128) { + self.hasher.write_i128(v) + } + + fn emit_i64(&mut self, v: i64) { + self.hasher.write_i64(v) + } + + fn emit_i32(&mut self, v: i32) { + self.hasher.write_i32(v) + } + + fn emit_i16(&mut self, v: i16) { + self.hasher.write_i16(v) + } + + fn emit_raw_bytes(&mut self, s: &[u8]) { + self.hasher.write(s) + } +} + +impl SpanEncoder for HashEncoder { + fn encode_span(&mut self, span: Span) { + span.hash(&mut self.hasher) + } + + fn encode_symbol(&mut self, symbol: Symbol) { + symbol.hash(&mut self.hasher) + } + + fn encode_byte_symbol(&mut self, byte_sym: ByteSymbol) { + byte_sym.hash(&mut self.hasher); + } + + fn encode_expn_id(&mut self, expn_id: rustc_span::ExpnId) { + expn_id.hash(&mut self.hasher) + } + + fn encode_syntax_context(&mut self, syntax_context: rustc_span::SyntaxContext) { + syntax_context.hash(&mut self.hasher) + } + + fn encode_crate_num(&mut self, crate_num: CrateNum) { + crate_num.hash(&mut self.hasher) + } + + fn encode_def_index(&mut self, def_index: DefIndex) { + def_index.hash(&mut self.hasher) + } + + fn encode_def_id(&mut self, def_id: rustc_span::def_id::DefId) { + def_id.hash(&mut self.hasher) + } +} + +impl Hash for TokenStream { + fn hash(&self, state: &mut H) { + Encodable::encode(self, &mut HashEncoder { hasher: state }); + } +} + /// Indicates whether a token can join with the following token to form a /// compound token. Used for conversions to `proc_macro::Spacing`. Also used to /// guide pretty-printing, which is where the `JointHidden` value (which isn't diff --git a/compiler/rustc_expand/Cargo.toml b/compiler/rustc_expand/Cargo.toml index f897833d85c00..8a65fdaac0a4e 100644 --- a/compiler/rustc_expand/Cargo.toml +++ b/compiler/rustc_expand/Cargo.toml @@ -21,6 +21,7 @@ rustc_hir = { path = "../rustc_hir" } rustc_lexer = { path = "../rustc_lexer" } rustc_lint_defs = { path = "../rustc_lint_defs" } rustc_macros = { path = "../rustc_macros" } +rustc_middle = { path = "../rustc_middle" } rustc_parse = { path = "../rustc_parse" } # We must use the proc_macro version that we will compile proc-macros against, # not the one from our own sysroot. diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs index b54dabbb8e26f..d95ad7dec5a36 100644 --- a/compiler/rustc_expand/src/lib.rs +++ b/compiler/rustc_expand/src/lib.rs @@ -31,4 +31,8 @@ pub mod module; #[allow(rustc::untranslatable_diagnostic)] pub mod proc_macro; +pub fn provide(providers: &mut rustc_middle::util::Providers) { + providers.derive_macro_expansion = proc_macro::provide_derive_macro_expansion; +} + rustc_fluent_macro::fluent_messages! { "../messages.ftl" } diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 9bfda8764f552..110e7ce1c46ce 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -1,9 +1,15 @@ +use std::cell::Cell; +use std::ptr::NonNull; + use rustc_ast::tokenstream::TokenStream; +use rustc_data_structures::svh::Svh; use rustc_errors::ErrorGuaranteed; +use rustc_middle::ty::{self, TyCtxt}; use rustc_parse::parser::{ForceCollect, Parser}; +use rustc_session::Session; use rustc_session::config::ProcMacroExecutionStrategy; -use rustc_span::Span; use rustc_span::profiling::SpannedEventArgRecorder; +use rustc_span::{LocalExpnId, Span}; use {rustc_ast as ast, rustc_proc_macro as pm}; use crate::base::{self, *}; @@ -30,9 +36,9 @@ impl pm::bridge::server::MessagePipe for MessagePipe { } } -fn exec_strategy(ecx: &ExtCtxt<'_>) -> impl pm::bridge::server::ExecutionStrategy + 'static { +pub fn exec_strategy(sess: &Session) -> impl pm::bridge::server::ExecutionStrategy + 'static { pm::bridge::server::MaybeCrossThread::>::new( - ecx.sess.opts.unstable_opts.proc_macro_execution_strategy + sess.opts.unstable_opts.proc_macro_execution_strategy == ProcMacroExecutionStrategy::CrossThread, ) } @@ -54,7 +60,7 @@ impl base::BangProcMacro for BangProcMacro { }); let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; - let strategy = exec_strategy(ecx); + let strategy = exec_strategy(ecx.sess); let server = proc_macro_server::Rustc::new(ecx); self.client.run(&strategy, server, input, proc_macro_backtrace).map_err(|e| { ecx.dcx().emit_err(errors::ProcMacroPanicked { @@ -85,7 +91,7 @@ impl base::AttrProcMacro for AttrProcMacro { }); let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; - let strategy = exec_strategy(ecx); + let strategy = exec_strategy(ecx.sess); let server = proc_macro_server::Rustc::new(ecx); self.client.run(&strategy, server, annotation, annotated, proc_macro_backtrace).map_err( |e| { @@ -113,6 +119,13 @@ impl MultiItemModifier for DeriveProcMacro { item: Annotatable, _is_derive_const: bool, ) -> ExpandResult, Annotatable> { + let _timer = ecx.sess.prof.generic_activity_with_arg_recorder( + "expand_derive_proc_macro_outer", + |recorder| { + recorder.record_arg_with_span(ecx.sess.source_map(), ecx.expansion_descr(), span); + }, + ); + // We need special handling for statement items // (e.g. `fn foo() { #[derive(Debug)] struct Bar; }`) let is_stmt = matches!(item, Annotatable::Stmt(..)); @@ -123,36 +136,39 @@ impl MultiItemModifier for DeriveProcMacro { // altogether. See #73345. crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess.psess); let input = item.to_tokens(); - let stream = { - let _timer = - ecx.sess.prof.generic_activity_with_arg_recorder("expand_proc_macro", |recorder| { - recorder.record_arg_with_span( - ecx.sess.source_map(), - ecx.expansion_descr(), - span, - ); - }); - let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; - let strategy = exec_strategy(ecx); - let server = proc_macro_server::Rustc::new(ecx); - match self.client.run(&strategy, server, input, proc_macro_backtrace) { - Ok(stream) => stream, - Err(e) => { - ecx.dcx().emit_err({ - errors::ProcMacroDerivePanicked { - span, - message: e.as_str().map(|message| { - errors::ProcMacroDerivePanickedHelp { message: message.into() } - }), - } - }); - return ExpandResult::Ready(vec![]); - } + let res = ty::tls::with(|tcx| { + let input = tcx.arena.alloc(input) as &TokenStream; + let invoc_id = ecx.current_expansion.id; + let invoc_expn_data = invoc_id.expn_data(); + + assert_eq!(invoc_expn_data.call_site, span); + + // FIXME(pr-time): Is this the correct way to check for incremental compilation (as + // well as for `cache_proc_macros`)? + if tcx.sess.opts.incremental.is_some() && tcx.sess.opts.unstable_opts.cache_proc_macros + { + // FIXME(pr-time): Just using the crate hash to notice when the proc-macro code has + // changed. How to *correctly* depend on exactly the macro definition? + // I.e., depending on the crate hash is just a HACK, and ideally the dependency would be + // more narrow. + let macro_def_id = invoc_expn_data.macro_def_id.unwrap(); + let proc_macro_crate_hash = tcx.crate_hash(macro_def_id.krate); + + let key = (invoc_id, proc_macro_crate_hash, input); + + enter_context((ecx, self.client), move || tcx.derive_macro_expansion(key).cloned()) + } else { + expand_derive_macro(tcx, invoc_id, input, ecx, self.client).cloned() } + }); + + let Ok(output) = res else { + // error will already have been emitted + return ExpandResult::Ready(vec![]); }; let error_count_before = ecx.dcx().err_count(); - let mut parser = Parser::new(&ecx.sess.psess, stream, Some("proc-macro derive")); + let mut parser = Parser::new(&ecx.sess.psess, output, Some("proc-macro derive")); let mut items = vec![]; loop { @@ -180,3 +196,102 @@ impl MultiItemModifier for DeriveProcMacro { ExpandResult::Ready(items) } } + +pub(super) fn provide_derive_macro_expansion<'tcx>( + tcx: TyCtxt<'tcx>, + key: (LocalExpnId, Svh, &'tcx TokenStream), +) -> Result<&'tcx TokenStream, ()> { + let (invoc_id, _macro_crate_hash, input) = key; + + with_context(|(ecx, client)| expand_derive_macro(tcx, invoc_id, input, ecx, *client)) +} + +type CLIENT = pm::bridge::client::Client; + +fn expand_derive_macro<'tcx>( + tcx: TyCtxt<'tcx>, + invoc_id: LocalExpnId, + input: &'tcx TokenStream, + ecx: &mut ExtCtxt<'_>, + client: CLIENT, +) -> Result<&'tcx TokenStream, ()> { + let invoc_expn_data = invoc_id.expn_data(); + let span = invoc_expn_data.call_site; + let event_arg = invoc_expn_data.kind.descr(); + let _timer = tcx.sess.prof.generic_activity_with_arg_recorder( + "expand_derive_proc_macro_inner", + |recorder| { + recorder.record_arg_with_span(tcx.sess.source_map(), event_arg.clone(), span); + }, + ); + + let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; + let strategy = crate::proc_macro::exec_strategy(tcx.sess); + let server = crate::proc_macro_server::Rustc::new(ecx); + + match client.run(&strategy, server, input.clone(), proc_macro_backtrace) { + Ok(stream) => Ok(tcx.arena.alloc(stream) as &TokenStream), + Err(e) => { + tcx.dcx().emit_err({ + errors::ProcMacroDerivePanicked { + span, + message: e.as_str().map(|message| errors::ProcMacroDerivePanickedHelp { + message: message.into(), + }), + } + }); + Err(()) + } + } +} + +// based on rust/compiler/rustc_middle/src/ty/context/tls.rs +thread_local! { + /// A thread local variable that stores a pointer to the current `CONTEXT`. + static TLV: Cell<(*mut (), Option)> = const { Cell::new((std::ptr::null_mut(), None)) }; +} + +/// Sets `context` as the new current `CONTEXT` for the duration of the function `f`. +#[inline] +pub(crate) fn enter_context<'a, F, R>(context: (&mut ExtCtxt<'a>, CLIENT), f: F) -> R +where + F: FnOnce() -> R, +{ + let (ectx, client) = context; + let erased = (ectx as *mut _ as *mut (), Some(client)); + TLV.with(|tlv| { + let old = tlv.replace(erased); + let _reset = rustc_data_structures::defer(move || tlv.set(old)); + f() + }) +} + +/// Allows access to the current `CONTEXT`. +/// Panics if there is no `CONTEXT` available. +#[inline] +#[track_caller] +fn with_context(f: F) -> R +where + F: for<'a, 'b> FnOnce(&'b mut (&mut ExtCtxt<'a>, CLIENT)) -> R, +{ + let (ectx, client_opt) = TLV.get(); + let ectx = NonNull::new(ectx).expect("no CONTEXT stored in tls"); + + // We could get an `CONTEXT` pointer from another thread. + // Ensure that `CONTEXT` is `DynSync`. + // FIXME(pr-time): we should not be able to? + // sync::assert_dyn_sync::>(); + + // prevent double entering, as that would allow creating two `&mut ExtCtxt`s + // FIXME(pr-time): probably use a RefCell instead (which checks this properly)? + TLV.with(|tlv| { + let old = tlv.replace((std::ptr::null_mut(), None)); + let _reset = rustc_data_structures::defer(move || tlv.set(old)); + let ectx = { + let mut casted = ectx.cast::>(); + unsafe { casted.as_mut() } + }; + + f(&mut (ectx, client_opt.unwrap())) + }) +} diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 3ba224723e365..1a69067efe836 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -881,6 +881,7 @@ pub static DEFAULT_QUERY_PROVIDERS: LazyLock = LazyLock::new(|| { providers.env_var_os = env_var_os; limits::provide(providers); proc_macro_decls::provide(providers); + rustc_expand::provide(providers); rustc_const_eval::provide(providers); rustc_middle::hir::provide(providers); rustc_borrowck::provide(providers); diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs index 4b6e38cd52dd3..4f0af1a53dace 100644 --- a/compiler/rustc_middle/src/arena.rs +++ b/compiler/rustc_middle/src/arena.rs @@ -116,6 +116,7 @@ macro_rules! arena_types { [decode] specialization_graph: rustc_middle::traits::specialization_graph::Graph, [] crate_inherent_impls: rustc_middle::ty::CrateInherentImpls, [] hir_owner_nodes: rustc_hir::OwnerNodes<'tcx>, + [decode] token_stream: rustc_ast::tokenstream::TokenStream, ]); ) } diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs index a8b357bf105b8..36f6f2176a101 100644 --- a/compiler/rustc_middle/src/query/erase.rs +++ b/compiler/rustc_middle/src/query/erase.rs @@ -2,6 +2,7 @@ use std::ffi::OsStr; use std::intrinsics::transmute_unchecked; use std::mem::MaybeUninit; +use rustc_ast::tokenstream::TokenStream; use rustc_span::ErrorGuaranteed; use crate::mir::interpret::EvalToValTreeResult; @@ -170,6 +171,10 @@ impl EraseType for Result>, CyclePlaceholder> { type Result = [u8; size_of::>, CyclePlaceholder>>()]; } +impl EraseType for Result<&'_ TokenStream, ()> { + type Result = [u8; size_of::>()]; +} + impl EraseType for Option<&'_ T> { type Result = [u8; size_of::>()]; } diff --git a/compiler/rustc_middle/src/query/keys.rs b/compiler/rustc_middle/src/query/keys.rs index 4d914c42cfc62..11f47fe8b42ae 100644 --- a/compiler/rustc_middle/src/query/keys.rs +++ b/compiler/rustc_middle/src/query/keys.rs @@ -2,11 +2,13 @@ use std::ffi::OsStr; +use rustc_ast::tokenstream::TokenStream; +use rustc_data_structures::svh::Svh; use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId, LocalModDefId, ModDefId}; use rustc_hir::hir_id::{HirId, OwnerId}; use rustc_query_system::dep_graph::DepNodeIndex; use rustc_query_system::query::{DefIdCache, DefaultCache, SingleCache, VecCache}; -use rustc_span::{DUMMY_SP, Ident, Span, Symbol}; +use rustc_span::{DUMMY_SP, Ident, LocalExpnId, Span, Symbol}; use crate::infer::canonical::CanonicalQueryInput; use crate::mir::mono::CollectionMode; @@ -616,6 +618,19 @@ impl Key for (LocalDefId, HirId) { } } +impl<'tcx> Key for (LocalExpnId, Svh, &'tcx TokenStream) { + type Cache = DefaultCache; + + fn default_span(&self, _tcx: TyCtxt<'_>) -> Span { + self.0.expn_data().call_site + } + + #[inline(always)] + fn key_as_def_id(&self) -> Option { + None + } +} + impl<'tcx> Key for (ValidityRequirement, ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>) { type Cache = DefaultCache; diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index f4d0120a2e7b9..3129bb8df13e5 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -70,6 +70,7 @@ use std::sync::Arc; use rustc_abi::Align; use rustc_arena::TypedArena; use rustc_ast::expand::allocator::AllocatorKind; +use rustc_ast::tokenstream::TokenStream; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_data_structures::sorted_map::SortedMap; @@ -99,7 +100,7 @@ use rustc_session::cstore::{ use rustc_session::lint::LintExpectationId; use rustc_span::def_id::LOCAL_CRATE; use rustc_span::source_map::Spanned; -use rustc_span::{DUMMY_SP, Span, Symbol}; +use rustc_span::{DUMMY_SP, LocalExpnId, Span, Symbol}; use rustc_target::spec::PanicStrategy; use {rustc_abi as abi, rustc_ast as ast, rustc_hir as hir}; @@ -164,6 +165,13 @@ pub use plumbing::{IntoQueryParam, TyCtxtAt, TyCtxtEnsureDone, TyCtxtEnsureOk}; // Queries marked with `fatal_cycle` do not need the latter implementation, // as they will raise an fatal error on query cycles instead. rustc_queries! { + query derive_macro_expansion(key: (LocalExpnId, Svh, &'tcx TokenStream)) -> Result<&'tcx TokenStream, ()> { + // eval_always + // no_hash + desc { "expanding a derive (proc) macro" } + cache_on_disk_if { true } + } + /// This exists purely for testing the interactions between delayed bugs and incremental. query trigger_delayed_bug(key: DefId) { desc { "triggering a delayed bug for testing incremental" } diff --git a/compiler/rustc_middle/src/query/on_disk_cache.rs b/compiler/rustc_middle/src/query/on_disk_cache.rs index a7ac34428986e..0c667a1eb5be1 100644 --- a/compiler/rustc_middle/src/query/on_disk_cache.rs +++ b/compiler/rustc_middle/src/query/on_disk_cache.rs @@ -785,6 +785,13 @@ impl<'a, 'tcx> Decodable> } } +impl<'a, 'tcx> Decodable> for &'tcx rustc_ast::tokenstream::TokenStream { + #[inline] + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + macro_rules! impl_ref_decoder { (<$tcx:tt> $($ty:ty,)*) => { $(impl<'a, $tcx> Decodable> for &$tcx [$ty] { diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 7c18fd8909808..3da22701ae341 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2157,6 +2157,8 @@ options! { "set options for branch target identification and pointer authentication on AArch64"), build_sdylib_interface: bool = (false, parse_bool, [UNTRACKED], "whether the stable interface is being built"), + cache_derive_macros: bool = (false, parse_bool, [TRACKED], + "cache the results of derive proc macro invocations (potentially unsound!) (default: no"), cf_protection: CFProtection = (CFProtection::None, parse_cfprotection, [TRACKED], "instrument control-flow architecture protection"), check_cfg_all_expected: bool = (false, parse_bool, [UNTRACKED], diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index 19494ffc37eaf..0c65943e30989 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -1558,3 +1558,9 @@ impl HashStable for ExpnId { hash.hash_stable(ctx, hasher); } } + +impl HashStable for LocalExpnId { + fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) { + self.to_expn_id().hash_stable(hcx, hasher); + } +} diff --git a/tests/incremental/derive_macro_expansion/auxiliary/derive_nothing.rs b/tests/incremental/derive_macro_expansion/auxiliary/derive_nothing.rs new file mode 100644 index 0000000000000..ee0fe7ea10023 --- /dev/null +++ b/tests/incremental/derive_macro_expansion/auxiliary/derive_nothing.rs @@ -0,0 +1,20 @@ +//@ force-host +//@ no-prefer-dynamic + +#![crate_type = "proc-macro"] + +extern crate proc_macro; +use proc_macro::TokenStream; + +#[proc_macro_derive(Nothing)] +pub fn derive(_input: TokenStream) -> TokenStream { + eprintln!("invoked"); + + return r#" + pub mod nothing_mod { + pub fn nothing() { + eprintln!("nothing"); + } + } + "#.parse().unwrap(); +} diff --git a/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs b/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs new file mode 100644 index 0000000000000..ad98c9f789f47 --- /dev/null +++ b/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs @@ -0,0 +1,39 @@ +// This test tests that derive-macro execution is cached. +// HOWEVER, this test can currently only be checked manually, +// by running it (through compiletest) with `-- --nocapture --verbose`. +// The proc-macro (for `Nothing`) prints a message to stderr when invoked, +// and this message should only be present during the first invocation, +// because the cached result should be used for the second invocation. +// FIXME(pr-time): Properly have the test check this, but how? UI-test that tests for `.stderr`? + +//@ aux-build:derive_nothing.rs +//@ revisions:cfail1 cfail2 +//@ compile-flags: -Z query-dep-graph -Zcache-proc-macros=true +//@ build-pass + +#![feature(rustc_attrs)] +#![feature(stmt_expr_attributes)] +#![allow(dead_code)] +#![crate_type = "rlib"] + +#![rustc_partition_codegened(module="proc_macro_unchanged-foo", cfg="cfail1")] +// #![rustc_partition_codegened(module="proc_macro_unchanged-foo", cfg="cfail2")] + +// `foo::nothing_mod` is created by the derive macro and doesn't change +// BUG: this yields the same result with `-Zcache-proc-macros=false` (i.e., uncached), +// not sure how to do this correctly. +#![rustc_partition_reused(module="proc_macro_unchanged-foo-nothing_mod", cfg="cfail2")] + + #[macro_use] + extern crate derive_nothing; + +pub mod foo { + #[derive(Nothing)] + pub struct Foo; + + pub fn use_foo(_f: Foo) { + nothing_mod::nothing(); + + eprintln!("foo used"); + } +} From 5220403e34bc4a2562dea2c40465e74ee8394e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 13 Aug 2025 13:14:33 +0200 Subject: [PATCH 2/6] Add some comments and rename the option to `-Zcache-derive-macros` --- compiler/rustc_ast/src/tokenstream.rs | 2 ++ compiler/rustc_expand/src/proc_macro.rs | 4 +++- compiler/rustc_middle/src/query/mod.rs | 9 +++++++-- .../derive_macro_expansion/proc_macro_unchanged.rs | 13 ++++++------- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 1d4e7cddc8f6a..e6066438efd80 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -650,6 +650,8 @@ impl SpanEncoder for HashEncoder { } } +/// TokenStream needs to be hashable because it is used as a query key for caching derive macro +/// expansions. impl Hash for TokenStream { fn hash(&self, state: &mut H) { Encodable::encode(self, &mut HashEncoder { hasher: state }); diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index 110e7ce1c46ce..afd6e98b09e1d 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -145,7 +145,8 @@ impl MultiItemModifier for DeriveProcMacro { // FIXME(pr-time): Is this the correct way to check for incremental compilation (as // well as for `cache_proc_macros`)? - if tcx.sess.opts.incremental.is_some() && tcx.sess.opts.unstable_opts.cache_proc_macros + if tcx.sess.opts.incremental.is_some() + && tcx.sess.opts.unstable_opts.cache_derive_macros { // FIXME(pr-time): Just using the crate hash to notice when the proc-macro code has // changed. How to *correctly* depend on exactly the macro definition? @@ -197,6 +198,7 @@ impl MultiItemModifier for DeriveProcMacro { } } +/// Provide a query for computing the output of a derive macro. pub(super) fn provide_derive_macro_expansion<'tcx>( tcx: TyCtxt<'tcx>, key: (LocalExpnId, Svh, &'tcx TokenStream), diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 3129bb8df13e5..33baca2487db1 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -165,9 +165,14 @@ pub use plumbing::{IntoQueryParam, TyCtxtAt, TyCtxtEnsureDone, TyCtxtEnsureOk}; // Queries marked with `fatal_cycle` do not need the latter implementation, // as they will raise an fatal error on query cycles instead. rustc_queries! { + /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`. + /// The key is: + /// - A unique key corresponding to the invocation of a macro. + /// - Strict Version Hash of a crate. + /// - Token stream which serves as an input to the macro. + /// + /// The output is the token stream generated by the proc macro. query derive_macro_expansion(key: (LocalExpnId, Svh, &'tcx TokenStream)) -> Result<&'tcx TokenStream, ()> { - // eval_always - // no_hash desc { "expanding a derive (proc) macro" } cache_on_disk_if { true } } diff --git a/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs b/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs index ad98c9f789f47..0cb3cd8ae3efe 100644 --- a/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs +++ b/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs @@ -8,24 +8,23 @@ //@ aux-build:derive_nothing.rs //@ revisions:cfail1 cfail2 -//@ compile-flags: -Z query-dep-graph -Zcache-proc-macros=true +//@ compile-flags: -Z query-dep-graph -Zcache-derive-macros=true //@ build-pass #![feature(rustc_attrs)] #![feature(stmt_expr_attributes)] #![allow(dead_code)] #![crate_type = "rlib"] - -#![rustc_partition_codegened(module="proc_macro_unchanged-foo", cfg="cfail1")] +#![rustc_partition_codegened(module = "proc_macro_unchanged-foo", cfg = "cfail1")] // #![rustc_partition_codegened(module="proc_macro_unchanged-foo", cfg="cfail2")] // `foo::nothing_mod` is created by the derive macro and doesn't change -// BUG: this yields the same result with `-Zcache-proc-macros=false` (i.e., uncached), +// BUG: this yields the same result with `-Zcache-derive-macros=false` (i.e., uncached), // not sure how to do this correctly. -#![rustc_partition_reused(module="proc_macro_unchanged-foo-nothing_mod", cfg="cfail2")] +#![rustc_partition_reused(module = "proc_macro_unchanged-foo-nothing_mod", cfg = "cfail2")] - #[macro_use] - extern crate derive_nothing; +#[macro_use] +extern crate derive_nothing; pub mod foo { #[derive(Nothing)] From f26f394b09327efa9cba694667942fdeff1fdcd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 13 Aug 2025 14:14:26 +0200 Subject: [PATCH 3/6] Refactor TLS access --- Cargo.lock | 1 + compiler/rustc_expand/Cargo.toml | 1 + compiler/rustc_expand/src/proc_macro.rs | 164 +++++++++++------------- 3 files changed, 80 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3bcdb034126e8..bdb8373e7efcf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3839,6 +3839,7 @@ dependencies = [ "rustc_serialize", "rustc_session", "rustc_span", + "scoped-tls", "smallvec", "thin-vec", "tracing", diff --git a/compiler/rustc_expand/Cargo.toml b/compiler/rustc_expand/Cargo.toml index 8a65fdaac0a4e..a18506c42afcf 100644 --- a/compiler/rustc_expand/Cargo.toml +++ b/compiler/rustc_expand/Cargo.toml @@ -29,6 +29,7 @@ rustc_proc_macro = { path = "../rustc_proc_macro" } rustc_serialize = { path = "../rustc_serialize" } rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } +scoped-tls = "1.0" smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } thin-vec = "0.2.12" tracing = "0.1" diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs index afd6e98b09e1d..3d303a702d5a1 100644 --- a/compiler/rustc_expand/src/proc_macro.rs +++ b/compiler/rustc_expand/src/proc_macro.rs @@ -1,6 +1,3 @@ -use std::cell::Cell; -use std::ptr::NonNull; - use rustc_ast::tokenstream::TokenStream; use rustc_data_structures::svh::Svh; use rustc_errors::ErrorGuaranteed; @@ -36,7 +33,7 @@ impl pm::bridge::server::MessagePipe for MessagePipe { } } -pub fn exec_strategy(sess: &Session) -> impl pm::bridge::server::ExecutionStrategy + 'static { +fn exec_strategy(sess: &Session) -> impl pm::bridge::server::ExecutionStrategy + 'static { pm::bridge::server::MaybeCrossThread::>::new( sess.opts.unstable_opts.proc_macro_execution_strategy == ProcMacroExecutionStrategy::CrossThread, @@ -107,7 +104,7 @@ impl base::AttrProcMacro for AttrProcMacro { } pub struct DeriveProcMacro { - pub client: pm::bridge::client::Client, + pub client: DeriveClient, } impl MultiItemModifier for DeriveProcMacro { @@ -136,32 +133,31 @@ impl MultiItemModifier for DeriveProcMacro { // altogether. See #73345. crate::base::ann_pretty_printing_compatibility_hack(&item, &ecx.sess.psess); let input = item.to_tokens(); - let res = ty::tls::with(|tcx| { - let input = tcx.arena.alloc(input) as &TokenStream; - let invoc_id = ecx.current_expansion.id; - let invoc_expn_data = invoc_id.expn_data(); - - assert_eq!(invoc_expn_data.call_site, span); - - // FIXME(pr-time): Is this the correct way to check for incremental compilation (as - // well as for `cache_proc_macros`)? - if tcx.sess.opts.incremental.is_some() - && tcx.sess.opts.unstable_opts.cache_derive_macros - { + + let invoc_id = ecx.current_expansion.id; + + let res = if ecx.sess.opts.incremental.is_some() + && ecx.sess.opts.unstable_opts.cache_derive_macros + { + ty::tls::with(|tcx| { // FIXME(pr-time): Just using the crate hash to notice when the proc-macro code has // changed. How to *correctly* depend on exactly the macro definition? // I.e., depending on the crate hash is just a HACK, and ideally the dependency would be // more narrow. + let invoc_expn_data = invoc_id.expn_data(); let macro_def_id = invoc_expn_data.macro_def_id.unwrap(); let proc_macro_crate_hash = tcx.crate_hash(macro_def_id.krate); + let input = tcx.arena.alloc(input) as &TokenStream; let key = (invoc_id, proc_macro_crate_hash, input); - enter_context((ecx, self.client), move || tcx.derive_macro_expansion(key).cloned()) - } else { - expand_derive_macro(tcx, invoc_id, input, ecx, self.client).cloned() - } - }); + QueryDeriveExpandCtx::enter(ecx, self.client, move || { + tcx.derive_macro_expansion(key).cloned() + }) + }) + } else { + expand_derive_macro(invoc_id, input, ecx, self.client) + }; let Ok(output) = res else { // error will already have been emitted @@ -205,36 +201,38 @@ pub(super) fn provide_derive_macro_expansion<'tcx>( ) -> Result<&'tcx TokenStream, ()> { let (invoc_id, _macro_crate_hash, input) = key; - with_context(|(ecx, client)| expand_derive_macro(tcx, invoc_id, input, ecx, *client)) + eprintln!("Expanding derive macro in a query"); + + QueryDeriveExpandCtx::with(|ecx, client| { + expand_derive_macro(invoc_id, input.clone(), ecx, client) + .map(|ts| tcx.arena.alloc(ts) as &TokenStream) + }) } -type CLIENT = pm::bridge::client::Client; +type DeriveClient = pm::bridge::client::Client; -fn expand_derive_macro<'tcx>( - tcx: TyCtxt<'tcx>, +fn expand_derive_macro( invoc_id: LocalExpnId, - input: &'tcx TokenStream, + input: TokenStream, ecx: &mut ExtCtxt<'_>, - client: CLIENT, -) -> Result<&'tcx TokenStream, ()> { + client: DeriveClient, +) -> Result { let invoc_expn_data = invoc_id.expn_data(); let span = invoc_expn_data.call_site; let event_arg = invoc_expn_data.kind.descr(); - let _timer = tcx.sess.prof.generic_activity_with_arg_recorder( - "expand_derive_proc_macro_inner", - |recorder| { - recorder.record_arg_with_span(tcx.sess.source_map(), event_arg.clone(), span); - }, - ); + let _timer = + ecx.sess.prof.generic_activity_with_arg_recorder("expand_proc_macro", |recorder| { + recorder.record_arg_with_span(ecx.sess.source_map(), event_arg.clone(), span); + }); let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace; - let strategy = crate::proc_macro::exec_strategy(tcx.sess); - let server = crate::proc_macro_server::Rustc::new(ecx); + let strategy = exec_strategy(ecx.sess); + let server = proc_macro_server::Rustc::new(ecx); - match client.run(&strategy, server, input.clone(), proc_macro_backtrace) { - Ok(stream) => Ok(tcx.arena.alloc(stream) as &TokenStream), + match client.run(&strategy, server, input, proc_macro_backtrace) { + Ok(stream) => Ok(stream), Err(e) => { - tcx.dcx().emit_err({ + ecx.dcx().emit_err({ errors::ProcMacroDerivePanicked { span, message: e.as_str().map(|message| errors::ProcMacroDerivePanickedHelp { @@ -247,53 +245,47 @@ fn expand_derive_macro<'tcx>( } } -// based on rust/compiler/rustc_middle/src/ty/context/tls.rs -thread_local! { - /// A thread local variable that stores a pointer to the current `CONTEXT`. - static TLV: Cell<(*mut (), Option)> = const { Cell::new((std::ptr::null_mut(), None)) }; -} - -/// Sets `context` as the new current `CONTEXT` for the duration of the function `f`. -#[inline] -pub(crate) fn enter_context<'a, F, R>(context: (&mut ExtCtxt<'a>, CLIENT), f: F) -> R -where - F: FnOnce() -> R, -{ - let (ectx, client) = context; - let erased = (ectx as *mut _ as *mut (), Some(client)); - TLV.with(|tlv| { - let old = tlv.replace(erased); - let _reset = rustc_data_structures::defer(move || tlv.set(old)); - f() - }) +/// Stores the context necessary to expand a derive proc macro via a query. +struct QueryDeriveExpandCtx { + /// Type-erased version of `&mut ExtCtxt` + expansion_ctx: *mut (), + client: DeriveClient, } -/// Allows access to the current `CONTEXT`. -/// Panics if there is no `CONTEXT` available. -#[inline] -#[track_caller] -fn with_context(f: F) -> R -where - F: for<'a, 'b> FnOnce(&'b mut (&mut ExtCtxt<'a>, CLIENT)) -> R, -{ - let (ectx, client_opt) = TLV.get(); - let ectx = NonNull::new(ectx).expect("no CONTEXT stored in tls"); - - // We could get an `CONTEXT` pointer from another thread. - // Ensure that `CONTEXT` is `DynSync`. - // FIXME(pr-time): we should not be able to? - // sync::assert_dyn_sync::>(); - - // prevent double entering, as that would allow creating two `&mut ExtCtxt`s - // FIXME(pr-time): probably use a RefCell instead (which checks this properly)? - TLV.with(|tlv| { - let old = tlv.replace((std::ptr::null_mut(), None)); - let _reset = rustc_data_structures::defer(move || tlv.set(old)); - let ectx = { - let mut casted = ectx.cast::>(); - unsafe { casted.as_mut() } - }; +impl QueryDeriveExpandCtx { + /// Store the extension context and the client into the thread local value. + /// It will be accessible via the `with` method while `f` is active. + fn enter(ecx: &mut ExtCtxt<'_>, client: DeriveClient, f: F) -> R + where + F: FnOnce() -> R, + { + // We need erasure to get rid of the lifetime + let ctx = Self { expansion_ctx: ecx as *mut _ as *mut (), client }; + DERIVE_EXPAND_CTX.set(&ctx, || f()) + } - f(&mut (ectx, client_opt.unwrap())) - }) + /// Accesses the thread local value of the derive expansion context. + /// Must be called while the `enter` function is active. + fn with(f: F) -> R + where + F: for<'a, 'b> FnOnce(&'b mut ExtCtxt<'a>, DeriveClient) -> R, + { + DERIVE_EXPAND_CTX.with(|ctx| { + let ectx = { + let casted = ctx.expansion_ctx.cast::>(); + // SAFETY: We can only get the value from `with` while the `enter` function + // is active (on the callstack), and that function's signature ensures that the + // lifetime is valid. + // If `with` is called at some other time, it will panic due to usage of + // `scoped_tls::with`. + unsafe { casted.as_mut().unwrap() } + }; + + f(ectx, ctx.client) + }) + } } + +// When we invoke a query to expand a derive proc macro, we need to provide it with the expansion +// context and derive Client. We do that using a thread-local. +scoped_tls::scoped_thread_local!(static DERIVE_EXPAND_CTX: QueryDeriveExpandCtx); From 637d70437f96e8a40fd99e567698c9d7436d28a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 13 Aug 2025 14:58:34 +0200 Subject: [PATCH 4/6] Update test --- .../auxiliary/derive_nothing.rs | 8 +---- .../proc_macro_unchanged.rs | 31 ++++++------------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/tests/incremental/derive_macro_expansion/auxiliary/derive_nothing.rs b/tests/incremental/derive_macro_expansion/auxiliary/derive_nothing.rs index ee0fe7ea10023..e75b78dd92460 100644 --- a/tests/incremental/derive_macro_expansion/auxiliary/derive_nothing.rs +++ b/tests/incremental/derive_macro_expansion/auxiliary/derive_nothing.rs @@ -10,11 +10,5 @@ use proc_macro::TokenStream; pub fn derive(_input: TokenStream) -> TokenStream { eprintln!("invoked"); - return r#" - pub mod nothing_mod { - pub fn nothing() { - eprintln!("nothing"); - } - } - "#.parse().unwrap(); + TokenStream::new() } diff --git a/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs b/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs index 0cb3cd8ae3efe..9f3be27001764 100644 --- a/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs +++ b/tests/incremental/derive_macro_expansion/proc_macro_unchanged.rs @@ -7,32 +7,21 @@ // FIXME(pr-time): Properly have the test check this, but how? UI-test that tests for `.stderr`? //@ aux-build:derive_nothing.rs -//@ revisions:cfail1 cfail2 -//@ compile-flags: -Z query-dep-graph -Zcache-derive-macros=true -//@ build-pass +//@ revisions:rpass1 rpass2 +//@ compile-flags: -Zquery-dep-graph -Zcache-derive-macros #![feature(rustc_attrs)] -#![feature(stmt_expr_attributes)] -#![allow(dead_code)] -#![crate_type = "rlib"] -#![rustc_partition_codegened(module = "proc_macro_unchanged-foo", cfg = "cfail1")] -// #![rustc_partition_codegened(module="proc_macro_unchanged-foo", cfg="cfail2")] - -// `foo::nothing_mod` is created by the derive macro and doesn't change -// BUG: this yields the same result with `-Zcache-derive-macros=false` (i.e., uncached), -// not sure how to do this correctly. -#![rustc_partition_reused(module = "proc_macro_unchanged-foo-nothing_mod", cfg = "cfail2")] #[macro_use] extern crate derive_nothing; -pub mod foo { - #[derive(Nothing)] - pub struct Foo; +#[cfg(rpass1)] +#[derive(Nothing)] +pub struct Foo; - pub fn use_foo(_f: Foo) { - nothing_mod::nothing(); +#[cfg(rpass2)] +#[derive(Nothing)] +#[rustc_clean(cfg = "rpass2", loaded_from_disk = "derive_macro_expansion")] +pub struct Foo; - eprintln!("foo used"); - } -} +fn main() {} From 2f37ce24e25a71f65e379c1c6f2d0b8aa2d7bf00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 13 Aug 2025 15:01:46 +0200 Subject: [PATCH 5/6] WIP: temporarily enable `cache-derive-macros` by default for rustc-perf --- compiler/rustc_session/src/options.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 3da22701ae341..a67e548cb9f73 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2157,8 +2157,8 @@ options! { "set options for branch target identification and pointer authentication on AArch64"), build_sdylib_interface: bool = (false, parse_bool, [UNTRACKED], "whether the stable interface is being built"), - cache_derive_macros: bool = (false, parse_bool, [TRACKED], - "cache the results of derive proc macro invocations (potentially unsound!) (default: no"), + cache_derive_macros: bool = (true, parse_bool, [TRACKED], + "cache the results of derive proc macro invocations (potentially unsound!) (default: yes"), cf_protection: CFProtection = (CFProtection::None, parse_cfprotection, [TRACKED], "instrument control-flow architecture protection"), check_cfg_all_expected: bool = (false, parse_bool, [UNTRACKED], From ff86ff46fd65a5a6870fc8cb10cdd42e1a247da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Wed, 13 Aug 2025 16:14:27 +0200 Subject: [PATCH 6/6] Derive `Hash` for `TokenStream` --- compiler/rustc_ast/src/ast.rs | 2 +- compiler/rustc_ast/src/token.rs | 16 ++-- compiler/rustc_ast/src/tokenstream.rs | 111 ++----------------------- compiler/rustc_middle/src/query/mod.rs | 2 +- 4 files changed, 17 insertions(+), 114 deletions(-) diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 87c9c797ea5b4..7a16464a09176 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -3243,7 +3243,7 @@ impl UseTree { /// Distinguishes between `Attribute`s that decorate items and Attributes that /// are contained as statements within items. These two cases need to be /// distinguished for pretty-printing. -#[derive(Clone, PartialEq, Encodable, Decodable, Debug, Copy, HashStable_Generic, Walkable)] +#[derive(Clone, PartialEq, Encodable, Decodable, Hash, Debug, Copy, HashStable_Generic, Walkable)] pub enum AttrStyle { Outer, Inner, diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index fc816f2cb7922..70467d7c5ff54 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -15,14 +15,14 @@ use rustc_span::{Ident, Symbol}; use crate::ast; use crate::util::case::Case; -#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] +#[derive(Clone, Copy, PartialEq, Hash, Encodable, Decodable, Debug, HashStable_Generic)] pub enum CommentKind { Line, Block, } // This type must not implement `Hash` due to the unusual `PartialEq` impl below. -#[derive(Copy, Clone, Debug, Encodable, Decodable, HashStable_Generic)] +#[derive(Copy, Clone, Debug, Hash, Encodable, Decodable, HashStable_Generic)] pub enum InvisibleOrigin { // From the expansion of a metavariable in a declarative macro. MetaVar(MetaVarKind), @@ -113,7 +113,7 @@ impl fmt::Display for MetaVarKind { /// Describes how a sequence of token trees is delimited. /// Cannot use `proc_macro::Delimiter` directly because this /// structure should implement some additional traits. -#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)] +#[derive(Copy, Clone, Debug, PartialEq, Hash, Encodable, Decodable, HashStable_Generic)] pub enum Delimiter { /// `( ... )` Parenthesis, @@ -175,7 +175,7 @@ impl Delimiter { // type. This means that float literals like `1f32` are classified by this type // as `Int`. Only upon conversion to `ast::LitKind` will such a literal be // given the `Float` kind. -#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] +#[derive(Clone, Copy, PartialEq, Hash, Encodable, Decodable, Debug, HashStable_Generic)] pub enum LitKind { Bool, // AST only, must never appear in a `Token` Byte, @@ -192,7 +192,7 @@ pub enum LitKind { } /// A literal token. -#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] +#[derive(Clone, Copy, PartialEq, Hash, Encodable, Decodable, Debug, HashStable_Generic)] pub struct Lit { pub kind: LitKind, pub symbol: Symbol, @@ -338,7 +338,7 @@ fn ident_can_begin_type(name: Symbol, span: Span, is_raw: IdentIsRaw) -> bool { .contains(&name) } -#[derive(PartialEq, Encodable, Decodable, Debug, Copy, Clone, HashStable_Generic)] +#[derive(PartialEq, Encodable, Decodable, Hash, Debug, Copy, Clone, HashStable_Generic)] pub enum IdentIsRaw { No, Yes, @@ -356,7 +356,7 @@ impl From for bool { } } -#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] +#[derive(Clone, Copy, PartialEq, Hash, Encodable, Decodable, Debug, HashStable_Generic)] pub enum TokenKind { /* Expression-operator symbols. */ /// `=` @@ -506,7 +506,7 @@ pub enum TokenKind { Eof, } -#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] +#[derive(Clone, Copy, PartialEq, Hash, Encodable, Decodable, Debug, HashStable_Generic)] pub struct Token { pub kind: TokenKind, pub span: Span, diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index e6066438efd80..5e427d3bc31e6 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -22,9 +22,8 @@ use std::{cmp, fmt, iter, mem}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync; use rustc_macros::{Decodable, Encodable, HashStable_Generic, Walkable}; -use rustc_serialize::{Decodable, Encodable, Encoder}; -use rustc_span::def_id::{CrateNum, DefIndex}; -use rustc_span::{ByteSymbol, DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym}; +use rustc_serialize::{Decodable, Encodable}; +use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym}; use thin_vec::ThinVec; use crate::ast::AttrStyle; @@ -33,7 +32,7 @@ use crate::token::{self, Delimiter, Token, TokenKind}; use crate::{AttrVec, Attribute}; /// Part of a `TokenStream`. -#[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)] +#[derive(Debug, Clone, PartialEq, Hash, Encodable, Decodable, HashStable_Generic)] pub enum TokenTree { /// A single token. Should never be `OpenDelim` or `CloseDelim`, because /// delimiters are implicitly represented by `Delimited`. @@ -559,110 +558,14 @@ pub struct AttrsTarget { } /// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s. -#[derive(Clone, Debug, Default, Encodable, Decodable)] +#[derive(Clone, Debug, Default, Hash, Encodable, Decodable)] pub struct TokenStream(pub(crate) Arc>); -struct HashEncoder { - hasher: H, -} - -impl Encoder for HashEncoder { - fn emit_usize(&mut self, v: usize) { - self.hasher.write_usize(v) - } - - fn emit_u128(&mut self, v: u128) { - self.hasher.write_u128(v) - } - - fn emit_u64(&mut self, v: u64) { - self.hasher.write_u64(v) - } - - fn emit_u32(&mut self, v: u32) { - self.hasher.write_u32(v) - } - - fn emit_u16(&mut self, v: u16) { - self.hasher.write_u16(v) - } - - fn emit_u8(&mut self, v: u8) { - self.hasher.write_u8(v) - } - - fn emit_isize(&mut self, v: isize) { - self.hasher.write_isize(v) - } - - fn emit_i128(&mut self, v: i128) { - self.hasher.write_i128(v) - } - - fn emit_i64(&mut self, v: i64) { - self.hasher.write_i64(v) - } - - fn emit_i32(&mut self, v: i32) { - self.hasher.write_i32(v) - } - - fn emit_i16(&mut self, v: i16) { - self.hasher.write_i16(v) - } - - fn emit_raw_bytes(&mut self, s: &[u8]) { - self.hasher.write(s) - } -} - -impl SpanEncoder for HashEncoder { - fn encode_span(&mut self, span: Span) { - span.hash(&mut self.hasher) - } - - fn encode_symbol(&mut self, symbol: Symbol) { - symbol.hash(&mut self.hasher) - } - - fn encode_byte_symbol(&mut self, byte_sym: ByteSymbol) { - byte_sym.hash(&mut self.hasher); - } - - fn encode_expn_id(&mut self, expn_id: rustc_span::ExpnId) { - expn_id.hash(&mut self.hasher) - } - - fn encode_syntax_context(&mut self, syntax_context: rustc_span::SyntaxContext) { - syntax_context.hash(&mut self.hasher) - } - - fn encode_crate_num(&mut self, crate_num: CrateNum) { - crate_num.hash(&mut self.hasher) - } - - fn encode_def_index(&mut self, def_index: DefIndex) { - def_index.hash(&mut self.hasher) - } - - fn encode_def_id(&mut self, def_id: rustc_span::def_id::DefId) { - def_id.hash(&mut self.hasher) - } -} - -/// TokenStream needs to be hashable because it is used as a query key for caching derive macro -/// expansions. -impl Hash for TokenStream { - fn hash(&self, state: &mut H) { - Encodable::encode(self, &mut HashEncoder { hasher: state }); - } -} - /// Indicates whether a token can join with the following token to form a /// compound token. Used for conversions to `proc_macro::Spacing`. Also used to /// guide pretty-printing, which is where the `JointHidden` value (which isn't /// part of `proc_macro::Spacing`) comes in useful. -#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)] +#[derive(Clone, Copy, Debug, PartialEq, Hash, Encodable, Decodable, HashStable_Generic)] pub enum Spacing { /// The token cannot join with the following token to form a compound /// token. @@ -1075,7 +978,7 @@ impl TokenCursor { } } -#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic, Walkable)] +#[derive(Debug, Copy, Clone, PartialEq, Hash, Encodable, Decodable, HashStable_Generic, Walkable)] pub struct DelimSpan { pub open: Span, pub close: Span, @@ -1099,7 +1002,7 @@ impl DelimSpan { } } -#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)] +#[derive(Copy, Clone, Debug, PartialEq, Hash, Encodable, Decodable, HashStable_Generic)] pub struct DelimSpacing { pub open: Spacing, pub close: Spacing, diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 33baca2487db1..cb369c65b5efa 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -168,7 +168,7 @@ rustc_queries! { /// Caches the expansion of a derive proc macro, e.g. `#[derive(Serialize)]`. /// The key is: /// - A unique key corresponding to the invocation of a macro. - /// - Strict Version Hash of a crate. + /// - Strict Version Hash of the crate defining the proc macro. /// - Token stream which serves as an input to the macro. /// /// The output is the token stream generated by the proc macro.