Skip to content

Commit 90340b7

Browse files
borsbors
committed
Auto merge of #14405 - Veykril:proc-macro-loading, r=Veykril
feat: Load proc-macros asynchronously Closes #8646
2 parents 71b2336 + e5f24a6 commit 90340b7

File tree

23 files changed

+288
-227
lines changed

23 files changed

+288
-227
lines changed

‎crates/base-db/src/change.rs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@ use std::{fmt, sync::Arc};
66
use salsa::Durability;
77
use vfs::FileId;
88

9-
use crate::{CrateGraph, SourceDatabaseExt, SourceRoot, SourceRootId};
9+
use crate::{CrateGraph, ProcMacros, SourceDatabaseExt, SourceRoot, SourceRootId};
1010

1111
/// Encapsulate a bunch of raw `.set` calls on the database.
1212
#[derive(Default)]
1313
pub struct Change {
1414
pub roots: Option<Vec<SourceRoot>>,
1515
pub files_changed: Vec<(FileId, Option<Arc<String>>)>,
1616
pub crate_graph: Option<CrateGraph>,
17+
pub proc_macros: Option<ProcMacros>,
1718
}
1819

1920
impl fmt::Debug for Change {
@@ -49,6 +50,10 @@ impl Change {
4950
self.crate_graph = Some(graph);
5051
}
5152

53+
pub fn set_proc_macros(&mut self, proc_macros: ProcMacros) {
54+
self.proc_macros = Some(proc_macros);
55+
}
56+
5257
pub fn apply(self, db: &mut dyn SourceDatabaseExt) {
5358
let _p = profile::span("RootDatabase::apply_change");
5459
if let Some(roots) = self.roots {
@@ -73,6 +78,9 @@ impl Change {
7378
if let Some(crate_graph) = self.crate_graph {
7479
db.set_crate_graph_with_durability(Arc::new(crate_graph), Durability::HIGH)
7580
}
81+
if let Some(proc_macros) = self.proc_macros {
82+
db.set_proc_macros_with_durability(Arc::new(proc_macros), Durability::HIGH)
83+
}
7684
}
7785
}
7886

‎crates/base-db/src/fixture.rs

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ use vfs::{file_set::FileSet, VfsPath};
1212
use crate::{
1313
input::{CrateName, CrateOrigin, LangCrateOrigin},
1414
Change, CrateDisplayName, CrateGraph, CrateId, Dependency, Edition, Env, FileId, FilePosition,
15-
FileRange, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, SourceDatabaseExt,
16-
SourceRoot, SourceRootId,
15+
FileRange, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, ProcMacros,
16+
SourceDatabaseExt, SourceRoot, SourceRootId,
1717
};
1818

1919
pub const WORKSPACE: SourceRootId = SourceRootId(0);
@@ -100,7 +100,7 @@ impl ChangeFixture {
100100

101101
pub fn parse_with_proc_macros(
102102
ra_fixture: &str,
103-
mut proc_macros: Vec<(String, ProcMacro)>,
103+
mut proc_macro_defs: Vec<(String, ProcMacro)>,
104104
) -> ChangeFixture {
105105
let (mini_core, proc_macro_names, fixture) = Fixture::parse(ra_fixture);
106106
let mut change = Change::new();
@@ -160,7 +160,6 @@ impl ChangeFixture {
160160
meta.cfg.clone(),
161161
meta.cfg,
162162
meta.env,
163-
Ok(Vec::new()),
164163
false,
165164
origin,
166165
meta.target_data_layout
@@ -200,7 +199,6 @@ impl ChangeFixture {
200199
default_cfg.clone(),
201200
default_cfg,
202201
Env::default(),
203-
Ok(Vec::new()),
204202
false,
205203
CrateOrigin::CratesIo { repo: None, name: None },
206204
default_target_data_layout
@@ -244,7 +242,6 @@ impl ChangeFixture {
244242
CfgOptions::default(),
245243
CfgOptions::default(),
246244
Env::default(),
247-
Ok(Vec::new()),
248245
false,
249246
CrateOrigin::Lang(LangCrateOrigin::Core),
250247
target_layout.clone(),
@@ -257,12 +254,13 @@ impl ChangeFixture {
257254
}
258255
}
259256

257+
let mut proc_macros = ProcMacros::default();
260258
if !proc_macro_names.is_empty() {
261259
let proc_lib_file = file_id;
262260
file_id.0 += 1;
263261

264-
proc_macros.extend(default_test_proc_macros());
265-
let (proc_macro, source) = filter_test_proc_macros(&proc_macro_names, proc_macros);
262+
proc_macro_defs.extend(default_test_proc_macros());
263+
let (proc_macro, source) = filter_test_proc_macros(&proc_macro_names, proc_macro_defs);
266264
let mut fs = FileSet::default();
267265
fs.insert(
268266
proc_lib_file,
@@ -282,11 +280,11 @@ impl ChangeFixture {
282280
CfgOptions::default(),
283281
CfgOptions::default(),
284282
Env::default(),
285-
Ok(proc_macro),
286283
true,
287284
CrateOrigin::CratesIo { repo: None, name: None },
288285
target_layout,
289286
);
287+
proc_macros.insert(proc_macros_crate, Ok(proc_macro));
290288

291289
for krate in all_crates {
292290
crate_graph
@@ -305,6 +303,7 @@ impl ChangeFixture {
305303
roots.push(root);
306304
change.set_roots(roots);
307305
change.set_crate_graph(crate_graph);
306+
change.set_proc_macros(proc_macros);
308307

309308
ChangeFixture { file_position, files, change }
310309
}

‎crates/base-db/src/input.rs

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,17 @@
66
//! actual IO. See `vfs` and `project_model` in the `rust-analyzer` crate for how
77
//! actual IO is done and lowered to input.
88
9-
use std::{fmt, ops, panic::RefUnwindSafe, str::FromStr, sync::Arc};
9+
use std::{fmt, mem, ops, panic::RefUnwindSafe, str::FromStr, sync::Arc};
1010

1111
use cfg::CfgOptions;
1212
use rustc_hash::FxHashMap;
1313
use stdx::hash::{NoHashHashMap, NoHashHashSet};
1414
use syntax::SmolStr;
1515
use tt::token_id::Subtree;
16-
use vfs::{file_set::FileSet, AnchoredPath, FileId, VfsPath};
16+
use vfs::{file_set::FileSet, AbsPathBuf, AnchoredPath, FileId, VfsPath};
17+
18+
pub type ProcMacroPaths = FxHashMap<CrateId, Result<(Option<String>, AbsPathBuf), String>>;
19+
pub type ProcMacros = FxHashMap<CrateId, ProcMacroLoadResult>;
1720

1821
/// Files are grouped into source roots. A source root is a directory on the
1922
/// file systems which is watched for changes. Typically it corresponds to a
@@ -269,7 +272,6 @@ pub struct CrateData {
269272
pub target_layout: TargetLayoutLoadResult,
270273
pub env: Env,
271274
pub dependencies: Vec<Dependency>,
272-
pub proc_macro: ProcMacroLoadResult,
273275
pub origin: CrateOrigin,
274276
pub is_proc_macro: bool,
275277
}
@@ -322,7 +324,6 @@ impl CrateGraph {
322324
cfg_options: CfgOptions,
323325
potential_cfg_options: CfgOptions,
324326
env: Env,
325-
proc_macro: ProcMacroLoadResult,
326327
is_proc_macro: bool,
327328
origin: CrateOrigin,
328329
target_layout: Result<Arc<str>, Arc<str>>,
@@ -335,7 +336,6 @@ impl CrateGraph {
335336
cfg_options,
336337
potential_cfg_options,
337338
env,
338-
proc_macro,
339339
dependencies: Vec::new(),
340340
origin,
341341
target_layout,
@@ -456,11 +456,11 @@ impl CrateGraph {
456456
}
457457

458458
/// Extends this crate graph by adding a complete disjoint second crate
459-
/// graph.
459+
/// graph and adjust the ids in the [`ProcMacroPaths`] accordingly.
460460
///
461461
/// The ids of the crates in the `other` graph are shifted by the return
462462
/// amount.
463-
pub fn extend(&mut self, other: CrateGraph) -> u32 {
463+
pub fn extend(&mut self, other: CrateGraph, proc_macros: &mut ProcMacroPaths) -> u32 {
464464
let start = self.arena.len() as u32;
465465
self.arena.extend(other.arena.into_iter().map(|(id, mut data)| {
466466
let new_id = id.shift(start);
@@ -469,6 +469,11 @@ impl CrateGraph {
469469
}
470470
(new_id, data)
471471
}));
472+
473+
*proc_macros = mem::take(proc_macros)
474+
.into_iter()
475+
.map(|(id, macros)| (id.shift(start), macros))
476+
.collect();
472477
start
473478
}
474479

@@ -645,7 +650,6 @@ mod tests {
645650
CfgOptions::default(),
646651
CfgOptions::default(),
647652
Env::default(),
648-
Ok(Vec::new()),
649653
false,
650654
CrateOrigin::CratesIo { repo: None, name: None },
651655
Err("".into()),
@@ -658,7 +662,6 @@ mod tests {
658662
CfgOptions::default(),
659663
CfgOptions::default(),
660664
Env::default(),
661-
Ok(Vec::new()),
662665
false,
663666
CrateOrigin::CratesIo { repo: None, name: None },
664667
Err("".into()),
@@ -671,7 +674,6 @@ mod tests {
671674
CfgOptions::default(),
672675
CfgOptions::default(),
673676
Env::default(),
674-
Ok(Vec::new()),
675677
false,
676678
CrateOrigin::CratesIo { repo: None, name: None },
677679
Err("".into()),
@@ -698,7 +700,6 @@ mod tests {
698700
CfgOptions::default(),
699701
CfgOptions::default(),
700702
Env::default(),
701-
Ok(Vec::new()),
702703
false,
703704
CrateOrigin::CratesIo { repo: None, name: None },
704705
Err("".into()),
@@ -711,7 +712,6 @@ mod tests {
711712
CfgOptions::default(),
712713
CfgOptions::default(),
713714
Env::default(),
714-
Ok(Vec::new()),
715715
false,
716716
CrateOrigin::CratesIo { repo: None, name: None },
717717
Err("".into()),
@@ -735,7 +735,6 @@ mod tests {
735735
CfgOptions::default(),
736736
CfgOptions::default(),
737737
Env::default(),
738-
Ok(Vec::new()),
739738
false,
740739
CrateOrigin::CratesIo { repo: None, name: None },
741740
Err("".into()),
@@ -748,7 +747,6 @@ mod tests {
748747
CfgOptions::default(),
749748
CfgOptions::default(),
750749
Env::default(),
751-
Ok(Vec::new()),
752750
false,
753751
CrateOrigin::CratesIo { repo: None, name: None },
754752
Err("".into()),
@@ -761,7 +759,6 @@ mod tests {
761759
CfgOptions::default(),
762760
CfgOptions::default(),
763761
Env::default(),
764-
Ok(Vec::new()),
765762
false,
766763
CrateOrigin::CratesIo { repo: None, name: None },
767764
Err("".into()),
@@ -785,7 +782,6 @@ mod tests {
785782
CfgOptions::default(),
786783
CfgOptions::default(),
787784
Env::default(),
788-
Ok(Vec::new()),
789785
false,
790786
CrateOrigin::CratesIo { repo: None, name: None },
791787
Err("".into()),
@@ -798,7 +794,6 @@ mod tests {
798794
CfgOptions::default(),
799795
CfgOptions::default(),
800796
Env::default(),
801-
Ok(Vec::new()),
802797
false,
803798
CrateOrigin::CratesIo { repo: None, name: None },
804799
Err("".into()),

‎crates/base-db/src/lib.rs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ pub use crate::{
1616
input::{
1717
CrateData, CrateDisplayName, CrateGraph, CrateId, CrateName, CrateOrigin, Dependency,
1818
Edition, Env, LangCrateOrigin, ProcMacro, ProcMacroExpander, ProcMacroExpansionError,
19-
ProcMacroId, ProcMacroKind, ProcMacroLoadResult, SourceRoot, SourceRootId,
20-
TargetLayoutLoadResult,
19+
ProcMacroId, ProcMacroKind, ProcMacroLoadResult, ProcMacroPaths, ProcMacros, SourceRoot,
20+
SourceRootId, TargetLayoutLoadResult,
2121
},
2222
};
2323
pub use salsa::{self, Cancelled};
@@ -73,6 +73,10 @@ pub trait SourceDatabase: FileLoader + std::fmt::Debug {
7373
/// The crate graph.
7474
#[salsa::input]
7575
fn crate_graph(&self) -> Arc<CrateGraph>;
76+
77+
/// The crate graph.
78+
#[salsa::input]
79+
fn proc_macros(&self) -> Arc<ProcMacros>;
7680
}
7781

7882
fn parse_query(db: &dyn SourceDatabase, file_id: FileId) -> Parse<ast::SourceFile> {

‎crates/hir-def/src/nameres/collector.rs

Lines changed: 27 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -78,25 +78,35 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, mut def_map: DefMap, tree_id: T
7878
}
7979

8080
let cfg_options = &krate.cfg_options;
81-
let proc_macros = match &krate.proc_macro {
82-
Ok(proc_macros) => {
83-
proc_macros
84-
.iter()
85-
.enumerate()
86-
.map(|(idx, it)| {
87-
// FIXME: a hacky way to create a Name from string.
88-
let name =
89-
tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() };
90-
(name.as_name(), ProcMacroExpander::new(base_db::ProcMacroId(idx as u32)))
91-
})
92-
.collect()
93-
}
94-
Err(e) => {
95-
def_map.proc_macro_loading_error = Some(e.clone().into_boxed_str());
96-
Vec::new()
81+
82+
let is_proc_macro = krate.is_proc_macro;
83+
let proc_macros = if is_proc_macro {
84+
match db.proc_macros().get(&def_map.krate) {
85+
Some(Ok(proc_macros)) => {
86+
proc_macros
87+
.iter()
88+
.enumerate()
89+
.map(|(idx, it)| {
90+
// FIXME: a hacky way to create a Name from string.
91+
let name =
92+
tt::Ident { text: it.name.clone(), span: tt::TokenId::unspecified() };
93+
(name.as_name(), ProcMacroExpander::new(base_db::ProcMacroId(idx as u32)))
94+
})
95+
.collect()
96+
}
97+
Some(Err(e)) => {
98+
def_map.proc_macro_loading_error = Some(e.clone().into_boxed_str());
99+
Vec::new()
100+
}
101+
None => {
102+
def_map.proc_macro_loading_error =
103+
Some("No proc-macros present for crate".to_owned().into_boxed_str());
104+
Vec::new()
105+
}
97106
}
107+
} else {
108+
vec![]
98109
};
99-
let is_proc_macro = krate.is_proc_macro;
100110

101111
let mut collector = DefCollector {
102112
db,

‎crates/hir-expand/src/proc_macro.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,10 @@ impl ProcMacroExpander {
3333
) -> ExpandResult<tt::Subtree> {
3434
match self.proc_macro_id {
3535
Some(id) => {
36-
let krate_graph = db.crate_graph();
37-
let proc_macros = match &krate_graph[def_crate].proc_macro {
38-
Ok(proc_macros) => proc_macros,
39-
Err(_) => {
36+
let proc_macros = db.proc_macros();
37+
let proc_macros = match proc_macros.get(&def_crate) {
38+
Some(Ok(proc_macros)) => proc_macros,
39+
Some(Err(_)) | None => {
4040
never!("Non-dummy expander even though there are no proc macros");
4141
return ExpandResult::with_err(
4242
tt::Subtree::empty(),
@@ -59,6 +59,7 @@ impl ProcMacroExpander {
5959
}
6060
};
6161

62+
let krate_graph = db.crate_graph();
6263
// Proc macros have access to the environment variables of the invoking crate.
6364
let env = &krate_graph[calling_crate].env;
6465
match proc_macro.expander.expand(tt, attr_arg, env) {

‎crates/ide-db/src/apply_change.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ impl RootDatabase {
6464
// SourceDatabase
6565
base_db::ParseQuery
6666
base_db::CrateGraphQuery
67+
base_db::ProcMacrosQuery
6768

6869
// SourceDatabaseExt
6970
base_db::FileTextQuery

‎crates/ide-db/src/lib.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ impl RootDatabase {
137137
pub fn new(lru_capacity: Option<usize>) -> RootDatabase {
138138
let mut db = RootDatabase { storage: ManuallyDrop::new(salsa::Storage::default()) };
139139
db.set_crate_graph_with_durability(Default::default(), Durability::HIGH);
140+
db.set_proc_macros_with_durability(Default::default(), Durability::HIGH);
140141
db.set_local_roots_with_durability(Default::default(), Durability::HIGH);
141142
db.set_library_roots_with_durability(Default::default(), Durability::HIGH);
142143
db.set_enable_proc_attr_macros(false);

‎crates/ide/src/lib.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,6 @@ impl Analysis {
235235
cfg_options.clone(),
236236
cfg_options,
237237
Env::default(),
238-
Ok(Vec::new()),
239238
false,
240239
CrateOrigin::CratesIo { repo: None, name: None },
241240
Err("Analysis::from_single_file has no target layout".into()),

0 commit comments

Comments
 (0)