Auto merge of rust-lang#105462 - oli-obk:feeding_full, r=cjgillot,petrochenkov

give the resolver access to TyCtxt

The resolver is now created after TyCtxt is created. Then macro expansion and name resolution are run and the results fed into queries just like before this PR.

Since the resolver had (before this PR) mutable access to the `CStore` and the source span table, these two datastructures are now behind a `RwLock`. To ensure that these are not mutated anymore after the resolver is done, a read lock to them is leaked right after the resolver finishes.

### PRs split out of this one and leading up to it:

* rust-lang#105423
* rust-lang#105357
* rust-lang#105603
* rust-lang#106776
* rust-lang#106810
* rust-lang#106812
* rust-lang#108032

Author: bors

Cargo.lock

@@ -4180,6 +4180,7 @@ dependencies = [
  "rustc_hir_analysis",
  "rustc_hir_typeck",
  "rustc_incremental",
+ "rustc_index",
  "rustc_lint",
  "rustc_macros",
  "rustc_metadata",

compiler/rustc_codegen_ssa/src/back/metadata.rs

@@ -33,6 +33,7 @@ use rustc_target::spec::{RelocModel, Target};
 /// <dt>dylib</dt>
 /// <dd>The metadata can be found in the `.rustc` section of the shared library.</dd>
 /// </dl>
+#[derive(Debug)]
 pub struct DefaultMetadataLoader;
 
 fn load_metadata_with(

compiler/rustc_driver_impl/src/lib.rs

@@ -320,7 +320,7 @@ fn run_compiler(
             }
 
             // Make sure name resolution and macro expansion is run.
-            queries.global_ctxt()?;
+            queries.global_ctxt()?.enter(|tcx| tcx.resolver_for_lowering(()));
 
             if callbacks.after_expansion(compiler, queries) == Compilation::Stop {
                 return early_exit();

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -16,7 +16,6 @@ use rustc_ast_pretty::pprust;
 use rustc_attr::{self as attr, TransparencyError};
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_errors::{Applicability, ErrorGuaranteed};
-use rustc_feature::Features;
 use rustc_lint_defs::builtin::{
     RUST_2021_INCOMPATIBLE_OR_PATTERNS, SEMICOLON_IN_EXPRESSIONS_FROM_MACROS,
 };
@@ -379,7 +378,6 @@ pub(super) fn try_match_macro<'matcher, T: Tracker<'matcher>>(
 /// Converts a macro item into a syntax extension.
 pub fn compile_declarative_macro(
     sess: &Session,
-    features: &Features,
     def: &ast::Item,
     edition: Edition,
 ) -> (SyntaxExtension, Vec<(usize, Span)>) {
@@ -508,7 +506,7 @@ pub fn compile_declarative_macro(
                         true,
                         &sess.parse_sess,
                         def.id,
-                        features,
+                        sess.features_untracked(),
                         edition,
                     )
                     .pop()
@@ -532,7 +530,7 @@ pub fn compile_declarative_macro(
                         false,
                         &sess.parse_sess,
                         def.id,
-                        features,
+                        sess.features_untracked(),
                         edition,
                     )
                     .pop()

compiler/rustc_interface/Cargo.toml

@@ -24,6 +24,7 @@ rustc_middle = { path = "../rustc_middle" }
 rustc_ast_lowering = { path = "../rustc_ast_lowering" }
 rustc_ast_passes = { path = "../rustc_ast_passes" }
 rustc_incremental = { path = "../rustc_incremental" }
+rustc_index = { path = "../rustc_index" }
 rustc_traits = { path = "../rustc_traits" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }

compiler/rustc_interface/src/passes.rs

@@ -8,11 +8,12 @@ use rustc_ast::{self as ast, visit};
 use rustc_borrowck as mir_borrowck;
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_data_structures::parallel;
+use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{Lrc, OnceCell, WorkerLocal};
-use rustc_errors::{ErrorGuaranteed, PResult};
+use rustc_errors::PResult;
 use rustc_expand::base::{ExtCtxt, LintStoreExpand, ResolverExpand};
 use rustc_hir::def_id::{StableCrateId, LOCAL_CRATE};
-use rustc_lint::{BufferedEarlyLint, EarlyCheckNode, LintStore};
+use rustc_lint::{unerased_lint_store, BufferedEarlyLint, EarlyCheckNode, LintStore};
 use rustc_metadata::creader::CStore;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
@@ -171,14 +172,12 @@ impl LintStoreExpand for LintStoreExpandImpl<'_> {
 /// syntax expansion, secondary `cfg` expansion, synthesis of a test
 /// harness if one is to be provided, injection of a dependency on the
 /// standard library and prelude, and name resolution.
-pub fn configure_and_expand(
-    sess: &Session,
-    lint_store: &LintStore,
-    mut krate: ast::Crate,
-    crate_name: Symbol,
-    resolver: &mut Resolver<'_, '_>,
-) -> Result<ast::Crate> {
-    trace!("configure_and_expand");
+#[instrument(level = "trace", skip(krate, resolver))]
+fn configure_and_expand(mut krate: ast::Crate, resolver: &mut Resolver<'_, '_>) -> ast::Crate {
+    let tcx = resolver.tcx();
+    let sess = tcx.sess;
+    let lint_store = unerased_lint_store(tcx);
+    let crate_name = tcx.crate_name(LOCAL_CRATE);
     pre_expansion_lint(sess, lint_store, resolver.registered_tools(), &krate, crate_name);
     rustc_builtin_macros::register_builtin_macros(resolver);
 
@@ -249,20 +248,19 @@ pub fn configure_and_expand(
             ecx.check_unused_macros();
         });
 
-        let recursion_limit_hit = ecx.reduced_recursion_limit.is_some();
+        // If we hit a recursion limit, exit early to avoid later passes getting overwhelmed
+        // with a large AST
+        if ecx.reduced_recursion_limit.is_some() {
+            sess.abort_if_errors();
+            unreachable!();
+        }
 
         if cfg!(windows) {
             env::set_var("PATH", &old_path);
         }
 
-        if recursion_limit_hit {
-            // If we hit a recursion limit, exit early to avoid later passes getting overwhelmed
-            // with a large AST
-            Err(ErrorGuaranteed::unchecked_claim_error_was_emitted())
-        } else {
-            Ok(krate)
-        }
-    })?;
+        krate
+    });
 
     sess.time("maybe_building_test_harness", || {
         rustc_builtin_macros::test_harness::inject(sess, resolver, &mut krate)
@@ -365,7 +363,7 @@ pub fn configure_and_expand(
         )
     });
 
-    Ok(krate)
+    krate
 }
 
 // Returns all the paths that correspond to generated files.
@@ -564,6 +562,28 @@ fn write_out_deps(
     }
 }
 
+fn resolver_for_lowering<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    (): (),
+) -> &'tcx Steal<(ty::ResolverAstLowering, Lrc<ast::Crate>)> {
+    let arenas = Resolver::arenas();
+    let krate = tcx.crate_for_resolver(()).steal();
+    let mut resolver = Resolver::new(tcx, &krate, &arenas);
+    let krate = configure_and_expand(krate, &mut resolver);
+
+    // Make sure we don't mutate the cstore from here on.
+    tcx.untracked().cstore.leak();
+
+    let ty::ResolverOutputs {
+        global_ctxt: untracked_resolutions,
+        ast_lowering: untracked_resolver_for_lowering,
+    } = resolver.into_outputs();
+
+    let feed = tcx.feed_unit_query();
+    feed.resolutions(tcx.arena.alloc(untracked_resolutions));
+    tcx.arena.alloc(Steal::new((untracked_resolver_for_lowering, Lrc::new(krate))))
+}
+
 fn output_filenames(tcx: TyCtxt<'_>, (): ()) -> Arc<OutputFilenames> {
     let sess = tcx.sess;
     let _timer = sess.timer("prepare_outputs");
@@ -597,7 +617,7 @@ fn output_filenames(tcx: TyCtxt<'_>, (): ()) -> Arc<OutputFilenames> {
         }
     }
 
-    write_out_deps(sess, tcx.cstore_untracked(), &outputs, &output_paths);
+    write_out_deps(sess, &*tcx.cstore_untracked(), &outputs, &output_paths);
 
     let only_dep_info = sess.opts.output_types.contains_key(&OutputType::DepInfo)
         && sess.opts.output_types.len() == 1;
@@ -618,6 +638,7 @@ pub static DEFAULT_QUERY_PROVIDERS: LazyLock<Providers> = LazyLock::new(|| {
     providers.analysis = analysis;
     providers.hir_crate = rustc_ast_lowering::lower_to_hir;
     providers.output_filenames = output_filenames;
+    providers.resolver_for_lowering = resolver_for_lowering;
     proc_macro_decls::provide(providers);
     rustc_const_eval::provide(providers);
     rustc_middle::hir::provide(providers);

compiler/rustc_interface/src/queries.rs

@@ -7,16 +7,19 @@ use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::CodegenResults;
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::{Lrc, OnceCell, WorkerLocal};
-use rustc_hir::def_id::LOCAL_CRATE;
+use rustc_data_structures::sync::{Lrc, OnceCell, RwLock, WorkerLocal};
+use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE};
+use rustc_hir::definitions::Definitions;
 use rustc_incremental::DepGraphFuture;
+use rustc_index::vec::IndexVec;
 use rustc_lint::LintStore;
+use rustc_metadata::creader::CStore;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
-use rustc_middle::ty::{self, GlobalCtxt, TyCtxt};
+use rustc_middle::ty::{GlobalCtxt, TyCtxt};
 use rustc_query_impl::Queries as TcxQueries;
-use rustc_resolve::Resolver;
 use rustc_session::config::{self, OutputFilenames, OutputType};
+use rustc_session::cstore::Untracked;
 use rustc_session::{output::find_crate_name, Session};
 use rustc_span::symbol::sym;
 use rustc_span::Symbol;
@@ -187,35 +190,18 @@ impl<'tcx> Queries<'tcx> {
         self.gcx.compute(|| {
             let crate_name = *self.crate_name()?.borrow();
             let (krate, lint_store) = self.register_plugins()?.steal();
-            let (krate, resolver_outputs) = {
-                let _timer = self.session().timer("configure_and_expand");
-                let sess = self.session();
-
-                let arenas = Resolver::arenas();
-                let mut resolver = Resolver::new(
-                    sess,
-                    &krate,
-                    crate_name,
-                    self.codegen_backend().metadata_loader(),
-                    &arenas,
-                );
-                let krate = passes::configure_and_expand(
-                    sess,
-                    &lint_store,
-                    krate,
-                    crate_name,
-                    &mut resolver,
-                )?;
-                (Lrc::new(krate), resolver.into_outputs())
-            };
-
-            let ty::ResolverOutputs {
-                untracked,
-                global_ctxt: untracked_resolutions,
-                ast_lowering: untracked_resolver_for_lowering,
-            } = resolver_outputs;
 
-            let gcx = passes::create_global_ctxt(
+            let sess = self.session();
+
+            let cstore = RwLock::new(Box::new(CStore::new(sess)) as _);
+            let definitions = RwLock::new(Definitions::new(sess.local_stable_crate_id()));
+            let mut source_span = IndexVec::default();
+            let _id = source_span.push(krate.spans.inner_span);
+            debug_assert_eq!(_id, CRATE_DEF_ID);
+            let source_span = RwLock::new(source_span);
+            let untracked = Untracked { cstore, source_span, definitions };
+
+            let qcx = passes::create_global_ctxt(
                 self.compiler,
                 lint_store,
                 self.dep_graph()?.steal(),
@@ -226,17 +212,18 @@ impl<'tcx> Queries<'tcx> {
                 &self.hir_arena,
             );
 
-            gcx.enter(|tcx| {
+            qcx.enter(|tcx| {
+                let feed = tcx.feed_local_crate();
+                feed.crate_name(crate_name);
+
                 let feed = tcx.feed_unit_query();
-                feed.resolver_for_lowering(
-                    tcx.arena.alloc(Steal::new((untracked_resolver_for_lowering, krate))),
+                feed.crate_for_resolver(tcx.arena.alloc(Steal::new(krate)));
+                feed.metadata_loader(
+                    tcx.arena.alloc(Steal::new(self.codegen_backend().metadata_loader())),
                 );
-                feed.resolutions(tcx.arena.alloc(untracked_resolutions));
                 feed.features_query(tcx.sess.features_untracked());
-                let feed = tcx.feed_local_crate();
-                feed.crate_name(crate_name);
             });
-            Ok(gcx)
+            Ok(qcx)
         })
     }