Auto merge of #114669 - cjgillot:metadata-wp, r=<try>

Make metadata a workproduct and reuse it

This PR aims to skip the generation of metadata by reusing the infrastructure that already exists for compiled codegen-units, namely "workproducts".

This can yield substantial gains (~10%) when we can demonstrate that metadata does not change between an incremental session and the next. This is the case if the crate is unchanged, or if all the changes are in upstream crates and have no effect on it. This latter case is most interesting, as it arises regularly for users with several crates in their workspace.

TODO:
- [x] Materialize the fact that metadata encoding relies on the relative order of definitions;
- [x] Refactor the handling of doc links.

Author: bors

Cargo.lock

@@ -4059,6 +4059,7 @@ dependencies = [
  "rustc_fs_util",
  "rustc_hir",
  "rustc_hir_pretty",
+ "rustc_incremental",
  "rustc_index",
  "rustc_macros",
  "rustc_middle",

compiler/rustc_interface/src/queries.rs

@@ -45,11 +45,24 @@ impl Linker {
     }
 
     pub fn link(self, sess: &Session, codegen_backend: &dyn CodegenBackend) {
-        let (codegen_results, work_products) = sess.time("finish_ongoing_codegen", || {
+        let (codegen_results, mut work_products) = sess.time("finish_ongoing_codegen", || {
             codegen_backend.join_codegen(self.ongoing_codegen, sess, &self.output_filenames)
         });
         sess.timings.end_section(sess.dcx(), TimingSection::Codegen);
 
+        if sess.opts.incremental.is_some()
+            && let Some(path) = self.metadata.path()
+            && let Some((id, product)) =
+                rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
+                    sess,
+                    "metadata",
+                    &[("rmeta", path)],
+                    &[],
+                )
+        {
+            work_products.insert(id, product);
+        }
+
         sess.dcx().abort_if_errors();
 
         let _timer = sess.timer("link");

compiler/rustc_metadata/Cargo.toml

@@ -20,6 +20,7 @@ rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
+rustc_incremental = { path = "../rustc_incremental" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1596,6 +1596,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 let table = tcx.associated_types_for_impl_traits_in_associated_fn(def_id);
                 record_defaulted_array!(self.tables.associated_types_for_impl_traits_in_associated_fn[def_id] <- table);
             }
+            if let DefKind::Mod = tcx.def_kind(def_id) {
+                record!(self.tables.doc_link_resolutions[def_id] <- tcx.doc_link_resolutions(def_id));
+                record_array!(self.tables.doc_link_traits_in_scope[def_id] <- tcx.doc_link_traits_in_scope(def_id));
+            }
         }
 
         for (def_id, impls) in &tcx.crate_inherent_impls(()).0.inherent_impls {
@@ -1604,14 +1608,6 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 def_id.index
             }));
         }
-
-        for (def_id, res_map) in &tcx.resolutions(()).doc_link_resolutions {
-            record!(self.tables.doc_link_resolutions[def_id.to_def_id()] <- res_map);
-        }
-
-        for (def_id, traits) in &tcx.resolutions(()).doc_link_traits_in_scope {
-            record_array!(self.tables.doc_link_traits_in_scope[def_id.to_def_id()] <- traits);
-        }
     }
 
     #[instrument(level = "trace", skip(self))]
@@ -2283,6 +2279,8 @@ pub struct EncodedMetadata {
     // This is an optional stub metadata containing only the crate header.
     // The header should be very small, so we load it directly into memory.
     stub_metadata: Option<Vec<u8>>,
+    // The path containing the metadata, to record as work product.
+    path: Option<Box<Path>>,
     // We need to carry MaybeTempDir to avoid deleting the temporary
     // directory while accessing the Mmap.
     _temp_dir: Option<MaybeTempDir>,
@@ -2298,14 +2296,24 @@ impl EncodedMetadata {
         let file = std::fs::File::open(&path)?;
         let file_metadata = file.metadata()?;
         if file_metadata.len() == 0 {
-            return Ok(Self { full_metadata: None, stub_metadata: None, _temp_dir: None });
+            return Ok(Self {
+                full_metadata: None,
+                stub_metadata: None,
+                path: None,
+                _temp_dir: None,
+            });
         }
         let full_mmap = unsafe { Some(Mmap::map(file)?) };
 
         let stub =
             if let Some(stub_path) = stub_path { Some(std::fs::read(stub_path)?) } else { None };
 
-        Ok(Self { full_metadata: full_mmap, stub_metadata: stub, _temp_dir: temp_dir })
+        Ok(Self {
+            full_metadata: full_mmap,
+            stub_metadata: stub,
+            path: Some(path.into()),
+            _temp_dir: temp_dir,
+        })
     }
 
     #[inline]
@@ -2317,6 +2325,11 @@ impl EncodedMetadata {
     pub fn stub_or_full(&self) -> &[u8] {
         self.stub_metadata.as_deref().unwrap_or(self.full())
     }
+
+    #[inline]
+    pub fn path(&self) -> Option<&Path> {
+        self.path.as_deref()
+    }
 }
 
 impl<S: Encoder> Encodable<S> for EncodedMetadata {
@@ -2341,11 +2354,47 @@ impl<D: Decoder> Decodable<D> for EncodedMetadata {
             None
         };
 
-        Self { full_metadata, stub_metadata: stub, _temp_dir: None }
+        Self { full_metadata, stub_metadata: stub, path: None, _temp_dir: None }
     }
 }
 
+#[instrument(level = "trace", skip(tcx))]
 pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) {
+    if let Some(ref_path) = ref_path {
+        let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata_stub");
+
+        with_encode_metadata_header(tcx, ref_path, |ecx| {
+            let header: LazyValue<CrateHeader> = ecx.lazy(CrateHeader {
+                name: tcx.crate_name(LOCAL_CRATE),
+                triple: tcx.sess.opts.target_triple.clone(),
+                hash: tcx.crate_hash(LOCAL_CRATE),
+                is_proc_macro_crate: false,
+                is_stub: true,
+            });
+            header.position.get()
+        });
+    }
+
+    let dep_node = tcx.metadata_dep_node();
+
+    if tcx.dep_graph.is_fully_enabled()
+        && let work_product_id = &rustc_middle::dep_graph::WorkProductId::from_cgu_name("metadata")
+        && let Some(work_product) = tcx.dep_graph.previous_work_product(work_product_id)
+        && tcx.try_mark_green(&dep_node)
+    {
+        let saved_path = &work_product.saved_files["rmeta"];
+        let incr_comp_session_dir = tcx.sess.incr_comp_session_dir_opt().unwrap();
+        let source_file = rustc_incremental::in_incr_comp_dir(&incr_comp_session_dir, saved_path);
+        debug!("copying preexisting metadata from {source_file:?} to {path:?}");
+        match rustc_fs_util::link_or_copy(&source_file, path) {
+            Ok(_) => {}
+            Err(err) => {
+                tcx.dcx().emit_fatal(FailCreateFileEncoder { err });
+            }
+        };
+        return;
+    };
+
     let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata");
 
     // Since encoding metadata is not in a query, and nothing is cached,
@@ -2359,35 +2408,30 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) {
         join(|| prefetch_mir(tcx), || tcx.exported_symbols(LOCAL_CRATE));
     }
 
-    with_encode_metadata_header(tcx, path, |ecx| {
-        // Encode all the entries and extra information in the crate,
-        // culminating in the `CrateRoot` which points to all of it.
-        let root = ecx.encode_crate_root();
-
-        // Flush buffer to ensure backing file has the correct size.
-        ecx.opaque.flush();
-        // Record metadata size for self-profiling
-        tcx.prof.artifact_size(
-            "crate_metadata",
-            "crate_metadata",
-            ecx.opaque.file().metadata().unwrap().len(),
-        );
-
-        root.position.get()
-    });
+    tcx.dep_graph.with_task(
+        dep_node,
+        tcx,
+        path,
+        |tcx, path| {
+            with_encode_metadata_header(tcx, path, |ecx| {
+                // Encode all the entries and extra information in the crate,
+                // culminating in the `CrateRoot` which points to all of it.
+                let root = ecx.encode_crate_root();
+
+                // Flush buffer to ensure backing file has the correct size.
+                ecx.opaque.flush();
+                // Record metadata size for self-profiling
+                tcx.prof.artifact_size(
+                    "crate_metadata",
+                    "crate_metadata",
+                    ecx.opaque.file().metadata().unwrap().len(),
+                );
 
-    if let Some(ref_path) = ref_path {
-        with_encode_metadata_header(tcx, ref_path, |ecx| {
-            let header: LazyValue<CrateHeader> = ecx.lazy(CrateHeader {
-                name: tcx.crate_name(LOCAL_CRATE),
-                triple: tcx.sess.opts.target_triple.clone(),
-                hash: tcx.crate_hash(LOCAL_CRATE),
-                is_proc_macro_crate: false,
-                is_stub: true,
+                root.position.get()
             });
-            header.position.get()
-        });
-    }
+        },
+        None,
+    );
 }
 
 fn with_encode_metadata_header(

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -98,6 +98,7 @@ rustc_with_all_queries!(define_dep_nodes![
     [] fn TraitSelect() -> (),
     [] fn CompileCodegenUnit() -> (),
     [] fn CompileMonoItem() -> (),
+    [] fn Metadata() -> (),
 ]);
 
 // WARNING: `construct` is generic and does not know that `CompileCodegenUnit` takes `Symbol`s as keys.
@@ -115,6 +116,12 @@ pub(crate) fn make_compile_mono_item<'tcx>(
     DepNode::construct(tcx, dep_kinds::CompileMonoItem, mono_item)
 }
 
+// WARNING: `construct` is generic and does not know that `Metadata` takes `()`s as keys.
+// Be very careful changing this type signature!
+pub(crate) fn make_metadata(tcx: TyCtxt<'_>) -> DepNode {
+    DepNode::construct(tcx, dep_kinds::Metadata, &())
+}
+
 pub trait DepNodeExt: Sized {
     fn extract_def_id(&self, tcx: TyCtxt<'_>) -> Option<DefId>;
 

compiler/rustc_middle/src/dep_graph/mod.rs

@@ -9,7 +9,7 @@ use crate::ty::{self, TyCtxt};
 mod dep_node;
 
 pub use dep_node::{DepKind, DepNode, DepNodeExt, dep_kinds, label_strs};
-pub(crate) use dep_node::{make_compile_codegen_unit, make_compile_mono_item};
+pub(crate) use dep_node::{make_compile_codegen_unit, make_compile_mono_item, make_metadata};
 pub use rustc_query_system::dep_graph::debug::{DepNodeFilter, EdgeFilter};
 pub use rustc_query_system::dep_graph::{
     DepContext, DepGraphQuery, DepNodeIndex, Deps, SerializedDepGraph, SerializedDepNodeIndex,

compiler/rustc_middle/src/query/mod.rs

@@ -204,6 +204,16 @@ rustc_queries! {
         desc { "getting the resolver for lowering" }
     }
 
+    /// Named module children from all kinds of items, including imports.
+    /// In addition to regular items this list also includes struct and variant constructors, and
+    /// items inside `extern {}` blocks because all of them introduce names into parent module.
+    ///
+    /// Module here is understood in name resolution sense - it can be a `mod` item,
+    /// or a crate root, or an enum, or a trait.
+    query module_children_local(key: LocalDefId) -> &'tcx [ModChild] {
+        desc { |tcx| "module exports for `{}`", tcx.def_path_str(key) }
+    }
+
     /// Return the span for a definition.
     ///
     /// Contrary to `def_span` below, this query returns the full absolute span of the definition.

compiler/rustc_middle/src/ty/context.rs

@@ -63,7 +63,6 @@ use crate::arena::Arena;
 use crate::dep_graph::{DepGraph, DepKindStruct};
 use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarKind, CanonicalVarKinds};
 use crate::lint::lint_level;
-use crate::metadata::ModChild;
 use crate::middle::codegen_fn_attrs::{CodegenFnAttrs, TargetFeature};
 use crate::middle::{resolve_bound_vars, stability};
 use crate::mir::interpret::{self, Allocation, ConstAllocation};
@@ -2089,9 +2088,8 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn iter_local_def_id(self) -> impl Iterator<Item = LocalDefId> {
-        // Create a dependency to the red node to be sure we re-execute this when the amount of
-        // definitions change.
-        self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
+        // Depend on the `analysis` query to ensure compilation if finished.
+        self.ensure_ok().analysis(());
 
         let definitions = &self.untracked.definitions;
         gen {
@@ -2111,9 +2109,8 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn def_path_table(self) -> &'tcx rustc_hir::definitions::DefPathTable {
-        // Create a dependency to the crate to be sure we re-execute this when the amount of
-        // definitions change.
-        self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
+        // Depend on the `analysis` query to ensure compilation if finished.
+        self.ensure_ok().analysis(());
 
         // Freeze definitions once we start iterating on them, to prevent adding new ones
         // while iterating. If some query needs to add definitions, it should be `ensure`d above.
@@ -3380,23 +3377,14 @@ impl<'tcx> TyCtxt<'tcx> {
         self.opt_rpitit_info(def_id).is_some()
     }
 
-    /// Named module children from all kinds of items, including imports.
-    /// In addition to regular items this list also includes struct and variant constructors, and
-    /// items inside `extern {}` blocks because all of them introduce names into parent module.
-    ///
-    /// Module here is understood in name resolution sense - it can be a `mod` item,
-    /// or a crate root, or an enum, or a trait.
-    ///
-    /// This is not a query, making it a query causes perf regressions
-    /// (probably due to hashing spans in `ModChild`ren).
-    pub fn module_children_local(self, def_id: LocalDefId) -> &'tcx [ModChild] {
-        self.resolutions(()).module_children.get(&def_id).map_or(&[], |v| &v[..])
-    }
-
     pub fn resolver_for_lowering(self) -> &'tcx Steal<(ty::ResolverAstLowering, Arc<ast::Crate>)> {
         self.resolver_for_lowering_raw(()).0
     }
 
+    pub fn metadata_dep_node(self) -> crate::dep_graph::DepNode {
+        crate::dep_graph::make_metadata(self)
+    }
+
     /// Given an `impl_id`, return the trait it implements.
     /// Return `None` if this is an inherent impl.
     pub fn impl_trait_ref(
@@ -3444,6 +3432,8 @@ pub struct DeducedParamAttrs {
 }
 
 pub fn provide(providers: &mut Providers) {
+    providers.module_children_local =
+        |tcx, def_id| tcx.resolutions(()).module_children.get(&def_id).map_or(&[], |v| &v[..]);
     providers.maybe_unused_trait_imports =
         |tcx, ()| &tcx.resolutions(()).maybe_unused_trait_imports;
     providers.names_imported_by_glob_use = |tcx, id| {

compiler/rustc_query_impl/src/plumbing.rs

@@ -923,6 +923,17 @@ macro_rules! define_queries {
                 }
             }
 
+            pub(crate) fn Metadata<'tcx>() -> DepKindStruct<'tcx> {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Unit,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                    name: &"Metadata",
+                }
+            }
+
             $(pub(crate) fn $name<'tcx>()-> DepKindStruct<'tcx> {
                 $crate::plumbing::query_callback::<query_impl::$name::QueryType<'tcx>>(
                     is_anon!([$($modifiers)*]),

compiler/rustc_resolve/src/late.rs

@@ -1559,12 +1559,15 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
     }
 
     fn with_mod_rib<T>(&mut self, id: NodeId, f: impl FnOnce(&mut Self) -> T) -> T {
-        let module = self.r.expect_module(self.r.local_def_id(id).to_def_id());
+        let def_id = self.r.local_def_id(id);
+        let module = self.r.expect_module(def_id.to_def_id());
         // Move down in the graph.
         let orig_module = replace(&mut self.parent_scope.module, module);
         self.with_rib(ValueNS, RibKind::Module(module), |this| {
             this.with_rib(TypeNS, RibKind::Module(module), |this| {
                 let ret = f(this);
+                this.r.doc_link_resolutions.entry(def_id).or_default();
+                this.r.doc_link_traits_in_scope.entry(def_id).or_default();
                 this.parent_scope.module = orig_module;
                 ret
             })
@@ -5247,6 +5250,8 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                 BuiltinLintDiag::UnusedLabel,
             );
         }
+        self.doc_link_resolutions.entry(CRATE_DEF_ID).or_default();
+        self.doc_link_traits_in_scope.entry(CRATE_DEF_ID).or_default();
     }
 }