Auto merge of rust-lang#138705 - oli-obk:hir-split, r=<try>

[perf] Decouple directly accessing a HIR owner from ast lowering

r? `@ghost`

cc rust-lang#95004

Author: bors

compiler/rustc_ast_lowering/src/lib.rs

@@ -429,7 +429,7 @@ fn compute_hir_hash(
     })
 }
 
-pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
+pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate {
     let sess = tcx.sess;
     // Queries that borrow `resolver_for_lowering`.
     tcx.ensure_done().output_filenames(());
@@ -466,7 +466,12 @@ pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
     // Don't hash unless necessary, because it's expensive.
     let opt_hir_hash =
         if tcx.needs_crate_hash() { Some(compute_hir_hash(tcx, &owners)) } else { None };
-    hir::Crate { owners, opt_hir_hash }
+
+    for (def_id, owner) in owners.drain_enumerated(..) {
+        tcx.super_duper_perf_hack_experiment(def_id).hir_owner(owner)
+    }
+
+    hir::Crate { opt_hir_hash }
 }
 
 #[derive(Copy, Clone, PartialEq, Debug)]

compiler/rustc_hir/src/definitions.rs

@@ -88,6 +88,10 @@ impl DefPathTable {
         DefPathHash::new(self.stable_crate_id, hash)
     }
 
+    pub fn def_keys(&self) -> impl Iterator<Item = DefIndex> + ExactSizeIterator {
+        self.index_to_key.iter_enumerated().map(|(index, _)| index)
+    }
+
     pub fn enumerated_keys_and_path_hashes(
         &self,
     ) -> impl Iterator<Item = (DefIndex, &DefKey, DefPathHash)> + ExactSizeIterator {

compiler/rustc_hir/src/hir.rs

@@ -1462,11 +1462,15 @@ pub struct AttributeMap<'tcx> {
 }
 
 impl<'tcx> AttributeMap<'tcx> {
-    pub const EMPTY: &'static AttributeMap<'static> = &AttributeMap {
-        map: SortedMap::new(),
-        opt_hash: Some(Fingerprint::ZERO),
-        define_opaque: None,
-    };
+    pub const EMPTY: &'static AttributeMap<'static> = &Self::empty();
+
+    pub const fn empty() -> AttributeMap<'static> {
+        AttributeMap {
+            map: SortedMap::new(),
+            opt_hash: Some(Fingerprint::ZERO),
+            define_opaque: None,
+        }
+    }
 
     #[inline]
     pub fn get(&self, id: ItemLocalId) -> &'tcx [Attribute] {
@@ -1567,8 +1571,7 @@ impl<'tcx> MaybeOwner<'tcx> {
 ///
 /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/hir.html
 #[derive(Debug)]
-pub struct Crate<'hir> {
-    pub owners: IndexVec<LocalDefId, MaybeOwner<'hir>>,
+pub struct Crate {
     // Only present when incr. comp. is enabled.
     pub opt_hir_hash: Option<Fingerprint>,
 }

compiler/rustc_hir/src/stable_hash_impls.rs

@@ -111,9 +111,9 @@ impl<'tcx, HirCtx: crate::HashStableContext> HashStable<HirCtx> for AttributeMap
     }
 }
 
-impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for Crate<'_> {
+impl<HirCtx: crate::HashStableContext> HashStable<HirCtx> for Crate {
     fn hash_stable(&self, hcx: &mut HirCtx, hasher: &mut StableHasher) {
-        let Crate { owners: _, opt_hir_hash } = self;
+        let Crate { opt_hir_hash } = self;
         opt_hir_hash.unwrap().hash_stable(hcx, hasher)
     }
 }

compiler/rustc_middle/src/arena.rs

@@ -118,6 +118,7 @@ macro_rules! arena_types {
             [decode] specialization_graph: rustc_middle::traits::specialization_graph::Graph,
             [] crate_inherent_impls: rustc_middle::ty::CrateInherentImpls,
             [] hir_owner_nodes: rustc_hir::OwnerNodes<'tcx>,
+            [] owner_info: rustc_hir::OwnerInfo<'tcx>,
         ]);
     )
 }

compiler/rustc_middle/src/hir/map.rs

@@ -9,7 +9,7 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::{DynSend, DynSync, par_for_each_in, try_par_for_each_in};
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId, LocalModDefId};
+use rustc_hir::def_id::{CRATE_DEF_ID, DefId, LOCAL_CRATE, LocalDefId, LocalModDefId};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathHash};
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::*;
@@ -396,9 +396,8 @@ impl<'tcx> TyCtxt<'tcx> {
     where
         V: Visitor<'tcx>,
     {
-        let krate = self.hir_crate(());
-        for info in krate.owners.iter() {
-            if let MaybeOwner::Owner(info) = info {
+        for def_id in self.hir_crate_items(()).definitions().chain([CRATE_DEF_ID]) {
+            if let MaybeOwner::Owner(info) = self.hir_owner(def_id) {
                 for attrs in info.attrs.map.values() {
                     walk_list!(visitor, visit_attribute, *attrs);
                 }

compiler/rustc_middle/src/hir/mod.rs

@@ -181,19 +181,25 @@ pub fn provide(providers: &mut Providers) {
     providers.hir_crate_items = map::hir_crate_items;
     providers.crate_hash = map::crate_hash;
     providers.hir_module_items = map::hir_module_items;
-    providers.local_def_id_to_hir_id = |tcx, def_id| match tcx.hir_crate(()).owners[def_id] {
-        MaybeOwner::Owner(_) => HirId::make_owner(def_id),
-        MaybeOwner::NonOwner(hir_id) => hir_id,
-        MaybeOwner::Phantom => bug!("No HirId for {:?}", def_id),
+    providers.local_def_id_to_hir_id = |tcx, def_id| {
+        tcx.ensure_ok().hir_crate(());
+        match tcx.hir_owner(def_id) {
+            MaybeOwner::Owner(_) => HirId::make_owner(def_id),
+            MaybeOwner::NonOwner(hir_id) => hir_id,
+            MaybeOwner::Phantom => bug!("No HirId for {:?}", def_id),
+        }
+    };
+    providers.opt_hir_owner_nodes = |tcx, id| {
+        tcx.ensure_ok().hir_crate(());
+        tcx.hir_owner(id).as_owner().map(|i| &i.nodes)
     };
-    providers.opt_hir_owner_nodes =
-        |tcx, id| tcx.hir_crate(()).owners.get(id)?.as_owner().map(|i| &i.nodes);
     providers.hir_owner_parent = |tcx, owner_id| {
         tcx.opt_local_parent(owner_id.def_id).map_or(CRATE_HIR_ID, |parent_def_id| {
             let parent_owner_id = tcx.local_def_id_to_hir_id(parent_def_id).owner;
             HirId {
                 owner: parent_owner_id,
-                local_id: tcx.hir_crate(()).owners[parent_owner_id.def_id]
+                local_id: tcx
+                    .hir_owner(parent_owner_id.def_id)
                     .unwrap()
                     .parenting
                     .get(&owner_id.def_id)
@@ -203,7 +209,8 @@ pub fn provide(providers: &mut Providers) {
         })
     };
     providers.hir_attr_map = |tcx, id| {
-        tcx.hir_crate(()).owners[id.def_id].as_owner().map_or(AttributeMap::EMPTY, |o| &o.attrs)
+        tcx.ensure_ok().hir_crate(());
+        tcx.hir_owner(id.def_id).as_owner().map_or(AttributeMap::EMPTY, |o| &o.attrs)
     };
     providers.def_span = |tcx, def_id| tcx.hir_span(tcx.local_def_id_to_hir_id(def_id));
     providers.def_ident_span = |tcx, def_id| {
@@ -236,7 +243,6 @@ pub fn provide(providers: &mut Providers) {
         |tcx, trait_id| tcx.resolutions(()).trait_impls.get(&trait_id).map_or(&[], |xs| &xs[..]);
     providers.expn_that_defined =
         |tcx, id| tcx.resolutions(()).expn_that_defined.get(&id).copied().unwrap_or(ExpnId::root());
-    providers.in_scope_traits_map = |tcx, id| {
-        tcx.hir_crate(()).owners[id.def_id].as_owner().map(|owner_info| &owner_info.trait_map)
-    };
+    providers.in_scope_traits_map =
+        |tcx, id| tcx.hir_owner(id.def_id).as_owner().map(|owner_info| &owner_info.trait_map);
 }

compiler/rustc_middle/src/query/mod.rs

@@ -166,12 +166,19 @@ rustc_queries! {
     /// query gives you access to all other items. To avoid this fate, do not
     /// call `tcx.hir_crate(())`; instead, prefer wrappers like
     /// [`TyCtxt::hir_visit_all_item_likes_in_crate`].
-    query hir_crate(key: ()) -> &'tcx Crate<'tcx> {
+    query hir_crate(key: ()) -> &'tcx Crate {
         arena_cache
         eval_always
         desc { "getting the crate HIR" }
     }
 
+    /// A query decoupling the `hir_crate` query from everything else
+    query hir_owner(key: LocalDefId) -> rustc_hir::MaybeOwner<'tcx> {
+        no_hash
+        desc { |tcx| "getting HIR of `{}`", tcx.def_path_str(key) }
+        feedable
+    }
+
     /// All items in the crate.
     query hir_crate_items(_: ()) -> &'tcx rustc_middle::hir::ModuleItems {
         arena_cache
@@ -192,7 +199,6 @@ rustc_queries! {
     /// Returns HIR ID for the given `LocalDefId`.
     query local_def_id_to_hir_id(key: LocalDefId) -> hir::HirId {
         desc { |tcx| "getting HIR ID of `{}`", tcx.def_path_str(key) }
-        feedable
     }
 
     /// Gives access to the HIR node's parent for the HIR owner `key`.
@@ -209,7 +215,6 @@ rustc_queries! {
     /// Avoid calling this query directly.
     query opt_hir_owner_nodes(key: LocalDefId) -> Option<&'tcx hir::OwnerNodes<'tcx>> {
         desc { |tcx| "getting HIR owner items in `{}`", tcx.def_path_str(key) }
-        feedable
     }
 
     /// Gives access to the HIR attributes inside the HIR owner `key`.
@@ -218,7 +223,6 @@ rustc_queries! {
     /// Avoid calling this query directly.
     query hir_attr_map(key: hir::OwnerId) -> &'tcx hir::AttributeMap<'tcx> {
         desc { |tcx| "getting HIR owner attributes in `{}`", tcx.def_path_str(key) }
-        feedable
     }
 
     /// Returns the *default* of the const pararameter given by `DefId`.

compiler/rustc_middle/src/ty/context.rs

@@ -37,7 +37,7 @@ use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId};
 use rustc_hir::definitions::{DefPathData, Definitions, DisambiguatorState};
 use rustc_hir::intravisit::VisitorExt;
 use rustc_hir::lang_items::LangItem;
-use rustc_hir::{self as hir, Attribute, HirId, Node, TraitCandidate};
+use rustc_hir::{self as hir, Attribute, HirId, MaybeOwner, Node, OwnerInfo, TraitCandidate};
 use rustc_index::IndexVec;
 use rustc_macros::{HashStable, TyDecodable, TyEncodable};
 use rustc_query_system::cache::WithDepNode;
@@ -1272,6 +1272,11 @@ impl<'tcx> TyCtxt<'tcx> {
         TyCtxtFeed { tcx: self, key: () }
     }
 
+    pub fn super_duper_perf_hack_experiment(self, key: LocalDefId) -> TyCtxtFeed<'tcx, LocalDefId> {
+        self.dep_graph.assert_eval_always();
+        TyCtxtFeed { tcx: self, key }
+    }
+
     /// Only used in the resolver to register the `CRATE_DEF_ID` `DefId` and feed
     /// some queries for it. It will panic if used twice.
     pub fn create_local_crate_def_id(self, span: Span) -> TyCtxtFeed<'tcx, LocalDefId> {
@@ -1326,24 +1331,25 @@ impl<'tcx> TyCtxtFeed<'tcx, LocalDefId> {
 
     // Fills in all the important parts needed by HIR queries
     pub fn feed_hir(&self) {
-        self.local_def_id_to_hir_id(HirId::make_owner(self.def_id()));
-
         let node = hir::OwnerNode::Synthetic;
         let bodies = Default::default();
-        let attrs = hir::AttributeMap::EMPTY;
-
+        let attrs = hir::AttributeMap::empty();
         let (opt_hash_including_bodies, _) =
             self.tcx.hash_owner_nodes(node, &bodies, &attrs.map, attrs.define_opaque);
         let node = node.into();
-        self.opt_hir_owner_nodes(Some(self.tcx.arena.alloc(hir::OwnerNodes {
-            opt_hash_including_bodies,
-            nodes: IndexVec::from_elem_n(
-                hir::ParentedNode { parent: hir::ItemLocalId::INVALID, node },
-                1,
-            ),
-            bodies,
+        self.hir_owner(MaybeOwner::Owner(self.tcx.arena.alloc(OwnerInfo {
+            nodes: hir::OwnerNodes {
+                opt_hash_including_bodies,
+                nodes: IndexVec::from_elem_n(
+                    hir::ParentedNode { parent: hir::ItemLocalId::INVALID, node },
+                    1,
+                ),
+                bodies,
+            },
+            parenting: Default::default(),
+            attrs,
+            trait_map: Default::default(),
         })));
-        self.feed_owner_id().hir_attr_map(attrs);
     }
 }
 

compiler/rustc_query_system/src/dep_graph/graph.rs

@@ -200,6 +200,18 @@ impl<D: Deps> DepGraph<D> {
         }
     }
 
+    pub fn assert_eval_always(&self) {
+        if let Some(..) = self.data {
+            D::read_deps(|task_deps| {
+                assert_matches!(
+                    task_deps,
+                    TaskDepsRef::EvalAlways,
+                    "expected no task dependency tracking"
+                );
+            })
+        }
+    }
+
     pub fn with_ignore<OP, R>(&self, op: OP) -> R
     where
         OP: FnOnce() -> R,