Use horde's SyncTable for default query caches and CtxtInterners

Cargo.lock

@@ -1709,6 +1709,15 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "horde"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3d7c629771c2e116e71d8e7bbc6e6e0450a8817766a7230bb0c98f81311df34"
+dependencies = [
+ "parking_lot",
+]
+
 [[package]]
 name = "html-checker"
 version = "0.1.0"
@@ -3803,6 +3812,7 @@ dependencies = [
  "elsa",
  "ena",
  "hashbrown 0.16.1",
+ "horde",
  "indexmap",
  "jobserver",
  "libc",

compiler/rustc_data_structures/Cargo.toml

@@ -10,6 +10,7 @@ bitflags = "2.4.1"
 either = "1.0"
 elsa = "1.11.0"
 ena = "0.14.4"
+horde = { version = "0.1.2", features = ["nightly"] }
 indexmap = "2.12.1"
 jobserver_crate = { version = "0.1.28", package = "jobserver" }
 measureme = "12.0.1"
@@ -33,7 +34,7 @@ tracing = "0.1"
 [dependencies.hashbrown]
 version = "0.16.1"
 default-features = false
-features = ["nightly"] # for may_dangle
+features = ["nightly"]   # for may_dangle
 
 [target.'cfg(windows)'.dependencies.windows]
 version = "0.61.0"

compiler/rustc_data_structures/src/sync.rs

@@ -25,6 +25,7 @@
 use std::collections::HashMap;
 use std::hash::{BuildHasher, Hash};
 
+pub use horde::collect;
 pub use parking_lot::{
     MappedRwLockReadGuard as MappedReadGuard, MappedRwLockWriteGuard as MappedWriteGuard,
     RwLockReadGuard as ReadGuard, RwLockWriteGuard as WriteGuard,
@@ -39,13 +40,15 @@ pub use self::parallel::{
     broadcast, par_fns, par_for_each_in, par_join, par_map, parallel_guard, spawn,
     try_par_for_each_in,
 };
+pub use self::sync_table::{LockedWrite, Read, SyncTable};
 pub use self::vec::{AppendOnlyIndexVec, AppendOnlyVec};
 pub use self::worker_local::{Registry, WorkerLocal};
 pub use crate::marker::*;
 
 mod freeze;
 mod lock;
 mod parallel;
+mod sync_table;
 mod vec;
 mod worker_local;
 

compiler/rustc_data_structures/src/sync/lock.rs

@@ -112,6 +112,11 @@ impl<T> Lock<T> {
         self.data.get_mut()
     }
 
+    #[inline(always)]
+    pub fn mode(&self) -> Mode {
+        self.mode
+    }
+
     #[inline(always)]
     pub fn try_lock(&self) -> Option<LockGuard<'_, T>> {
         let mode = self.mode;

compiler/rustc_data_structures/src/sync/sync_table.rs

@@ -0,0 +1,168 @@
+use std::borrow::Borrow;
+use std::hash::{BuildHasher, Hash, Hasher};
+use std::ops::{Deref, DerefMut};
+
+use horde::collect::{Pin, pin};
+pub use horde::sync_table::Read;
+use horde::sync_table::Write;
+use rustc_hash::FxBuildHasher;
+
+use crate::sharded::IntoPointer;
+use crate::sync::{DynSync, Lock, LockGuard, Mode};
+
+pub struct SyncTable<K, V> {
+    // We use this lock to protect `table` instead of the internal mutex in `horde::SyncTable`
+    // as it's faster when synchronization is disabled.
+    lock: Lock<()>,
+
+    table: horde::SyncTable<K, V, FxBuildHasher>,
+}
+
+// Memory reclamation can move elements to other threads for dropping,
+// so we require `Sync` instead of `DynSync` here
+unsafe impl<K: Sync, V: Sync> DynSync for SyncTable<K, V> where FxBuildHasher: Sync {}
+
+impl<K, V> Default for SyncTable<K, V> {
+    fn default() -> Self {
+        Self { lock: Lock::default(), table: horde::SyncTable::default() }
+    }
+}
+
+impl<K, V> SyncTable<K, V> {
+    /// Creates a [Read] handle from a pinned region.
+    ///
+    /// Use [horde::collect::pin] to get a `Pin` instance.
+    #[inline]
+    pub fn read<'a>(&'a self, pin: Pin<'a>) -> Read<'a, K, V, FxBuildHasher> {
+        self.table.read(pin)
+    }
+
+    /// Creates a [LockedWrite] handle by taking the underlying mutex that protects writes.
+    #[inline]
+    pub fn lock(&self) -> LockedWrite<'_, K, V> {
+        LockedWrite {
+            _guard: self.lock.lock(),
+            table: {
+                // SAFETY: We ensure there's only 1 writer at a time using our own lock
+                unsafe { self.table.unsafe_write() }
+            },
+        }
+    }
+
+    /// Hashes a key with the table's hasher.
+    #[inline]
+    pub fn hash_key<Q>(&self, key: &Q) -> u64
+    where
+        K: Borrow<Q>,
+        Q: ?Sized + Hash,
+    {
+        self.table.hash_key::<Q>(key)
+    }
+
+    pub fn len(&self) -> usize {
+        pin(|pin| self.read(pin).len())
+    }
+
+    pub fn with_capacity(cap: usize) -> Self {
+        Self { lock: Lock::new(()), table: horde::SyncTable::new_with(FxBuildHasher, cap) }
+    }
+}
+
+/// A handle to a [SyncTable] with write access protected by a lock.
+pub struct LockedWrite<'a, K, V> {
+    table: Write<'a, K, V, FxBuildHasher>,
+    _guard: LockGuard<'a, ()>,
+}
+
+impl<'a, K, V> Deref for LockedWrite<'a, K, V> {
+    type Target = Write<'a, K, V, FxBuildHasher>;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        &self.table
+    }
+}
+
+impl<'a, K, V> DerefMut for LockedWrite<'a, K, V> {
+    #[inline]
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.table
+    }
+}
+
+impl<K: Eq + Hash + Copy + Send> SyncTable<K, ()> {
+    pub fn contains_pointer_to<T: Hash + IntoPointer>(&self, value: &T) -> bool
+    where
+        K: IntoPointer,
+    {
+        pin(|pin| {
+            let mut state = FxBuildHasher.build_hasher();
+            value.hash(&mut state);
+            let hash = state.finish();
+            let value = value.into_pointer();
+            self.read(pin).get_from_hash(hash, |entry| entry.into_pointer() == value).is_some()
+        })
+    }
+
+    #[inline]
+    pub fn intern_ref<Q: ?Sized>(&self, value: &Q, make: impl FnOnce() -> K) -> K
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq,
+    {
+        pin(|pin| {
+            let hash = self.hash_key(value);
+
+            let entry = self.read(pin).get(value, Some(hash));
+            if let Some(entry) = entry {
+                return *entry.0;
+            }
+
+            let mut write = self.lock();
+
+            if self.lock.mode() == Mode::Sync {
+                let entry = self.read(pin).get(value, Some(hash));
+                if let Some(entry) = entry {
+                    return *entry.0;
+                }
+            }
+
+            let result = make();
+
+            write.insert_new(result, (), Some(hash));
+
+            result
+        })
+    }
+
+    #[inline]
+    pub fn intern<Q>(&self, value: Q, make: impl FnOnce(Q) -> K) -> K
+    where
+        K: Borrow<Q>,
+        Q: Hash + Eq,
+    {
+        pin(|pin| {
+            let hash = self.hash_key(&value);
+
+            let entry = self.read(pin).get(&value, Some(hash));
+            if let Some(entry) = entry {
+                return *entry.0;
+            }
+
+            let mut write = self.lock();
+
+            if self.lock.mode() == Mode::Sync {
+                let entry = self.read(pin).get(&value, Some(hash));
+                if let Some(entry) = entry {
+                    return *entry.0;
+                }
+            }
+
+            let result = make(value);
+
+            write.insert_new(result, (), Some(hash));
+
+            result
+        })
+    }
+}

compiler/rustc_interface/src/util.rs

@@ -14,7 +14,7 @@ use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::{CompiledModules, CrateInfo, TargetConfig};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::jobserver::Proxy;
-use rustc_data_structures::sync;
+use rustc_data_structures::sync::{self, collect};
 use rustc_metadata::{DylibError, EncodedMetadata, load_symbol_from_dylib};
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::ty::{CurrentGcx, TyCtxt};
@@ -216,7 +216,10 @@ pub(crate) fn run_in_thread_pool_with_globals<
     let builder = rustc_thread_pool::ThreadPoolBuilder::new()
         .thread_name(|_| "rustc".to_string())
         .acquire_thread_handler(move || proxy_.acquire_thread())
-        .release_thread_handler(move || proxy__.release_thread())
+        .release_thread_handler(move || {
+            collect::release();
+            proxy__.release_thread()
+        })
         .num_threads(threads)
         .deadlock_handler(move || {
             // On deadlock, creates a new thread and forwards information in thread

compiler/rustc_middle/src/query/caches.rs

@@ -1,6 +1,7 @@
 use std::sync::OnceLock;
 
-use rustc_data_structures::sharded::ShardedHashMap;
+use rustc_data_structures::sync::SyncTable;
+use rustc_data_structures::sync::collect::pin;
 pub use rustc_data_structures::vec_cache::VecCache;
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_index::Idx;
@@ -41,7 +42,7 @@ pub trait QueryCache: Sized {
 /// In-memory cache for queries whose keys aren't suitable for any of the
 /// more specialized kinds of cache. Backed by a sharded hashmap.
 pub struct DefaultCache<K, V> {
-    cache: ShardedHashMap<K, (V, DepNodeIndex)>,
+    cache: SyncTable<K, (V, DepNodeIndex)>,
 }
 
 impl<K, V> Default for DefaultCache<K, V> {
@@ -53,33 +54,34 @@ impl<K, V> Default for DefaultCache<K, V> {
 impl<K, V> QueryCache for DefaultCache<K, V>
 where
     K: QueryKey,
-    V: Copy,
+    V: Copy + Send,
 {
     type Key = K;
     type Value = V;
 
     #[inline(always)]
     fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
-        self.cache.get(key)
+        pin(|pin| {
+            let result = self.cache.read(pin).get(key, None);
+            if let Some((_, value)) = result { Some(*value) } else { None }
+        })
     }
 
     #[inline]
     fn complete(&self, key: K, value: V, index: DepNodeIndex) {
-        // We may be overwriting another value. This is all right, since the dep-graph
-        // will check that the value fingerprint matches.
-        self.cache.insert(key, (value, index));
+        self.cache.lock().insert_new(key, (value, index), None);
     }
 
     fn for_each(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
-        for shard in self.cache.lock_shards() {
-            for (k, v) in shard.iter() {
+        pin(|pin| {
+            for (k, v) in self.cache.read(pin).iter() {
                 f(k, &v.0, v.1);
             }
-        }
+        })
     }
 
     fn len(&self) -> usize {
-        self.cache.len()
+        pin(|pin| self.cache.read(pin).len())
     }
 }
 
@@ -142,7 +144,7 @@ impl<V> Default for DefIdCache<V> {
 
 impl<V> QueryCache for DefIdCache<V>
 where
-    V: Copy,
+    V: Copy + Send,
 {
     type Key = DefId;
     type Value = V;

compiler/rustc_middle/src/query/job.rs

@@ -4,6 +4,7 @@ use std::num::NonZero;
 use std::sync::Arc;
 
 use parking_lot::{Condvar, Mutex};
+use rustc_data_structures::sync::collect;
 use rustc_span::Span;
 
 use crate::query::plumbing::CycleError;
@@ -108,6 +109,7 @@ impl<'tcx> QueryLatch<'tcx> {
         // we have to be in the `wait` call. This is ensured by the deadlock handler
         // getting the self.info lock.
         rustc_thread_pool::mark_blocked();
+        collect::release();
         tcx.jobserver_proxy.release_thread();
         waiter.condvar.wait(&mut waiters_guard);
         // Release the lock before we potentially block in `acquire_thread`

compiler/rustc_middle/src/query/keys.rs

@@ -24,7 +24,8 @@ use crate::{mir, traits};
 #[derive(Copy, Clone, Debug)]
 pub struct LocalCrate;
 
-pub trait QueryKeyBounds = Copy + Debug + Eq + Hash + for<'a> HashStable<StableHashingContext<'a>>;
+pub trait QueryKeyBounds =
+    Copy + Debug + Eq + Hash + Send + for<'a> HashStable<StableHashingContext<'a>>;
 
 /// Controls what types can legally be used as the key for a query.
 pub trait QueryKey: Sized + QueryKeyBounds {