Skip to content

Commit ebdc942

Browse files
committed
Use horde's SyncTable for default query caches and CtxtInterners
1 parent 8db65c7 commit ebdc942

File tree

11 files changed

+216
-24
lines changed

11 files changed

+216
-24
lines changed

Cargo.lock

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1709,6 +1709,15 @@ dependencies = [
17091709
"windows-sys 0.61.2",
17101710
]
17111711

1712+
[[package]]
1713+
name = "horde"
1714+
version = "0.1.2"
1715+
source = "registry+https://github.com/rust-lang/crates.io-index"
1716+
checksum = "f3d7c629771c2e116e71d8e7bbc6e6e0450a8817766a7230bb0c98f81311df34"
1717+
dependencies = [
1718+
"parking_lot",
1719+
]
1720+
17121721
[[package]]
17131722
name = "html-checker"
17141723
version = "0.1.0"
@@ -3803,6 +3812,7 @@ dependencies = [
38033812
"elsa",
38043813
"ena",
38053814
"hashbrown 0.16.1",
3815+
"horde",
38063816
"indexmap",
38073817
"jobserver",
38083818
"libc",

compiler/rustc_data_structures/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ bitflags = "2.4.1"
1010
either = "1.0"
1111
elsa = "1.11.0"
1212
ena = "0.14.4"
13+
horde = { version = "0.1.2", features = ["nightly"] }
1314
indexmap = "2.12.1"
1415
jobserver_crate = { version = "0.1.28", package = "jobserver" }
1516
measureme = "12.0.1"
@@ -33,7 +34,7 @@ tracing = "0.1"
3334
[dependencies.hashbrown]
3435
version = "0.16.1"
3536
default-features = false
36-
features = ["nightly"] # for may_dangle
37+
features = ["nightly"] # for may_dangle
3738

3839
[target.'cfg(windows)'.dependencies.windows]
3940
version = "0.61.0"

compiler/rustc_data_structures/src/sync.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
use std::collections::HashMap;
2626
use std::hash::{BuildHasher, Hash};
2727

28+
pub use horde::collect;
2829
pub use parking_lot::{
2930
MappedRwLockReadGuard as MappedReadGuard, MappedRwLockWriteGuard as MappedWriteGuard,
3031
RwLockReadGuard as ReadGuard, RwLockWriteGuard as WriteGuard,
@@ -39,13 +40,15 @@ pub use self::parallel::{
3940
broadcast, par_fns, par_for_each_in, par_join, par_map, parallel_guard, spawn,
4041
try_par_for_each_in,
4142
};
43+
pub use self::sync_table::{LockedWrite, Read, SyncTable};
4244
pub use self::vec::{AppendOnlyIndexVec, AppendOnlyVec};
4345
pub use self::worker_local::{Registry, WorkerLocal};
4446
pub use crate::marker::*;
4547

4648
mod freeze;
4749
mod lock;
4850
mod parallel;
51+
mod sync_table;
4952
mod vec;
5053
mod worker_local;
5154

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
use std::borrow::Borrow;
2+
use std::hash::{BuildHasher, Hash, Hasher};
3+
use std::ops::{Deref, DerefMut};
4+
5+
use horde::collect::{Pin, pin};
6+
pub use horde::sync_table::Read;
7+
use horde::sync_table::Write;
8+
use rustc_hash::FxBuildHasher;
9+
10+
use crate::sharded::IntoPointer;
11+
use crate::sync::{DynSync, Lock, LockGuard};
12+
13+
pub struct SyncTable<K, V> {
14+
// We use this lock to protect `table` instead of the internal mutex in `horde::SyncTable`
15+
// as it's faster when synchronization is disabled.
16+
lock: Lock<()>,
17+
18+
table: horde::SyncTable<K, V, FxBuildHasher>,
19+
}
20+
21+
// Memory reclamation can move elements to other threads for dropping,
22+
// so we require `Sync` instead of `DynSync` here
23+
unsafe impl<K: Sync, V: Sync> DynSync for SyncTable<K, V> where FxBuildHasher: Sync {}
24+
25+
impl<K, V> Default for SyncTable<K, V> {
26+
fn default() -> Self {
27+
Self { lock: Lock::default(), table: horde::SyncTable::default() }
28+
}
29+
}
30+
31+
impl<K, V> SyncTable<K, V> {
32+
/// Creates a [Read] handle from a pinned region.
33+
///
34+
/// Use [horde::collect::pin] to get a `Pin` instance.
35+
#[inline]
36+
pub fn read<'a>(&'a self, pin: Pin<'a>) -> Read<'a, K, V, FxBuildHasher> {
37+
self.table.read(pin)
38+
}
39+
40+
/// Creates a [LockedWrite] handle by taking the underlying mutex that protects writes.
41+
#[inline]
42+
pub fn lock(&self) -> LockedWrite<'_, K, V> {
43+
LockedWrite {
44+
_guard: self.lock.lock(),
45+
table: {
46+
// SAFETY: We ensure there's only 1 writer at a time using our own lock
47+
unsafe { self.table.unsafe_write() }
48+
},
49+
}
50+
}
51+
52+
/// Hashes a key with the table's hasher.
53+
#[inline]
54+
pub fn hash_key<Q>(&self, key: &Q) -> u64
55+
where
56+
K: Borrow<Q>,
57+
Q: ?Sized + Hash,
58+
{
59+
self.table.hash_key::<Q>(key)
60+
}
61+
62+
pub fn len(&self) -> usize {
63+
pin(|pin| self.read(pin).len())
64+
}
65+
66+
pub fn with_capacity(cap: usize) -> Self {
67+
Self { lock: Lock::new(()), table: horde::SyncTable::new_with(FxBuildHasher, cap) }
68+
}
69+
}
70+
71+
/// A handle to a [SyncTable] with write access protected by a lock.
72+
pub struct LockedWrite<'a, K, V> {
73+
table: Write<'a, K, V, FxBuildHasher>,
74+
_guard: LockGuard<'a, ()>,
75+
}
76+
77+
impl<'a, K, V> Deref for LockedWrite<'a, K, V> {
78+
type Target = Write<'a, K, V, FxBuildHasher>;
79+
80+
#[inline]
81+
fn deref(&self) -> &Self::Target {
82+
&self.table
83+
}
84+
}
85+
86+
impl<'a, K, V> DerefMut for LockedWrite<'a, K, V> {
87+
#[inline]
88+
fn deref_mut(&mut self) -> &mut Self::Target {
89+
&mut self.table
90+
}
91+
}
92+
93+
impl<K: Eq + Hash + Copy + Send> SyncTable<K, ()> {
94+
pub fn contains_pointer_to<T: Hash + IntoPointer>(&self, value: &T) -> bool
95+
where
96+
K: IntoPointer,
97+
{
98+
pin(|pin| {
99+
let mut state = FxBuildHasher.build_hasher();
100+
value.hash(&mut state);
101+
let hash = state.finish();
102+
let value = value.into_pointer();
103+
self.read(pin).get_from_hash(hash, |entry| entry.into_pointer() == value).is_some()
104+
})
105+
}
106+
107+
#[inline]
108+
pub fn intern_ref<Q: ?Sized>(&self, value: &Q, make: impl FnOnce() -> K) -> K
109+
where
110+
K: Borrow<Q>,
111+
Q: Hash + Eq,
112+
{
113+
pin(|pin| {
114+
let hash = self.hash_key(value);
115+
116+
let entry = self.read(pin).get(value, Some(hash));
117+
if let Some(entry) = entry {
118+
return *entry.0;
119+
}
120+
121+
let mut write = self.lock();
122+
123+
let entry = self.read(pin).get(value, Some(hash));
124+
if let Some(entry) = entry {
125+
return *entry.0;
126+
}
127+
128+
let result = make();
129+
130+
write.insert_new(result, (), Some(hash));
131+
132+
result
133+
})
134+
}
135+
136+
#[inline]
137+
pub fn intern<Q>(&self, value: Q, make: impl FnOnce(Q) -> K) -> K
138+
where
139+
K: Borrow<Q>,
140+
Q: Hash + Eq,
141+
{
142+
pin(|pin| {
143+
let hash = self.hash_key(&value);
144+
145+
let entry = self.read(pin).get(&value, Some(hash));
146+
if let Some(entry) = entry {
147+
return *entry.0;
148+
}
149+
150+
let mut write = self.lock();
151+
152+
let entry = self.read(pin).get(&value, Some(hash));
153+
if let Some(entry) = entry {
154+
return *entry.0;
155+
}
156+
157+
let result = make(value);
158+
159+
write.insert_new(result, (), Some(hash));
160+
161+
result
162+
})
163+
}
164+
}

compiler/rustc_interface/src/util.rs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use rustc_codegen_ssa::traits::CodegenBackend;
1414
use rustc_codegen_ssa::{CompiledModules, CrateInfo, TargetConfig};
1515
use rustc_data_structures::fx::FxIndexMap;
1616
use rustc_data_structures::jobserver::Proxy;
17-
use rustc_data_structures::sync;
17+
use rustc_data_structures::sync::{self, collect};
1818
use rustc_metadata::{DylibError, EncodedMetadata, load_symbol_from_dylib};
1919
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
2020
use rustc_middle::ty::{CurrentGcx, TyCtxt};
@@ -216,7 +216,10 @@ pub(crate) fn run_in_thread_pool_with_globals<
216216
let builder = rustc_thread_pool::ThreadPoolBuilder::new()
217217
.thread_name(|_| "rustc".to_string())
218218
.acquire_thread_handler(move || proxy_.acquire_thread())
219-
.release_thread_handler(move || proxy__.release_thread())
219+
.release_thread_handler(move || {
220+
collect::release();
221+
proxy__.release_thread()
222+
})
220223
.num_threads(threads)
221224
.deadlock_handler(move || {
222225
// On deadlock, creates a new thread and forwards information in thread

compiler/rustc_middle/src/query/caches.rs

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use std::sync::OnceLock;
22

3-
use rustc_data_structures::sharded::ShardedHashMap;
3+
use rustc_data_structures::sync::SyncTable;
4+
use rustc_data_structures::sync::collect::pin;
45
pub use rustc_data_structures::vec_cache::VecCache;
56
use rustc_hir::def_id::LOCAL_CRATE;
67
use rustc_index::Idx;
@@ -41,7 +42,7 @@ pub trait QueryCache: Sized {
4142
/// In-memory cache for queries whose keys aren't suitable for any of the
4243
/// more specialized kinds of cache. Backed by a sharded hashmap.
4344
pub struct DefaultCache<K, V> {
44-
cache: ShardedHashMap<K, (V, DepNodeIndex)>,
45+
cache: SyncTable<K, (V, DepNodeIndex)>,
4546
}
4647

4748
impl<K, V> Default for DefaultCache<K, V> {
@@ -53,33 +54,34 @@ impl<K, V> Default for DefaultCache<K, V> {
5354
impl<K, V> QueryCache for DefaultCache<K, V>
5455
where
5556
K: QueryKey,
56-
V: Copy,
57+
V: Copy + Send,
5758
{
5859
type Key = K;
5960
type Value = V;
6061

6162
#[inline(always)]
6263
fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
63-
self.cache.get(key)
64+
pin(|pin| {
65+
let result = self.cache.read(pin).get(key, None);
66+
if let Some((_, value)) = result { Some(*value) } else { None }
67+
})
6468
}
6569

6670
#[inline]
6771
fn complete(&self, key: K, value: V, index: DepNodeIndex) {
68-
// We may be overwriting another value. This is all right, since the dep-graph
69-
// will check that the value fingerprint matches.
70-
self.cache.insert(key, (value, index));
72+
self.cache.lock().insert_new(key, (value, index), None);
7173
}
7274

7375
fn for_each(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {
74-
for shard in self.cache.lock_shards() {
75-
for (k, v) in shard.iter() {
76+
pin(|pin| {
77+
for (k, v) in self.cache.read(pin).iter() {
7678
f(k, &v.0, v.1);
7779
}
78-
}
80+
})
7981
}
8082

8183
fn len(&self) -> usize {
82-
self.cache.len()
84+
pin(|pin| self.cache.read(pin).len())
8385
}
8486
}
8587

@@ -142,7 +144,7 @@ impl<V> Default for DefIdCache<V> {
142144

143145
impl<V> QueryCache for DefIdCache<V>
144146
where
145-
V: Copy,
147+
V: Copy + Send,
146148
{
147149
type Key = DefId;
148150
type Value = V;

compiler/rustc_middle/src/query/job.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use std::num::NonZero;
44
use std::sync::Arc;
55

66
use parking_lot::{Condvar, Mutex};
7+
use rustc_data_structures::sync::collect;
78
use rustc_span::Span;
89

910
use crate::query::plumbing::CycleError;
@@ -108,6 +109,7 @@ impl<'tcx> QueryLatch<'tcx> {
108109
// we have to be in the `wait` call. This is ensured by the deadlock handler
109110
// getting the self.info lock.
110111
rustc_thread_pool::mark_blocked();
112+
collect::release();
111113
tcx.jobserver_proxy.release_thread();
112114
waiter.condvar.wait(&mut waiters_guard);
113115
// Release the lock before we potentially block in `acquire_thread`

compiler/rustc_middle/src/query/keys.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ use crate::{mir, traits};
2424
#[derive(Copy, Clone, Debug)]
2525
pub struct LocalCrate;
2626

27-
pub trait QueryKeyBounds = Copy + Debug + Eq + Hash + for<'a> HashStable<StableHashingContext<'a>>;
27+
pub trait QueryKeyBounds =
28+
Copy + Debug + Eq + Hash + Send + for<'a> HashStable<StableHashingContext<'a>>;
2829

2930
/// Controls what types can legally be used as the key for a query.
3031
pub trait QueryKey: Sized + QueryKeyBounds {

0 commit comments

Comments
 (0)