Skip to content

Commit 7aef5cd

Browse files
committed
feat(locking): Added build unit level locking
1 parent 4f15cc8 commit 7aef5cd

File tree

7 files changed

+386
-16
lines changed

7 files changed

+386
-16
lines changed

src/cargo/core/compiler/build_runner/compilation_files.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,13 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
277277
self.layout(unit.kind).build_dir().fingerprint(&dir)
278278
}
279279

280+
/// The path of the primary and secondary locks for a given build unit
281+
/// when fine grain locking is enabled.
282+
pub fn build_unit_lock(&self, unit: &Unit) -> (PathBuf, PathBuf) {
283+
let dir = self.pkg_dir(unit);
284+
self.layout(unit.kind).build_dir().build_unit_lock(&dir)
285+
}
286+
280287
/// Directory where incremental output for the given unit should go.
281288
pub fn incremental_dir(&self, unit: &Unit) -> &Path {
282289
self.layout(unit.kind).build_dir().incremental()

src/cargo/core/compiler/build_runner/mod.rs

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ use std::sync::{Arc, Mutex};
66

77
use crate::core::PackageId;
88
use crate::core::compiler::compilation::{self, UnitOutput};
9+
use crate::core::compiler::locking::LockingMode;
910
use crate::core::compiler::{self, Unit, artifact};
1011
use crate::util::cache_lock::CacheLockMode;
1112
use crate::util::errors::CargoResult;
@@ -89,6 +90,10 @@ pub struct BuildRunner<'a, 'gctx> {
8990
/// because the target has a type error. This is in an Arc<Mutex<..>>
9091
/// because it is continuously updated as the job progresses.
9192
pub failed_scrape_units: Arc<Mutex<HashSet<UnitHash>>>,
93+
94+
/// The locking mode to use for this build.
95+
/// We use fine grain by default, but fallback to coarse grain for some systems.
96+
pub locking_mode: LockingMode,
9297
}
9398

9499
impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
@@ -111,6 +116,12 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
111116
}
112117
};
113118

119+
let locking_mode = if bcx.gctx.cli_unstable().build_dir_new_layout {
120+
LockingMode::Fine
121+
} else {
122+
LockingMode::Coarse
123+
};
124+
114125
Ok(Self {
115126
bcx,
116127
compilation: Compilation::new(bcx)?,
@@ -128,6 +139,7 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
128139
lto: HashMap::new(),
129140
metadata_for_doc_units: HashMap::new(),
130141
failed_scrape_units: Arc::new(Mutex::new(HashSet::new())),
142+
locking_mode,
131143
})
132144
}
133145

@@ -360,11 +372,11 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
360372
#[tracing::instrument(skip_all)]
361373
pub fn prepare_units(&mut self) -> CargoResult<()> {
362374
let dest = self.bcx.profiles.get_dir_name();
363-
let host_layout = Layout::new(self.bcx.ws, None, &dest)?;
375+
let host_layout = Layout::new(self.bcx.ws, None, &dest, &self.locking_mode)?;
364376
let mut targets = HashMap::new();
365377
for kind in self.bcx.all_kinds.iter() {
366378
if let CompileKind::Target(target) = *kind {
367-
let layout = Layout::new(self.bcx.ws, Some(target), &dest)?;
379+
let layout = Layout::new(self.bcx.ws, Some(target), &dest, &self.locking_mode)?;
368380
targets.insert(target, layout);
369381
}
370382
}

src/cargo/core/compiler/layout.rs

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@
103103
104104
use crate::core::Workspace;
105105
use crate::core::compiler::CompileTarget;
106+
use crate::core::compiler::locking::LockingMode;
106107
use crate::util::{CargoResult, FileLock};
107108
use cargo_util::paths;
108109
use std::path::{Path, PathBuf};
@@ -126,6 +127,7 @@ impl Layout {
126127
ws: &Workspace<'_>,
127128
target: Option<CompileTarget>,
128129
dest: &str,
130+
locking_mode: &LockingMode,
129131
) -> CargoResult<Layout> {
130132
let is_new_layout = ws.gctx().cli_unstable().build_dir_new_layout;
131133
let mut root = ws.target_dir();
@@ -152,15 +154,18 @@ impl Layout {
152154
// For now we don't do any more finer-grained locking on the artifact
153155
// directory, so just lock the entire thing for the duration of this
154156
// compile.
155-
let artifact_dir_lock =
156-
dest.open_rw_exclusive_create(".cargo-lock", ws.gctx(), "build directory")?;
157+
let artifact_dir_lock = if matches!(locking_mode, LockingMode::Coarse) {
158+
dest.open_rw_exclusive_create(".cargo-lock", ws.gctx(), "build directory")?
159+
} else {
160+
dest.open_ro_shared_create(".cargo-lock", ws.gctx(), "build directory")?
161+
};
157162

158163
let build_dir_lock = if root != build_root {
159-
Some(build_dest.open_rw_exclusive_create(
160-
".cargo-lock",
161-
ws.gctx(),
162-
"build directory",
163-
)?)
164+
Some(if matches!(locking_mode, LockingMode::Coarse) {
165+
build_dest.open_rw_exclusive_create(".cargo-lock", ws.gctx(), "build directory")?
166+
} else {
167+
build_dest.open_ro_shared_create(".cargo-lock", ws.gctx(), "build directory")?
168+
})
164169
} else {
165170
None
166171
};
@@ -345,6 +350,11 @@ impl BuildDirLayout {
345350
self.build().join(pkg_dir)
346351
}
347352
}
353+
/// Fetch the lock paths for a build unit
354+
pub fn build_unit_lock(&self, pkg_dir: &str) -> (PathBuf, PathBuf) {
355+
let dir = self.build_unit(pkg_dir);
356+
(dir.join("primary.lock"), dir.join("secondary.lock"))
357+
}
348358
/// Fetch the artifact path.
349359
pub fn artifact(&self) -> &Path {
350360
&self.artifact

src/cargo/core/compiler/locking.rs

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
//! This module handles the locking logic during compilation.
2+
//!
3+
//! The locking scheme is based on build unit level locking.
4+
//! Each build unit consists of a primary and secondary lock used to represent multiple lock states.
5+
//!
6+
//! | State | Primary | Secondary |
7+
//! |------------------------|-------------|-------------|
8+
//! | Unlocked | `unlocked` | `unlocked` |
9+
//! | Building Exclusive | `exclusive` | `exclusive` |
10+
//! | Building Non-Exclusive | `shared` | `exclusive` |
11+
//! | Shared Partial | `shared` | `unlocked` |
12+
//! | Shared Full | `shared` | `shared` |
13+
//!
14+
//! Generally a build unit will full the following flow:
15+
//! 1. Acquire a "building exclusive" lock for the current build unit.
16+
//! 2. Acquire "shared" locks on all dependency build units.
17+
//! 3. Begin building with rustc
18+
//! 4. If we are building a library, downgrade to a "building non-exclusive" lock when the `.rmeta` has been generated.
19+
//! 5. Once complete release all locks.
20+
//!
21+
//! Most build units only require metadata (.rmeta) from dependencies, so they can begin building
22+
//! once the dependency units have produced the .rmeta. These units take a "shared partial" lock
23+
//! which can be taken while the dependency still holds the "build non-exclusive" lock.
24+
//!
25+
//! Note that some build unit types like bin and proc-macros require the full dependency build
26+
//! (.rlib). For these unit types they must take a "shared full" lock on dependency units which will
27+
//! block until the dependency unit is fully built.
28+
//!
29+
//! The primary reason for the complexity here it to enable fine grain locking while also allowing pipelined builds.
30+
//!
31+
//! [`CompilationLock`] is the primary interface for locking.
32+
33+
use std::{
34+
collections::HashSet,
35+
fs::{File, OpenOptions},
36+
path::{Path, PathBuf},
37+
};
38+
39+
use anyhow::Context;
40+
use itertools::Itertools;
41+
use tracing::{instrument, trace};
42+
43+
use crate::{
44+
CargoResult,
45+
core::compiler::{BuildRunner, Unit},
46+
};
47+
48+
/// The locking mode that will be used for output directories.
49+
#[derive(Debug)]
50+
pub enum LockingMode {
51+
/// Fine grain locking (Build unit level)
52+
Fine,
53+
/// Coarse grain locking (Profile level)
54+
Coarse,
55+
}
56+
57+
/// The type of lock to take when taking a shared lock.
58+
/// See the module documentation for more information about shared lock types.
59+
#[derive(Debug)]
60+
pub enum SharedLockType {
61+
/// A shared lock that might still be compiling a .rlib
62+
Partial,
63+
/// A shared lock that is guaranteed to not be compiling
64+
Full,
65+
}
66+
67+
/// A lock for compiling a build unit.
68+
///
69+
/// Internally this lock is made up of many [`UnitLock`]s for the unit and it's dependencies.
70+
pub struct CompilationLock {
71+
/// The path to the lock file of the unit to compile
72+
unit: UnitLock,
73+
/// The paths to lock files of the unit's dependencies
74+
dependency_units: Vec<UnitLock>,
75+
}
76+
77+
impl CompilationLock {
78+
pub fn new(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> Self {
79+
let unit_lock = build_runner.files().build_unit_lock(unit).into();
80+
81+
let dependency_units = all_dependency_units(build_runner, unit)
82+
.into_iter()
83+
.map(|unit| build_runner.files().build_unit_lock(&unit).into())
84+
.collect_vec();
85+
86+
Self {
87+
unit: unit_lock,
88+
dependency_units,
89+
}
90+
}
91+
92+
#[instrument(skip(self))]
93+
pub fn lock(&mut self, ty: &SharedLockType) -> CargoResult<()> {
94+
self.unit.lock_exclusive()?;
95+
96+
for d in self.dependency_units.iter_mut() {
97+
d.lock_shared(ty)?;
98+
}
99+
100+
trace!("acquired lock: {:?}", self.unit.primary.parent());
101+
102+
Ok(())
103+
}
104+
105+
pub fn rmeta_produced(&mut self) -> CargoResult<()> {
106+
trace!("downgrading lock: {:?}", self.unit.primary.parent());
107+
108+
// Downgrade the lock on the unit we are building so that we can unblock other units to
109+
// compile. We do not need to downgrade our dependency locks since they should always be a
110+
// shared lock.
111+
self.unit.downgrade()?;
112+
113+
Ok(())
114+
}
115+
}
116+
117+
/// A lock for a single build unit.
118+
struct UnitLock {
119+
primary: PathBuf,
120+
secondary: PathBuf,
121+
guard: Option<UnitLockGuard>,
122+
}
123+
124+
struct UnitLockGuard {
125+
primary: File,
126+
_secondary: Option<File>,
127+
}
128+
129+
impl UnitLock {
130+
pub fn lock_exclusive(&mut self) -> CargoResult<()> {
131+
assert!(self.guard.is_none());
132+
133+
let primary_lock = open_file(&self.primary)?;
134+
primary_lock.lock()?;
135+
136+
let secondary_lock = open_file(&self.secondary)?;
137+
secondary_lock.lock()?;
138+
139+
self.guard = Some(UnitLockGuard {
140+
primary: primary_lock,
141+
_secondary: Some(secondary_lock),
142+
});
143+
Ok(())
144+
}
145+
146+
pub fn lock_shared(&mut self, ty: &SharedLockType) -> CargoResult<()> {
147+
assert!(self.guard.is_none());
148+
149+
let primary_lock = open_file(&self.primary)?;
150+
primary_lock.lock_shared()?;
151+
152+
let secondary_lock = if matches!(ty, SharedLockType::Full) {
153+
let secondary_lock = open_file(&self.secondary)?;
154+
secondary_lock.lock_shared()?;
155+
Some(secondary_lock)
156+
} else {
157+
None
158+
};
159+
160+
self.guard = Some(UnitLockGuard {
161+
primary: primary_lock,
162+
_secondary: secondary_lock,
163+
});
164+
Ok(())
165+
}
166+
167+
pub fn downgrade(&mut self) -> CargoResult<()> {
168+
let guard = self
169+
.guard
170+
.as_ref()
171+
.context("guard was None while calling downgrade")?;
172+
173+
// NOTE:
174+
// > Subsequent flock() calls on an already locked file will convert an existing lock to the new lock mode.
175+
// https://man7.org/linux/man-pages/man2/flock.2.html
176+
//
177+
// However, the `std::file::File::lock/lock_shared` is allowed to change this in the
178+
// future. So its probably up to us if we are okay with using this or if we want to use a
179+
// different interface to flock.
180+
guard.primary.lock_shared()?;
181+
182+
Ok(())
183+
}
184+
}
185+
186+
impl From<(PathBuf, PathBuf)> for UnitLock {
187+
fn from((primary, secondary): (PathBuf, PathBuf)) -> Self {
188+
Self {
189+
primary,
190+
secondary,
191+
guard: None,
192+
}
193+
}
194+
}
195+
196+
fn open_file<T: AsRef<Path>>(f: T) -> CargoResult<File> {
197+
Ok(OpenOptions::new()
198+
.read(true)
199+
.create(true)
200+
.write(true)
201+
.append(true)
202+
.open(f)?)
203+
}
204+
205+
fn all_dependency_units<'a>(
206+
build_runner: &'a BuildRunner<'a, '_>,
207+
unit: &Unit,
208+
) -> HashSet<&'a Unit> {
209+
fn inner<'a>(
210+
build_runner: &'a BuildRunner<'a, '_>,
211+
unit: &Unit,
212+
results: &mut HashSet<&'a Unit>,
213+
) {
214+
for dep in build_runner.unit_deps(unit) {
215+
if results.insert(&dep.unit) {
216+
inner(&build_runner, &dep.unit, results);
217+
}
218+
}
219+
}
220+
221+
let mut results = HashSet::new();
222+
inner(build_runner, unit, &mut results);
223+
return results;
224+
}

0 commit comments

Comments
 (0)