Skip to content

Commit 74a62b2

Browse files
committed
revert: remove package graph cache (no wall-clock improvement)
The package graph cache saved ~160ms of graph construction but that work runs in parallel with SCM indexing (~688ms), so it never reduced wall clock time. Remove it to keep the codebase simpler. The actual performance win comes from the stat-mismatch resolution in the previous commit, which targets hash_scope on the critical path. https://claude.ai/code/session_01HXQRNfNhJQu26zPuR1VppH
1 parent 53b2cb0 commit 74a62b2

File tree

9 files changed

+42
-657
lines changed

9 files changed

+42
-657
lines changed

Cargo.lock

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

crates/turborepo-lib/src/run/builder.rs

Lines changed: 25 additions & 207 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,9 @@ use turborepo_errors::Spanned;
1717
use turborepo_process::ProcessManager;
1818
use turborepo_repository::{
1919
change_mapper::PackageInclusionReason,
20-
discovery::{LocalPackageDiscovery, PackageDiscovery},
21-
package_graph::{
22-
PackageGraph, PackageName,
23-
cache as graph_cache,
24-
lazy_lockfile::LazyLockfile,
25-
},
20+
package_graph::{PackageGraph, PackageName},
2621
package_json,
2722
package_json::PackageJson,
28-
package_manager::PackageManager,
2923
};
3024
use turborepo_run_summary::observability;
3125
use turborepo_scm::SCM;
@@ -153,161 +147,6 @@ impl RunBuilder {
153147
self.opts.run_opts.dry_run.is_none() && self.opts.run_opts.graph.is_none()
154148
}
155149

156-
/// Try to load a cached PackageGraph. Returns None on any failure.
157-
async fn try_load_cached_graph(
158-
&self,
159-
root_package_json: &PackageJson,
160-
) -> Option<PackageGraph> {
161-
let _span = tracing::info_span!("try_load_cached_graph").entered();
162-
163-
// Detect package manager
164-
let package_manager = PackageManager::get_package_manager(&self.repo_root, root_package_json)
165-
.ok()?;
166-
167-
// Run workspace discovery to get the current workspace paths
168-
let discovery = LocalPackageDiscovery::new(
169-
self.repo_root.clone(),
170-
package_manager.clone(),
171-
);
172-
let discovery_response = discovery.discover_packages().await.ok()?;
173-
174-
// Collect workspace package.json paths (sorted for deterministic fingerprint)
175-
let mut workspace_pkg_json_paths: Vec<AbsoluteSystemPathBuf> = discovery_response
176-
.workspaces
177-
.iter()
178-
.map(|w| w.package_json.clone())
179-
.collect();
180-
workspace_pkg_json_paths.sort();
181-
182-
// Collect turbo.json paths
183-
let mut turbo_json_paths: Vec<AbsoluteSystemPathBuf> = Vec::new();
184-
let root_turbo_json = self.opts.repo_opts.root_turbo_json_path.clone();
185-
if root_turbo_json.exists() {
186-
turbo_json_paths.push(root_turbo_json);
187-
}
188-
for ws in &discovery_response.workspaces {
189-
if let Some(ref tj) = ws.turbo_json {
190-
turbo_json_paths.push(tj.clone());
191-
}
192-
}
193-
turbo_json_paths.sort();
194-
195-
// Compute sorted workspace paths for the fingerprint
196-
let workspace_paths: Vec<String> = workspace_pkg_json_paths
197-
.iter()
198-
.filter_map(|p| self.repo_root.anchor(p).ok().map(|r| r.to_string()))
199-
.collect();
200-
201-
// Collect all input files and compute fingerprint
202-
let input_files = graph_cache::collect_input_files(
203-
&self.repo_root,
204-
&package_manager,
205-
&workspace_pkg_json_paths,
206-
&turbo_json_paths,
207-
);
208-
209-
let fingerprint = graph_cache::compute_fingerprint(
210-
&self.repo_root,
211-
&input_files,
212-
workspace_paths,
213-
)?;
214-
215-
// Try to load the cache
216-
let cache_dir = self.repo_root.join_components(&[".turbo", "cache"]);
217-
let cache_path = cache_dir.join_component(graph_cache::CACHE_FILE_NAME);
218-
219-
// Spawn lazy lockfile parsing in background (runs in parallel with cache load)
220-
let lazy_lockfile = LazyLockfile::spawn(
221-
package_manager.clone(),
222-
self.repo_root.clone(),
223-
root_package_json.clone(),
224-
);
225-
226-
let graph = graph_cache::try_load(
227-
&cache_path,
228-
&fingerprint,
229-
self.version,
230-
&self.repo_root,
231-
None, // lockfile loaded lazily
232-
);
233-
234-
match graph {
235-
Some(mut graph) => {
236-
// Resolve the lazy lockfile — it's been parsing in the background
237-
let lockfile = lazy_lockfile.resolve().await;
238-
graph.set_lockfile(lockfile);
239-
tracing::info!("package graph loaded from cache");
240-
Some(graph)
241-
}
242-
None => {
243-
// Cache miss — don't waste the lockfile parse though
244-
tracing::debug!("package graph cache miss");
245-
None
246-
}
247-
}
248-
}
249-
250-
/// Best-effort save of the PackageGraph to the cache file.
251-
fn save_graph_cache(&self, graph: &PackageGraph, _root_package_json: &PackageJson) {
252-
let _span = tracing::info_span!("save_graph_cache").entered();
253-
254-
let package_manager = graph.package_manager().clone();
255-
256-
// Collect workspace package.json paths from the graph
257-
let mut workspace_pkg_json_paths: Vec<AbsoluteSystemPathBuf> = graph
258-
.packages()
259-
.filter(|(name, _)| !matches!(name, PackageName::Root))
260-
.map(|(_, info)| self.repo_root.resolve(&info.package_json_path))
261-
.collect();
262-
workspace_pkg_json_paths.sort();
263-
264-
// Collect turbo.json paths
265-
let mut turbo_json_paths: Vec<AbsoluteSystemPathBuf> = Vec::new();
266-
let root_turbo_json = self.opts.repo_opts.root_turbo_json_path.clone();
267-
if root_turbo_json.exists() {
268-
turbo_json_paths.push(root_turbo_json);
269-
}
270-
// Check for workspace turbo.json files
271-
for (name, info) in graph.packages() {
272-
if matches!(name, PackageName::Root) {
273-
continue;
274-
}
275-
let ws_turbo_json = self.repo_root
276-
.resolve(info.package_path())
277-
.join_component("turbo.json");
278-
if ws_turbo_json.exists() {
279-
turbo_json_paths.push(ws_turbo_json);
280-
}
281-
}
282-
turbo_json_paths.sort();
283-
284-
let workspace_paths: Vec<String> = workspace_pkg_json_paths
285-
.iter()
286-
.filter_map(|p| self.repo_root.anchor(p).ok().map(|r| r.to_string()))
287-
.collect();
288-
289-
let input_files = graph_cache::collect_input_files(
290-
&self.repo_root,
291-
&package_manager,
292-
&workspace_pkg_json_paths,
293-
&turbo_json_paths,
294-
);
295-
296-
let fingerprint = match graph_cache::compute_fingerprint(
297-
&self.repo_root,
298-
&input_files,
299-
workspace_paths,
300-
) {
301-
Some(fp) => fp,
302-
None => return,
303-
};
304-
305-
let cache_dir = self.repo_root.join_components(&[".turbo", "cache"]);
306-
let cache_path = cache_dir.join_component(graph_cache::CACHE_FILE_NAME);
307-
308-
graph_cache::save(&cache_path, graph, fingerprint, self.version);
309-
}
310-
311150
pub fn calculate_filtered_packages(
312151
repo_root: &AbsoluteSystemPath,
313152
opts: &Opts,
@@ -429,52 +268,31 @@ impl RunBuilder {
429268
run_telemetry.track_daemon_init(DaemonInitStatus::Disabled);
430269

431270
let mut pkg_dep_graph = {
432-
let _span = tracing::info_span!("pkg_dep_graph_build").entered();
433-
434-
// Try to load from cache (only for multi-package repos)
435-
let cached_graph = if !self.opts.run_opts.single_package {
436-
self.try_load_cached_graph(&root_package_json).await
437-
} else {
438-
None
439-
};
440-
441-
match cached_graph {
442-
Some(graph) => graph,
443-
None => {
444-
// Cache miss or single-package mode — build from scratch
445-
let builder =
446-
PackageGraph::builder(&self.repo_root, root_package_json.clone())
447-
.with_single_package_mode(self.opts.run_opts.single_package)
448-
.with_allow_no_package_manager(
449-
self.opts.repo_opts.allow_no_package_manager,
450-
);
451-
452-
let graph = builder.build().await;
453-
454-
let graph = match graph {
455-
Ok(graph) => graph,
456-
Err(turborepo_repository::package_graph::Error::PackageJson(
457-
package_json::Error::Io(io),
458-
)) if io.kind() == ErrorKind::NotFound => {
459-
run_telemetry
460-
.track_error(TrackedErrors::InvalidPackageDiscovery);
461-
return Err(
462-
turborepo_repository::package_graph::Error::PackageJson(
463-
package_json::Error::Io(io),
464-
)
465-
.into(),
466-
);
467-
}
468-
Err(e) => return Err(e.into()),
469-
};
470-
471-
// Best-effort save to cache
472-
if !self.opts.run_opts.single_package {
473-
self.save_graph_cache(&graph, &root_package_json);
474-
}
475-
476-
graph
271+
let builder = PackageGraph::builder(&self.repo_root, root_package_json.clone())
272+
.with_single_package_mode(self.opts.run_opts.single_package)
273+
.with_allow_no_package_manager(self.opts.repo_opts.allow_no_package_manager);
274+
275+
let graph = builder
276+
.build()
277+
.instrument(tracing::info_span!("pkg_dep_graph_build"))
278+
.await;
279+
280+
match graph {
281+
Ok(graph) => graph,
282+
// if we can't find the package.json, it is a bug, and we should report it.
283+
// likely cause is that package discovery watching is not up to date.
284+
// note: there _is_ a false positive from a race condition that can occur
285+
// from toctou if the package.json is deleted, but we'd like to know
286+
Err(turborepo_repository::package_graph::Error::PackageJson(
287+
package_json::Error::Io(io),
288+
)) if io.kind() == ErrorKind::NotFound => {
289+
run_telemetry.track_error(TrackedErrors::InvalidPackageDiscovery);
290+
return Err(turborepo_repository::package_graph::Error::PackageJson(
291+
package_json::Error::Io(io),
292+
)
293+
.into());
477294
}
295+
Err(e) => return Err(e.into()),
478296
}
479297
};
480298

crates/turborepo-lockfiles/src/lib.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,14 @@ pub use error::Error;
3232
pub use npm::*;
3333
pub use pnpm::{PnpmLockfile, pnpm_global_change, pnpm_subgraph};
3434
use rayon::prelude::*;
35-
use serde::{Deserialize, Serialize};
35+
use serde::Serialize;
3636
use turbopath::RelativeUnixPathBuf;
3737
pub use yarn1::{Yarn1Lockfile, yarn_subgraph};
3838

3939
type ResolveCache = DashMap<String, Option<Package>>;
4040
type DepsCache = DashMap<String, Option<HashMap<String, String>>>;
4141

42-
#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash, Serialize, Deserialize)]
42+
#[derive(Debug, PartialEq, Eq, Clone, PartialOrd, Ord, Hash, Serialize)]
4343
pub struct Package {
4444
pub key: String,
4545
pub version: String,

crates/turborepo-repository/Cargo.toml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ turborepo-lockfiles = { workspace = true }
4040
turborepo-unescape = { workspace = true }
4141
wax = { workspace = true }
4242
which = { workspace = true }
43-
xxhash-rust = { version = "0.8", features = ["xxh64"] }
4443

4544
[dev-dependencies]
4645
insta = { workspace = true }

0 commit comments

Comments
 (0)