feat(build-dir): Reorganize build-dir layout

ranger-ross · ranger-ross · commit c849440047ec · 2025-10-08T00:35:50.000+09:00
diff --git a/src/cargo/core/compiler/build_runner/compilation_files.rs b/src/cargo/core/compiler/build_runner/compilation_files.rs
@@ -230,17 +230,25 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         self.export_dir.clone()
     }
 
-    /// Directory name to use for a package in the form `NAME-HASH`.
+    /// Directory name to use for a package in the form `{NAME}{SEPERATOR}{HASH}`.
+    ///
+    /// The seperator will usually be `/` making nested hashed directories, but will sometimes be
+    /// `-` for backwards compatiblity (artifact directory)
     ///
     /// Note that some units may share the same directory, so care should be
     /// taken in those cases!
-    fn pkg_dir(&self, unit: &Unit) -> String {
+    fn pkg_dir(&self, unit: &Unit, seperator: &str) -> String {
+        let seperator = match self.ws.gctx().cli_unstable().build_dir_new_layout {
+            true => seperator,
+            false => "-",
+        };
+
         let name = unit.pkg.package_id().name();
         let meta = self.metas[unit];
         if let Some(c_extra_filename) = meta.c_extra_filename() {
-            format!("{}-{}", name, c_extra_filename)
+            format!("{}{}{}", name, seperator, c_extra_filename)
         } else {
-            format!("{}-{}", name, self.target_short_hash(unit))
+            format!("{}{}{}", name, seperator, self.target_short_hash(unit))
         }
     }
 
@@ -256,20 +264,20 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
 
     /// Returns the host `deps` directory path.
     pub fn host_deps(&self, unit: &Unit) -> PathBuf {
-        let dir = self.pkg_dir(unit);
+        let dir = self.pkg_dir(unit, "/");
         self.host.deps(&dir)
     }
 
     /// Returns the directories where Rust crate dependencies are found for the
     /// specified unit.
     pub fn deps_dir(&self, unit: &Unit) -> PathBuf {
-        let dir = self.pkg_dir(unit);
+        let dir = self.pkg_dir(unit, "/");
         self.layout(unit.kind).deps(&dir)
     }
 
     /// Directory where the fingerprint for the given unit should go.
     pub fn fingerprint_dir(&self, unit: &Unit) -> PathBuf {
-        let dir = self.pkg_dir(unit);
+        let dir = self.pkg_dir(unit, "/");
         self.layout(unit.kind).fingerprint(&dir)
     }
 
@@ -309,7 +317,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         assert!(unit.target.is_custom_build());
         assert!(!unit.mode.is_run_custom_build());
         assert!(self.metas.contains_key(unit));
-        let dir = self.pkg_dir(unit);
+        let dir = self.pkg_dir(unit, "/");
         self.layout(CompileKind::Host).build_script(&dir)
     }
 
@@ -318,7 +326,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
     fn artifact_dir(&self, unit: &Unit) -> PathBuf {
         assert!(self.metas.contains_key(unit));
         assert!(unit.artifact.is_true());
-        let dir = self.pkg_dir(unit);
+        let dir = self.pkg_dir(unit, "-");
         let kind = match unit.target.kind() {
             TargetKind::Bin => "bin",
             TargetKind::Lib(lib_kinds) => match lib_kinds.as_slice() {
@@ -343,8 +351,8 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
     pub fn build_script_run_dir(&self, unit: &Unit) -> PathBuf {
         assert!(unit.target.is_custom_build());
         assert!(unit.mode.is_run_custom_build());
-        let dir = self.pkg_dir(unit);
-        self.layout(unit.kind).build().join(dir)
+        let dir = self.pkg_dir(unit, "/");
+        self.layout(unit.kind).build_script_execution(&dir)
     }
 
     /// Returns the "`OUT_DIR`" directory for running a build script.
diff --git a/src/cargo/core/compiler/build_runner/mod.rs b/src/cargo/core/compiler/build_runner/mod.rs
@@ -6,10 +6,11 @@ use std::sync::{Arc, Mutex};
 
 use crate::core::PackageId;
 use crate::core::compiler::compilation::{self, UnitOutput};
-use crate::core::compiler::{self, Unit, artifact};
+use crate::core::compiler::{self, CompileTarget, Unit, artifact};
 use crate::util::cache_lock::CacheLockMode;
 use crate::util::errors::CargoResult;
 use anyhow::{Context as _, bail};
+use cargo_util::paths;
 use filetime::FileTime;
 use itertools::Itertools;
 use jobserver::Client;
@@ -358,11 +359,18 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
     #[tracing::instrument(skip_all)]
     pub fn prepare_units(&mut self) -> CargoResult<()> {
         let dest = self.bcx.profiles.get_dir_name();
-        let host_layout = Layout::new(self.bcx.ws, None, &dest)?;
+        let host = &self.compilation.host;
+        let host_target = CompileTarget::new(&host)?;
+        let host_layout = if self.bcx.gctx.cli_unstable().build_dir_new_layout {
+            Layout::new(self.bcx.ws, Some(host_target), &dest, true)?
+        } else {
+            Layout::new(self.bcx.ws, None, &dest, true)?
+        };
         let mut targets = HashMap::new();
         for kind in self.bcx.all_kinds.iter() {
             if let CompileKind::Target(target) = *kind {
-                let layout = Layout::new(self.bcx.ws, Some(target), &dest)?;
+                let is_host = target == host_target;
+                let layout = Layout::new(self.bcx.ws, Some(target), &dest, is_host)?;
                 targets.insert(target, layout);
             }
         }
@@ -401,9 +409,17 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             self.compilation
                 .root_output
                 .insert(kind, layout.dest().to_path_buf());
-            self.compilation
-                .deps_output
-                .insert(kind, layout.legacy_deps().to_path_buf());
+            if self.bcx.gctx.cli_unstable().build_dir_new_layout {
+                for (unit, _) in self.bcx.unit_graph.iter() {
+                    let dep_dir = self.files().deps_dir(unit);
+                    paths::create_dir_all(&dep_dir)?;
+                    self.compilation.deps_output.insert(kind, dep_dir);
+                }
+            } else {
+                self.compilation
+                    .deps_output
+                    .insert(kind, layout.legacy_deps().to_path_buf());
+            }
         }
         Ok(())
     }
diff --git a/src/cargo/core/compiler/layout.rs b/src/cargo/core/compiler/layout.rs
@@ -142,6 +142,7 @@ pub struct Layout {
     /// Will be `None` when the build-dir and target-dir are the same path as we cannot
     /// lock the same path twice.
     _build_lock: Option<FileLock>,
+    is_new_layout: bool,
 }
 
 impl Layout {
@@ -155,12 +156,23 @@ impl Layout {
         ws: &Workspace<'_>,
         target: Option<CompileTarget>,
         dest: &str,
+        is_host_layout: bool,
     ) -> CargoResult<Layout> {
+        let is_new_layout = ws.gctx().cli_unstable().build_dir_new_layout;
         let mut root = ws.target_dir();
         let mut build_root = ws.build_dir();
-        if let Some(target) = target {
-            root.push(target.short_name());
-            build_root.push(target.short_name());
+        if is_new_layout {
+            assert!(target.is_some());
+            let short_name = target.as_ref().unwrap().short_name();
+            if !is_host_layout {
+                root.push(short_name);
+            }
+            build_root.push(short_name);
+        } else {
+            if let Some(target) = target {
+                root.push(target.short_name());
+                build_root.push(target.short_name());
+            }
         }
         let build_dest = build_root.join(dest);
         let dest = root.join(dest);
@@ -212,14 +224,17 @@ impl Layout {
             dest,
             _lock: lock,
             _build_lock: build_lock,
+            is_new_layout,
         })
     }
 
     /// Makes sure all directories stored in the Layout exist on the filesystem.
     pub fn prepare(&mut self) -> CargoResult<()> {
-        paths::create_dir_all(&self.deps)?;
+        if !self.is_new_layout {
+            paths::create_dir_all(&self.deps)?;
+            paths::create_dir_all(&self.fingerprint)?;
+        }
         paths::create_dir_all(&self.incremental)?;
-        paths::create_dir_all(&self.fingerprint)?;
         paths::create_dir_all(&self.examples)?;
         paths::create_dir_all(&self.build_examples)?;
         paths::create_dir_all(&self.build)?;
@@ -232,8 +247,12 @@ impl Layout {
         &self.dest
     }
     /// Fetch the deps path.
-    pub fn deps(&self, _pkg_dir: &str) -> PathBuf {
-        self.deps.clone()
+    pub fn deps(&self, pkg_dir: &str) -> PathBuf {
+        if self.is_new_layout {
+            self.build_unit(pkg_dir).join("deps")
+        } else {
+            self.deps.clone()
+        }
     }
     /// Fetch the deps path. (old layout)
     pub fn legacy_deps(&self) -> &Path {
@@ -261,7 +280,11 @@ impl Layout {
     }
     /// Fetch the fingerprint path.
     pub fn fingerprint(&self, pkg_dir: &str) -> PathBuf {
-        self.fingerprint.join(pkg_dir)
+        if self.is_new_layout {
+            self.build_unit(pkg_dir).join("fingerprint")
+        } else {
+            self.fingerprint.join(pkg_dir)
+        }
     }
     /// Fetch the fingerprint path. (old layout)
     pub fn legacy_fingerprint(&self) -> &Path {
@@ -273,16 +296,28 @@ impl Layout {
     }
     /// Fetch the build script path.
     pub fn build_script(&self, pkg_dir: &str) -> PathBuf {
-        self.build().join(pkg_dir)
+        if self.is_new_layout {
+            self.build_unit(pkg_dir).join("build-script")
+        } else {
+            self.build().join(pkg_dir)
+        }
     }
     /// Fetch the build script execution path.
     pub fn build_script_execution(&self, pkg_dir: &str) -> PathBuf {
-        self.build().join(pkg_dir)
+        if self.is_new_layout {
+            self.build_unit(pkg_dir).join("build-script-execution")
+        } else {
+            self.build().join(pkg_dir)
+        }
     }
     /// Fetch the artifact path.
     pub fn artifact(&self) -> &Path {
         &self.artifact
     }
+    /// Fetch the build unit path
+    pub fn build_unit(&self, pkg_dir: &str) -> PathBuf {
+        self.build().join(pkg_dir)
+    }
     /// Create and return the tmp path.
     pub fn prepare_tmp(&self) -> CargoResult<&Path> {
         paths::create_dir_all(&self.tmp)?;
diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs
@@ -56,7 +56,7 @@ pub mod unit_dependencies;
 pub mod unit_graph;
 
 use std::borrow::Cow;
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::env;
 use std::ffi::{OsStr, OsString};
 use std::fmt::Display;
@@ -69,6 +69,7 @@ use std::sync::{Arc, LazyLock};
 use annotate_snippets::{AnnotationKind, Group, Level, Renderer, Snippet};
 use anyhow::{Context as _, Error};
 use cargo_platform::{Cfg, Platform};
+use itertools::Itertools;
 use lazycell::LazyCell;
 use regex::Regex;
 use tracing::{debug, instrument, trace};
@@ -1664,18 +1665,35 @@ fn build_deps_args(
     unit: &Unit,
 ) -> CargoResult<()> {
     let bcx = build_runner.bcx;
-    cmd.arg("-L").arg(&{
-        let mut deps = OsString::from("dependency=");
-        deps.push(build_runner.files().deps_dir(unit));
-        deps
-    });
+    if build_runner.bcx.gctx.cli_unstable().build_dir_new_layout {
+        let mut map = BTreeMap::new();
+
+        // Recursively add all depenendency args to rustc process
+        add_dep_arg(&mut map, build_runner, unit);
+
+        let paths = map.into_iter().map(|(_, path)| path).sorted_unstable();
+
+        for path in paths {
+            cmd.arg("-L").arg(&{
+                let mut deps = OsString::from("dependency=");
+                deps.push(path);
+                deps
+            });
+        }
+    } else {
+        cmd.arg("-L").arg(&{
+            let mut deps = OsString::from("dependency=");
+            deps.push(build_runner.files().deps_dir(unit));
+            deps
+        });
+    }
 
     // Be sure that the host path is also listed. This'll ensure that proc macro
     // dependencies are correctly found (for reexported macros).
     if !unit.kind.is_host() {
         cmd.arg("-L").arg(&{
             let mut deps = OsString::from("dependency=");
-            deps.push(build_runner.files().host_deps(&unit));
+            deps.push(build_runner.files().host_deps(unit));
             deps
         });
     }
@@ -1751,6 +1769,21 @@ fn build_deps_args(
     Ok(())
 }
 
+fn add_dep_arg<'a, 'b: 'a>(
+    map: &mut BTreeMap<&'a Unit, PathBuf>,
+    build_runner: &'b BuildRunner<'b, '_>,
+    unit: &'a Unit,
+) {
+    if map.contains_key(&unit) {
+        return;
+    }
+    map.insert(&unit, build_runner.files().deps_dir(&unit));
+
+    for dep in build_runner.unit_deps(unit) {
+        add_dep_arg(map, build_runner, &dep.unit);
+    }
+}
+
 /// Adds extra rustc flags and environment variables collected from the output
 /// of a build-script to the command to execute, include custom environment
 /// variables and `cfg`.
diff --git a/src/cargo/ops/cargo_clean.rs b/src/cargo/ops/cargo_clean.rs
@@ -1,4 +1,4 @@
-use crate::core::compiler::{CompileKind, CompileMode, Layout, RustcTargetData};
+use crate::core::compiler::{CompileKind, CompileMode, CompileTarget, Layout, RustcTargetData};
 use crate::core::profiles::Profiles;
 use crate::core::{PackageIdSpec, PackageIdSpecQuery, TargetKind, Workspace};
 use crate::ops;
@@ -116,15 +116,22 @@ fn clean_specs(
     let target_data = RustcTargetData::new(ws, &requested_kinds)?;
     let (pkg_set, resolve) = ops::resolve_ws(ws, dry_run)?;
     let prof_dir_name = profiles.get_dir_name();
-    let host_layout = Layout::new(ws, None, &prof_dir_name)?;
+    let host_target = CompileTarget::new(target_data.short_name(&CompileKind::Host))?;
+    let host_layout = if clean_ctx.gctx.cli_unstable().build_dir_new_layout {
+        Layout::new(ws, Some(host_target), &prof_dir_name, true)?
+    } else {
+        Layout::new(ws, None, &prof_dir_name, true)?
+    };
     // Convert requested kinds to a Vec of layouts.
     let target_layouts: Vec<(CompileKind, Layout)> = requested_kinds
         .into_iter()
         .filter_map(|kind| match kind {
-            CompileKind::Target(target) => match Layout::new(ws, Some(target), &prof_dir_name) {
-                Ok(layout) => Some(Ok((kind, layout))),
-                Err(e) => Some(Err(e)),
-            },
+            CompileKind::Target(target) => {
+                match Layout::new(ws, Some(target), &prof_dir_name, false) {
+                    Ok(layout) => Some(Ok((kind, layout))),
+                    Err(e) => Some(Err(e)),
+                }
+            }
             CompileKind::Host => None,
         })
         .collect::<CargoResult<_>>()?;
@@ -228,6 +235,10 @@ fn clean_specs(
                 for (compile_kind, layout) in &layouts {
                     let triple = target_data.short_name(compile_kind);
 
+                    if clean_ctx.gctx.cli_unstable().build_dir_new_layout {
+                        let dir = layout.build_unit(&pkg.name());
+                        clean_ctx.rm_rf_glob(&dir)?;
+                    }
                     let (file_types, _unsupported) = target_data
                         .info(*compile_kind)
                         .rustc_outputs(mode, target.kind(), triple, clean_ctx.gctx)?;
diff --git a/tests/testsuite/build_dir.rs b/tests/testsuite/build_dir.rs
