Add ToolTarget to bootstrap

src/bootstrap/README.md

@@ -105,7 +105,7 @@ build/
       debuginfo/
       ...
 
-    # Bootstrap host tools (which are always compiled with the stage0 compiler)
+    # Host tools (which are always compiled with the stage0 compiler)
     # are stored here.
     bootstrap-tools/
 

src/bootstrap/src/core/build_steps/check.rs

@@ -4,7 +4,9 @@ use crate::core::build_steps::compile::{
     add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make,
 };
 use crate::core::build_steps::tool;
-use crate::core::build_steps::tool::{COMPILETEST_ALLOW_FEATURES, SourceType, prepare_tool_cargo};
+use crate::core::build_steps::tool::{
+    COMPILETEST_ALLOW_FEATURES, SourceType, get_tool_target_compiler, prepare_tool_cargo,
+};
 use crate::core::builder::{
     self, Alias, Builder, Kind, RunConfig, ShouldRun, Step, StepMetadata, crate_description,
 };
@@ -247,8 +249,10 @@ fn prepare_compiler_for_check(
     mode: Mode,
 ) -> Compiler {
     let host = builder.host_target;
+
     match mode {
         Mode::ToolBootstrap => builder.compiler(0, host),
+        Mode::ToolTarget => get_tool_target_compiler(builder, target),
         Mode::ToolStd => {
             // These tools require the local standard library to be checked
             let build_compiler = builder.compiler(builder.top_stage, host);

src/bootstrap/src/core/build_steps/tool.rs

@@ -365,6 +365,17 @@ pub(crate) fn get_tool_rustc_compiler(
     builder.compiler(target_compiler.stage.saturating_sub(1), builder.config.host_target)
 }
 
+/// Returns a compiler that is able to compile a `ToolTarget` tool for the given `target`.
+pub(crate) fn get_tool_target_compiler(builder: &Builder<'_>, target: TargetSelection) -> Compiler {
+    todo!("FIX");
+    if builder.host_target == target {
+        builder.compiler(0, builder.host_target)
+    } else {
+        // FIXME: should this be builder.top_stage to avoid rebuilds?
+        builder.compiler(1, target)
+    }
+}
+
 /// Links a built tool binary with the given `name` from the build directory to the
 /// tools directory.
 fn copy_link_tool_bin(

src/bootstrap/src/core/builder/cargo.rs

@@ -537,23 +537,25 @@ impl Builder<'_> {
             }
         }
 
-        let stage = if compiler.stage == 0 && self.local_rebuild {
+        let build_compiler_stage = if compiler.stage == 0 && self.local_rebuild {
             // Assume the local-rebuild rustc already has stage1 features.
             1
         } else {
             compiler.stage
         };
 
         // We synthetically interpret a stage0 compiler used to build tools as a
-        // "raw" compiler in that it's the exact snapshot we download. Normally
-        // the stage0 build means it uses libraries build by the stage0
-        // compiler, but for tools we just use the precompiled libraries that
-        // we've downloaded
-        let use_snapshot = mode == Mode::ToolBootstrap;
-        assert!(!use_snapshot || stage == 0 || self.local_rebuild);
-
-        let maybe_sysroot = self.sysroot(compiler);
-        let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot };
+        // "raw" compiler in that it's the exact snapshot we download. For things like
+        // ToolRustc, we would have to use the artificial stage0-sysroot compiler instead.
+        let use_snapshot =
+            mode == Mode::ToolBootstrap || (mode == Mode::ToolTarget && build_compiler_stage == 0);
+        assert!(!use_snapshot || build_compiler_stage == 0 || self.local_rebuild);
+
+        let sysroot = if use_snapshot {
+            self.rustc_snapshot_sysroot().to_path_buf()
+        } else {
+            self.sysroot(compiler)
+        };
         let libdir = self.rustc_libdir(compiler);
 
         let sysroot_str = sysroot.as_os_str().to_str().expect("sysroot should be UTF-8");
@@ -562,7 +564,7 @@ impl Builder<'_> {
         }
 
         let mut rustflags = Rustflags::new(target);
-        if stage != 0 {
+        if build_compiler_stage != 0 {
             if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") {
                 cargo.args(s.split_whitespace());
             }
@@ -604,7 +606,7 @@ impl Builder<'_> {
         // sysroot. Passing this cfg enables raw-dylib support instead, which makes the native
         // library unnecessary. This can be removed when windows-rs enables raw-dylib
         // unconditionally.
-        if let Mode::Rustc | Mode::ToolRustc | Mode::ToolBootstrap = mode {
+        if let Mode::Rustc | Mode::ToolRustc | Mode::ToolBootstrap | Mode::ToolTarget = mode {
             rustflags.arg("--cfg=windows_raw_dylib");
         }
 
@@ -657,7 +659,7 @@ impl Builder<'_> {
         // FIXME(rust-lang/cargo#5754) we shouldn't be using special command arguments
         // to the host invocation here, but rather Cargo should know what flags to pass rustc
         // itself.
-        if stage == 0 {
+        if build_compiler_stage == 0 {
             hostflags.arg("--cfg=bootstrap");
         }
 
@@ -666,7 +668,7 @@ impl Builder<'_> {
         // #71458.
         let mut rustdocflags = rustflags.clone();
         rustdocflags.propagate_cargo_env("RUSTDOCFLAGS");
-        if stage == 0 {
+        if build_compiler_stage == 0 {
             rustdocflags.env("RUSTDOCFLAGS_BOOTSTRAP");
         } else {
             rustdocflags.env("RUSTDOCFLAGS_NOT_BOOTSTRAP");
@@ -677,7 +679,7 @@ impl Builder<'_> {
         }
 
         match mode {
-            Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {}
+            Mode::Std | Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolTarget => {}
             Mode::Rustc | Mode::Codegen | Mode::ToolRustc => {
                 // Build proc macros both for the host and the target unless proc-macros are not
                 // supported by the target.
@@ -719,7 +721,7 @@ impl Builder<'_> {
         // feature on the rustc side.
         cargo.arg("-Zbinary-dep-depinfo");
         let allow_features = match mode {
-            Mode::ToolBootstrap | Mode::ToolStd => {
+            Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolTarget => {
                 // Restrict the allowed features so we don't depend on nightly
                 // accidentally.
                 //
@@ -827,7 +829,7 @@ impl Builder<'_> {
         cargo
             .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target))
             .env("RUSTC_REAL", self.rustc(compiler))
-            .env("RUSTC_STAGE", stage.to_string())
+            .env("RUSTC_STAGE", build_compiler_stage.to_string())
             .env("RUSTC_SYSROOT", sysroot)
             .env("RUSTC_LIBDIR", libdir)
             .env("RUSTDOC", self.bootstrap_out.join("rustdoc"))
@@ -872,7 +874,7 @@ impl Builder<'_> {
         let debuginfo_level = match mode {
             Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc,
             Mode::Std => self.config.rust_debuginfo_level_std,
-            Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => {
+            Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc | Mode::ToolTarget => {
                 self.config.rust_debuginfo_level_tools
             }
         };
@@ -884,11 +886,10 @@ impl Builder<'_> {
             profile_var("DEBUG_ASSERTIONS"),
             match mode {
                 Mode::Std => self.config.std_debug_assertions,
-                Mode::Rustc => self.config.rustc_debug_assertions,
-                Mode::Codegen => self.config.rustc_debug_assertions,
-                Mode::ToolBootstrap => self.config.tools_debug_assertions,
-                Mode::ToolStd => self.config.tools_debug_assertions,
-                Mode::ToolRustc => self.config.tools_debug_assertions,
+                Mode::Rustc | Mode::Codegen => self.config.rustc_debug_assertions,
+                Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc | Mode::ToolTarget => {
+                    self.config.tools_debug_assertions
+                }
             }
             .to_string(),
         );
@@ -959,7 +960,11 @@ impl Builder<'_> {
                     cargo.env("CFG_VIRTUAL_RUSTC_DEV_SOURCE_BASE_DIR", map_to);
                 }
             }
-            Mode::Std | Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd => {
+            Mode::Std
+            | Mode::ToolBootstrap
+            | Mode::ToolRustc
+            | Mode::ToolStd
+            | Mode::ToolTarget => {
                 if let Some(ref map_to) =
                     self.build.debuginfo_map_to(GitRepo::Rustc, RemapScheme::NonCompiler)
                 {
@@ -1274,7 +1279,7 @@ impl Builder<'_> {
             };
 
             if let Some(limit) = limit
-                && (stage == 0
+                && (build_compiler_stage == 0
                     || self.config.default_codegen_backend(target).unwrap_or_default() == "llvm")
             {
                 rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}"));

src/bootstrap/src/core/builder/mod.rs

@@ -959,6 +959,7 @@ impl<'a> Builder<'a> {
                 tool::RemoteTestServer,
                 tool::RemoteTestClient,
                 tool::RustInstaller,
+                tool::FeaturesStatusDump,
                 tool::Cargo,
                 tool::RustAnalyzer,
                 tool::RustAnalyzerProcMacroSrv,

src/bootstrap/src/lib.rs

@@ -253,12 +253,24 @@ pub enum Mode {
     /// These tools are intended to be only executed on the host system that
     /// invokes bootstrap, and they thus cannot be cross-compiled.
     ///
-    /// They are always built using the stage0 compiler, and typically they
+    /// They are always built using the stage0 compiler, and they
     /// can be compiled with stable Rust.
     ///
     /// These tools also essentially do not participate in staging.
     ToolBootstrap,
 
+    /// Build a cross-compilable helper tool. These tools do not depend on unstable features or
+    /// compiler internals, but they might be cross-compilable (so we cannot build them using the
+    /// stage0 compiler, unlike `ToolBootstrap`).
+    ///
+    /// Some of these tools are also shipped in our `dist` archives.
+    /// While we could compile them using the stage0 compiler when not cross-compiling, we instead
+    /// use the in-tree compiler (and std) to build them, so that we can ship e.g. std security
+    /// fixes and avoid depending fully on stage0 for the artifacts that we ship.
+    ///
+    /// This mode is used e.g. for linkers and linker tools invoked by rustc on its host target.
+    ToolTarget,
+
     /// Build a tool which uses the locally built std, placing output in the
     /// "stageN-tools" directory. Its usage is quite rare, mainly used by
     /// compiletest which needs libtest.
@@ -273,11 +285,21 @@ pub enum Mode {
 
 impl Mode {
     pub fn is_tool(&self) -> bool {
-        matches!(self, Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd)
+        match self {
+            Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd | Mode::ToolTarget => true,
+            Mode::Std | Mode::Codegen | Mode::Rustc => false,
+        }
     }
 
     pub fn must_support_dlopen(&self) -> bool {
-        matches!(self, Mode::Std | Mode::Codegen)
+        match self {
+            Mode::Std | Mode::Codegen => true,
+            Mode::ToolBootstrap
+            | Mode::ToolRustc
+            | Mode::ToolStd
+            | Mode::ToolTarget
+            | Mode::Rustc => false,
+        }
     }
 }
 
@@ -804,17 +826,39 @@ impl Build {
     /// stage when running with a particular host compiler.
     ///
     /// The mode indicates what the root directory is for.
-    fn stage_out(&self, compiler: Compiler, mode: Mode) -> PathBuf {
-        let suffix = match mode {
-            Mode::Std => "-std",
-            Mode::Rustc => "-rustc",
-            Mode::Codegen => "-codegen",
-            Mode::ToolBootstrap => {
-                return self.out.join(compiler.host).join("bootstrap-tools");
+    fn stage_out(&self, build_compiler: Compiler, mode: Mode) -> PathBuf {
+        use std::fmt::Write;
+
+        fn bootstrap_tool() -> (Option<u32>, &'static str) {
+            (None, "bootstrap-tools")
+        }
+        fn staged_tool(build_compiler: Compiler) -> (Option<u32>, &'static str) {
+            (Some(build_compiler.stage), "tools")
+        }
+
+        let (stage, suffix) = match mode {
+            Mode::Std => (Some(build_compiler.stage), "std"),
+            Mode::Rustc => (Some(build_compiler.stage), "rustc"),
+            Mode::Codegen => (Some(build_compiler.stage), "codegen"),
+            Mode::ToolBootstrap => bootstrap_tool(),
+            Mode::ToolStd | Mode::ToolRustc => (Some(build_compiler.stage), "tools"),
+            Mode::ToolTarget => {
+                // If we're not cross-compiling (the common case), share the target directory with
+                // bootstrap tools to reuse the build cache.
+                if build_compiler.stage == 0 {
+                    bootstrap_tool()
+                } else {
+                    staged_tool(build_compiler)
+                }
             }
-            Mode::ToolStd | Mode::ToolRustc => "-tools",
         };
-        self.out.join(compiler.host).join(format!("stage{}{}", compiler.stage, suffix))
+        let path = self.out.join(build_compiler.host);
+        let mut dir_name = String::new();
+        if let Some(stage) = stage {
+            write!(dir_name, "stage{stage}-").unwrap();
+        }
+        dir_name.push_str(suffix);
+        path.join(dir_name)
     }
 
     /// Returns the root output directory for all Cargo output in a given stage,