diff --git a/src/bootstrap/src/core/build_steps/check.rs b/src/bootstrap/src/core/build_steps/check.rs index 49d12b64da510..da5e743924a7d 100644 --- a/src/bootstrap/src/core/build_steps/check.rs +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -1,10 +1,7 @@ //! Implementation of compiling the compiler and standard library, in "check"-based modes. -use std::fs; -use std::path::{Path, PathBuf}; - use crate::core::build_steps::compile::{ - add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make, + run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make, }; use crate::core::build_steps::tool; use crate::core::build_steps::tool::{ @@ -16,7 +13,7 @@ use crate::core::builder::{ }; use crate::core::config::TargetSelection; use crate::utils::build_stamp::{self, BuildStamp}; -use crate::{CodegenBackendKind, Compiler, Mode, Subcommand, t}; +use crate::{BuiltArtifactsDir, CodegenBackendKind, Compiler, Mode, Subcommand}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Std { @@ -162,43 +159,6 @@ impl Step for Std { } } -/// Represents a proof that rustc was **checked**. -/// Contains directories with .rmeta files generated by checking rustc for a specific -/// target. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct RmetaSysroot { - host_dir: PathBuf, - target_dir: PathBuf, -} - -impl RmetaSysroot { - /// Copy rmeta artifacts from the given `stamp` into a sysroot located at `directory`. - fn from_stamp( - builder: &Builder<'_>, - stamp: BuildStamp, - target: TargetSelection, - directory: &Path, - ) -> Self { - let host_dir = directory.join("host"); - let target_dir = directory.join(target); - let _ = fs::remove_dir_all(directory); - t!(fs::create_dir_all(directory)); - add_to_sysroot(builder, &target_dir, &host_dir, &stamp); - - Self { host_dir, target_dir } - } - - /// Configure the given cargo invocation so that the compiled crate will be able to use - /// rustc .rmeta artifacts that were previously generated. - fn configure_cargo(&self, cargo: &mut Cargo) { - cargo.append_to_env( - "RUSTC_ADDITIONAL_SYSROOT_PATHS", - format!("{},{}", self.host_dir.to_str().unwrap(), self.target_dir.to_str().unwrap()), - ",", - ); - } -} - /// Checks rustc using the given `build_compiler` for the given `target`, and produces /// a sysroot in the build directory that stores the generated .rmeta files. /// @@ -218,7 +178,7 @@ impl PrepareRustcRmetaSysroot { } impl Step for PrepareRustcRmetaSysroot { - type Output = RmetaSysroot; + type Output = BuiltArtifactsDir; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.never() @@ -239,7 +199,7 @@ impl Step for PrepareRustcRmetaSysroot { .out .join(build_compiler.host) .join(format!("stage{}-rustc-rmeta-artifacts", build_compiler.stage + 1)); - RmetaSysroot::from_stamp(builder, stamp, self.target, &dir) + BuiltArtifactsDir::from_stamp(builder, stamp, self.target, &dir) } } @@ -262,7 +222,7 @@ impl PrepareStdRmetaSysroot { } impl Step for PrepareStdRmetaSysroot { - type Output = RmetaSysroot; + type Output = BuiltArtifactsDir; fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { run.never() @@ -282,7 +242,7 @@ impl Step for PrepareStdRmetaSysroot { .join(self.build_compiler.host) .join(format!("stage{}-std-rmeta-artifacts", self.build_compiler.stage)); - RmetaSysroot::from_stamp(builder, stamp, self.target, &dir) + BuiltArtifactsDir::from_stamp(builder, stamp, self.target, &dir) } } @@ -398,8 +358,8 @@ impl Step for Rustc { #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CompilerForCheck { build_compiler: Compiler, - rustc_rmeta_sysroot: Option, - std_rmeta_sysroot: Option, + rustc_rmeta_sysroot: Option, + std_rmeta_sysroot: Option, } impl CompilerForCheck { @@ -425,7 +385,7 @@ fn prepare_std( builder: &Builder<'_>, build_compiler: Compiler, target: TargetSelection, -) -> Option { +) -> Option { // We need to build the host stdlib even if we only check, to compile build scripts and proc // macros builder.std(build_compiler, builder.host_target); diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 0b75e85772f86..78df5c5518763 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -36,8 +36,8 @@ use crate::utils::helpers::{ exe, get_clang_cl_resource_dir, is_debug_info, is_dylib, symlink_dir, t, up_to_date, }; use crate::{ - CLang, CodegenBackendKind, Compiler, DependencyType, FileType, GitRepo, LLVM_TOOLS, Mode, - debug, trace, + BuiltArtifactsDir, CLang, CodegenBackendKind, Compiler, DependencyType, FileType, GitRepo, + LLVM_TOOLS, Mode, debug, trace, }; /// Build a standard library for the given `target` using the given `build_compiler`. @@ -955,6 +955,11 @@ pub struct BuiltRustc { /// This can be different from the *build_compiler* passed to the `Rustc` step because of /// uplifting. pub build_compiler: Compiler, + /// Build stamp that stores the rustc rlibs that were built by `build_compiler`. + /// + /// FIXME: this shouldn't be an option, ideally get rid of the Option once download-ci-rustc + /// is refactored. + pub stamp: Option, } /// Build rustc using the passed `build_compiler`. @@ -1028,6 +1033,8 @@ impl Step for Rustc { let build_compiler = self.build_compiler; let target = self.target; + let stamp = build_stamp::librustc_stamp(builder, build_compiler, target); + // NOTE: the ABI of the stage0 compiler is different from the ABI of the downloaded compiler, // so its artifacts can't be reused. if builder.download_rustc() && build_compiler.stage != 0 { @@ -1040,7 +1047,7 @@ impl Step for Rustc { &sysroot, builder.config.ci_rustc_dev_contents(), ); - return BuiltRustc { build_compiler }; + return BuiltRustc { build_compiler, stamp: None }; } // Build a standard library for `target` using the `build_compiler`. @@ -1052,9 +1059,11 @@ impl Step for Rustc { builder.info("WARNING: Using a potentially old librustc. This may not behave well."); builder.info("WARNING: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); - builder.ensure(RustcLink::from_rustc(self)); - return BuiltRustc { build_compiler }; + return BuiltRustc { + build_compiler, + stamp: Some(stamp.load_from_disk().expect("Rustc stamp was not found on disk")), + }; } // The stage of the compiler that we're building @@ -1072,25 +1081,16 @@ impl Step for Rustc { // be uplifting. We cannot uplift stage 1, as it has a different ABI than stage 2+, // so we always uplift the stage2 compiler (compiled with stage 1). let uplift_build_compiler = builder.compiler(1, build_compiler.host); + let stamp = build_stamp::librustc_stamp(builder, uplift_build_compiler, target) + .load_from_disk() + .expect("Rustc stamp was not found on disk"); let msg = format!("Uplifting rustc from stage2 to stage{stage})"); builder.info(&msg); - // Here the compiler that built the rlibs (`uplift_build_compiler`) can be different - // from the compiler whose sysroot should be modified in this step. So we need to copy - // the (previously built) rlibs into the correct sysroot. - builder.ensure(RustcLink::from_build_compiler_and_sysroot( - // This is the compiler that actually built the rustc rlibs - uplift_build_compiler, - // We copy the rlibs into the sysroot of `build_compiler` - build_compiler, - target, - self.crates, - )); - // Here we have performed an uplift, so we return the actual build compiler that "built" // this rustc. - return BuiltRustc { build_compiler: uplift_build_compiler }; + return BuiltRustc { build_compiler: uplift_build_compiler, stamp: Some(stamp) }; } // Build a standard library for the current host target using the `build_compiler`. @@ -1133,7 +1133,6 @@ impl Step for Rustc { build_compiler, target, ); - let stamp = build_stamp::librustc_stamp(builder, build_compiler, target); run_cargo( builder, cargo, @@ -1163,8 +1162,7 @@ impl Step for Rustc { strip_debug(builder, target, &target_root_dir.join("rustc-main")); } - builder.ensure(RustcLink::from_rustc(self)); - BuiltRustc { build_compiler } + BuiltRustc { build_compiler, stamp: Some(stamp) } } fn metadata(&self) -> Option { @@ -1471,74 +1469,6 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect } } -/// `RustcLink` copies compiler rlibs from a rustc build into a compiler sysroot. -/// It works with (potentially up to) three compilers: -/// - `build_compiler` is a compiler that built rustc rlibs -/// - `sysroot_compiler` is a compiler into whose sysroot we will copy the rlibs -/// - In most situations, `build_compiler` == `sysroot_compiler` -/// - `target_compiler` is the compiler whose rlibs were built. It is not represented explicitly -/// in this step, rather we just read the rlibs from a rustc build stamp of `build_compiler`. -/// -/// This is necessary for tools using `rustc_private`, where the previous compiler will build -/// a tool against the next compiler. -/// To build a tool against a compiler, the rlibs of that compiler that it links against -/// must be in the sysroot of the compiler that's doing the compiling. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct RustcLink { - /// This compiler **built** some rustc, whose rlibs we will copy into a sysroot. - build_compiler: Compiler, - /// This is the compiler into whose sysroot we want to copy the built rlibs. - /// In most cases, it will correspond to `build_compiler`. - sysroot_compiler: Compiler, - target: TargetSelection, - /// Not actually used; only present to make sure the cache invalidation is correct. - crates: Vec, -} - -impl RustcLink { - /// Copy rlibs from the build compiler that build this `rustc` into the sysroot of that - /// build compiler. - fn from_rustc(rustc: Rustc) -> Self { - Self { - build_compiler: rustc.build_compiler, - sysroot_compiler: rustc.build_compiler, - target: rustc.target, - crates: rustc.crates, - } - } - - /// Copy rlibs **built** by `build_compiler` into the sysroot of `sysroot_compiler`. - fn from_build_compiler_and_sysroot( - build_compiler: Compiler, - sysroot_compiler: Compiler, - target: TargetSelection, - crates: Vec, - ) -> Self { - Self { build_compiler, sysroot_compiler, target, crates } - } -} - -impl Step for RustcLink { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Same as `std_link`, only for librustc - fn run(self, builder: &Builder<'_>) { - let build_compiler = self.build_compiler; - let sysroot_compiler = self.sysroot_compiler; - let target = self.target; - add_to_sysroot( - builder, - &builder.sysroot_target_libdir(sysroot_compiler, target), - &builder.sysroot_target_libdir(sysroot_compiler, sysroot_compiler.host), - &build_stamp::librustc_stamp(builder, build_compiler, target), - ); - } -} - /// Output of the `compile::GccCodegenBackend` step. /// It includes the path to the libgccjit library on which this backend depends. #[derive(Clone)] @@ -1601,6 +1531,7 @@ impl Step for GccCodegenBackend { ); cargo.arg("--manifest-path").arg(builder.src.join("compiler/rustc_codegen_gcc/Cargo.toml")); rustc_cargo_env(builder, &mut cargo, target); + self.compilers.configure_cargo(&mut cargo); add_cg_gcc_cargo_flags(&mut cargo, &gcc); @@ -1675,6 +1606,7 @@ impl Step for CraneliftCodegenBackend { .arg("--manifest-path") .arg(builder.src.join("compiler/rustc_codegen_cranelift/Cargo.toml")); rustc_cargo_env(builder, &mut cargo, target); + self.compilers.configure_cargo(&mut cargo); let _guard = builder.msg( Kind::Build, @@ -2190,7 +2122,7 @@ impl Step for Assemble { ); // It is possible that an uplift has happened, so we override build_compiler here. - let BuiltRustc { build_compiler } = + let BuiltRustc { build_compiler, stamp: _ } = builder.ensure(Rustc::new(build_compiler, target_compiler.host)); let stage = target_compiler.stage; @@ -2279,6 +2211,7 @@ impl Step for Assemble { let prepare_compilers = || { RustcPrivateCompilers::from_build_and_target_compiler( + builder, build_compiler, target_compiler, ) @@ -2374,6 +2307,40 @@ impl Step for Assemble { } } +/// Compiles rustc using `build_compiler` for the given `target`, and +/// outputs a directory with the generated compiler rlibs. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PrepareRlibSysroot { + build_compiler: Compiler, + target: TargetSelection, +} + +impl PrepareRlibSysroot { + pub fn new(build_compiler: Compiler, target: TargetSelection) -> Self { + Self { build_compiler, target } + } +} + +impl Step for PrepareRlibSysroot { + type Output = Option; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + // Build rustc + let rustc = builder.ensure(Rustc::new(self.build_compiler, self.target)); + + // Copy the generated rlib artifacts to a separate directory + let dir = builder + .out + .join(self.build_compiler.host) + .join(format!("stage{}-rustc-rlib-artifacts", self.build_compiler.stage + 1)); + rustc.stamp.map(|stamp| BuiltArtifactsDir::from_stamp(builder, stamp, self.target, &dir)) + } +} + /// Link some files into a rustc sysroot. /// /// For a particular stage this will link the file listed in `stamp` into the diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 820dda5a6524c..862c9307b9f51 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1406,8 +1406,8 @@ impl Step for Clippy { // Prepare the image directory // We expect clippy to build, because we've exited this step above if tool // state for clippy isn't testing. - let clippy = builder.ensure(tool::Clippy::from_compilers(self.compilers)); - let cargoclippy = builder.ensure(tool::CargoClippy::from_compilers(self.compilers)); + let clippy = builder.ensure(tool::Clippy::from_compilers(self.compilers.clone())); + let cargoclippy = builder.ensure(tool::CargoClippy::from_compilers(self.compilers.clone())); let mut tarball = Tarball::new(builder, "clippy", &target.triple); tarball.set_overlay(OverlayKind::Clippy); @@ -1454,7 +1454,7 @@ impl Step for Miri { return None; } - let miri = builder.ensure(tool::Miri::from_compilers(self.compilers)); + let miri = builder.ensure(tool::Miri::from_compilers(self.compilers.clone())); let cargomiri = builder.ensure(tool::CargoMiri::from_compilers(self.compilers)); let mut tarball = Tarball::new(builder, "miri", &self.target.triple); @@ -1522,7 +1522,8 @@ impl Step for CraneliftCodegenBackend { tarball.add_legal_and_readme_to("share/doc/rustc_codegen_cranelift"); let compilers = self.compilers; - let stamp = builder.ensure(compile::CraneliftCodegenBackend { compilers }); + let stamp = + builder.ensure(compile::CraneliftCodegenBackend { compilers: compilers.clone() }); if builder.config.dry_run() { return None; @@ -1581,7 +1582,7 @@ impl Step for Rustfmt { } fn run(self, builder: &Builder<'_>) -> Option { - let rustfmt = builder.ensure(tool::Rustfmt::from_compilers(self.compilers)); + let rustfmt = builder.ensure(tool::Rustfmt::from_compilers(self.compilers.clone())); let cargofmt = builder.ensure(tool::Cargofmt::from_compilers(self.compilers)); let mut tarball = Tarball::new(builder, "rustfmt", &self.target.triple); @@ -1660,14 +1661,14 @@ impl Step for Extended { // Std stage N is documented with compiler stage N add_component!("rust-json-docs" => JsonDocs { build_compiler: target_compiler, target }); add_component!("cargo" => Cargo { build_compiler, target }); - add_component!("rustfmt" => Rustfmt { compilers: rustc_private_compilers, target }); - add_component!("rust-analyzer" => RustAnalyzer { compilers: rustc_private_compilers, target }); + add_component!("rustfmt" => Rustfmt { compilers: rustc_private_compilers.clone(), target }); + add_component!("rust-analyzer" => RustAnalyzer { compilers: rustc_private_compilers.clone(), target }); add_component!("llvm-components" => LlvmTools { target }); - add_component!("clippy" => Clippy { compilers: rustc_private_compilers, target }); - add_component!("miri" => Miri { compilers: rustc_private_compilers, target }); + add_component!("clippy" => Clippy { compilers: rustc_private_compilers.clone(), target }); + add_component!("miri" => Miri { compilers: rustc_private_compilers.clone(), target }); add_component!("analysis" => Analysis { build_compiler, target }); add_component!("rustc-codegen-cranelift" => CraneliftCodegenBackend { - compilers: rustc_private_compilers, + compilers: rustc_private_compilers.clone(), target }); add_component!("llvm-bitcode-linker" => LlvmBitcodeLinker { diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs index 9f7248b80f763..1dfe888f7245d 100644 --- a/src/bootstrap/src/core/build_steps/run.rs +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -149,7 +149,7 @@ impl Step for Miri { let compilers = self.compilers; let target = self.target; - builder.ensure(tool::Miri::from_compilers(compilers)); + builder.ensure(tool::Miri::from_compilers(compilers.clone())); // Get a target sysroot for Miri. let miri_sysroot = diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index cb81d738666a3..190c7438f6481 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -511,7 +511,7 @@ impl Step for Rustfmt { /// Runs `cargo test` for rustfmt. fn run(self, builder: &Builder<'_>) { - let tool_result = builder.ensure(tool::Rustfmt::from_compilers(self.compilers)); + let tool_result = builder.ensure(tool::Rustfmt::from_compilers(self.compilers.clone())); let build_compiler = tool_result.build_compiler; let target = self.compilers.target(); @@ -615,13 +615,13 @@ impl Step for Miri { // This compiler runs on the host, we'll just use it for the target. let compilers = RustcPrivateCompilers::new(builder, stage, host); + let target_compiler = compilers.target_compiler(); + // Build our tools. - let miri = builder.ensure(tool::Miri::from_compilers(compilers)); + let miri = builder.ensure(tool::Miri::from_compilers(compilers.clone())); // the ui tests also assume cargo-miri has been built builder.ensure(tool::CargoMiri::from_compilers(compilers)); - let target_compiler = compilers.target_compiler(); - // We also need sysroots, for Miri and for the host (the latter for build scripts). // This is for the tests so everything is done with the target compiler. let miri_sysroot = Miri::build_miri_sysroot(builder, target_compiler, target); diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index c5308034fe307..3698e79f2efce 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -13,7 +13,7 @@ use std::ffi::OsStr; use std::path::PathBuf; use std::{env, fs}; -use crate::core::build_steps::compile::is_lto_stage; +use crate::core::build_steps::compile::{PrepareRlibSysroot, is_lto_stage}; use crate::core::build_steps::toolstate::ToolState; use crate::core::build_steps::{compile, llvm}; use crate::core::builder; @@ -23,7 +23,7 @@ use crate::core::builder::{ use crate::core::config::{DebuginfoLevel, RustcLto, TargetSelection}; use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{add_dylib_path, exe, t}; -use crate::{Compiler, FileType, Kind, Mode}; +use crate::{BuiltArtifactsDir, Compiler, FileType, Kind, Mode}; #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub enum SourceType { @@ -37,14 +37,38 @@ pub enum ToolArtifactKind { Library, } +/// A compiler used to compile a tool. +/// We need `RustcPrivateCompilers` to build `rustc_private` tools, so we represent these +/// separately. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +enum ToolCompiler { + RustcPrivate(RustcPrivateCompilers), + Other(Compiler, Mode), +} + +impl ToolCompiler { + fn mode(&self) -> Mode { + match self { + ToolCompiler::RustcPrivate(_) => Mode::ToolRustcPrivate, + ToolCompiler::Other(_, mode) => *mode, + } + } + + fn build_compiler(&self) -> Compiler { + match self { + ToolCompiler::RustcPrivate(compiler) => compiler.build_compiler(), + ToolCompiler::Other(compiler, _) => *compiler, + } + } +} + #[derive(Debug, Clone, Hash, PartialEq, Eq)] struct ToolBuild { /// Compiler that will build this tool. - build_compiler: Compiler, + build_compiler: ToolCompiler, target: TargetSelection, tool: &'static str, path: &'static str, - mode: Mode, source_type: SourceType, extra_features: Vec, /// Nightly-only features that are allowed (comma-separated list). @@ -81,18 +105,21 @@ impl Step for ToolBuild { let mut tool = self.tool; let path = self.path; - match self.mode { + let mode = self.build_compiler.mode(); + let build_compiler = self.build_compiler.build_compiler(); + + match mode { Mode::ToolRustcPrivate => { // FIXME: remove this, it's only needed for download-rustc... - if !self.build_compiler.is_forced_compiler() && builder.download_rustc() { - builder.std(self.build_compiler, self.build_compiler.host); - builder.ensure(compile::Rustc::new(self.build_compiler, target)); + if !build_compiler.is_forced_compiler() && builder.download_rustc() { + builder.std(build_compiler, build_compiler.host); + builder.ensure(compile::Rustc::new(build_compiler, target)); } } Mode::ToolStd => { // If compiler was forced, its artifacts should have been prepared earlier. - if !self.build_compiler.is_forced_compiler() { - builder.std(self.build_compiler, target); + if !build_compiler.is_forced_compiler() { + builder.std(build_compiler, target); } } Mode::ToolBootstrap | Mode::ToolTarget => {} // uses downloaded stage0 compiler libs @@ -101,21 +128,27 @@ impl Step for ToolBuild { let mut cargo = prepare_tool_cargo( builder, - self.build_compiler, - self.mode, + build_compiler, + mode, target, Kind::Build, path, self.source_type, &self.extra_features, ); + match &self.build_compiler { + ToolCompiler::RustcPrivate(compiler) => { + compiler.configure_cargo(&mut cargo); + } + ToolCompiler::Other(_, _) => {} + } // The stage0 compiler changes infrequently and does not directly depend on code // in the current working directory. Therefore, caching it with sccache should be // useful. // This is only performed for non-incremental builds, as ccache cannot deal with these. if let Some(ref ccache) = builder.config.ccache - && matches!(self.mode, Mode::ToolBootstrap) + && matches!(mode, Mode::ToolBootstrap) && !builder.config.incremental { cargo.env("RUSTC_WRAPPER", ccache); @@ -123,7 +156,7 @@ impl Step for ToolBuild { // Rustc tools (miri, clippy, cargo, rustfmt, rust-analyzer) // could use the additional optimizations. - if self.mode == Mode::ToolRustcPrivate && is_lto_stage(&self.build_compiler) { + if mode == Mode::ToolRustcPrivate && is_lto_stage(&build_compiler) { let lto = match builder.config.rust_lto { RustcLto::Off => Some("off"), RustcLto::Thin => Some("thin"), @@ -141,8 +174,7 @@ impl Step for ToolBuild { cargo.args(self.cargo_args); - let _guard = - builder.msg(Kind::Build, self.tool, self.mode, self.build_compiler, self.target); + let _guard = builder.msg(Kind::Build, self.tool, mode, build_compiler, self.target); // we check this below let build_success = compile::stream_cargo(builder, cargo, vec![], &mut |_| {}); @@ -163,14 +195,14 @@ impl Step for ToolBuild { } let tool_path = match self.artifact_kind { ToolArtifactKind::Binary => { - copy_link_tool_bin(builder, self.build_compiler, self.target, self.mode, tool) + copy_link_tool_bin(builder, build_compiler, self.target, mode, tool) } ToolArtifactKind::Library => builder - .cargo_out(self.build_compiler, self.mode, self.target) + .cargo_out(build_compiler, mode, self.target) .join(format!("lib{tool}.rlib")), }; - ToolBuildResult { tool_path, build_compiler: self.build_compiler } + ToolBuildResult { tool_path, build_compiler } } } } @@ -430,16 +462,16 @@ macro_rules! bootstrap_tool { let is_unstable = false $(|| $unstable)*; let compiletest_wants_stage0 = $tool_name == "compiletest" && builder.config.compiletest_use_stage0_libtest; + let mode = if is_unstable && !compiletest_wants_stage0 { + // use in-tree libraries for unstable features + Mode::ToolStd + } else { + Mode::ToolBootstrap + }; builder.ensure(ToolBuild { - build_compiler: self.compiler, + build_compiler: ToolCompiler::Other(self.compiler, mode), target: self.target, tool: $tool_name, - mode: if is_unstable && !compiletest_wants_stage0 { - // use in-tree libraries for unstable features - Mode::ToolStd - } else { - Mode::ToolBootstrap - }, path: $path, source_type: if false $(|| $external)* { SourceType::Submodule @@ -538,11 +570,11 @@ impl Step for RustcPerf { // We need to ensure the rustc-perf submodule is initialized. builder.require_submodule("src/tools/rustc-perf", None); + let mode = Mode::ToolBootstrap; let tool = ToolBuild { - build_compiler: self.compiler, + build_compiler: ToolCompiler::Other(self.compiler, mode), target: self.target, tool: "collector", - mode: Mode::ToolBootstrap, path: "src/tools/rustc-perf", source_type: SourceType::Submodule, extra_features: Vec::new(), @@ -555,7 +587,7 @@ impl Step for RustcPerf { let res = builder.ensure(tool.clone()); // We also need to symlink the `rustc-fake` binary to the corresponding directory, // because `collector` expects it in the same directory. - copy_link_tool_bin(builder, tool.build_compiler, tool.target, tool.mode, "rustc-fake"); + copy_link_tool_bin(builder, self.compiler, tool.target, mode, "rustc-fake"); res } @@ -568,11 +600,12 @@ pub struct ErrorIndex { impl ErrorIndex { pub fn command(builder: &Builder<'_>, compilers: RustcPrivateCompilers) -> BootstrapCommand { + let target_compiler = compilers.target_compiler(); + // Error-index-generator links with the rustdoc library, so we need to add `rustc_lib_paths` // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc. let mut cmd = command(builder.ensure(ErrorIndex { compilers }).tool_path); - let target_compiler = compilers.target_compiler(); let mut dylib_paths = builder.rustc_lib_paths(target_compiler); dylib_paths.push(builder.sysroot_target_libdir(target_compiler, target_compiler.host)); add_dylib_path(dylib_paths, &mut cmd); @@ -603,11 +636,11 @@ impl Step for ErrorIndex { } fn run(self, builder: &Builder<'_>) -> ToolBuildResult { + let target = self.compilers.target(); builder.ensure(ToolBuild { - build_compiler: self.compilers.build_compiler, - target: self.compilers.target(), + build_compiler: ToolCompiler::RustcPrivate(self.compilers), + target, tool: "error_index_generator", - mode: Mode::ToolRustcPrivate, path: "src/tools/error_index_generator", source_type: SourceType::InTree, extra_features: Vec::new(), @@ -650,10 +683,9 @@ impl Step for RemoteTestServer { fn run(self, builder: &Builder<'_>) -> ToolBuildResult { builder.ensure(ToolBuild { - build_compiler: self.build_compiler, + build_compiler: ToolCompiler::Other(self.build_compiler, Mode::ToolTarget), target: self.target, tool: "remote-test-server", - mode: Mode::ToolTarget, path: "src/tools/remote-test-server", source_type: SourceType::InTree, extra_features: Vec::new(), @@ -753,13 +785,12 @@ impl Step for Rustdoc { let compilers = RustcPrivateCompilers::from_target_compiler(builder, target_compiler); let tool_path = builder .ensure(ToolBuild { - build_compiler: compilers.build_compiler, + build_compiler: ToolCompiler::RustcPrivate(compilers), target, // Cargo adds a number of paths to the dylib search path on windows, which results in // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" // rustdoc a different name. tool: "rustdoc_tool_binary", - mode: Mode::ToolRustcPrivate, path: "src/tools/rustdoc", source_type: SourceType::InTree, extra_features, @@ -830,10 +861,9 @@ impl Step for Cargo { builder.std(self.build_compiler, self.target); builder.ensure(ToolBuild { - build_compiler: self.build_compiler, + build_compiler: ToolCompiler::Other(self.build_compiler, Mode::ToolTarget), target: self.target, tool: "cargo", - mode: Mode::ToolTarget, path: "src/tools/cargo", source_type: SourceType::Submodule, extra_features: Vec::new(), @@ -901,10 +931,9 @@ impl Step for LldWrapper { fn run(self, builder: &Builder<'_>) -> Self::Output { let lld_dir = builder.ensure(llvm::Lld { target: self.target }); let tool = builder.ensure(ToolBuild { - build_compiler: self.build_compiler, + build_compiler: ToolCompiler::Other(self.build_compiler, Mode::ToolTarget), target: self.target, tool: "lld-wrapper", - mode: Mode::ToolTarget, path: "src/tools/lld-wrapper", source_type: SourceType::InTree, extra_features: Vec::new(), @@ -992,10 +1021,9 @@ impl Step for WasmComponentLd { fn run(self, builder: &Builder<'_>) -> ToolBuildResult { builder.ensure(ToolBuild { - build_compiler: self.build_compiler, + build_compiler: ToolCompiler::Other(self.build_compiler, Mode::ToolTarget), target: self.target, tool: "wasm-component-ld", - mode: Mode::ToolTarget, path: "src/tools/wasm-component-ld", source_type: SourceType::InTree, extra_features: vec![], @@ -1042,13 +1070,11 @@ impl Step for RustAnalyzer { } fn run(self, builder: &Builder<'_>) -> ToolBuildResult { - let build_compiler = self.compilers.build_compiler; let target = self.compilers.target(); builder.ensure(ToolBuild { - build_compiler, + build_compiler: ToolCompiler::RustcPrivate(self.compilers), target, tool: "rust-analyzer", - mode: Mode::ToolRustcPrivate, path: "src/tools/rust-analyzer", extra_features: vec!["in-rust-tree".to_owned()], source_type: SourceType::InTree, @@ -1101,11 +1127,12 @@ impl Step for RustAnalyzerProcMacroSrv { } fn run(self, builder: &Builder<'_>) -> Self::Output { + let target = self.compilers.target(); + let target_compiler = self.compilers.target_compiler(); let tool_result = builder.ensure(ToolBuild { - build_compiler: self.compilers.build_compiler, - target: self.compilers.target(), + build_compiler: ToolCompiler::RustcPrivate(self.compilers), + target, tool: "rust-analyzer-proc-macro-srv", - mode: Mode::ToolRustcPrivate, path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", extra_features: vec!["in-rust-tree".to_owned()], source_type: SourceType::InTree, @@ -1116,7 +1143,7 @@ impl Step for RustAnalyzerProcMacroSrv { // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` // so that r-a can use it. - let libexec_path = builder.sysroot(self.compilers.target_compiler).join("libexec"); + let libexec_path = builder.sysroot(target_compiler).join("libexec"); t!(fs::create_dir_all(&libexec_path)); builder.copy_link( &tool_result.tool_path, @@ -1188,10 +1215,9 @@ impl Step for LlvmBitcodeLinker { fn run(self, builder: &Builder<'_>) -> ToolBuildResult { builder.ensure(ToolBuild { - build_compiler: self.build_compiler, + build_compiler: ToolCompiler::Other(self.build_compiler, Mode::ToolTarget), target: self.target, tool: "llvm-bitcode-linker", - mode: Mode::ToolTarget, path: "src/tools/llvm-bitcode-linker", source_type: SourceType::InTree, extra_features: vec![], @@ -1280,13 +1306,20 @@ impl Step for LibcxxVersionTool { /// /// Eventually, this could also be used for .rmetas and check builds, but so far we only deal with /// normal builds here. -#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] +#[derive(Clone, Debug, Hash, PartialEq, Eq)] pub struct RustcPrivateCompilers { /// Compiler that builds the tool and that builds `target_compiler`. build_compiler: Compiler, /// Compiler to which .rlib artifacts the tool links to. /// The host target of this compiler corresponds to the target of the tool. + /// + /// Ideally, we wouldn't use target_compiler at all, and just prepare the rlibs here. + /// But there are many places in bootstrap that require a `Compiler` to e.g. resolve + /// sysroot/lib paths, so we need to actually materialize `target_compiler`. target_compiler: Compiler, + /// Sysroot directory that contains rustc rlib artifacts of `target_compiler`, built with + /// `build_compiler`. + rlib_sysroot: Option, } impl RustcPrivateCompilers { @@ -1299,14 +1332,18 @@ impl RustcPrivateCompilers { // FIXME: make 100% sure that `target_compiler` was indeed built with `build_compiler`... let target_compiler = builder.compiler(build_compiler.stage + 1, target); - Self { build_compiler, target_compiler } + Self::from_build_and_target_compiler(builder, build_compiler, target_compiler) } pub fn from_build_and_target_compiler( + builder: &Builder<'_>, build_compiler: Compiler, target_compiler: Compiler, ) -> Self { - Self { build_compiler, target_compiler } + let target = target_compiler.host; + + let rlib_sysroot = builder.ensure(PrepareRlibSysroot::new(build_compiler, target)); + Self { build_compiler, target_compiler, rlib_sysroot } } /// Create rustc tool compilers from the build compiler. @@ -1316,15 +1353,13 @@ impl RustcPrivateCompilers { target: TargetSelection, ) -> Self { let target_compiler = builder.compiler(build_compiler.stage + 1, target); - Self { build_compiler, target_compiler } + Self::from_build_and_target_compiler(builder, build_compiler, target_compiler) } /// Create rustc tool compilers from the target compiler. pub fn from_target_compiler(builder: &Builder<'_>, target_compiler: Compiler) -> Self { - Self { - build_compiler: Self::build_compiler_from_stage(builder, target_compiler.stage), - target_compiler, - } + let build_compiler = Self::build_compiler_from_stage(builder, target_compiler.stage); + Self::from_build_and_target_compiler(builder, build_compiler, target_compiler) } fn build_compiler_from_stage(builder: &Builder<'_>, stage: u32) -> Compiler { @@ -1350,6 +1385,12 @@ impl RustcPrivateCompilers { pub fn target(&self) -> TargetSelection { self.target_compiler.host } + + pub fn configure_cargo(&self, cargo: &mut builder::Cargo) { + if let Some(rlib_sysroot) = &self.rlib_sysroot { + rlib_sysroot.configure_cargo(cargo); + } + } } /// Creates a step that builds an extended `Mode::ToolRustcPrivate` tool @@ -1462,11 +1503,11 @@ fn build_extended_rustc_tool( } let build_compiler = compilers.build_compiler; + let target_compiler = compilers.target_compiler; let ToolBuildResult { tool_path, .. } = builder.ensure(ToolBuild { - build_compiler, + build_compiler: ToolCompiler::RustcPrivate(compilers), target, tool: tool_name, - mode: Mode::ToolRustcPrivate, path, extra_features, source_type: SourceType::InTree, @@ -1475,7 +1516,6 @@ fn build_extended_rustc_tool( artifact_kind: ToolArtifactKind::Binary, }); - let target_compiler = compilers.target_compiler; if let Some(add_bins_to_sysroot) = add_bins_to_sysroot && !add_bins_to_sysroot.is_empty() { diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index 627085df812c5..332a37b50edb3 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -1613,7 +1613,7 @@ Alternatively, you can set `build.local-rebuild=true` and use a stage0 compiler assert_eq!(run_compiler, compilers.target_compiler()); // Prepare the tools - let miri = self.ensure(tool::Miri::from_compilers(compilers)); + let miri = self.ensure(tool::Miri::from_compilers(compilers.clone())); let cargo_miri = self.ensure(tool::CargoMiri::from_compilers(compilers)); // Invoke cargo-miri, make sure it can find miri and cargo. let mut cmd = command(cargo_miri.tool_path); @@ -1651,7 +1651,7 @@ Alternatively, you can set `build.local-rebuild=true` and use a stage0 compiler // for RustcPrivateCompilers. We will use build compiler stage N-1 to build Clippy stage N. let compilers = RustcPrivateCompilers::from_target_compiler(self, build_compiler); - let _ = self.ensure(tool::Clippy::from_compilers(compilers)); + let _ = self.ensure(tool::Clippy::from_compilers(compilers.clone())); let cargo_clippy = self.ensure(tool::CargoClippy::from_compilers(compilers)); let mut dylib_path = helpers::dylib_path(); dylib_path.insert(0, self.sysroot(build_compiler).join("lib")); diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index a2aeed20948e5..c61b6061304f2 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -34,7 +34,7 @@ use utils::channel::GitInfo; use utils::exec::ExecutionContext; use crate::core::builder; -use crate::core::builder::Kind; +use crate::core::builder::{Builder, Cargo, Kind}; use crate::core::config::{DryRun, LldMode, LlvmLibunwind, TargetSelection, flags}; use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{self, dir_is_empty, exe, libdir, set_file_times, split_debuginfo}; @@ -57,6 +57,7 @@ pub use utils::helpers::{PanicTracker, symlink_dir}; #[cfg(feature = "tracing")] pub use utils::tracing::setup_tracing; +use crate::core::build_steps::compile::add_to_sysroot; use crate::core::build_steps::vendor::VENDOR_DIR; const LLVM_TOOLS: &[&str] = &[ @@ -2155,3 +2156,41 @@ pub fn prepare_behaviour_dump_dir(build: &Build) { t!(INITIALIZED.set(true)); } } + +/// This directory contains built artifacts of something (rustc or std) that are stored separately +/// so that they do not have to be copied into the sysroot of some build compiler. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct BuiltArtifactsDir { + /// Subdirectory with host artifacts. + host_dir: PathBuf, + /// Subdirectory with target artifacts. + target_dir: PathBuf, +} + +impl BuiltArtifactsDir { + /// Copy artifacts from the given `stamp` into the given `directory`. + pub fn from_stamp( + builder: &Builder<'_>, + stamp: BuildStamp, + target: TargetSelection, + directory: &Path, + ) -> Self { + let host_dir = directory.join("host"); + let target_dir = directory.join(target); + let _ = fs::remove_dir_all(directory); + t!(fs::create_dir_all(directory)); + add_to_sysroot(builder, &target_dir, &host_dir, &stamp); + + Self { host_dir, target_dir } + } + + /// Configure the given cargo invocation so that the compiled crate will be able to use + /// the built artifacts that were previously generated. + pub fn configure_cargo(&self, cargo: &mut Cargo) { + cargo.append_to_env( + "RUSTC_ADDITIONAL_SYSROOT_PATHS", + format!("{},{}", self.host_dir.to_str().unwrap(), self.target_dir.to_str().unwrap()), + ",", + ); + } +} diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index 4c35388a181aa..df5eb6571d72c 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -17,7 +17,7 @@ mod tests; /// Manages a stamp file to track build state. The file is created in the given /// directory and can have custom content and name. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct BuildStamp { path: PathBuf, stamp: String, @@ -34,6 +34,15 @@ impl BuildStamp { Self { path: dir.join(".stamp"), stamp: String::new() } } + /// Loads the contents of the stamp from disk. + /// Returns `None` if the file does not exist on disk. + pub fn load_from_disk(self) -> Option { + let Ok(stamp) = std::fs::read_to_string(&self.path) else { + return None; + }; + Some(Self { path: self.path, stamp }) + } + /// Returns path of the stamp file. pub fn path(&self) -> &Path { &self.path