Download beta compiler toolchain in bootstrap if it doesn't yet exist

This is needed for when the shell scripts bypass python altogether and run the downloaded bootstrap directly

Author: jyn514

src/bootstrap/config.rs

@@ -55,8 +55,7 @@ pub enum DryRun {
 /// filled out from the decoded forms of the structs below. For documentation
 /// each field, see the corresponding fields in
 /// `config.toml.example`.
-#[derive(Default)]
-#[cfg_attr(test, derive(Clone))]
+#[derive(Clone, Default)]
 pub struct Config {
     pub changelog_seen: Option<usize>,
     pub ccache: Option<String>,
@@ -217,36 +216,41 @@ pub struct Config {
     pub reuse: Option<PathBuf>,
     pub cargo_native_static: bool,
     pub configure_args: Vec<String>,
+    pub out: PathBuf,
+    pub rust_info: channel::GitInfo,
 
     // These are either the stage0 downloaded binaries or the locally installed ones.
     pub initial_cargo: PathBuf,
     pub initial_rustc: PathBuf,
+
     #[cfg(not(test))]
     initial_rustfmt: RefCell<RustfmtState>,
     #[cfg(test)]
     pub initial_rustfmt: RefCell<RustfmtState>,
-    pub out: PathBuf,
-    pub rust_info: channel::GitInfo,
 }
 
-#[derive(Default, Deserialize)]
-#[cfg_attr(test, derive(Clone))]
+#[derive(Clone, Default, Deserialize)]
 pub struct Stage0Metadata {
+    pub compiler: CompilerMetadata,
     pub config: Stage0Config,
     pub checksums_sha256: HashMap<String, String>,
     pub rustfmt: Option<RustfmtMetadata>,
 }
-#[derive(Default, Deserialize)]
-#[cfg_attr(test, derive(Clone))]
+#[derive(Clone, Default, Deserialize)]
+pub struct CompilerMetadata {
+    pub date: String,
+    pub version: String,
+}
+
+#[derive(Clone, Default, Deserialize)]
 pub struct Stage0Config {
     pub dist_server: String,
     pub artifacts_server: String,
     pub artifacts_with_llvm_assertions_server: String,
     pub git_merge_commit_email: String,
     pub nightly_branch: String,
 }
-#[derive(Default, Deserialize)]
-#[cfg_attr(test, derive(Clone))]
+#[derive(Clone, Default, Deserialize)]
 pub struct RustfmtMetadata {
     pub date: String,
     pub version: String,
@@ -422,8 +426,7 @@ impl PartialEq<&str> for TargetSelection {
 }
 
 /// Per-target configuration stored in the global configuration structure.
-#[derive(Default)]
-#[cfg_attr(test, derive(Clone))]
+#[derive(Clone, Default)]
 pub struct Target {
     /// Some(path to llvm-config) if using an external LLVM.
     pub llvm_config: Option<PathBuf>,
@@ -979,10 +982,10 @@ impl Config {
             config.out = crate::util::absolute(&config.out);
         }
 
-        config.initial_rustc = build
-            .rustc
-            .map(PathBuf::from)
-            .unwrap_or_else(|| config.out.join(config.build.triple).join("stage0/bin/rustc"));
+        config.initial_rustc = build.rustc.map(PathBuf::from).unwrap_or_else(|| {
+            config.download_beta_toolchain();
+            config.out.join(config.build.triple).join("stage0/bin/rustc")
+        });
         config.initial_cargo = build
             .cargo
             .map(PathBuf::from)

src/bootstrap/download.rs

@@ -361,26 +361,69 @@ impl Config {
 
     pub(crate) fn download_ci_rustc(&self, commit: &str) {
         self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})"));
+
         let version = self.artifact_version_part(commit);
+        // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the
+        // `rustc_private` crates for tools.
+        let extra_components = ["rustc-dev"];
+
+        self.download_toolchain(
+            &version,
+            "ci-rustc",
+            commit,
+            &extra_components,
+            Self::download_ci_component,
+        );
+    }
+
+    pub(crate) fn download_beta_toolchain(&self) {
+        self.verbose(&format!("downloading stage0 beta artifacts"));
+
+        let date = &self.stage0_metadata.compiler.date;
+        let version = &self.stage0_metadata.compiler.version;
+        let extra_components = ["cargo"];
+
+        let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| {
+            config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0")
+        };
+
+        self.download_toolchain(
+            version,
+            "stage0",
+            date,
+            &extra_components,
+            download_beta_component,
+        );
+    }
+
+    fn download_toolchain(
+        &self,
+        channel: &str,
+        sysroot: &str,
+        stamp_key: &str,
+        extra_components: &[&str],
+        download_component: fn(&Config, String, &str, &str),
+    ) {
         let host = self.build.triple;
-        let bin_root = self.out.join(host).join("ci-rustc");
+        let bin_root = self.out.join(host).join(sysroot);
         let rustc_stamp = bin_root.join(".rustc-stamp");
 
-        if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit)
+        if !bin_root.join("bin").join("rustc").exists()
+            || program_out_of_date(&rustc_stamp, stamp_key)
         {
             if bin_root.exists() {
                 t!(fs::remove_dir_all(&bin_root));
             }
-            let filename = format!("rust-std-{version}-{host}.tar.xz");
+            let filename = format!("rust-std-{channel}-{host}.tar.xz");
             let pattern = format!("rust-std-{host}");
-            self.download_ci_component(filename, &pattern, commit);
-            let filename = format!("rustc-{version}-{host}.tar.xz");
-            self.download_ci_component(filename, "rustc", commit);
-            // download-rustc doesn't need its own cargo, it can just use beta's.
-            let filename = format!("rustc-dev-{version}-{host}.tar.xz");
-            self.download_ci_component(filename, "rustc-dev", commit);
-            let filename = format!("rust-src-{version}.tar.xz");
-            self.download_ci_component(filename, "rust-src", commit);
+            download_component(self, filename, &pattern, stamp_key);
+            let filename = format!("rustc-{channel}-{host}.tar.xz");
+            download_component(self, filename, "rustc", stamp_key);
+
+            for component in extra_components {
+                let filename = format!("{component}-{channel}-{host}.tar.xz");
+                download_component(self, filename, component, stamp_key);
+            }
 
             if self.should_fix_bins_and_dylibs() {
                 self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
@@ -397,7 +440,7 @@ impl Config {
                 }
             }
 
-            t!(fs::write(rustc_stamp, commit));
+            t!(fs::write(rustc_stamp, stamp_key));
         }
     }
 

src/bootstrap/flags.rs

@@ -82,8 +82,7 @@ pub struct Flags {
     pub llvm_bolt_profile_use: Option<String>,
 }
 
-#[derive(Debug)]
-#[cfg_attr(test, derive(Clone))]
+#[derive(Clone, Debug)]
 pub enum Subcommand {
     Build {
         paths: Vec<PathBuf>,

src/bootstrap/test.rs

@@ -1120,7 +1120,7 @@ impl Step for Tidy {
         if builder.config.channel == "dev" || builder.config.channel == "nightly" {
             builder.info("fmt check");
             if builder.initial_rustfmt().is_none() {
-                let inferred_rustfmt_dir = builder.config.initial_rustc.parent().unwrap();
+                let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap();
                 eprintln!(
                     "\
 error: no `rustfmt` binary found in {PATH}