Rollup merge of rust-lang#113373 - jyn514:download-rustc-fixes, r=albertlarsan68

various download-rustc fixes

separated out from rust-lang#112143 because it keeps getting stuck in limbo.

best reviewed commit-by-commit

Author: workingjubilee

src/bootstrap/compile.rs

@@ -688,8 +688,8 @@ fn cp_rustc_component_to_ci_sysroot(
     contents: Vec<String>,
 ) {
     let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false });
+    let ci_rustc_dir = builder.config.ci_rustc_dir();
 
-    let ci_rustc_dir = builder.out.join(&*builder.build.build.triple).join("ci-rustc");
     for file in contents {
         let src = ci_rustc_dir.join(&file);
         let dst = sysroot.join(file);
@@ -1424,7 +1424,7 @@ impl Step for Sysroot {
                 // FIXME: this is wrong when compiler.host != build, but we don't support that today
                 OsStr::new(std::env::consts::DLL_EXTENSION),
             ];
-            let ci_rustc_dir = builder.ci_rustc_dir(builder.config.build);
+            let ci_rustc_dir = builder.config.ci_rustc_dir();
             builder.cp_filtered(&ci_rustc_dir, &sysroot, &|path| {
                 if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) {
                     return true;

src/bootstrap/config.rs

@@ -1375,6 +1375,25 @@ impl Config {
         let mut omit_git_hash = None;
 
         if let Some(rust) = toml.rust {
+            set(&mut config.channel, rust.channel);
+
+            config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc);
+            // This list is incomplete, please help by expanding it!
+            if config.download_rustc_commit.is_some() {
+                // We need the channel used by the downloaded compiler to match the one we set for rustdoc;
+                // otherwise rustdoc-ui tests break.
+                let ci_channel = t!(fs::read_to_string(config.src.join("src/ci/channel")));
+                let ci_channel = ci_channel.trim_end();
+                if config.channel != ci_channel
+                    && !(config.channel == "dev" && ci_channel == "nightly")
+                {
+                    panic!(
+                        "setting rust.channel={} is incompatible with download-rustc",
+                        config.channel
+                    );
+                }
+            }
+
             debug = rust.debug;
             debug_assertions = rust.debug_assertions;
             debug_assertions_std = rust.debug_assertions_std;
@@ -1386,6 +1405,7 @@ impl Config {
             debuginfo_level_std = rust.debuginfo_level_std;
             debuginfo_level_tools = rust.debuginfo_level_tools;
             debuginfo_level_tests = rust.debuginfo_level_tests;
+
             config.rust_split_debuginfo = rust
                 .split_debuginfo
                 .as_deref()
@@ -1401,7 +1421,6 @@ impl Config {
             set(&mut config.jemalloc, rust.jemalloc);
             set(&mut config.test_compare_mode, rust.test_compare_mode);
             set(&mut config.backtrace, rust.backtrace);
-            set(&mut config.channel, rust.channel);
             config.description = rust.description;
             set(&mut config.rust_dist_src, rust.dist_src);
             set(&mut config.verbose_tests, rust.verbose_tests);
@@ -1442,8 +1461,6 @@ impl Config {
             config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
             config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
             config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
-            config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc);
-
             config.rust_lto = rust
                 .lto
                 .as_deref()
@@ -1555,6 +1572,11 @@ impl Config {
                 let mut target = Target::from_triple(&triple);
 
                 if let Some(ref s) = cfg.llvm_config {
+                    if config.download_rustc_commit.is_some() && triple == &*config.build.triple {
+                        panic!(
+                            "setting llvm_config for the host is incompatible with download-rustc"
+                        );
+                    }
                     target.llvm_config = Some(config.src.join(s));
                 }
                 target.llvm_has_rust_patches = cfg.llvm_has_rust_patches;
@@ -1825,6 +1847,12 @@ impl Config {
         self.out.join(&*self.build.triple).join("ci-llvm")
     }
 
+    /// Directory where the extracted `rustc-dev` component is stored.
+    pub(crate) fn ci_rustc_dir(&self) -> PathBuf {
+        assert!(self.download_rustc());
+        self.out.join(self.build.triple).join("ci-rustc")
+    }
+
     /// Determine whether llvm should be linked dynamically.
     ///
     /// If `false`, llvm should be linked statically.
@@ -1860,11 +1888,11 @@ impl Config {
         self.download_rustc_commit().is_some()
     }
 
-    pub(crate) fn download_rustc_commit(&self) -> Option<&'static str> {
+    pub(crate) fn download_rustc_commit(&self) -> Option<&str> {
         static DOWNLOAD_RUSTC: OnceCell<Option<String>> = OnceCell::new();
         if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() {
             // avoid trying to actually download the commit
-            return None;
+            return self.download_rustc_commit.as_deref();
         }
 
         DOWNLOAD_RUSTC

src/bootstrap/download.rs

@@ -402,7 +402,11 @@ impl Config {
 
     fn ci_component_contents(&self, stamp_file: &str) -> Vec<String> {
         assert!(self.download_rustc());
-        let ci_rustc_dir = self.out.join(&*self.build.triple).join("ci-rustc");
+        if self.dry_run() {
+            return vec![];
+        }
+
+        let ci_rustc_dir = self.ci_rustc_dir();
         let stamp_file = ci_rustc_dir.join(stamp_file);
         let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string());
         t!(BufReader::new(contents_file).lines().collect())
@@ -419,7 +423,7 @@ impl Config {
         self.download_toolchain(
             &version,
             "ci-rustc",
-            commit,
+            &format!("{commit}-{}", self.llvm_assertions),
             &extra_components,
             Self::download_ci_component,
         );
@@ -495,8 +499,15 @@ impl Config {
 
     /// Download a single component of a CI-built toolchain (not necessarily a published nightly).
     // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
-    fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) {
-        Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc")
+    fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) {
+        Self::download_component(
+            self,
+            DownloadSource::CI,
+            filename,
+            prefix,
+            commit_with_assertions,
+            "ci-rustc",
+        )
     }
 
     fn download_component(
@@ -516,11 +527,18 @@ impl Config {
         let bin_root = self.out.join(self.build.triple).join(destination);
         let tarball = cache_dir.join(&filename);
         let (base_url, url, should_verify) = match mode {
-            DownloadSource::CI => (
-                self.stage0_metadata.config.artifacts_server.clone(),
-                format!("{key}/{filename}"),
-                false,
-            ),
+            DownloadSource::CI => {
+                let dist_server = if self.llvm_assertions {
+                    self.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone()
+                } else {
+                    self.stage0_metadata.config.artifacts_server.clone()
+                };
+                let url = format!(
+                    "{}/{filename}",
+                    key.strip_suffix(&format!("-{}", self.llvm_assertions)).unwrap()
+                );
+                (dist_server, url, false)
+            }
             DownloadSource::Dist => {
                 let dist_server = env::var("RUSTUP_DIST_SERVER")
                     .unwrap_or(self.stage0_metadata.config.dist_server.to_string());

src/bootstrap/lib.rs

@@ -822,11 +822,6 @@ impl Build {
         self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir())
     }
 
-    /// Directory where the extracted `rustc-dev` component is stored.
-    fn ci_rustc_dir(&self, target: TargetSelection) -> PathBuf {
-        self.out.join(&*target.triple).join("ci-rustc")
-    }
-
     /// Root output directory for LLVM compiled for `target`
     ///
     /// Note that if LLVM is configured externally then the directory returned

src/bootstrap/test.rs

@@ -2348,7 +2348,7 @@ impl Step for Crate {
                 // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`,
                 // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`.
                 // Override it.
-                if builder.download_rustc() {
+                if builder.download_rustc() && compiler.stage > 0 {
                     let sysroot = builder
                         .out
                         .join(compiler.host.triple)

src/tools/lint-docs/src/groups.rs

@@ -39,7 +39,6 @@ impl<'a> LintExtractor<'a> {
     fn collect_groups(&self) -> Result<LintGroups, Box<dyn Error>> {
         let mut result = BTreeMap::new();
         let mut cmd = Command::new(self.rustc_path);
-        cmd.env_remove("LD_LIBRARY_PATH");
         cmd.arg("-Whelp");
         let output = cmd.output().map_err(|e| format!("failed to run command {:?}\n{}", cmd, e))?;
         if !output.status.success() {

src/tools/lint-docs/src/lib.rs

@@ -403,12 +403,6 @@ impl<'a> LintExtractor<'a> {
         fs::write(&tempfile, source)
             .map_err(|e| format!("failed to write {}: {}", tempfile.display(), e))?;
         let mut cmd = Command::new(self.rustc_path);
-        // NOTE: bootstrap sets `LD_LIBRARY_PATH` for building lint-docs itself.
-        // Unfortunately, lint-docs is a bootstrap tool while rustc is built from source,
-        // and sometimes the paths conflict. In particular, when using `download-rustc`,
-        // the LLVM versions can differ between `ci-llvm` and `ci-rustc-sysroot`.
-        // Unset LD_LIBRARY_PATH here so it doesn't interfere with running the compiler.
-        cmd.env_remove("LD_LIBRARY_PATH");
         if options.contains(&"edition2015") {
             cmd.arg("--edition=2015");
         } else {

tests/ui-fulldeps/missing-rustc-driver-error.rs

@@ -1,8 +1,8 @@
 // Test that we get the following hint when trying to use a compiler crate without rustc_driver.
 // error-pattern: try adding `extern crate rustc_driver;` at the top level of this crate
 // compile-flags: --emit link
-// The exactly list of required crates depends on the target. as such only test Unix targets.
-// only-unix
+// normalize-stderr-test ".*crate .* required.*\n\n" -> ""
+// normalize-stderr-test: "aborting due to [0-9]+" -> "aborting due to NUMBER"
 
 #![feature(rustc_private)]
 

tests/ui-fulldeps/missing-rustc-driver-error.stderr

@@ -2,15 +2,5 @@ error: crate `rustc_serialize` required to be available in rlib format, but was
    |
    = help: try adding `extern crate rustc_driver;` at the top level of this crate
 
-error: crate `smallvec` required to be available in rlib format, but was not found in this form
-
-error: crate `thin_vec` required to be available in rlib format, but was not found in this form
-
-error: crate `indexmap` required to be available in rlib format, but was not found in this form
-
-error: crate `hashbrown` required to be available in rlib format, but was not found in this form
-
-error: crate `equivalent` required to be available in rlib format, but was not found in this form
-
-error: aborting due to 6 previous errors
+error: aborting due to NUMBER previous errors
 

tests/ui/sanitize/address.rs

@@ -1,11 +1,12 @@
 // needs-sanitizer-support
 // needs-sanitizer-address
+// ignore-cross-compile
 //
 // compile-flags: -Z sanitizer=address -O -g
 //
 // run-fail
 // error-pattern: AddressSanitizer: stack-buffer-overflow
-// error-pattern: 'xs' (line 13) <== Memory access at offset
+// error-pattern: 'xs' (line 14) <== Memory access at offset
 
 use std::hint::black_box;