Rollup merge of #145763 - Kobzol:llvm-bindir-cross, r=Mark-Simulacrum

Ship LLVM tools for the correct target when cross-compiling

The LLVM config returned from the `Llvm` step in bootstrap was always the *host* LLVM config (as we cannot execute the cross-compiled LLVM config). But this wasn't obvious in bootstrap before (there was just a comment about it, but that's it), which caused a bug where bootstrap was copying LLVM tools from the host target to the cross-compiled rustc sysroot. This was probably happening for quite a long time, we just haven't noticed before.

Note that I consider this to be mostly a hotfix, I plan to refactor the LLVM handling in bootstrap soon-ish to make it harder to misuse and be better in general.

Fixes: #145699

Author: jhpratt

src/bootstrap/src/core/build_steps/compile.rs

@@ -1394,8 +1394,8 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect
     if builder.config.llvm_enzyme {
         cargo.env("LLVM_ENZYME", "1");
     }
-    let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target });
-    cargo.env("LLVM_CONFIG", &llvm_config);
+    let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target });
+    cargo.env("LLVM_CONFIG", &host_llvm_config);
 
     // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script
     // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by
@@ -2001,14 +2001,52 @@ impl Step for Assemble {
         if builder.config.llvm_enabled(target_compiler.host) {
             trace!("target_compiler.host" = ?target_compiler.host, "LLVM enabled");
 
-            let llvm::LlvmResult { llvm_config, .. } =
-                builder.ensure(llvm::Llvm { target: target_compiler.host });
+            let target = target_compiler.host;
+            let llvm::LlvmResult { host_llvm_config, .. } = builder.ensure(llvm::Llvm { target });
             if !builder.config.dry_run() && builder.config.llvm_tools_enabled {
                 trace!("LLVM tools enabled");
 
-                let llvm_bin_dir =
-                    command(llvm_config).arg("--bindir").run_capture_stdout(builder).stdout();
-                let llvm_bin_dir = Path::new(llvm_bin_dir.trim());
+                let host_llvm_bin_dir = command(&host_llvm_config)
+                    .arg("--bindir")
+                    .run_capture_stdout(builder)
+                    .stdout()
+                    .trim()
+                    .to_string();
+
+                let llvm_bin_dir = if target == builder.host_target {
+                    PathBuf::from(host_llvm_bin_dir)
+                } else {
+                    // If we're cross-compiling, we cannot run the target llvm-config in order to
+                    // figure out where binaries are located. We thus have to guess.
+                    let external_llvm_config = builder
+                        .config
+                        .target_config
+                        .get(&target)
+                        .and_then(|t| t.llvm_config.clone());
+                    if let Some(external_llvm_config) = external_llvm_config {
+                        // If we have an external LLVM, just hope that the bindir is the directory
+                        // where the LLVM config is located
+                        external_llvm_config.parent().unwrap().to_path_buf()
+                    } else {
+                        // If we have built LLVM locally, then take the path of the host bindir
+                        // relative to its output build directory, and then apply it to the target
+                        // LLVM output build directory.
+                        let host_llvm_out = builder.llvm_out(builder.host_target);
+                        let target_llvm_out = builder.llvm_out(target);
+                        if let Ok(relative_path) =
+                            Path::new(&host_llvm_bin_dir).strip_prefix(host_llvm_out)
+                        {
+                            target_llvm_out.join(relative_path)
+                        } else {
+                            // This is the most desperate option, just replace the host target with
+                            // the actual target in the directory path...
+                            PathBuf::from(
+                                host_llvm_bin_dir
+                                    .replace(&*builder.host_target.triple, &target.triple),
+                            )
+                        }
+                    }
+                };
 
                 // Since we've already built the LLVM tools, install them to the sysroot.
                 // This is the equivalent of installing the `llvm-tools-preview` component via

src/bootstrap/src/core/build_steps/dist.rs

@@ -2230,11 +2230,12 @@ fn maybe_install_llvm(
             builder.install(&llvm_dylib_path, dst_libdir, FileType::NativeLibrary);
         }
         !builder.config.dry_run()
-    } else if let llvm::LlvmBuildStatus::AlreadyBuilt(llvm::LlvmResult { llvm_config, .. }) =
-        llvm::prebuilt_llvm_config(builder, target, true)
+    } else if let llvm::LlvmBuildStatus::AlreadyBuilt(llvm::LlvmResult {
+        host_llvm_config, ..
+    }) = llvm::prebuilt_llvm_config(builder, target, true)
     {
         trace!("LLVM already built, installing LLVM files");
-        let mut cmd = command(llvm_config);
+        let mut cmd = command(host_llvm_config);
         cmd.arg("--libfiles");
         builder.verbose(|| println!("running {cmd:?}"));
         let files = cmd.run_capture_stdout(builder).stdout();

src/bootstrap/src/core/build_steps/llvm.rs

@@ -29,7 +29,7 @@ use crate::{CLang, GitRepo, Kind, trace};
 pub struct LlvmResult {
     /// Path to llvm-config binary.
     /// NB: This is always the host llvm-config!
-    pub llvm_config: PathBuf,
+    pub host_llvm_config: PathBuf,
     /// Path to LLVM cmake directory for the target.
     pub llvm_cmake_dir: PathBuf,
 }
@@ -109,14 +109,14 @@ pub fn prebuilt_llvm_config(
         && let Some(ref s) = config.llvm_config
     {
         check_llvm_version(builder, s);
-        let llvm_config = s.to_path_buf();
-        let mut llvm_cmake_dir = llvm_config.clone();
+        let host_llvm_config = s.to_path_buf();
+        let mut llvm_cmake_dir = host_llvm_config.clone();
         llvm_cmake_dir.pop();
         llvm_cmake_dir.pop();
         llvm_cmake_dir.push("lib");
         llvm_cmake_dir.push("cmake");
         llvm_cmake_dir.push("llvm");
-        return LlvmBuildStatus::AlreadyBuilt(LlvmResult { llvm_config, llvm_cmake_dir });
+        return LlvmBuildStatus::AlreadyBuilt(LlvmResult { host_llvm_config, llvm_cmake_dir });
     }
 
     if handle_submodule_when_needed {
@@ -141,7 +141,7 @@ pub fn prebuilt_llvm_config(
     };
 
     let llvm_cmake_dir = out_dir.join("lib/cmake/llvm");
-    let res = LlvmResult { llvm_config: build_llvm_config, llvm_cmake_dir };
+    let res = LlvmResult { host_llvm_config: build_llvm_config, llvm_cmake_dir };
 
     static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
     let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| {
@@ -483,11 +483,11 @@ impl Step for Llvm {
 
         // https://llvm.org/docs/HowToCrossCompileLLVM.html
         if !builder.config.is_host_target(target) {
-            let LlvmResult { llvm_config, .. } =
+            let LlvmResult { host_llvm_config, .. } =
                 builder.ensure(Llvm { target: builder.config.host_target });
             if !builder.config.dry_run() {
                 let llvm_bindir =
-                    command(&llvm_config).arg("--bindir").run_capture_stdout(builder).stdout();
+                    command(&host_llvm_config).arg("--bindir").run_capture_stdout(builder).stdout();
                 let host_bin = Path::new(llvm_bindir.trim());
                 cfg.define(
                     "LLVM_TABLEGEN",
@@ -496,7 +496,7 @@ impl Step for Llvm {
                 // LLVM_NM is required for cross compiling using MSVC
                 cfg.define("LLVM_NM", host_bin.join("llvm-nm").with_extension(EXE_EXTENSION));
             }
-            cfg.define("LLVM_CONFIG_PATH", llvm_config);
+            cfg.define("LLVM_CONFIG_PATH", host_llvm_config);
             if builder.config.llvm_clang {
                 let build_bin =
                     builder.llvm_out(builder.config.host_target).join("build").join("bin");
@@ -538,7 +538,7 @@ impl Step for Llvm {
 
         // Helper to find the name of LLVM's shared library on darwin and linux.
         let find_llvm_lib_name = |extension| {
-            let major = get_llvm_version_major(builder, &res.llvm_config);
+            let major = get_llvm_version_major(builder, &res.host_llvm_config);
             match &llvm_version_suffix {
                 Some(version_suffix) => format!("libLLVM-{major}{version_suffix}.{extension}"),
                 None => format!("libLLVM-{major}.{extension}"),
@@ -915,7 +915,7 @@ impl Step for Enzyme {
         }
         let target = self.target;
 
-        let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: self.target });
+        let LlvmResult { host_llvm_config, .. } = builder.ensure(Llvm { target: self.target });
 
         static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
         let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| {
@@ -969,7 +969,7 @@ impl Step for Enzyme {
 
         cfg.out_dir(&out_dir)
             .profile(profile)
-            .env("LLVM_CONFIG_REAL", &llvm_config)
+            .env("LLVM_CONFIG_REAL", &host_llvm_config)
             .define("LLVM_ENABLE_ASSERTIONS", "ON")
             .define("ENZYME_EXTERNAL_SHARED_LIB", "ON")
             .define("ENZYME_BC_LOADER", "OFF")
@@ -1006,13 +1006,13 @@ impl Step for Lld {
         }
         let target = self.target;
 
-        let LlvmResult { llvm_config, llvm_cmake_dir } = builder.ensure(Llvm { target });
+        let LlvmResult { host_llvm_config, llvm_cmake_dir } = builder.ensure(Llvm { target });
 
         // The `dist` step packages LLD next to LLVM's binaries for download-ci-llvm. The root path
         // we usually expect here is `./build/$triple/ci-llvm/`, with the binaries in its `bin`
         // subfolder. We check if that's the case, and if LLD's binary already exists there next to
         // `llvm-config`: if so, we can use it instead of building LLVM/LLD from source.
-        let ci_llvm_bin = llvm_config.parent().unwrap();
+        let ci_llvm_bin = host_llvm_config.parent().unwrap();
         if ci_llvm_bin.is_dir() && ci_llvm_bin.file_name().unwrap() == "bin" {
             let lld_path = ci_llvm_bin.join(exe("lld", target));
             if lld_path.exists() {
@@ -1095,7 +1095,7 @@ impl Step for Lld {
             // Use the host llvm-tblgen binary.
             cfg.define(
                 "LLVM_TABLEGEN_EXE",
-                llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION),
+                host_llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION),
             );
         }
 
@@ -1136,7 +1136,7 @@ impl Step for Sanitizers {
             return runtimes;
         }
 
-        let LlvmResult { llvm_config, .. } =
+        let LlvmResult { host_llvm_config, .. } =
             builder.ensure(Llvm { target: builder.config.host_target });
 
         static STAMP_HASH_MEMO: OnceLock<String> = OnceLock::new();
@@ -1176,7 +1176,7 @@ impl Step for Sanitizers {
         cfg.define("COMPILER_RT_BUILD_XRAY", "OFF");
         cfg.define("COMPILER_RT_DEFAULT_TARGET_ONLY", "ON");
         cfg.define("COMPILER_RT_USE_LIBCXX", "OFF");
-        cfg.define("LLVM_CONFIG_PATH", &llvm_config);
+        cfg.define("LLVM_CONFIG_PATH", &host_llvm_config);
 
         if self.target.contains("ohos") {
             cfg.define("COMPILER_RT_USE_BUILTINS_LIBRARY", "ON");

src/bootstrap/src/core/build_steps/test.rs

@@ -2038,12 +2038,14 @@ HELP: You can add it into `bootstrap.toml` in `rust.codegen-backends = [{name:?}
         let mut llvm_components_passed = false;
         let mut copts_passed = false;
         if builder.config.llvm_enabled(compiler.host) {
-            let llvm::LlvmResult { llvm_config, .. } =
+            let llvm::LlvmResult { host_llvm_config, .. } =
                 builder.ensure(llvm::Llvm { target: builder.config.host_target });
             if !builder.config.dry_run() {
-                let llvm_version = get_llvm_version(builder, &llvm_config);
-                let llvm_components =
-                    command(&llvm_config).arg("--components").run_capture_stdout(builder).stdout();
+                let llvm_version = get_llvm_version(builder, &host_llvm_config);
+                let llvm_components = command(&host_llvm_config)
+                    .arg("--components")
+                    .run_capture_stdout(builder)
+                    .stdout();
                 // Remove trailing newline from llvm-config output.
                 cmd.arg("--llvm-version")
                     .arg(llvm_version.trim())
@@ -2061,7 +2063,7 @@ HELP: You can add it into `bootstrap.toml` in `rust.codegen-backends = [{name:?}
             // rustc args as a workaround.
             if !builder.config.dry_run() && suite.ends_with("fulldeps") {
                 let llvm_libdir =
-                    command(&llvm_config).arg("--libdir").run_capture_stdout(builder).stdout();
+                    command(&host_llvm_config).arg("--libdir").run_capture_stdout(builder).stdout();
                 let link_llvm = if target.is_msvc() {
                     format!("-Clink-arg=-LIBPATH:{llvm_libdir}")
                 } else {
@@ -2075,7 +2077,7 @@ HELP: You can add it into `bootstrap.toml` in `rust.codegen-backends = [{name:?}
                 // tools. Pass the path to run-make tests so they can use them.
                 // (The coverage-run tests also need these tools to process
                 // coverage reports.)
-                let llvm_bin_path = llvm_config
+                let llvm_bin_path = host_llvm_config
                     .parent()
                     .expect("Expected llvm-config to be contained in directory");
                 assert!(llvm_bin_path.is_dir());

src/bootstrap/src/core/builder/mod.rs

@@ -1647,11 +1647,15 @@ You have to build a stage1 compiler for `{}` first, and then use it to build a s
     ///
     /// Note that this returns `None` if LLVM is disabled, or if we're in a
     /// check build or dry-run, where there's no need to build all of LLVM.
+    ///
+    /// FIXME(@kobzol)
+    /// **WARNING**: This actually returns the **HOST** LLVM config, not LLVM config for the given
+    /// *target*.
     pub fn llvm_config(&self, target: TargetSelection) -> Option<PathBuf> {
         if self.config.llvm_enabled(target) && self.kind != Kind::Check && !self.config.dry_run() {
-            let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target });
-            if llvm_config.is_file() {
-                return Some(llvm_config);
+            let llvm::LlvmResult { host_llvm_config, .. } = self.ensure(llvm::Llvm { target });
+            if host_llvm_config.is_file() {
+                return Some(host_llvm_config);
             }
         }
         None

src/bootstrap/src/core/builder/tests.rs

@@ -434,14 +434,14 @@ fn test_prebuilt_llvm_config_path_resolution() {
         false,
     )
     .llvm_result()
-    .llvm_config
+    .host_llvm_config
     .clone();
     let actual = drop_win_disk_prefix_if_present(actual);
     assert_eq!(expected, actual);
 
     let actual = prebuilt_llvm_config(&builder, builder.config.host_target, false)
         .llvm_result()
-        .llvm_config
+        .host_llvm_config
         .clone();
     let actual = drop_win_disk_prefix_if_present(actual);
     assert_eq!(expected, actual);
@@ -459,7 +459,7 @@ fn test_prebuilt_llvm_config_path_resolution() {
 
     let actual = prebuilt_llvm_config(&builder, builder.config.host_target, false)
         .llvm_result()
-        .llvm_config
+        .host_llvm_config
         .clone();
     let expected = builder
         .out
@@ -482,7 +482,7 @@ fn test_prebuilt_llvm_config_path_resolution() {
 
         let actual = prebuilt_llvm_config(&builder, builder.config.host_target, false)
             .llvm_result()
-            .llvm_config
+            .host_llvm_config
             .clone();
         let expected = builder
             .out