feat: Query rustc for clang target triples instead of hardcoding them (#1004)

* Add new workspace crate `gen-target-info`

For generating `src/target_info.rs`, and use it to simplify
riscv target arch mapping logic in `Build::add_default_flags`

Signed-off-by: Jiahao XU <[email protected]>

* Fix msrv CI: Pass `--locked` to `cargo` to use constructed lockfile

Signed-off-by: Jiahao XU <[email protected]>

* Fix msrv: Use edition 2018 in workspace gen-target-info

Signed-off-by: Jiahao XU <[email protected]>

* Fix `gen-target-info`: Generate formatted rust code

Signed-off-by: Jiahao XU <[email protected]>

* Fix `gen-target-info`

Signed-off-by: Jiahao XU <[email protected]>

* Format `Cargo.toml`

Signed-off-by: Jiahao XU <[email protected]>

* Rename `write_string_mapping_to_file` to `write_target_tuple_mapping`

Signed-off-by: Jiahao XU <[email protected]>

* Refactor: Extract new fn `generate_riscv_arch_mapping`

Signed-off-by: Jiahao XU <[email protected]>

* Add doc for the `target_info.rs` to warn against manually editing the file

Signed-off-by: Jiahao XU <[email protected]>

---------

Signed-off-by: Jiahao XU <[email protected]>

Author: NobodyXu

.github/workflows/main.yml

@@ -170,8 +170,8 @@ jobs:
       run: cargo +nightly update -Zminimal-versions
     - name: Cache downloaded crates since 1.53 is really slow in fetching
       uses: Swatinem/rust-cache@v2
-    - run: cargo check --lib -p cc
-    - run: cargo check --lib -p cc --all-features
+    - run: cargo check --lib -p cc --locked
+    - run: cargo check --lib -p cc --locked --all-features
 
   clippy:
     name: Clippy

Cargo.toml

@@ -36,5 +36,6 @@ tempfile = "3"
 [workspace]
 members = [
     "dev-tools/cc-test",
+    "dev-tools/gen-target-info",
     "dev-tools/gen-windows-sys-binding",
 ]

dev-tools/gen-target-info/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "gen-target-info"
+version = "0.1.0"
+edition = "2018"
+publish = false
+
+[dependencies]
+serde = { version = "1.0.163", features = ["derive"] }
+serde-tuple-vec-map = "1.0.1"
+serde_json = "1.0.107"

dev-tools/gen-target-info/src/lib.rs

@@ -0,0 +1,8 @@
+mod target_specs;
+pub use target_specs::*;
+
+mod read;
+pub use read::get_target_specs_from_json;
+
+mod write;
+pub use write::*;

dev-tools/gen-target-info/src/main.rs

@@ -0,0 +1,40 @@
+use gen_target_info::{get_target_specs_from_json, write_target_tuple_mapping, RustcTargetSpecs};
+use std::{fs::File, io::Write as _};
+
+const PRELUDE: &str = r#"//! This file is generated code. Please edit the generator
+//! in dev-tools/gen-target-info if you need to make changes.
+
+"#;
+
+fn generate_riscv_arch_mapping(f: &mut File, target_specs: &RustcTargetSpecs) {
+    let mut riscv_target_mapping = target_specs
+        .0
+        .iter()
+        .filter_map(|(target, target_spec)| {
+            let arch = target.split_once('-').unwrap().0;
+            (arch.contains("riscv") && arch != &target_spec.arch)
+                .then_some((arch, &*target_spec.arch))
+        })
+        .collect::<Vec<_>>();
+    riscv_target_mapping.sort_unstable_by_key(|(arch, _)| &**arch);
+    riscv_target_mapping.dedup();
+    write_target_tuple_mapping(f, "RISCV_ARCH_MAPPING", &riscv_target_mapping);
+}
+
+fn main() {
+    let target_specs = get_target_specs_from_json();
+
+    // Open file to write to
+    let manifest_dir = env!("CARGO_MANIFEST_DIR");
+
+    let path = format!("{manifest_dir}/../../src/target_info.rs");
+    let mut f = File::create(path).expect("failed to create src/target_info.rs");
+
+    f.write_all(PRELUDE.as_bytes()).unwrap();
+
+    // Start generating
+    generate_riscv_arch_mapping(&mut f, &target_specs);
+
+    // Flush the data onto disk
+    f.flush().unwrap();
+}

dev-tools/gen-target-info/src/read.rs

@@ -0,0 +1,22 @@
+use std::process;
+
+use crate::RustcTargetSpecs;
+
+pub fn get_target_specs_from_json() -> RustcTargetSpecs {
+    let mut cmd = process::Command::new("rustc");
+    cmd.args([
+        "+nightly",
+        "-Zunstable-options",
+        "--print",
+        "all-target-specs-json",
+    ])
+    .stdout(process::Stdio::piped());
+
+    let process::Output { status, stdout, .. } = cmd.output().unwrap();
+
+    if !status.success() {
+        panic!("{:?} failed with non-zero exit status: {}", cmd, status)
+    }
+
+    serde_json::from_slice(&stdout).unwrap()
+}

dev-tools/gen-target-info/src/target_specs.rs

@@ -0,0 +1,33 @@
+use serde::Deserialize;
+
+#[derive(Debug, Deserialize)]
+#[serde(transparent)]
+pub struct PreLinkArgs(
+    /// First field in the linker name,
+    /// second field is the args.
+    #[serde(with = "tuple_vec_map")]
+    pub Vec<(String, Vec<String>)>,
+);
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all(deserialize = "kebab-case"))]
+pub struct TargetSpec {
+    pub arch: String,
+    pub llvm_target: String,
+    /// link env to remove, mostly for apple
+    pub link_env_remove: Option<Vec<String>>,
+    /// link env to set, mostly for apple, e.g. `ZERO_AR_DATE=1`
+    pub link_env: Option<Vec<String>>,
+    pub os: Option<String>,
+    /// `apple`, `pc`
+    pub vendor: Option<String>,
+    pub pre_link_args: Option<PreLinkArgs>,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(transparent)]
+pub struct RustcTargetSpecs(
+    /// First field in the tuple is the rustc target
+    #[serde(with = "tuple_vec_map")]
+    pub Vec<(String, TargetSpec)>,
+);

dev-tools/gen-target-info/src/write.rs

@@ -0,0 +1,14 @@
+use std::{fmt::Write as _, fs, io::Write as _};
+
+pub fn write_target_tuple_mapping(f: &mut fs::File, variable_name: &str, data: &[(&str, &str)]) {
+    let mut content = format!("pub const {variable_name}: &[(&str, &str)] = &[\n");
+
+    for (f1, f2) in data {
+        write!(&mut content, r#"    ("{f1}", "{f2}"),"#).unwrap();
+        content.push('\n');
+    }
+
+    content.push_str("];\n");
+
+    f.write_all(content.as_bytes()).unwrap();
+}

src/lib.rs

@@ -247,6 +247,8 @@ mod tool;
 pub use tool::Tool;
 use tool::ToolFamily;
 
+mod target_info;
+
 /// A builder for compilation of a native library.
 ///
 /// A `Build` is the main type of the `cc` crate and is used to control all the
@@ -312,6 +314,8 @@ enum ErrorKind {
     ToolNotFound,
     /// One of the function arguments failed validation.
     InvalidArgument,
+    /// Invalid target
+    InvalidTarget,
     #[cfg(feature = "parallel")]
     /// jobserver helpthread failure
     JobserverHelpThreadError,
@@ -1900,6 +1904,13 @@ impl Build {
                     && !(target.contains("android")
                         && android_clang_compiler_uses_target_arg_internally(&cmd.path))
                 {
+                    let (arch, rest) = target.split_once('-').ok_or_else(|| {
+                        Error::new(
+                            ErrorKind::InvalidTarget,
+                            format!("Invalid target `{}`: no `-` in it", target),
+                        )
+                    })?;
+
                     if target.contains("darwin") {
                         if let Some(arch) =
                             map_darwin_target_from_rust_to_compiler_architecture(target)
@@ -1983,39 +1994,17 @@ impl Build {
                                 format!("--target={}-apple-tvos{}", arch, deployment_target).into(),
                             );
                         }
-                    } else if target.starts_with("riscv64gc-") {
-                        cmd.args.push(
-                            format!("--target={}", target.replace("riscv64gc", "riscv64")).into(),
-                        );
-                    } else if target.starts_with("riscv64imac-") {
-                        cmd.args.push(
-                            format!("--target={}", target.replace("riscv64imac", "riscv64")).into(),
-                        );
-                    } else if target.starts_with("riscv32gc-") {
+                    } else if let Ok(index) = target_info::RISCV_ARCH_MAPPING
+                        .binary_search_by_key(&arch, |(arch, _)| &arch)
+                    {
                         cmd.args.push(
-                            format!("--target={}", target.replace("riscv32gc", "riscv32")).into(),
+                            format!(
+                                "--target={}-{}",
+                                target_info::RISCV_ARCH_MAPPING[index].1,
+                                rest
+                            )
+                            .into(),
                         );
-                    } else if target.starts_with("riscv32i-") {
-                        cmd.args.push(
-                            format!("--target={}", target.replace("riscv32i", "riscv32")).into(),
-                        )
-                    } else if target.starts_with("riscv32im-") {
-                        cmd.args.push(
-                            format!("--target={}", target.replace("riscv32im", "riscv32")).into(),
-                        )
-                    } else if target.starts_with("riscv32imc-") {
-                        cmd.args.push(
-                            format!("--target={}", target.replace("riscv32imc", "riscv32")).into(),
-                        )
-                    } else if target.starts_with("riscv32imac-") {
-                        cmd.args.push(
-                            format!("--target={}", target.replace("riscv32imac", "riscv32")).into(),
-                        )
-                    } else if target.starts_with("riscv32imafc-") {
-                        cmd.args.push(
-                            format!("--target={}", target.replace("riscv32imafc", "riscv32"))
-                                .into(),
-                        )
                     } else if target.contains("uefi") {
                         if target.contains("x86_64") {
                             cmd.args.push("--target=x86_64-unknown-windows-gnu".into());

src/target_info.rs

@@ -0,0 +1,13 @@
+//! This file is generated code. Please edit the generator
+//! in dev-tools/gen-target-info if you need to make changes.
+
+pub const RISCV_ARCH_MAPPING: &[(&str, &str)] = &[
+    ("riscv32gc", "riscv32"),
+    ("riscv32i", "riscv32"),
+    ("riscv32im", "riscv32"),
+    ("riscv32imac", "riscv32"),
+    ("riscv32imafc", "riscv32"),
+    ("riscv32imc", "riscv32"),
+    ("riscv64gc", "riscv64"),
+    ("riscv64imac", "riscv64"),
+];