diff --git a/Cargo.toml b/Cargo.toml index c10feff4..474b7846 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,10 @@ exclude = [ "crates/optix/examples/common", ] -[profile.dev.package.rustc_codegen_nvvm] +[profile.dev.package.rustc_codegen_nvvm_v7] +opt-level = 3 + +[profile.dev.package.rustc_codegen_nvvm_v19] opt-level = 3 [workspace.dependencies] diff --git a/container/ubuntu22-cuda12/Dockerfile b/container/ubuntu22-cuda12/Dockerfile index 7d66ff9c..90e3cc7e 100644 --- a/container/ubuntu22-cuda12/Dockerfile +++ b/container/ubuntu22-cuda12/Dockerfile @@ -1,4 +1,5 @@ FROM nvcr.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04 +# TODO: needs to change to 12.9.0 for llvm-v19 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \ build-essential \ diff --git a/container/ubuntu24-cuda12/Dockerfile b/container/ubuntu24-cuda12/Dockerfile index 01365070..e87c2f87 100644 --- a/container/ubuntu24-cuda12/Dockerfile +++ b/container/ubuntu24-cuda12/Dockerfile @@ -1,4 +1,5 @@ FROM nvcr.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04 +# TODO: needs to change to 12.9.0 for llvm-v19 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \ build-essential \ diff --git a/crates/cuda_builder/Cargo.toml b/crates/cuda_builder/Cargo.toml index 8095b79e..d39077f4 100644 --- a/crates/cuda_builder/Cargo.toml +++ b/crates/cuda_builder/Cargo.toml @@ -8,8 +8,14 @@ description = "Builder for easily building rustc_codegen_nvvm crates" repository = "https://github.com/Rust-GPU/Rust-CUDA" readme = "../../README.md" +[features] +default = ["nvvm-v7"] +nvvm-v7 = ["dep:rustc_codegen_nvvm_v7"] +nvvm-v19 = ["dep:rustc_codegen_nvvm_v19"] + [dependencies] -rustc_codegen_nvvm = { version = "0.3", path = "../rustc_codegen_nvvm" } +rustc_codegen_nvvm_v7 = { version = "0.3", path = "../rustc_codegen_nvvm_v7", optional = true } +rustc_codegen_nvvm_v19 = { version = "0.3", path = "../rustc_codegen_nvvm_v19", optional = true } nvvm = { path = "../nvvm", version = "0.1" } serde = { version = "1.0.217", features = ["derive"] } -serde_json = "1.0.138" +serde_json = "1.0.138" \ No newline at end of file diff --git a/crates/cuda_builder/src/lib.rs b/crates/cuda_builder/src/lib.rs index 3977b7a2..affe355a 100644 --- a/crates/cuda_builder/src/lib.rs +++ b/crates/cuda_builder/src/lib.rs @@ -50,12 +50,14 @@ impl DebugInfo { } } +#[derive(Debug, Clone, Copy, PartialEq)] pub enum EmitOption { LlvmIr, Bitcode, } /// A builder for easily compiling Rust GPU crates in build.rs +#[derive(Debug)] pub struct CudaBuilder { path_to_crate: PathBuf, /// Whether to compile the gpu crate for release. @@ -200,13 +202,23 @@ pub struct CudaBuilder { impl CudaBuilder { pub fn new(path_to_crate_root: impl AsRef) -> Self { + let path_to_crate = path_to_crate_root.as_ref().to_owned(); + + println!("cargo:warning=Creating CudaBuilder for crate at: {}", path_to_crate.display()); + Self { - path_to_crate: path_to_crate_root.as_ref().to_owned(), + path_to_crate, release: true, ptx_file_copy_path: None, generate_line_info: true, nvvm_opts: true, - arch: NvvmArch::Compute61, + arch: if cfg!(feature = "nvvm-v19") { + NvvmArch::Compute120 + } else if cfg!(feature = "nvvm-v7") { + NvvmArch::default() + } else { + panic!("No NVVM version feature enabled. Enable either 'nvvm-v7' or 'nvvm-v19'"); + }, ftz: false, fast_sqrt: false, fast_div: false, @@ -225,17 +237,20 @@ impl CudaBuilder { pub fn build_args(mut self, args: &[impl AsRef]) -> Self { self.build_args .extend(args.iter().map(|s| s.as_ref().to_owned())); + println!("cargo:warning=Added build args: {:?}", self.build_args); self } /// Whether to generate any debug info and what level of info to generate. - pub fn debug(mut self, debug: DebugInfo) -> Self { - self.debug = debug; + pub fn debug(mut self, debug_info: DebugInfo) -> Self { + println!("cargo:warning=Setting debug info level: {:?}", debug_info); + self.debug = debug_info; self } /// Whether to compile the gpu crate for release. pub fn release(mut self, release: bool) -> Self { + println!("cargo:warning=Setting release mode: {}", release); self.release = release; self.nvvm_opts = release; self @@ -245,6 +260,7 @@ impl CudaBuilder { /// This defaults to `true`, but nothing will be generated /// if the gpu crate is built as release. pub fn generate_line_info(mut self, generate_line_info: bool) -> Self { + println!("cargo:warning=Setting generate_line_info: {}", generate_line_info); self.generate_line_info = generate_line_info; self } @@ -252,6 +268,7 @@ impl CudaBuilder { /// Whether to run libnvvm optimizations. This defaults to `false` /// but will be set to `true` if release is specified. pub fn nvvm_opts(mut self, nvvm_opts: bool) -> Self { + println!("cargo:warning=Setting nvvm_opts: {}", nvvm_opts); self.nvvm_opts = nvvm_opts; self } @@ -275,49 +292,58 @@ impl CudaBuilder { /// The chosen architecture enables target features for conditional compilation. /// See the documentation on the `arch` field for more details. pub fn arch(mut self, arch: NvvmArch) -> Self { + println!("cargo:warning=Setting target architecture: {:?}", arch); self.arch = arch; self } /// Flush denormal values to zero when performing single-precision floating point operations. pub fn ftz(mut self, ftz: bool) -> Self { + println!("cargo:warning=Setting flush-to-zero: {}", ftz); self.ftz = ftz; self } /// Use a fast approximation for single-precision floating point square root. pub fn fast_sqrt(mut self, fast_sqrt: bool) -> Self { + println!("cargo:warning=Setting fast sqrt: {}", fast_sqrt); self.fast_sqrt = fast_sqrt; self } /// Use a fast approximation for single-precision floating point division. pub fn fast_div(mut self, fast_div: bool) -> Self { + println!("cargo:warning=Setting fast div: {}", fast_div); self.fast_div = fast_div; self } /// Enable FMA (fused multiply-add) contraction. pub fn fma_contraction(mut self, fma_contraction: bool) -> Self { + println!("cargo:warning=Setting FMA contraction: {}", fma_contraction); self.fma_contraction = fma_contraction; self } /// Emit LLVM IR, the exact same as rustc's `--emit=llvm-ir`. pub fn emit_llvm_ir(mut self, emit_llvm_ir: bool) -> Self { + println!("cargo:warning=Setting emit LLVM IR: {}", emit_llvm_ir); self.emit = emit_llvm_ir.then_some(EmitOption::LlvmIr); self } /// Emit LLVM Bitcode, the exact same as rustc's `--emit=llvm-bc`. pub fn emit_llvm_bitcode(mut self, emit_llvm_bitcode: bool) -> Self { + println!("cargo:warning=Setting emit LLVM bitcode: {}", emit_llvm_bitcode); self.emit = emit_llvm_bitcode.then_some(EmitOption::Bitcode); self } /// Copy the final ptx file to this location once finished building. pub fn copy_to(mut self, path: impl AsRef) -> Self { - self.ptx_file_copy_path = Some(path.as_ref().to_path_buf()); + let copy_path = path.as_ref().to_path_buf(); + println!("cargo:warning=Setting PTX copy destination: {}", copy_path.display()); + self.ptx_file_copy_path = Some(copy_path); self } @@ -329,6 +355,7 @@ impl CudaBuilder { /// /// Code compiled with this option should always work under CUDA, but it might not be the most efficient or practical. pub fn optix(mut self, optix: bool) -> Self { + println!("cargo:warning=Setting OptiX mode: {}", optix); self.optix = optix; self } @@ -341,6 +368,7 @@ impl CudaBuilder { /// However, this means the overriden functions are likely to not be deterministic, so if you rely on strict /// determinism in things like `rapier`, then it may be helpful to disable such a feature. pub fn override_libm(mut self, override_libm: bool) -> Self { + println!("cargo:warning=Setting libm override: {}", override_libm); self.override_libm = override_libm; self } @@ -359,6 +387,7 @@ impl CudaBuilder { /// Future versions may support smarter placement and user-controlled /// packing/spilling strategies. pub fn use_constant_memory_space(mut self, use_constant_memory_space: bool) -> Self { + println!("cargo:warning=Setting constant memory space usage: {}", use_constant_memory_space); self.use_constant_memory_space = use_constant_memory_space; self } @@ -366,18 +395,31 @@ impl CudaBuilder { /// An optional path where to dump LLVM IR of the final output the codegen will feed to libnvvm. Usually /// used for debugging. pub fn final_module_path(mut self, path: impl AsRef) -> Self { - self.final_module_path = Some(path.as_ref().to_path_buf()); + let module_path = path.as_ref().to_path_buf(); + println!("cargo:warning=Setting final module path: {}", module_path.display()); + self.final_module_path = Some(module_path); self } /// Runs rustc to build the codegen and codegens the gpu crate, returning the path of the final /// ptx file. If [`ptx_file_copy_path`](Self::ptx_file_copy_path) is set, this returns the copied path. pub fn build(self) -> Result { + println!("cargo:warning=Starting CUDA build process"); + println!("cargo:warning=Build configuration: {:?}", self); + println!("cargo:rerun-if-changed={}", self.path_to_crate.display()); + let path = invoke_rustc(&self)?; - if let Some(copy_path) = self.ptx_file_copy_path { - std::fs::copy(path, ©_path).map_err(CudaBuilderError::FailedToCopyPtxFile)?; - Ok(copy_path) + println!("cargo:warning=Build completed successfully, PTX file generated at: {}", path.display()); + + if let Some(copy_path) = &self.ptx_file_copy_path { + println!("cargo:warning=Copying PTX file from {} to {}", path.display(), copy_path.display()); + std::fs::copy(&path, copy_path).map_err(|e| { + println!("cargo:warning=Failed to copy PTX file: {}", e); + CudaBuilderError::FailedToCopyPtxFile(e) + })?; + println!("cargo:warning=PTX file copied successfully"); + Ok(copy_path.clone()) } else { Ok(path) } @@ -403,14 +445,34 @@ fn dylib_path() -> Vec { } fn find_rustc_codegen_nvvm() -> PathBuf { + println!("cargo:warning=Looking for rustc_codegen_nvvm library"); + + // Determine which version to look for based on enabled features + let version_suffix = if cfg!(feature = "nvvm-v19") { + "_v19" + } else if cfg!(feature = "nvvm-v7") { + "_v7" + } else { + panic!("No NVVM version feature enabled. Enable either 'nvvm-v7' or 'nvvm-v19'"); + }; + let filename = format!( - "{}rustc_codegen_nvvm{}", + "{}rustc_codegen_nvvm{}{}", env::consts::DLL_PREFIX, + version_suffix, env::consts::DLL_SUFFIX ); - for mut path in dylib_path() { + + println!("cargo:warning=Searching for library: {}", filename); + + let paths = dylib_path(); + println!("cargo:warning=Library search paths: {:?}", paths); + + for mut path in paths { path.push(&filename); + println!("cargo:warning=Checking path: {}", path.display()); if path.is_file() { + println!("cargo:warning=Found rustc_codegen_nvvm at: {}", path.display()); return path; } } @@ -427,6 +489,8 @@ fn join_checking_for_separators(strings: Vec>, sep: &str) -> St } fn invoke_rustc(builder: &CudaBuilder) -> Result { + println!("cargo:warning=Invoking rustc for CUDA compilation"); + // see https://github.com/EmbarkStudios/rust-gpu/blob/main/crates/spirv-builder/src/lib.rs#L385-L392 // on what this does let rustc_codegen_nvvm = find_rustc_codegen_nvvm(); @@ -448,47 +512,59 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result { } let mut llvm_args = vec![NvvmOption::Arch(builder.arch).to_string()]; + println!("cargo:warning=Base LLVM arg - arch: {}", llvm_args[0]); if !builder.nvvm_opts { llvm_args.push("-opt=0".to_string()); + println!("cargo:warning=Added LLVM arg: -opt=0"); } if builder.ftz { llvm_args.push("-ftz=1".to_string()); + println!("cargo:warning=Added LLVM arg: -ftz=1"); } if builder.fast_sqrt { llvm_args.push("-prec-sqrt=0".to_string()); + println!("cargo:warning=Added LLVM arg: -prec-sqrt=0"); } if builder.fast_div { llvm_args.push("-prec-div=0".to_string()); + println!("cargo:warning=Added LLVM arg: -prec-div=0"); } if !builder.fma_contraction { llvm_args.push("-fma=0".to_string()); + println!("cargo:warning=Added LLVM arg: -fma=0"); } if builder.override_libm { llvm_args.push("--override-libm".to_string()); + println!("cargo:warning=Added LLVM arg: --override-libm"); } if let Some(path) = &builder.final_module_path { llvm_args.push("--final-module-path".to_string()); llvm_args.push(path.to_str().unwrap().to_string()); + println!("cargo:warning=Added final module path: {}", path.display()); } if builder.debug != DebugInfo::None { let (nvvm_flag, rustc_flag) = builder.debug.into_nvvm_and_rustc_options(); - llvm_args.push(nvvm_flag); - rustflags.push(rustc_flag); + llvm_args.push(nvvm_flag.clone()); + rustflags.push(rustc_flag.clone()); + println!("cargo:warning=Added debug flags - NVVM: {}, rustc: {}", nvvm_flag, rustc_flag); } let llvm_args = llvm_args.join(" "); if !llvm_args.is_empty() { rustflags.push(["-Cllvm-args=", &llvm_args].concat()); + println!("cargo:warning=Final LLVM args: {}", llvm_args); } + println!("cargo:warning=Final rustflags: {:?}", rustflags); + let mut cargo = Command::new("cargo"); cargo.args([ "build", @@ -499,9 +575,13 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result { ]); cargo.args(&builder.build_args); + println!("cargo:warning=Added cargo build args: {:?}", builder.build_args); if builder.release { cargo.arg("--release"); + println!("cargo:warning=Building in release mode"); + } else { + println!("cargo:warning=Building in debug mode"); } // TODO(RDambrosio016): Remove this once we can get meaningful error messages in panic to work. @@ -513,6 +593,7 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result { cargo.arg("-Zunstable-options"); cargo.arg("--config"); cargo.arg("optix=\"1\""); + println!("cargo:warning=OptiX mode enabled"); } // If we're nested in `cargo` invocation, use a different `--target-dir`, @@ -532,15 +613,22 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result { && dir.ends_with(profile) && dir.pop() { - cargo.arg("--target-dir").arg(dir.join("cuda-builder")); + let target_dir = dir.join("cuda-builder"); + cargo.arg("--target-dir").arg(&target_dir); + println!("cargo:warning=Using custom target directory: {}", target_dir.display()); } } let arch = format!("{:?}0", builder.arch); - cargo.env("CUDA_ARCH", arch.strip_prefix("Compute").unwrap()); + let cuda_arch = arch.strip_prefix("Compute").unwrap(); + cargo.env("CUDA_ARCH", cuda_arch); + println!("cargo:warning=Set CUDA_ARCH environment variable: {}", cuda_arch); let cargo_encoded_rustflags = join_checking_for_separators(rustflags, "\x1f"); + println!("cargo:warning=Executing cargo command: {:?}", cargo); + println!("cargo:warning=Starting cargo build..."); + let build = cargo .stderr(Stdio::inherit()) .current_dir(&builder.path_to_crate) @@ -552,10 +640,24 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result { // we do that even in case of an error, to let through any useful messages // that ended up on stdout instead of stderr. let stdout = String::from_utf8(build.stdout).unwrap(); + println!("cargo:warning=Cargo stdout length: {} bytes", stdout.len()); + let artifact = get_last_artifact(&stdout); + if build.status.success() { - Ok(artifact.expect("Artifact created when compilation succeeded (Did you forget to mark the crate-type as lib/rlib?)")) + println!("cargo:warning=Cargo build completed successfully"); + match artifact { + Some(path) => { + println!("cargo:warning=Generated PTX artifact: {}", path.display()); + Ok(path) + } + None => { + println!("cargo:warning=No PTX artifact found despite successful build - did you forget to mark the crate-type as lib/rlib?"); + Err(CudaBuilderError::BuildFailed) + } + } } else { + println!("cargo:warning=Cargo build failed with exit code: {:?}", build.status.code()); Err(CudaBuilderError::BuildFailed) } } @@ -567,7 +669,9 @@ struct RustcOutput { } fn get_last_artifact(out: &str) -> Option { - let last = out + println!("cargo:warning=Parsing cargo output for artifacts"); + + let artifacts: Vec<_> = out .lines() .filter_map(|line| match serde_json::from_str::(line) { Ok(line) => Some(line), @@ -578,15 +682,27 @@ fn get_last_artifact(out: &str) -> Option { } }) .filter(|line| line.reason == "compiler-artifact") - .next_back() - .expect("Did not find output file in rustc output"); + .collect(); + + println!("cargo:warning=Found {} compiler artifacts", artifacts.len()); + + let last = artifacts.into_iter().next_back()?; let mut filenames = last .filenames .unwrap() .into_iter() - .filter(|v| v.ends_with(".ptx")); + .filter(|v| { + let is_ptx = v.ends_with(".ptx"); + if is_ptx { + println!("cargo:warning=Found PTX file: {}", v); + } + is_ptx + }); + let filename = filenames.next()?; assert_eq!(filenames.next(), None, "Crate had multiple .ptx artifacts"); + + println!("cargo:warning=Selected PTX artifact: {}", filename); Some(filename.into()) } diff --git a/crates/cuda_std/Cargo.toml b/crates/cuda_std/Cargo.toml index 209929ce..e27e7563 100644 --- a/crates/cuda_std/Cargo.toml +++ b/crates/cuda_std/Cargo.toml @@ -14,3 +14,8 @@ cuda_std_macros = { version = "0.2", path = "../cuda_std_macros" } half = "2.4.1" bitflags = "2.8" paste = "1.0.15" + +[features] +default = ["nvvm-v7"] +nvvm-v7 = [] +nvvm-v19 = [] diff --git a/crates/cuda_std/src/atomic/intrinsics.rs b/crates/cuda_std/src/atomic/intrinsics.rs index 7a4fc2a8..fa91d217 100644 --- a/crates/cuda_std/src/atomic/intrinsics.rs +++ b/crates/cuda_std/src/atomic/intrinsics.rs @@ -68,6 +68,7 @@ macro_rules! load { #[allow(clippy::missing_safety_doc)] #[doc = concat!("Performs a ", stringify!($ordering), " atomic load at the ", stringify!($scope), " level with a width of ", stringify!($width), " bits")] pub unsafe fn [](ptr: *const []) -> [] { + // TODO: llvm-v19 needs $scope_asm instead of $scope let mut out; asm!( concat!("ld.", stringify!($ordering), load_scope!($ordering, $scope), ".", stringify!([]), " {}, [{}];"), @@ -115,6 +116,7 @@ macro_rules! store { #[allow(clippy::missing_safety_doc)] #[doc = concat!("Performs a ", stringify!($ordering), " atomic store at the ", stringify!($scope), " level with a width of ", stringify!($width), " bits")] pub unsafe fn [](ptr: *mut [], val: []) { + // TODO: llvm-v19 needs $scope_asm instead of $scope asm!( concat!("st.", stringify!($ordering), load_scope!($ordering, $scope), ".", stringify!([]), " [{}], {};"), in(reg64) ptr, diff --git a/crates/cuda_std/src/cfg.rs b/crates/cuda_std/src/cfg.rs index b029e776..36224cd1 100644 --- a/crates/cuda_std/src/cfg.rs +++ b/crates/cuda_std/src/cfg.rs @@ -16,6 +16,12 @@ pub enum ComputeCapability { Compute72, Compute75, Compute80, + Compute86, + Compute87, + Compute89, + Compute90, + Compute100, + Compute120, } impl ComputeCapability { @@ -42,6 +48,12 @@ impl ComputeCapability { "720" => ComputeCapability::Compute72, "750" => ComputeCapability::Compute75, "800" => ComputeCapability::Compute80, + "860" => ComputeCapability::Compute86, // Ampere (RTX 30 series, A100) + "870" => ComputeCapability::Compute87, // Ampere (Jetson AGX Orin) + "890" => ComputeCapability::Compute89, // Ada Lovelace (RTX 40 series) + "900" => ComputeCapability::Compute90, // Hopper (H100) + "1000" => ComputeCapability::Compute100, // Blackwell (RTX 50 series, H200, B100, CUDA 12.6 and later) + "1200" => ComputeCapability::Compute120, // Blackwell (RTX 50 series, H200, B100, CUDA 12.8 and later) _ => panic!("CUDA_ARCH had an invalid value"), } } diff --git a/crates/cust/src/module.rs b/crates/cust/src/module.rs index 772f4a18..68f2eb75 100644 --- a/crates/cust/src/module.rs +++ b/crates/cust/src/module.rs @@ -56,6 +56,11 @@ pub enum JitTarget { Compute75 = 75, Compute80 = 80, Compute86 = 86, + Compute87 = 87, + Compute89 = 89, + Compute90 = 90, + Compute100 = 100, + Compute120 = 120, } /// How to handle cases where a loaded module's data does not contain an exact match for the diff --git a/crates/nvvm/src/lib.rs b/crates/nvvm/src/lib.rs index 70f81740..186096eb 100644 --- a/crates/nvvm/src/lib.rs +++ b/crates/nvvm/src/lib.rs @@ -4,7 +4,6 @@ use std::{ ffi::{CStr, CString}, fmt::Display, mem::MaybeUninit, - ptr::null_mut, str::FromStr, }; @@ -339,7 +338,252 @@ impl Display for NvvmArch { impl Default for NvvmArch { fn default() -> Self { - Self::Compute52 + Self::Compute61 + } +} + +impl NvvmArch { + /// Get the numeric capability value (e.g., 35 for Compute35) + pub fn capability_value(&self) -> u32 { + match self { + Self::Compute35 => 35, + Self::Compute37 => 37, + Self::Compute50 => 50, + Self::Compute52 => 52, + Self::Compute53 => 53, + Self::Compute60 => 60, + Self::Compute61 => 61, + Self::Compute62 => 62, + Self::Compute70 => 70, + Self::Compute72 => 72, + Self::Compute75 => 75, + Self::Compute80 => 80, + Self::Compute86 => 86, + Self::Compute87 => 87, + Self::Compute89 => 89, + Self::Compute90 => 90, + Self::Compute90a => 90, + Self::Compute100 => 100, + Self::Compute100f => 100, + Self::Compute100a => 100, + Self::Compute101 => 101, + Self::Compute101f => 101, + Self::Compute101a => 101, + Self::Compute103 => 103, + Self::Compute103f => 103, + Self::Compute103a => 103, + Self::Compute120 => 120, + Self::Compute120f => 120, + Self::Compute120a => 120, + Self::Compute121 => 121, + Self::Compute121f => 121, + Self::Compute121a => 121, + } + } + + /// Get the major version number (e.g., 7 for Compute70) + pub fn major_version(&self) -> u32 { + self.capability_value() / 10 + } + + /// Get the minor version number (e.g., 5 for Compute75) + pub fn minor_version(&self) -> u32 { + self.capability_value() % 10 + } + + /// Get the target feature string (e.g., "compute_35" for Compute35, "compute_90a" for Compute90a) + pub fn target_feature(&self) -> String { + match self { + Self::Compute35 => "compute_35".to_string(), + Self::Compute37 => "compute_37".to_string(), + Self::Compute50 => "compute_50".to_string(), + Self::Compute52 => "compute_52".to_string(), + Self::Compute53 => "compute_53".to_string(), + Self::Compute60 => "compute_60".to_string(), + Self::Compute61 => "compute_61".to_string(), + Self::Compute62 => "compute_62".to_string(), + Self::Compute70 => "compute_70".to_string(), + Self::Compute72 => "compute_72".to_string(), + Self::Compute75 => "compute_75".to_string(), + Self::Compute80 => "compute_80".to_string(), + Self::Compute86 => "compute_86".to_string(), + Self::Compute87 => "compute_87".to_string(), + Self::Compute89 => "compute_89".to_string(), + Self::Compute90 => "compute_90".to_string(), + Self::Compute90a => "compute_90a".to_string(), + Self::Compute100 => "compute_100".to_string(), + Self::Compute100f => "compute_100f".to_string(), + Self::Compute100a => "compute_100a".to_string(), + Self::Compute101 => "compute_101".to_string(), + Self::Compute101f => "compute_101f".to_string(), + Self::Compute101a => "compute_101a".to_string(), + Self::Compute103 => "compute_103".to_string(), + Self::Compute103f => "compute_103f".to_string(), + Self::Compute103a => "compute_103a".to_string(), + Self::Compute120 => "compute_120".to_string(), + Self::Compute120f => "compute_120f".to_string(), + Self::Compute120a => "compute_120a".to_string(), + Self::Compute121 => "compute_121".to_string(), + Self::Compute121f => "compute_121f".to_string(), + Self::Compute121a => "compute_121a".to_string(), + } + } + + /// Get all target features up to and including this architecture. + /// + /// # PTX Forward-Compatibility Rules (per NVIDIA documentation): + /// + /// - **No suffix** (compute_XX): PTX is forward-compatible across all future architectures. + /// Example: compute_70 runs on CC 7.0, 8.x, 9.x, 10.x, 12.x, and all future GPUs. + /// + /// - **Family-specific 'f' suffix** (compute_XXf): Forward-compatible within the same major + /// version family. Supports devices with same major CC and equal or higher minor CC. + /// Example: compute_100f runs on CC 10.0, 10.3, and future 10.x devices, but NOT on 11.x. + /// + /// - **Architecture-specific 'a' suffix** (compute_XXa): The code only runs on GPUs of that + /// specific CC and no others. No forward or backward compatibility whatsoever. + /// These features are primarily related to Tensor Core programming. + /// Example: compute_100a ONLY runs on CC 10.0, not on 10.3, 10.1, 9.0, or any other version. + /// + /// For more details on family and architecture-specific features, see: + /// + pub fn all_target_features(&self) -> Vec { + let mut features: Vec = if self.is_architecture_variant() { + // 'a' variants: include all available instructions for the architecture + // This means: all base variants up to same version, all 'f' variants with same major and <= minor, plus itself + let base_features: Vec = NvvmArch::iter() + .filter(|arch| { + arch.is_base_variant() && arch.capability_value() <= self.capability_value() + }) + .map(|arch| arch.target_feature()) + .collect(); + + let family_features: Vec = NvvmArch::iter() + .filter(|arch| { + arch.is_family_variant() + && arch.major_version() == self.major_version() + && arch.minor_version() <= self.minor_version() + }) + .map(|arch| arch.target_feature()) + .collect(); + + base_features + .into_iter() + .chain(family_features) + .chain(std::iter::once(self.target_feature())) + .collect() + } else if self.is_family_variant() { + // 'f' variants: same major version with equal or higher minor version + NvvmArch::iter() + .filter(|arch| { + // Include base variants with same major and >= minor version + arch.is_base_variant() + && arch.major_version() == self.major_version() + && arch.minor_version() >= self.minor_version() + }) + .map(|arch| arch.target_feature()) + .chain(std::iter::once(self.target_feature())) // Add the 'f' variant itself + .collect() + } else { + // Base variants: all base architectures from lower or equal versions + NvvmArch::iter() + .filter(|arch| { + arch.is_base_variant() && arch.capability_value() <= self.capability_value() + }) + .map(|arch| arch.target_feature()) + .collect() + }; + + features.sort(); + features + } + + /// Create an iterator over all architectures from Compute35 up to and including this one + pub fn iter_up_to(&self) -> impl Iterator { + let current = self.capability_value(); + NvvmArch::iter().filter(move |arch| arch.capability_value() <= current) + } + + /// Check if this architecture is a base variant (no suffix) + pub fn is_base_variant(&self) -> bool { + let feature = self.target_feature(); + // A base variant doesn't end with any letter suffix + !feature + .chars() + .last() + .is_some_and(|c| c.is_ascii_alphabetic()) + } + + /// Check if this architecture is a family-specific variant (f suffix) + /// Family-specific features are supported across devices within the same major compute capability + pub fn is_family_variant(&self) -> bool { + self.target_feature().ends_with('f') + } + + /// Check if this architecture is an architecture-specific variant (a suffix) + /// Architecture-specific features are locked to that exact compute capability only + pub fn is_architecture_variant(&self) -> bool { + self.target_feature().ends_with('a') + } + + /// Get the base architecture for this variant (strips f/a suffix if present) + pub fn base_architecture(&self) -> Self { + match self { + // Already base variants + Self::Compute35 + | Self::Compute37 + | Self::Compute50 + | Self::Compute52 + | Self::Compute53 + | Self::Compute60 + | Self::Compute61 + | Self::Compute62 + | Self::Compute70 + | Self::Compute72 + | Self::Compute75 + | Self::Compute80 + | Self::Compute86 + | Self::Compute87 + | Self::Compute89 + | Self::Compute90 + | Self::Compute100 + | Self::Compute101 + | Self::Compute103 + | Self::Compute120 + | Self::Compute121 => *self, + + // Family-specific variants + Self::Compute100f => Self::Compute100, + Self::Compute101f => Self::Compute101, + Self::Compute103f => Self::Compute103, + Self::Compute120f => Self::Compute120, + Self::Compute121f => Self::Compute121, + + // Architecture-specific variants + Self::Compute90a => Self::Compute90, + Self::Compute100a => Self::Compute100, + Self::Compute101a => Self::Compute101, + Self::Compute103a => Self::Compute103, + Self::Compute120a => Self::Compute120, + Self::Compute121a => Self::Compute121, + } + } + + /// Get all available variants for the same base architecture (including the base) + pub fn get_variants(&self) -> Vec { + let base = self.base_architecture(); + let base_value = base.capability_value(); + + NvvmArch::iter() + .filter(|arch| arch.capability_value() == base_value) + .collect() + } + + /// Get all available variants for a given capability value + pub fn variants_for_capability(capability: u32) -> Vec { + NvvmArch::iter() + .filter(|arch| arch.capability_value() == capability) + .collect() } } @@ -693,8 +937,21 @@ impl NvvmProgram { /// Verify the program without actually compiling it. In the case of invalid IR, you can find /// more detailed error info by calling [`compiler_log`](Self::compiler_log). - pub fn verify(&self) -> Result<(), NvvmError> { - unsafe { nvvm_sys::nvvmVerifyProgram(self.raw, 0, null_mut()).to_result() } + pub fn verify(&self, options: &[NvvmOption]) -> Result<(), NvvmError> { + let option_strings: Vec<_> = options.iter().map(|opt| opt.to_string()).collect(); + let option_cstrings: Vec<_> = option_strings.iter() + .map(|s| std::ffi::CString::new(s.as_str()).unwrap()) + .collect(); + let mut option_ptrs: Vec<_> = option_cstrings.iter() + .map(|cs| cs.as_ptr()) + .collect(); + unsafe { + nvvm_sys::nvvmVerifyProgram( + self.raw, + option_ptrs.len() as i32, + option_ptrs.as_mut_ptr() + ).to_result() + } } } @@ -1029,6 +1286,12 @@ mod tests { "-arch=compute_72", "-arch=compute_75", "-arch=compute_80", + "-arch=compute_86", + "-arch=compute_87", + "-arch=compute_89", + "-arch=compute_90", + "-arch=compute_100", + "-arch=compute_120", "-ftz=1", "-prec-sqrt=0", "-prec-div=0", @@ -1050,6 +1313,12 @@ mod tests { Arch(Compute72), Arch(Compute75), Arch(Compute80), + Arch(Compute86), + Arch(Compute87), + Arch(Compute89), + Arch(Compute90), + Arch(Compute100), + Arch(Compute120), Ftz, FastSqrt, FastDiv, diff --git a/crates/rustc_codegen_nvvm/CHANGELOG.md b/crates/rustc_codegen_nvvm_v19/CHANGELOG.md similarity index 100% rename from crates/rustc_codegen_nvvm/CHANGELOG.md rename to crates/rustc_codegen_nvvm_v19/CHANGELOG.md diff --git a/crates/rustc_codegen_nvvm_v19/Cargo.toml b/crates/rustc_codegen_nvvm_v19/Cargo.toml new file mode 100644 index 00000000..fe8f1e34 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "rustc_codegen_nvvm_v19" +version = "0.3.0" +authors = [ + "Riccardo D'Ambrosio ", + "The Rust Project Developers", +] +edition = "2024" +license = "MIT OR Apache-2.0" +description = "A codegen backend for Rustc which targets the libnvvm CUDA library" +repository = "https://github.com/Rust-GPU/Rust-CUDA" +readme = "../../README.md" + +[lib] +crate-type = ["dylib"] + +[dependencies] +nvvm = { version = "0.1", path = "../nvvm" } +rustc-demangle = "0.1.24" +libc = "0.2.169" +libloading = "0.8.0" +tar = "0.4.43" +object = "0.36.7" +bitflags = "2.8.0" +# To avoid duplicate dependencies, this should match the version of gimli used +# by `rustc_codegen_ssa` via its `thorin-dwp` dependency. +gimli = "0.30" +tracing = { version = "0.1.41", features = ["release_max_level_debug"] } +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +rustc_codegen_nvvm_macros = { version = "0.1", path = "../rustc_codegen_nvvm_macros" } +smallvec = { version = "1.14.0", features = ["union", "may_dangle"] } +itertools = "0.14.0" + +[build-dependencies] +build-helper = "0.1.1" +cc = { version = "1.0", features = ["parallel"] } +xz = "0.1.0" +tar = "0.4.37" +curl = "0.4.40" + +[package.metadata.rust-analyzer] +rustc_private = true diff --git a/crates/rustc_codegen_nvvm_v19/build.rs b/crates/rustc_codegen_nvvm_v19/build.rs new file mode 100644 index 00000000..fd135a06 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/build.rs @@ -0,0 +1,407 @@ +use std::{ + env, + ffi::{OsStr, OsString}, + fmt::Display, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use curl::easy::Easy; +use tar::Archive; +use xz::read::XzDecoder; + +static PREBUILT_LLVM_URL: &str = + "https://github.com/rust-gpu/rustc_codegen_nvvm-llvm/releases/download/LLVM-19.1.7/"; + +static REQUIRED_MAJOR_LLVM_VERSION: u8 = 19; + +fn main() { + rustc_llvm_build(); + + // this is set by cuda_builder, but in case somebody is using the codegen + // manually, default to 1200. + if option_env!("CUDA_ARCH").is_none() { + println!("cargo:rustc-env=CUDA_ARCH=1200") + } +} + +fn fail(s: &str) -> ! { + println!("\n\n{}\n\n", s); + std::process::exit(1); +} + +#[track_caller] +pub fn output(cmd: &mut Command) -> String { + let output = match cmd.stderr(Stdio::inherit()).output() { + Ok(status) => status, + Err(e) => fail(&format!( + "failed to execute command: {:?}\nerror: {}", + cmd, e + )), + }; + assert!( + output.status.success(), + "command did not execute successfully: {:?}\n\ + expected success, got: {}", + cmd, + output.status + ); + + String::from_utf8(output.stdout).unwrap() +} + +fn target_to_llvm_prebuilt(target: &str) -> String { + let base = match target { + "x86_64-pc-windows-msvc" => "windows-x86_64", + // NOTE(RDambrosio016): currently disabled because of weird issues with segfaults and building the C++ shim + // "x86_64-unknown-linux-gnu" => "linux-x86_64", + _ => panic!( + "Unsupported target with no matching prebuilt LLVM: `{}`, install LLVM and set LLVM_CONFIG", + target + ), + }; + format!("{}.tar.xz", base) +} + +fn find_llvm_config(target: &str) -> PathBuf { + // first, if LLVM_CONFIG is set then see if its llvm version if 7.x, if so, use that. + let config_env = tracked_env_var_os("LLVM_CONFIG_19"); + // if LLVM_CONFIG is not set, try using llvm-config as a normal app in PATH. + let path_to_try = config_env.unwrap_or_else(|| "llvm-config-19".into()); + + // if USE_PREBUILT_LLVM is set to 1 then download prebuilt llvm without trying llvm-config + if tracked_env_var_os("USE_PREBUILT_LLVM_19") != Some("1".into()) { + let cmd = Command::new(&path_to_try).arg("--version").output(); + + if let Ok(out) = cmd { + let version = String::from_utf8(out.stdout).unwrap(); + if version.starts_with(&REQUIRED_MAJOR_LLVM_VERSION.to_string()) { + return PathBuf::from(path_to_try); + } + println!( + "cargo:warning=Prebuilt llvm-config version does not start with {}", + REQUIRED_MAJOR_LLVM_VERSION + ); + } else { + println!("cargo:warning=Failed to run prebuilt llvm-config"); + } + } + + // otherwise, download prebuilt LLVM. + println!("cargo:warning=Downloading prebuilt LLVM"); + let mut url = tracked_env_var_os("PREBUILT_LLVM_URL") + .map(|x| x.to_string_lossy().to_string()) + .unwrap_or_else(|| PREBUILT_LLVM_URL.to_string()); + + let prebuilt_name = target_to_llvm_prebuilt(target); + url = format!("{}{}", url, prebuilt_name); + + let out = env::var("OUT_DIR").expect("OUT_DIR was not set"); + let mut easy = Easy::new(); + + easy.url(&url).unwrap(); + easy.follow_location(true).unwrap(); + let mut xz_encoded = Vec::with_capacity(20_000_000); // 20mb + { + let mut transfer = easy.transfer(); + transfer + .write_function(|data| { + xz_encoded.extend_from_slice(data); + Ok(data.len()) + }) + .expect("Failed to download prebuilt LLVM"); + transfer + .perform() + .expect("Failed to download prebuilt LLVM"); + } + + let decompressor = XzDecoder::new(xz_encoded.as_slice()); + let mut ar = Archive::new(decompressor); + + ar.unpack(&out).expect("Failed to unpack LLVM to LLVM dir"); + let out_path = PathBuf::from(out).join(prebuilt_name.strip_suffix(".tar.xz").unwrap()); + + println!("cargo:rerun-if-changed={}", out_path.display()); + + out_path + .join("bin") + .join(format!("llvm-config-19{}", std::env::consts::EXE_SUFFIX)) +} + +fn detect_llvm_link() -> (&'static str, &'static str) { + // Force the link mode we want, preferring static by default, but + // possibly overridden by `configure --enable-llvm-link-shared`. + if tracked_env_var_os("LLVM_LINK_SHARED").is_some() { + ("dylib", "--link-shared") + } else { + ("static", "--link-static") + } +} + +pub fn tracked_env_var_os + Display>(key: K) -> Option { + println!("cargo:rerun-if-env-changed={}", key); + env::var_os(key) +} + +fn run_llvm_as() { + // Check if libintrinsics.ll exists + let libintrinsics_path = Path::new("libintrinsics.ll"); + if !libintrinsics_path.exists() { + fail("libintrinsics.ll not found"); + } + + println!("cargo:rerun-if-changed=libintrinsics.ll"); + + let mut cmd = Command::new("llvm-as-19"); + cmd.arg("libintrinsics.ll"); + + let output = match cmd.stderr(Stdio::inherit()).output() { + Ok(status) => status, + Err(e) => fail(&format!( + "failed to execute llvm-as: {:?}\nerror: {}", + cmd, e + )), + }; + + if !output.status.success() { + fail(&format!( + "llvm-as failed: {:?}\nstatus: {}", + cmd, output.status + )); + } +} + +fn rustc_llvm_build() { + let target = env::var("TARGET").expect("TARGET was not set"); + let llvm_config = find_llvm_config(&target); + + let required_components = &[ + "ipo", + "bitreader", + "bitwriter", + "lto", + "nvptx", + ]; + + let components = output(Command::new(&llvm_config).arg("--components")); + let mut components = components.split_whitespace().collect::>(); + components.retain(|c| required_components.contains(c)); + + for component in required_components { + assert!( + components.contains(component), + "require llvm component {} but wasn't found", + component + ); + } + + for component in components.iter() { + println!("cargo:rustc-cfg=llvm_component=\"{}\"", component); + } + + // Run llvm-as on libintrinsics.ll + run_llvm_as(); + + // Link in our own LLVM shims, compiled with the same flags as LLVM + let mut cmd = Command::new(&llvm_config); + cmd.arg("--cxxflags"); + let cxxflags = output(&mut cmd); + let mut cfg = cc::Build::new(); + cfg.warnings(false); + for flag in cxxflags.split_whitespace() { + if flag.starts_with("-flto") { + continue; + } + // ignore flags that aren't supported in gcc 8 + if flag == "-Wcovered-switch-default" { + continue; + } + if flag == "-Wstring-conversion" { + continue; + } + if flag == "-Werror=unguarded-availability-new" { + continue; + } + + cfg.flag(flag); + } + + for component in &components { + let mut flag = String::from("LLVM_COMPONENT_"); + flag.push_str(&component.to_uppercase()); + cfg.define(&flag, None); + } + + if tracked_env_var_os("LLVM_RUSTLLVM").is_some() { + cfg.define("LLVM_RUSTLLVM", None); + } + + build_helper::rerun_if_changed(Path::new("rustc_llvm_wrapper")); + cfg.file("rustc_llvm_wrapper/RustWrapper.cpp") + .file("rustc_llvm_wrapper/PassWrapper.cpp") + .include("rustc_llvm_wrapper") + .cpp(true) + .cpp_link_stdlib(None) // we handle this below + .compile("llvm-wrapper"); + + let (llvm_kind, llvm_link_arg) = detect_llvm_link(); + + // Link in all LLVM libraries, if we're using the "wrong" llvm-config then + // we don't pick up system libs because unfortunately they're for the host + // of llvm-config, not the target that we're attempting to link. + let mut cmd = Command::new(&llvm_config); + cmd.arg(llvm_link_arg).arg("--libs"); + + if target.contains("windows-gnu") { + println!("cargo:rustc-link-lib=shell32"); + println!("cargo:rustc-link-lib=uuid"); + } else if target.contains("netbsd") || target.contains("haiku") { + println!("cargo:rustc-link-lib=z"); + } + cmd.args(&components); + + for lib in output(&mut cmd).split_whitespace() { + let name = if let Some(stripped) = lib.strip_prefix("-l") { + stripped + } else if let Some(stripped) = lib.strip_prefix('-') { + stripped + } else if Path::new(lib).exists() { + // On MSVC llvm-config will print the full name to libraries, but + // we're only interested in the name part + let name = Path::new(lib).file_name().unwrap().to_str().unwrap(); + name.trim_end_matches(".lib") + } else if lib.ends_with(".lib") { + // Some MSVC libraries just come up with `.lib` tacked on, so chop + // that off + lib.trim_end_matches(".lib") + } else { + continue; + }; + + // Don't need or want this library, but LLVM's CMake build system + // doesn't provide a way to disable it, so filter it here even though we + // may or may not have built it. We don't reference anything from this + // library and it otherwise may just pull in extra dependencies on + // libedit which we don't want + if name == "LLVMLineEditor" { + continue; + } + + let kind = if name.starts_with("LLVM") { + llvm_kind + } else { + "dylib" + }; + println!("cargo:rustc-link-lib={}={}", kind, name); + } + + // Link in the system libraries that LLVM depends on + #[cfg(not(target_os = "windows"))] + link_llvm_system_libs(&llvm_config, required_components); + + // LLVM ldflags + // + // If we're a cross-compile of LLVM then unfortunately we can't trust these + // ldflags (largely where all the LLVM libs are located). Currently just + // hack around this by replacing the host triple with the target and pray + // that those -L directories are the same! + let mut cmd = Command::new(&llvm_config); + cmd.arg(llvm_link_arg).arg("--ldflags"); + for lib in output(&mut cmd).split_whitespace() { + if let Some(stripped) = lib.strip_prefix("-LIBPATH:") { + println!("cargo:rustc-link-search=native={}", stripped); + } else if let Some(stripped) = lib.strip_prefix("-l") { + println!("cargo:rustc-link-lib={}", stripped); + } else if let Some(stripped) = lib.strip_prefix("-L") { + println!("cargo:rustc-link-search=native={}", stripped); + } + } + + // Some LLVM linker flags (-L and -l) may be needed even when linking + // rustc_llvm, for example when using static libc++, we may need to + // manually specify the library search path and -ldl -lpthread as link + // dependencies. + let llvm_linker_flags = tracked_env_var_os("LLVM_LINKER_FLAGS"); + if let Some(s) = llvm_linker_flags { + for lib in s.into_string().unwrap().split_whitespace() { + if let Some(stripped) = lib.strip_prefix("-l") { + println!("cargo:rustc-link-lib={}", stripped); + } else if let Some(stripped) = lib.strip_prefix("-L") { + println!("cargo:rustc-link-search=native={}", stripped); + } + } + } + + let llvm_static_stdcpp = tracked_env_var_os("LLVM_STATIC_STDCPP"); + let llvm_use_libcxx = tracked_env_var_os("LLVM_USE_LIBCXX"); + + let stdcppname = if target.contains("openbsd") { + if target.contains("sparc64") { + "estdc++" + } else { + "c++" + } + } else if target.contains("freebsd") || target.contains("darwin") { + "c++" + } else if target.contains("netbsd") && llvm_static_stdcpp.is_some() { + // NetBSD uses a separate library when relocation is required + "stdc++_pic" + } else if llvm_use_libcxx.is_some() { + "c++" + } else { + "stdc++" + }; + + // RISC-V requires libatomic for sub-word atomic operations + if target.starts_with("riscv") { + println!("cargo:rustc-link-lib=atomic"); + } + + // C++ runtime library + if !target.contains("msvc") { + if let Some(s) = llvm_static_stdcpp { + assert!(!cxxflags.contains("stdlib=libc++")); + let path = PathBuf::from(s); + println!( + "cargo:rustc-link-search=native={}", + path.parent().unwrap().display() + ); + if target.contains("windows") { + println!("cargo:rustc-link-lib=static-nobundle={}", stdcppname); + } else { + println!("cargo:rustc-link-lib=static={}", stdcppname); + } + } else if cxxflags.contains("stdlib=libc++") { + println!("cargo:rustc-link-lib=c++"); + } else { + println!("cargo:rustc-link-lib={}", stdcppname); + } + } + + // Libstdc++ depends on pthread which Rust doesn't link on MinGW + // since nothing else requires it. + if target.contains("windows-gnu") { + println!("cargo:rustc-link-lib=static-nobundle=pthread"); + } +} + +#[cfg(not(target_os = "windows"))] +fn link_llvm_system_libs(llvm_config: &Path, components: &[&str]) { + let (_, llvm_link_arg) = detect_llvm_link(); + let mut cmd: Command = Command::new(llvm_config); + cmd.arg(llvm_link_arg).arg("--system-libs"); + + for comp in components { + cmd.arg(comp); + } + + for lib in output(&mut cmd).split_whitespace() { + let name = if let Some(stripped) = lib.strip_prefix("-l") { + stripped + } else { + continue; + }; + + println!("cargo:rustc-link-lib=dylib={}", name); + } +} diff --git a/crates/rustc_codegen_nvvm_v19/libintrinsics.bc b/crates/rustc_codegen_nvvm_v19/libintrinsics.bc new file mode 100644 index 00000000..50871a4d Binary files /dev/null and b/crates/rustc_codegen_nvvm_v19/libintrinsics.bc differ diff --git a/crates/rustc_codegen_nvvm_v19/libintrinsics.ll b/crates/rustc_codegen_nvvm_v19/libintrinsics.ll new file mode 100644 index 00000000..427ff8d9 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/libintrinsics.ll @@ -0,0 +1,332 @@ +; This is a hand-written llvm ir module which contains extra functions +; that are easier to write. They mostly contain nvvm intrinsics that are wrapped in new +; functions so that rustc does not think they are llvm intrinsics and so you don't need to always use nightly for that. +; +; if you update this make sure to update libintrinsics.bc by running llvm-as (make sure you are using llvm-7 or it won't work when +; loaded into libnvvm). +source_filename = "libintrinsics" +target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64-a:8:8" +target triple = "nvptx64-nvidia-cuda" + +; thread ---- + +define i32 @__nvvm_thread_idx_x() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.tid.x() + ret i32 %0 +} + +define i32 @__nvvm_thread_idx_y() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.tid.y() + ret i32 %0 +} + +define i32 @__nvvm_thread_idx_z() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.tid.z() + ret i32 %0 +} + +; block dimension ---- + +define i32 @__nvvm_block_dim_x() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.x() + ret i32 %0 +} + +define i32 @__nvvm_block_dim_y() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.y() + ret i32 %0 +} + +define i32 @__nvvm_block_dim_z() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.ntid.z() + ret i32 %0 +} + +; block idx ---- + +define i32 @__nvvm_block_idx_x() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() + ret i32 %0 +} + +define i32 @__nvvm_block_idx_y() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.y() + ret i32 %0 +} + +define i32 @__nvvm_block_idx_z() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.z() + ret i32 %0 +} + +; grid dimension ---- + +define i32 @__nvvm_grid_dim_x() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.x() + ret i32 %0 +} + +define i32 @__nvvm_grid_dim_y() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.y() + ret i32 %0 +} + +define i32 @__nvvm_grid_dim_z() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.nctaid.z() + ret i32 %0 +} + +; warp ---- + +define i32 @__nvvm_warp_size() #0 { +start: + %0 = call i32 @llvm.nvvm.read.ptx.sreg.warpsize() + ret i32 %0 +} + +declare i32 @llvm.nvvm.read.ptx.sreg.tid.x() +declare i32 @llvm.nvvm.read.ptx.sreg.tid.y() +declare i32 @llvm.nvvm.read.ptx.sreg.tid.z() +declare i32 @llvm.nvvm.read.ptx.sreg.ntid.x() +declare i32 @llvm.nvvm.read.ptx.sreg.ntid.y() +declare i32 @llvm.nvvm.read.ptx.sreg.ntid.z() +declare i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() +declare i32 @llvm.nvvm.read.ptx.sreg.ctaid.y() +declare i32 @llvm.nvvm.read.ptx.sreg.ctaid.z() +declare i32 @llvm.nvvm.read.ptx.sreg.nctaid.x() +declare i32 @llvm.nvvm.read.ptx.sreg.nctaid.y() +declare i32 @llvm.nvvm.read.ptx.sreg.nctaid.z() +declare i32 @llvm.nvvm.read.ptx.sreg.warpsize() + +; other ---- + +define void @__nvvm_block_barrier() #1 { +start: + call void @llvm.nvvm.barrier0() + ret void +} + +declare void @llvm.nvvm.barrier0() + +define void @__nvvm_grid_fence() #1 { +start: + call void @llvm.nvvm.membar.cta() + ret void +} + +declare void @llvm.nvvm.membar.cta() + +define void @__nvvm_device_fence() #1 { +start: + call void @llvm.nvvm.membar.gl() + ret void +} + +declare void @llvm.nvvm.membar.gl() + +define void @__nvvm_system_fence() #1 { +start: + call void @llvm.nvvm.membar.sys() + ret void +} + +declare void @llvm.nvvm.membar.sys() + +define void @__nvvm_trap() #1 { +start: + call void @llvm.trap() + unreachable + ret void +} + +declare void @llvm.trap() + +; math stuff ------------- + +define {i8, i1} @__nvvm_i8_addo(i8, i8) #0 { +start: + %2 = sext i8 %0 to i16 + %3 = sext i8 %1 to i16 + %4 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %2, i16 %3) + %5 = extractvalue {i16, i1} %4, 0 + %6 = extractvalue {i16, i1} %4, 1 + %7 = trunc i16 %5 to i8 + %8 = insertvalue {i8, i1} undef, i8 %7, 0 + %9 = insertvalue {i8, i1} %8, i1 %6, 1 + ret {i8, i1} %9 +} +declare {i16, i1} @llvm.sadd.with.overflow.i16(i16, i16) #0 + +define {i8, i1} @__nvvm_u8_addo(i8, i8) #0 { +start: + %2 = sext i8 %0 to i16 + %3 = sext i8 %1 to i16 + %4 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %2, i16 %3) + %5 = extractvalue {i16, i1} %4, 0 + %6 = extractvalue {i16, i1} %4, 1 + %7 = trunc i16 %5 to i8 + %8 = insertvalue {i8, i1} undef, i8 %7, 0 + %9 = insertvalue {i8, i1} %8, i1 %6, 1 + ret {i8, i1} %9 +} +declare {i16, i1} @llvm.uadd.with.overflow.i16(i16, i16) #0 + +define {i8, i1} @__nvvm_i8_subo(i8, i8) #0 { +start: + %2 = sext i8 %0 to i16 + %3 = sext i8 %1 to i16 + %4 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 %2, i16 %3) + %5 = extractvalue {i16, i1} %4, 0 + %6 = extractvalue {i16, i1} %4, 1 + %7 = trunc i16 %5 to i8 + %8 = insertvalue {i8, i1} undef, i8 %7, 0 + %9 = insertvalue {i8, i1} %8, i1 %6, 1 + ret {i8, i1} %9 +} +declare {i16, i1} @llvm.ssub.with.overflow.i16(i16, i16) #0 + +define {i8, i1} @__nvvm_u8_subo(i8, i8) #0 { +start: + %2 = sext i8 %0 to i16 + %3 = sext i8 %1 to i16 + %4 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 %2, i16 %3) + %5 = extractvalue {i16, i1} %4, 0 + %6 = extractvalue {i16, i1} %4, 1 + %7 = trunc i16 %5 to i8 + %8 = insertvalue {i8, i1} undef, i8 %7, 0 + %9 = insertvalue {i8, i1} %8, i1 %6, 1 + ret {i8, i1} %9 +} +declare {i16, i1} @llvm.usub.with.overflow.i16(i16, i16) #0 + +define {i8, i1} @__nvvm_i8_mulo(i8, i8) #0 { +start: + %2 = sext i8 %0 to i16 + %3 = sext i8 %1 to i16 + %4 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %2, i16 %3) + %5 = extractvalue {i16, i1} %4, 0 + %6 = extractvalue {i16, i1} %4, 1 + %7 = trunc i16 %5 to i8 + %8 = insertvalue {i8, i1} undef, i8 %7, 0 + %9 = insertvalue {i8, i1} %8, i1 %6, 1 + ret {i8, i1} %9 +} +declare {i16, i1} @llvm.smul.with.overflow.i16(i16, i16) #0 + +define {i8, i1} @__nvvm_u8_mulo(i8, i8) #0 { +start: + %2 = sext i8 %0 to i16 + %3 = sext i8 %1 to i16 + %4 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %2, i16 %3) + %5 = extractvalue {i16, i1} %4, 0 + %6 = extractvalue {i16, i1} %4, 1 + %7 = trunc i16 %5 to i8 + %8 = insertvalue {i8, i1} undef, i8 %7, 0 + %9 = insertvalue {i8, i1} %8, i1 %6, 1 + ret {i8, i1} %9 +} +declare {i16, i1} @llvm.umul.with.overflow.i16(i16, i16) #0 + +; This is a bit weird, we need to use functions defined in rust crates (compiler_builtins) +; as intrinsics in the codegen, but we can't directly use their name, otherwise we will have +; really odd and incorrect behavior in the crate theyre defined in. So we need to make a wrapper for them that is opaque +; to the codegen, which is what this is doing. + +define {<2 x i64>, i1} @__nvvm_i128_addo(<2 x i64>, <2 x i64>) #0 { +start: + %2 = call {<2 x i64>, i1} @__rust_i128_addo(<2 x i64> %0, <2 x i64> %1) + ret {<2 x i64>, i1} %2 +} +declare {<2 x i64>, i1} @__rust_i128_addo(<2 x i64>, <2 x i64>) #0 + +define {<2 x i64>, i1} @__nvvm_u128_addo(<2 x i64>, <2 x i64>) #0 { +start: + %2 = call {<2 x i64>, i1} @__rust_u128_addo(<2 x i64> %0, <2 x i64> %1) + ret {<2 x i64>, i1} %2 +} +declare {<2 x i64>, i1} @__rust_u128_addo(<2 x i64>, <2 x i64>) #0 + +define {<2 x i64>, i1} @__nvvm_i128_subo(<2 x i64>, <2 x i64>) #0 { +start: + %2 = call {<2 x i64>, i1} @__rust_i128_subo(<2 x i64> %0, <2 x i64> %1) + ret {<2 x i64>, i1} %2 +} +declare {<2 x i64>, i1} @__rust_i128_subo(<2 x i64>, <2 x i64>) #0 + +define {<2 x i64>, i1} @__nvvm_u128_subo(<2 x i64>, <2 x i64>) #0 { +start: + %2 = call {<2 x i64>, i1} @__rust_u128_subo(<2 x i64> %0, <2 x i64> %1) + ret {<2 x i64>, i1} %2 +} +declare {<2 x i64>, i1} @__rust_u128_subo(<2 x i64>, <2 x i64>) #0 + +define {<2 x i64>, i1} @__nvvm_i128_mulo(<2 x i64>, <2 x i64>) #0 { +start: + %2 = call {<2 x i64>, i1} @__rust_i128_mulo(<2 x i64> %0, <2 x i64> %1) + ret {<2 x i64>, i1} %2 +} +declare {<2 x i64>, i1} @__rust_i128_mulo(<2 x i64>, <2 x i64>) #0 + +define {<2 x i64>, i1} @__nvvm_u128_mulo(<2 x i64>, <2 x i64>) #0 { +start: + %2 = call {<2 x i64>, i1} @__rust_u128_mulo(<2 x i64> %0, <2 x i64> %1) + ret {<2 x i64>, i1} %2 +} +declare {<2 x i64>, i1} @__rust_u128_mulo(<2 x i64>, <2 x i64>) #0 + +; Required because we need to explicitly generate { i32, i1 } for the following intrinsics +; except rustc will not generate them (it will make { i32, i8 }) which libnvvm rejects. + +define { i32, i8 } @__nvvm_warp_shuffle(i32, i32, i32, i32, i32) #1 { +start: + %5 = call { i32, i1 } @llvm.nvvm.shfl.sync.i32(i32 %0, i32 %1, i32 %2, i32 %3, i32 %4) + %6 = extractvalue { i32, i1 } %5, 1 + %7 = zext i1 %6 to i8 + %8 = extractvalue { i32, i1 } %5, 0 + %9 = insertvalue { i32, i8 } undef, i32 %8, 0 + %10 = insertvalue { i32, i8 } %9, i8 %7, 1 + ret { i32, i8 } %10 +} + +declare { i32, i1 } @llvm.nvvm.shfl.sync.i32(i32, i32, i32, i32, i32) #1 + +define { i32, i8 } @__nvvm_warp_match_all_32(i32, i32) { +start: + %2 = call { i32, i1 } @llvm.nvvm.match.all.sync.i32(i32 %0, i32 %1) + %3 = extractvalue { i32, i1 } %2, 1 + %4 = zext i1 %3 to i8 + %5 = extractvalue { i32, i1 } %2, 0 + %6 = insertvalue { i32, i8 } undef, i32 %5, 0 + %7 = insertvalue { i32, i8 } %6, i8 %4, 1 + ret { i32, i8 } %7 +} + +declare { i32, i1 } @llvm.nvvm.match.all.sync.i32(i32, i32) #1 + +define { i32, i8 } @__nvvm_warp_match_all_64(i32, i64) { +start: + %2 = call { i32, i1 } @llvm.nvvm.match.all.sync.i64(i32 %0, i64 %1) + %3 = extractvalue { i32, i1 } %2, 1 + %4 = zext i1 %3 to i8 + %5 = extractvalue { i32, i1 } %2, 0 + %6 = insertvalue { i32, i8 } undef, i32 %5, 0 + %7 = insertvalue { i32, i8 } %6, i8 %4, 1 + ret { i32, i8 } %7 +} + +declare { i32, i1 } @llvm.nvvm.match.all.sync.i64(i32, i64) #1 + +attributes #0 = { alwaysinline speculatable } +attributes #1 = { alwaysinline } diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/.editorconfig b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/.editorconfig similarity index 100% rename from crates/rustc_codegen_nvvm/rustc_llvm_wrapper/.editorconfig rename to crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/.editorconfig diff --git a/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/LLVMWrapper.h b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/LLVMWrapper.h new file mode 100644 index 00000000..db88ab4f --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/LLVMWrapper.h @@ -0,0 +1,49 @@ +#ifndef INCLUDED_RUSTC_LLVM_LLVMWRAPPER_H +#define INCLUDED_RUSTC_LLVM_LLVMWRAPPER_H + +#include "SuppressLLVMWarnings.h" + +#include "llvm/Config/llvm-config.h" // LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR +#include "llvm/Support/raw_ostream.h" // llvm::raw_ostream +#include // size_t etc +#include // uint64_t etc + +#define LLVM_VERSION_GE(major, minor) \ + (LLVM_VERSION_MAJOR > (major) || \ + LLVM_VERSION_MAJOR == (major) && LLVM_VERSION_MINOR >= (minor)) + +#define LLVM_VERSION_LT(major, minor) (!LLVM_VERSION_GE((major), (minor))) + +extern "C" void LLVMRustSetLastError(const char *); + +enum class LLVMRustResult { Success, Failure }; + +typedef struct OpaqueRustString *RustStringRef; +typedef struct LLVMOpaqueTwine *LLVMTwineRef; +typedef struct LLVMOpaqueSMDiagnostic *LLVMSMDiagnosticRef; + +extern "C" void LLVMRustStringWriteImpl(RustStringRef buf, + const char *slice_ptr, + size_t slice_len); + +class RawRustStringOstream : public llvm::raw_ostream { + RustStringRef Str; + uint64_t Pos; + + void write_impl(const char *Ptr, size_t Size) override { + LLVMRustStringWriteImpl(Str, Ptr, Size); + Pos += Size; + } + + uint64_t current_pos() const override { return Pos; } + +public: + explicit RawRustStringOstream(RustStringRef Str) : Str(Str), Pos(0) {} + + ~RawRustStringOstream() { + // LLVM requires this. + flush(); + } +}; + +#endif // INCLUDED_RUSTC_LLVM_LLVMWRAPPER_H \ No newline at end of file diff --git a/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/PassWrapper.cpp b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/PassWrapper.cpp new file mode 100644 index 00000000..94d61b12 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/PassWrapper.cpp @@ -0,0 +1,1805 @@ +#include "LLVMWrapper.h" + +#include "llvm-c/Core.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/Lint.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/Bitcode/BitcodeWriterPass.h" +#include "llvm/CodeGen/CommandFlags.h" +#include "llvm/IR/AssemblyAnnotationWriter.h" +#include "llvm/IR/AutoUpgrade.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/PassManager.h" +#include "llvm/IR/Verifier.h" +#include "llvm/IR/DebugInfo.h" // TODO: non-standard +#include "llvm/LTO/LTO.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/TargetRegistry.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Passes/PassPlugin.h" +#include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/Support/CBindingWrapping.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Signals.h" // TODO: non-standard +#include "llvm/Support/TimeProfiler.h" +#include "llvm/Support/VirtualFileSystem.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/TargetParser/Host.h" +#include "llvm/Transforms/IPO/FunctionImport.h" +#include "llvm/Transforms/IPO/Internalize.h" +#include "llvm/Transforms/IPO/LowerTypeTests.h" +#include "llvm/Transforms/IPO/ThinLTOBitcodeWriter.h" +#include "llvm/Transforms/Instrumentation/AddressSanitizer.h" +#include "llvm/Transforms/Instrumentation/DataFlowSanitizer.h" +#include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" +#include "llvm/Transforms/Instrumentation/InstrProfiling.h" +#include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" +#include "llvm/Transforms/Scalar/AnnotationRemarks.h" +#include "llvm/Transforms/Utils/CanonicalizeAliases.h" +#include "llvm/Transforms/Utils/FunctionImportUtils.h" +#include "llvm/Transforms/Utils/NameAnonGlobals.h" +#include +#include +#include + +// Conditional includes prevent clang-format from fully sorting the list, +// so keep them separate. +#if LLVM_VERSION_GE(19, 0) +#include "llvm/Support/PGOOptions.h" +#endif + +// version check +#if LLVM_VERSION_MAJOR != 19 +#error "This code requires LLVM major version 19" +#endif + +using namespace llvm; + +static codegen::RegisterCodeGenFlags CGF; + +typedef struct LLVMOpaquePass *LLVMPassRef; +typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef; + +DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef) +DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + +extern "C" void LLVMRustTimeTraceProfilerInitialize() { + timeTraceProfilerInitialize( + /* TimeTraceGranularity */ 0, + /* ProcName */ "rustc"); +} + +extern "C" void LLVMRustTimeTraceProfilerFinishThread() { + timeTraceProfilerFinishThread(); +} + +extern "C" void LLVMRustTimeTraceProfilerFinish(const char *FileName) { + auto FN = StringRef(FileName); + std::error_code EC; + auto OS = raw_fd_ostream(FN, EC, sys::fs::CD_CreateAlways); + + timeTraceProfilerWrite(OS); + timeTraceProfilerCleanup(); +} + +#ifdef LLVM_COMPONENT_X86 +#define SUBTARGET_X86 SUBTARGET(X86) +#else +#define SUBTARGET_X86 +#endif + +#ifdef LLVM_COMPONENT_ARM +#define SUBTARGET_ARM SUBTARGET(ARM) +#else +#define SUBTARGET_ARM +#endif + +#ifdef LLVM_COMPONENT_AARCH64 +#define SUBTARGET_AARCH64 SUBTARGET(AArch64) +#else +#define SUBTARGET_AARCH64 +#endif + +#ifdef LLVM_COMPONENT_AVR +#define SUBTARGET_AVR SUBTARGET(AVR) +#else +#define SUBTARGET_AVR +#endif + +#ifdef LLVM_COMPONENT_M68k +#define SUBTARGET_M68K SUBTARGET(M68k) +#else +#define SUBTARGET_M68K +#endif + +#ifdef LLVM_COMPONENT_CSKY +#define SUBTARGET_CSKY SUBTARGET(CSKY) +#else +#define SUBTARGET_CSKY +#endif + +#ifdef LLVM_COMPONENT_MIPS +#define SUBTARGET_MIPS SUBTARGET(Mips) +#else +#define SUBTARGET_MIPS +#endif + +#ifdef LLVM_COMPONENT_POWERPC +#define SUBTARGET_PPC SUBTARGET(PPC) +#else +#define SUBTARGET_PPC +#endif + +#ifdef LLVM_COMPONENT_SYSTEMZ +#define SUBTARGET_SYSTEMZ SUBTARGET(SystemZ) +#else +#define SUBTARGET_SYSTEMZ +#endif + +#ifdef LLVM_COMPONENT_MSP430 +#define SUBTARGET_MSP430 SUBTARGET(MSP430) +#else +#define SUBTARGET_MSP430 +#endif + +#ifdef LLVM_COMPONENT_RISCV +#define SUBTARGET_RISCV SUBTARGET(RISCV) +#else +#define SUBTARGET_RISCV +#endif + +#ifdef LLVM_COMPONENT_SPARC +#define SUBTARGET_SPARC SUBTARGET(Sparc) +#else +#define SUBTARGET_SPARC +#endif + +#ifdef LLVM_COMPONENT_XTENSA +#define SUBTARGET_XTENSA SUBTARGET(XTENSA) +#else +#define SUBTARGET_XTENSA +#endif + +#ifdef LLVM_COMPONENT_HEXAGON +#define SUBTARGET_HEXAGON SUBTARGET(Hexagon) +#else +#define SUBTARGET_HEXAGON +#endif + +#ifdef LLVM_COMPONENT_LOONGARCH +#define SUBTARGET_LOONGARCH SUBTARGET(LoongArch) +#else +#define SUBTARGET_LOONGARCH +#endif + +#define GEN_SUBTARGETS \ + SUBTARGET_X86 \ + SUBTARGET_ARM \ + SUBTARGET_AARCH64 \ + SUBTARGET_AVR \ + SUBTARGET_M68K \ + SUBTARGET_CSKY \ + SUBTARGET_MIPS \ + SUBTARGET_PPC \ + SUBTARGET_SYSTEMZ \ + SUBTARGET_MSP430 \ + SUBTARGET_SPARC \ + SUBTARGET_HEXAGON \ + SUBTARGET_XTENSA \ + SUBTARGET_RISCV \ + SUBTARGET_LOONGARCH + +#define SUBTARGET(x) \ + namespace llvm { \ + extern const SubtargetFeatureKV x##FeatureKV[]; \ + extern const SubtargetFeatureKV x##SubTypeKV[]; \ + } + +GEN_SUBTARGETS +#undef SUBTARGET + +// This struct and various functions are sort of a hack right now, but the +// problem is that we've got in-memory LLVM modules after we generate and +// optimize all codegen-units for one compilation in rustc. To be compatible +// with the LTO support above we need to serialize the modules plus their +// ThinLTO summary into memory. +// +// This structure is basically an owned version of a serialize module, with +// a ThinLTO summary attached. +struct LLVMRustThinLTOBuffer { + std::string data; + std::string thin_link_data; +}; + +extern "C" bool LLVMRustHasFeature(LLVMTargetMachineRef TM, + const char *Feature) { + TargetMachine *Target = unwrap(TM); + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + return MCInfo->checkFeatures(std::string("+") + Feature); +} + +enum class LLVMRustCodeModel { + Tiny, + Small, + Kernel, + Medium, + Large, + None, +}; + +static std::optional fromRust(LLVMRustCodeModel Model) { + switch (Model) { + case LLVMRustCodeModel::Tiny: + return CodeModel::Tiny; + case LLVMRustCodeModel::Small: + return CodeModel::Small; + case LLVMRustCodeModel::Kernel: + return CodeModel::Kernel; + case LLVMRustCodeModel::Medium: + return CodeModel::Medium; + case LLVMRustCodeModel::Large: + return CodeModel::Large; + case LLVMRustCodeModel::None: + return std::nullopt; + default: + report_fatal_error("Bad CodeModel."); + } +} + +enum class LLVMRustCodeGenOptLevel { + None, + Less, + Default, + Aggressive, +}; + +using CodeGenOptLevelEnum = llvm::CodeGenOptLevel; + +static CodeGenOptLevelEnum fromRust(LLVMRustCodeGenOptLevel Level) { + switch (Level) { + case LLVMRustCodeGenOptLevel::None: + return CodeGenOptLevelEnum::None; + case LLVMRustCodeGenOptLevel::Less: + return CodeGenOptLevelEnum::Less; + case LLVMRustCodeGenOptLevel::Default: + return CodeGenOptLevelEnum::Default; + case LLVMRustCodeGenOptLevel::Aggressive: + return CodeGenOptLevelEnum::Aggressive; + default: { + std::string error; + auto stream = llvm::raw_string_ostream(error); + stream << "Rust passed unsupported CodeGenOptLevel with value: " + << static_cast(Level) << "\n"; + + // Print stack trace + llvm::sys::PrintStackTrace(llvm::errs()); + + stream.flush(); + report_fatal_error(error.c_str()); + } + } +} + +enum class LLVMRustPassBuilderOptLevel { + O0, + O1, + O2, + O3, + Os, + Oz, +}; + +static OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) { + switch (Level) { + case LLVMRustPassBuilderOptLevel::O0: + return OptimizationLevel::O0; + case LLVMRustPassBuilderOptLevel::O1: + return OptimizationLevel::O1; + case LLVMRustPassBuilderOptLevel::O2: + return OptimizationLevel::O2; + case LLVMRustPassBuilderOptLevel::O3: + return OptimizationLevel::O3; + case LLVMRustPassBuilderOptLevel::Os: + return OptimizationLevel::Os; + case LLVMRustPassBuilderOptLevel::Oz: + return OptimizationLevel::Oz; + default: + report_fatal_error("Bad PassBuilderOptLevel."); + } +} + +enum class LLVMRustRelocModel { + Static, + PIC, + DynamicNoPic, + ROPI, + RWPI, + ROPIRWPI, +}; + +static Reloc::Model fromRust(LLVMRustRelocModel RustReloc) { + switch (RustReloc) { + case LLVMRustRelocModel::Static: + return Reloc::Static; + case LLVMRustRelocModel::PIC: + return Reloc::PIC_; + case LLVMRustRelocModel::DynamicNoPic: + return Reloc::DynamicNoPIC; + case LLVMRustRelocModel::ROPI: + return Reloc::ROPI; + case LLVMRustRelocModel::RWPI: + return Reloc::RWPI; + case LLVMRustRelocModel::ROPIRWPI: + return Reloc::ROPI_RWPI; + } + report_fatal_error("Bad RelocModel."); +} + +enum class LLVMRustFloatABI { + Default, + Soft, + Hard, +}; + +static FloatABI::ABIType fromRust(LLVMRustFloatABI RustFloatAbi) { + switch (RustFloatAbi) { + case LLVMRustFloatABI::Default: + return FloatABI::Default; + case LLVMRustFloatABI::Soft: + return FloatABI::Soft; + case LLVMRustFloatABI::Hard: + return FloatABI::Hard; + } + report_fatal_error("Bad FloatABI."); +} + +/// getLongestEntryLength - Return the length of the longest entry in the table. +template static size_t getLongestEntryLength(ArrayRef Table) { + size_t MaxLen = 0; + for (auto &I : Table) + MaxLen = std::max(MaxLen, std::strlen(I.Key)); + return MaxLen; +} + +extern "C" void LLVMRustPrintTargetCPUs(LLVMTargetMachineRef TM, + RustStringRef OutStr) { + ArrayRef CPUTable = + unwrap(TM)->getMCSubtargetInfo()->getAllProcessorDescriptions(); + auto OS = RawRustStringOstream(OutStr); + + // Just print a bare list of target CPU names, and let Rust-side code handle + // the full formatting of `--print=target-cpus`. + for (auto &CPU : CPUTable) { + OS << CPU.Key << "\n"; + } +} + +extern "C" size_t LLVMRustGetTargetFeaturesCount(LLVMTargetMachineRef TM) { + const TargetMachine *Target = unwrap(TM); + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + const ArrayRef FeatTable = + MCInfo->getAllProcessorFeatures(); + return FeatTable.size(); +} + +extern "C" void LLVMRustGetTargetFeature(LLVMTargetMachineRef TM, size_t Index, + const char **Feature, + const char **Desc) { + const TargetMachine *Target = unwrap(TM); + const MCSubtargetInfo *MCInfo = Target->getMCSubtargetInfo(); + const ArrayRef FeatTable = + MCInfo->getAllProcessorFeatures(); + const SubtargetFeatureKV Feat = FeatTable[Index]; + *Feature = Feat.Key; + *Desc = Feat.Desc; +} + +extern "C" const char *LLVMRustGetHostCPUName(size_t *OutLen) { + StringRef Name = sys::getHostCPUName(); + *OutLen = Name.size(); + return Name.data(); +} + +extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine( + const char *TripleStr, const char *CPU, const char *Feature, + const char *ABIStr, LLVMRustCodeModel RustCM, LLVMRustRelocModel RustReloc, + LLVMRustCodeGenOptLevel RustOptLevel, LLVMRustFloatABI RustFloatABIType, + bool FunctionSections, bool DataSections, bool UniqueSectionNames, + bool TrapUnreachable, bool Singlethread, bool VerboseAsm, + bool EmitStackSizeSection, bool RelaxELFRelocations, bool UseInitArray, + const char *SplitDwarfFile, const char *OutputObjFile, + const char *DebugInfoCompression, bool UseEmulatedTls, + const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) { + + auto OptLevel = fromRust(RustOptLevel); + auto RM = fromRust(RustReloc); + auto CM = fromRust(RustCM); + auto FloatABIType = fromRust(RustFloatABIType); + + std::string Error; + auto Trip = Triple(Triple::normalize(TripleStr)); + const llvm::Target *TheTarget = + TargetRegistry::lookupTarget(Trip.getTriple(), Error); + if (TheTarget == nullptr) { + LLVMRustSetLastError(Error.c_str()); + return nullptr; + } + + TargetOptions Options = codegen::InitTargetOptionsFromCodeGenFlags(Trip); + + Options.FloatABIType = FloatABIType; + Options.DataSections = DataSections; + Options.FunctionSections = FunctionSections; + Options.UniqueSectionNames = UniqueSectionNames; + Options.MCOptions.AsmVerbose = VerboseAsm; + // Always preserve comments that were written by the user + Options.MCOptions.PreserveAsmComments = true; + Options.MCOptions.ABIName = ABIStr; + if (SplitDwarfFile) { + Options.MCOptions.SplitDwarfFile = SplitDwarfFile; + } + if (OutputObjFile) { + Options.ObjectFilenameForDebug = OutputObjFile; + } + if (!strcmp("zlib", DebugInfoCompression) && + llvm::compression::zlib::isAvailable()) { +#if LLVM_VERSION_GE(19, 0) + Options.MCOptions.CompressDebugSections = DebugCompressionType::Zlib; +#else + Options.CompressDebugSections = DebugCompressionType::Zlib; +#endif + } else if (!strcmp("zstd", DebugInfoCompression) && + llvm::compression::zstd::isAvailable()) { +#if LLVM_VERSION_GE(19, 0) + Options.MCOptions.CompressDebugSections = DebugCompressionType::Zstd; +#else + Options.CompressDebugSections = DebugCompressionType::Zstd; +#endif + } else if (!strcmp("none", DebugInfoCompression)) { +#if LLVM_VERSION_GE(19, 0) + Options.MCOptions.CompressDebugSections = DebugCompressionType::None; +#else + Options.CompressDebugSections = DebugCompressionType::None; +#endif + } + +#if LLVM_VERSION_GE(19, 0) + Options.MCOptions.X86RelaxRelocations = RelaxELFRelocations; +#else + Options.RelaxELFRelocations = RelaxELFRelocations; +#endif + Options.UseInitArray = UseInitArray; + Options.EmulatedTLS = UseEmulatedTls; + + if (TrapUnreachable) { + // Tell LLVM to codegen `unreachable` into an explicit trap instruction. + // This limits the extent of possible undefined behavior in some cases, as + // it prevents control flow from "falling through" into whatever code + // happens to be laid out next in memory. + Options.TrapUnreachable = true; + // But don't emit traps after other traps or no-returns unnecessarily. + // ...except for when targeting WebAssembly, because the NoTrapAfterNoreturn + // option causes bugs in the LLVM WebAssembly backend. You should be able to + // remove this check when Rust's minimum supported LLVM version is >= 18 + // https://github.com/llvm/llvm-project/pull/65876 + if (!Trip.isWasm()) { + Options.NoTrapAfterNoreturn = true; + } + } + + if (Singlethread) { + Options.ThreadModel = ThreadModel::Single; + } + + Options.EmitStackSizeSection = EmitStackSizeSection; + + if (ArgsCstrBuff != nullptr) { +#if LLVM_VERSION_GE(20, 0) + int buffer_offset = 0; + assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); + auto Arg0 = std::string(ArgsCstrBuff); + buffer_offset = Arg0.size() + 1; + + std::string CommandlineArgs; + raw_string_ostream OS(CommandlineArgs); + ListSeparator LS(" "); + for (StringRef Arg : split(StringRef(ArgsCstrBuff + buffer_offset, + ArgsCstrBuffLen - buffer_offset), + '\0')) { + OS << LS; + sys::printArg(OS, Arg, /*Quote=*/true); + } + OS.flush(); + Options.MCOptions.Argv0 = Arg0; + Options.MCOptions.CommandlineArgs = CommandlineArgs; +#else + int buffer_offset = 0; + assert(ArgsCstrBuff[ArgsCstrBuffLen - 1] == '\0'); + + const size_t arg0_len = std::strlen(ArgsCstrBuff); + char *arg0 = new char[arg0_len + 1]; + memcpy(arg0, ArgsCstrBuff, arg0_len); + arg0[arg0_len] = '\0'; + buffer_offset += arg0_len + 1; + + const int num_cmd_arg_strings = std::count( + &ArgsCstrBuff[buffer_offset], &ArgsCstrBuff[ArgsCstrBuffLen], '\0'); + + std::string *cmd_arg_strings = new std::string[num_cmd_arg_strings]; + for (int i = 0; i < num_cmd_arg_strings; ++i) { + assert(buffer_offset < ArgsCstrBuffLen); + const int len = std::strlen(ArgsCstrBuff + buffer_offset); + cmd_arg_strings[i] = std::string(&ArgsCstrBuff[buffer_offset], len); + buffer_offset += len + 1; + } + + assert(buffer_offset == ArgsCstrBuffLen); + + Options.MCOptions.Argv0 = arg0; + Options.MCOptions.CommandLineArgs = + llvm::ArrayRef(cmd_arg_strings, num_cmd_arg_strings); +#endif + } + + TargetMachine *TM = TheTarget->createTargetMachine( + Trip.getTriple(), CPU, Feature, Options, RM, CM, OptLevel); + return wrap(TM); +} + +extern "C" void LLVMRustDisposeTargetMachine(LLVMTargetMachineRef TM) { +#if LLVM_VERSION_LT(20, 0) + MCTargetOptions &MCOptions = unwrap(TM)->Options.MCOptions; + delete[] MCOptions.Argv0; + delete[] MCOptions.CommandLineArgs.data(); +#endif + + delete unwrap(TM); +} + +// Unfortunately, the LLVM C API doesn't provide a way to create the +// TargetLibraryInfo pass, so we use this method to do so. +extern "C" void LLVMRustAddLibraryInfo(LLVMPassManagerRef PMR, LLVMModuleRef M, + bool DisableSimplifyLibCalls) { + auto TargetTriple = Triple(unwrap(M)->getTargetTriple()); + auto TLII = TargetLibraryInfoImpl(TargetTriple); + if (DisableSimplifyLibCalls) + TLII.disableAllFunctions(); + unwrap(PMR)->add(new TargetLibraryInfoWrapperPass(TLII)); +} + +extern "C" void LLVMRustSetLLVMOptions(int Argc, char **Argv) { + // Initializing the command-line options more than once is not allowed. So, + // check if they've already been initialized. (This could happen if we're + // being called from rustpkg, for example). If the arguments change, then + // that's just kinda unfortunate. + static bool Initialized = false; + if (Initialized) + return; + Initialized = true; + cl::ParseCommandLineOptions(Argc, Argv); +} + +enum class LLVMRustFileType { + AssemblyFile, + ObjectFile, +}; + +static CodeGenFileType fromRust(LLVMRustFileType Type) { + switch (Type) { + case LLVMRustFileType::AssemblyFile: + return CodeGenFileType::AssemblyFile; + case LLVMRustFileType::ObjectFile: + return CodeGenFileType::ObjectFile; + default: + report_fatal_error("Bad FileType."); + } +} + +extern "C" LLVMRustResult +LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR, + LLVMModuleRef M, const char *Path, const char *DwoPath, + LLVMRustFileType RustFileType, bool VerifyIR) { + llvm::legacy::PassManager *PM = unwrap(PMR); + auto FileType = fromRust(RustFileType); + + std::string ErrorInfo; + std::error_code EC; + auto OS = raw_fd_ostream(Path, EC, sys::fs::OF_None); + if (EC) + ErrorInfo = EC.message(); + if (ErrorInfo != "") { + LLVMRustSetLastError(ErrorInfo.c_str()); + return LLVMRustResult::Failure; + } + + auto BOS = buffer_ostream(OS); + if (DwoPath) { + auto DOS = raw_fd_ostream(DwoPath, EC, sys::fs::OF_None); + EC.clear(); + if (EC) + ErrorInfo = EC.message(); + if (ErrorInfo != "") { + LLVMRustSetLastError(ErrorInfo.c_str()); + return LLVMRustResult::Failure; + } + auto DBOS = buffer_ostream(DOS); + unwrap(Target)->addPassesToEmitFile(*PM, BOS, &DBOS, FileType, !VerifyIR); + PM->run(*unwrap(M)); + } else { + unwrap(Target)->addPassesToEmitFile(*PM, BOS, nullptr, FileType, !VerifyIR); + PM->run(*unwrap(M)); + } + + // Apparently `addPassesToEmitFile` adds a pointer to our on-the-stack output + // stream (OS), so the only real safe place to delete this is here? Don't we + // wish this was written in Rust? + LLVMDisposePassManager(PMR); + return LLVMRustResult::Success; +} + +extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)( + void *, // LlvmSelfProfiler + const char *, // pass name + const char *); // IR name +extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)( + void *); // LlvmSelfProfiler + +std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) { + if (const auto *Cast = any_cast(&WrappedIr)) + return (*Cast)->getName().str(); + if (const auto *Cast = any_cast(&WrappedIr)) + return (*Cast)->getName().str(); + if (const auto *Cast = any_cast(&WrappedIr)) + return (*Cast)->getName().str(); + if (const auto *Cast = any_cast(&WrappedIr)) + return (*Cast)->getName(); + return ""; +} + +void LLVMSelfProfileInitializeCallbacks( + PassInstrumentationCallbacks &PIC, void *LlvmSelfProfiler, + LLVMRustSelfProfileBeforePassCallback BeforePassCallback, + LLVMRustSelfProfileAfterPassCallback AfterPassCallback) { + PIC.registerBeforeNonSkippedPassCallback( + [LlvmSelfProfiler, BeforePassCallback](StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + }); + + PIC.registerAfterPassCallback( + [LlvmSelfProfiler, AfterPassCallback]( + StringRef Pass, llvm::Any IR, const PreservedAnalyses &Preserved) { + AfterPassCallback(LlvmSelfProfiler); + }); + + PIC.registerAfterPassInvalidatedCallback( + [LlvmSelfProfiler, + AfterPassCallback](StringRef Pass, const PreservedAnalyses &Preserved) { + AfterPassCallback(LlvmSelfProfiler); + }); + + PIC.registerBeforeAnalysisCallback( + [LlvmSelfProfiler, BeforePassCallback](StringRef Pass, llvm::Any Ir) { + std::string PassName = Pass.str(); + std::string IrName = LLVMRustwrappedIrGetName(Ir); + BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str()); + }); + + PIC.registerAfterAnalysisCallback( + [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) { + AfterPassCallback(LlvmSelfProfiler); + }); +} + +enum class LLVMRustOptStage { + PreLinkNoLTO, + PreLinkThinLTO, + PreLinkFatLTO, + ThinLTO, + FatLTO, +}; + +struct LLVMRustSanitizerOptions { + bool SanitizeAddress; + bool SanitizeAddressRecover; + bool SanitizeCFI; + bool SanitizeDataFlow; + char **SanitizeDataFlowABIList; + size_t SanitizeDataFlowABIListLen; + bool SanitizeKCFI; + bool SanitizeMemory; + bool SanitizeMemoryRecover; + int SanitizeMemoryTrackOrigins; + bool SanitizeThread; + bool SanitizeHWAddress; + bool SanitizeHWAddressRecover; + bool SanitizeKernelAddress; + bool SanitizeKernelAddressRecover; +}; + +// This symbol won't be available or used when Enzyme is not enabled. +// Always set AugmentPassBuilder to true, since it registers optimizations which +// will improve the performance for Enzyme. +#ifdef ENZYME +extern "C" void registerEnzymeAndPassPipeline(llvm::PassBuilder &PB, + /* augmentPassBuilder */ bool); +#endif + +extern "C" LLVMRustResult LLVMRustOptimize( + LLVMModuleRef ModuleRef, LLVMTargetMachineRef TMRef, + LLVMRustPassBuilderOptLevel OptLevelRust, LLVMRustOptStage OptStage, + bool IsLinkerPluginLTO, bool NoPrepopulatePasses, bool VerifyIR, + bool LintIR, LLVMRustThinLTOBuffer **ThinLTOBufferRef, bool EmitThinLTO, + bool EmitThinLTOSummary, bool MergeFunctions, bool UnrollLoops, + bool SLPVectorize, bool LoopVectorize, bool DisableSimplifyLibCalls, + bool EmitLifetimeMarkers, bool RunEnzyme, + LLVMRustSanitizerOptions *SanitizerOptions, const char *PGOGenPath, + const char *PGOUsePath, bool InstrumentCoverage, + const char *InstrProfileOutput, const char *PGOSampleUsePath, + bool DebugInfoForProfiling, void *LlvmSelfProfiler, + LLVMRustSelfProfileBeforePassCallback BeforePassCallback, + LLVMRustSelfProfileAfterPassCallback AfterPassCallback, + const char *ExtraPasses, size_t ExtraPassesLen, const char *LLVMPlugins, + size_t LLVMPluginsLen) { + Module *TheModule = unwrap(ModuleRef); + TargetMachine *TM = unwrap(TMRef); + OptimizationLevel OptLevel = fromRust(OptLevelRust); + + PipelineTuningOptions PTO; + PTO.LoopUnrolling = UnrollLoops; + PTO.LoopInterleaving = UnrollLoops; + PTO.LoopVectorization = LoopVectorize; + PTO.SLPVectorization = SLPVectorize; + PTO.MergeFunctions = MergeFunctions; + + PassInstrumentationCallbacks PIC; + + if (LlvmSelfProfiler) { + LLVMSelfProfileInitializeCallbacks(PIC, LlvmSelfProfiler, + BeforePassCallback, AfterPassCallback); + } + + std::optional PGOOpt; + auto FS = vfs::getRealFileSystem(); + if (PGOGenPath) { + assert(!PGOUsePath && !PGOSampleUsePath); + PGOOpt = PGOOptions(PGOGenPath, "", "", "", FS, PGOOptions::IRInstr, + PGOOptions::NoCSAction, +#if LLVM_VERSION_GE(19, 0) + PGOOptions::ColdFuncOpt::Default, +#endif + DebugInfoForProfiling); + } else if (PGOUsePath) { + assert(!PGOSampleUsePath); + PGOOpt = PGOOptions(PGOUsePath, "", "", "", FS, PGOOptions::IRUse, + PGOOptions::NoCSAction, +#if LLVM_VERSION_GE(19, 0) + PGOOptions::ColdFuncOpt::Default, +#endif + DebugInfoForProfiling); + } else if (PGOSampleUsePath) { + PGOOpt = PGOOptions(PGOSampleUsePath, "", "", "", FS, PGOOptions::SampleUse, + PGOOptions::NoCSAction, +#if LLVM_VERSION_GE(19, 0) + PGOOptions::ColdFuncOpt::Default, +#endif + DebugInfoForProfiling); + } else if (DebugInfoForProfiling) { + PGOOpt = PGOOptions("", "", "", "", FS, PGOOptions::NoAction, + PGOOptions::NoCSAction, +#if LLVM_VERSION_GE(19, 0) + PGOOptions::ColdFuncOpt::Default, +#endif + DebugInfoForProfiling); + } + + auto PB = PassBuilder(TM, PTO, PGOOpt, &PIC); + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + + StandardInstrumentations SI(TheModule->getContext(), + /*DebugLogging=*/false); + SI.registerCallbacks(PIC, &MAM); + + if (LLVMPluginsLen) { + auto PluginsStr = StringRef(LLVMPlugins, LLVMPluginsLen); + SmallVector Plugins; + PluginsStr.split(Plugins, ',', -1, false); + for (auto PluginPath : Plugins) { + auto Plugin = PassPlugin::Load(PluginPath.str()); + if (!Plugin) { + auto Err = Plugin.takeError(); + auto ErrMsg = llvm::toString(std::move(Err)); + LLVMRustSetLastError(ErrMsg.c_str()); + return LLVMRustResult::Failure; + } + Plugin->registerPassBuilderCallbacks(PB); + } + } + + FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); }); + + Triple TargetTriple(TheModule->getTargetTriple()); + std::unique_ptr TLII( + new TargetLibraryInfoImpl(TargetTriple)); + if (DisableSimplifyLibCalls) + TLII->disableAllFunctions(); + FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); }); + + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + // We manually collect pipeline callbacks so we can apply them at O0, where + // the PassBuilder does not create a pipeline. + std::vector> + PipelineStartEPCallbacks; +#if LLVM_VERSION_GE(20, 0) + std::vector> + OptimizerLastEPCallbacks; +#else + std::vector> + OptimizerLastEPCallbacks; +#endif + + if (!IsLinkerPluginLTO && SanitizerOptions && SanitizerOptions->SanitizeCFI && + !NoPrepopulatePasses) { + PipelineStartEPCallbacks.push_back( + [](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(LowerTypeTestsPass( + /*ExportSummary=*/nullptr, + /*ImportSummary=*/nullptr)); + }); + } + + if (VerifyIR) { + PipelineStartEPCallbacks.push_back( + [VerifyIR](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(VerifierPass()); + }); + } + + if (LintIR) { + PipelineStartEPCallbacks.push_back( + [](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(createModuleToFunctionPassAdaptor(LintPass())); + }); + } + + if (InstrumentCoverage) { + PipelineStartEPCallbacks.push_back( + [InstrProfileOutput](ModulePassManager &MPM, OptimizationLevel Level) { + InstrProfOptions Options; + if (InstrProfileOutput) { + Options.InstrProfileOutput = InstrProfileOutput; + } + // cargo run tests in multhreading mode by default + // so use atomics for coverage counters + Options.Atomic = true; + MPM.addPass(InstrProfilingLoweringPass(Options, false)); + }); + } + + if (SanitizerOptions) { + if (SanitizerOptions->SanitizeDataFlow) { + std::vector ABIListFiles( + SanitizerOptions->SanitizeDataFlowABIList, + SanitizerOptions->SanitizeDataFlowABIList + + SanitizerOptions->SanitizeDataFlowABIListLen); + OptimizerLastEPCallbacks.push_back( +#if LLVM_VERSION_GE(20, 0) + [ABIListFiles](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [ABIListFiles](ModulePassManager &MPM, OptimizationLevel Level) { +#endif + MPM.addPass(DataFlowSanitizerPass(ABIListFiles)); + }); + } + + if (SanitizerOptions->SanitizeMemory) { + MemorySanitizerOptions Options( + SanitizerOptions->SanitizeMemoryTrackOrigins, + SanitizerOptions->SanitizeMemoryRecover, + /*CompileKernel=*/false, + /*EagerChecks=*/true); + OptimizerLastEPCallbacks.push_back( +#if LLVM_VERSION_GE(20, 0) + [Options](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [Options](ModulePassManager &MPM, OptimizationLevel Level) { +#endif + MPM.addPass(MemorySanitizerPass(Options)); + }); + } + + if (SanitizerOptions->SanitizeThread) { + OptimizerLastEPCallbacks.push_back( +#if LLVM_VERSION_GE(20, 0) + [](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [](ModulePassManager &MPM, OptimizationLevel Level) { +#endif + MPM.addPass(ModuleThreadSanitizerPass()); + MPM.addPass( + createModuleToFunctionPassAdaptor(ThreadSanitizerPass())); + }); + } + + if (SanitizerOptions->SanitizeAddress || + SanitizerOptions->SanitizeKernelAddress) { + OptimizerLastEPCallbacks.push_back( +#if LLVM_VERSION_GE(20, 0) + [SanitizerOptions, TM](ModulePassManager &MPM, + OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [SanitizerOptions, TM](ModulePassManager &MPM, + OptimizationLevel Level) { +#endif + auto CompileKernel = SanitizerOptions->SanitizeKernelAddress; + AddressSanitizerOptions opts = AddressSanitizerOptions{ + CompileKernel, + SanitizerOptions->SanitizeAddressRecover || + SanitizerOptions->SanitizeKernelAddressRecover, + /*UseAfterScope=*/true, + AsanDetectStackUseAfterReturnMode::Runtime, + }; + MPM.addPass(AddressSanitizerPass( + opts, + /*UseGlobalGC*/ true, + // UseOdrIndicator should be false on windows machines + // https://reviews.llvm.org/D137227 + !TM->getTargetTriple().isOSWindows())); + }); + } + if (SanitizerOptions->SanitizeHWAddress) { + OptimizerLastEPCallbacks.push_back( +#if LLVM_VERSION_GE(20, 0) + [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level, + ThinOrFullLTOPhase phase) { +#else + [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) { +#endif + HWAddressSanitizerOptions opts( + /*CompileKernel=*/false, + SanitizerOptions->SanitizeHWAddressRecover, + /*DisableOptimization=*/false); + MPM.addPass(HWAddressSanitizerPass(opts)); + }); + } + } + + ModulePassManager MPM; + bool NeedThinLTOBufferPasses = EmitThinLTO; + auto ThinLTOBuffer = std::make_unique(); + raw_string_ostream ThinLTODataOS(ThinLTOBuffer->data); + raw_string_ostream ThinLinkDataOS(ThinLTOBuffer->thin_link_data); + if (!NoPrepopulatePasses) { + // The pre-link pipelines don't support O0 and require using + // buildO0DefaultPipeline() instead. At the same time, the LTO pipelines do + // support O0 and using them is required. + bool IsLTO = OptStage == LLVMRustOptStage::ThinLTO || + OptStage == LLVMRustOptStage::FatLTO; + if (OptLevel == OptimizationLevel::O0 && !IsLTO) { + for (const auto &C : PipelineStartEPCallbacks) + PB.registerPipelineStartEPCallback(C); + for (const auto &C : OptimizerLastEPCallbacks) + PB.registerOptimizerLastEPCallback(C); + + // We manually schedule ThinLTOBufferPasses below, so don't pass the value + // to enable it here. + MPM = PB.buildO0DefaultPipeline(OptLevel); + } else { + for (const auto &C : PipelineStartEPCallbacks) + PB.registerPipelineStartEPCallback(C); + for (const auto &C : OptimizerLastEPCallbacks) + PB.registerOptimizerLastEPCallback(C); + + switch (OptStage) { + case LLVMRustOptStage::PreLinkNoLTO: + if (ThinLTOBufferRef) { + // This is similar to LLVM's `buildFatLTODefaultPipeline`, where the + // bitcode for embedding is obtained after performing + // `ThinLTOPreLinkDefaultPipeline`. + MPM.addPass(PB.buildThinLTOPreLinkDefaultPipeline(OptLevel)); + if (EmitThinLTO) { + MPM.addPass(ThinLTOBitcodeWriterPass( + ThinLTODataOS, EmitThinLTOSummary ? &ThinLinkDataOS : nullptr)); + } else { + MPM.addPass(BitcodeWriterPass(ThinLTODataOS)); + } + *ThinLTOBufferRef = ThinLTOBuffer.release(); + MPM.addPass(PB.buildModuleOptimizationPipeline( + OptLevel, ThinOrFullLTOPhase::None)); + MPM.addPass( + createModuleToFunctionPassAdaptor(AnnotationRemarksPass())); + } else { + MPM = PB.buildPerModuleDefaultPipeline(OptLevel); + } + break; + case LLVMRustOptStage::PreLinkThinLTO: + MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel); + NeedThinLTOBufferPasses = false; + break; + case LLVMRustOptStage::PreLinkFatLTO: + MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel); + NeedThinLTOBufferPasses = false; + break; + case LLVMRustOptStage::ThinLTO: + // FIXME: Does it make sense to pass the ModuleSummaryIndex? + // It only seems to be needed for C++ specific optimizations. + MPM = PB.buildThinLTODefaultPipeline(OptLevel, nullptr); + break; + case LLVMRustOptStage::FatLTO: + MPM = PB.buildLTODefaultPipeline(OptLevel, nullptr); + break; + } + } + } else { + // We're not building any of the default pipelines but we still want to + // add the verifier, instrumentation, etc passes if they were requested + for (const auto &C : PipelineStartEPCallbacks) + C(MPM, OptLevel); + for (const auto &C : OptimizerLastEPCallbacks) +#if LLVM_VERSION_GE(20, 0) + C(MPM, OptLevel, ThinOrFullLTOPhase::None); +#else + C(MPM, OptLevel); +#endif + } + + if (ExtraPassesLen) { + if (auto Err = + PB.parsePassPipeline(MPM, StringRef(ExtraPasses, ExtraPassesLen))) { + std::string ErrMsg = toString(std::move(Err)); + LLVMRustSetLastError(ErrMsg.c_str()); + return LLVMRustResult::Failure; + } + } + + if (NeedThinLTOBufferPasses) { + MPM.addPass(CanonicalizeAliasesPass()); + MPM.addPass(NameAnonGlobalPass()); + } + // For `-Copt-level=0`, ThinLTO, or LTO. + if (ThinLTOBufferRef && *ThinLTOBufferRef == nullptr) { + if (EmitThinLTO) { + MPM.addPass(ThinLTOBitcodeWriterPass( + ThinLTODataOS, EmitThinLTOSummary ? &ThinLinkDataOS : nullptr)); + } else { + MPM.addPass(BitcodeWriterPass(ThinLTODataOS)); + } + *ThinLTOBufferRef = ThinLTOBuffer.release(); + } + + // now load "-enzyme" pass: +#ifdef ENZYME + if (RunEnzyme) { + registerEnzymeAndPassPipeline(PB, true); + if (auto Err = PB.parsePassPipeline(MPM, "enzyme")) { + std::string ErrMsg = toString(std::move(Err)); + LLVMRustSetLastError(ErrMsg.c_str()); + return LLVMRustResult::Failure; + } + } +#endif + + // Upgrade all calls to old intrinsics first. + for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;) + UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove + + MPM.run(*TheModule, MAM); + return LLVMRustResult::Success; +} + +// Callback to demangle function name +// Parameters: +// * name to be demangled +// * name len +// * output buffer +// * output buffer len +// Returns len of demangled string, or 0 if demangle failed. +typedef size_t (*DemangleFn)(const char *, size_t, char *, size_t); + +namespace { + +class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter { + DemangleFn Demangle; + std::vector Buf; + +public: + RustAssemblyAnnotationWriter(DemangleFn Demangle) : Demangle(Demangle) {} + + // Return empty string if demangle failed + // or if name does not need to be demangled + StringRef CallDemangle(StringRef name) { + if (!Demangle) { + return StringRef(); + } + + if (Buf.size() < name.size() * 2) { + // Semangled name usually shorter than mangled, + // but allocate twice as much memory just in case + Buf.resize(name.size() * 2); + } + + auto R = Demangle(name.data(), name.size(), Buf.data(), Buf.size()); + if (!R) { + // Demangle failed. + return StringRef(); + } + + auto Demangled = StringRef(Buf.data(), R); + if (Demangled == name) { + // Do not print anything if demangled name is equal to mangled. + return StringRef(); + } + + return Demangled; + } + + void emitFunctionAnnot(const Function *F, + formatted_raw_ostream &OS) override { + StringRef Demangled = CallDemangle(F->getName()); + if (Demangled.empty()) { + return; + } + + OS << "; " << Demangled << "\n"; + } + + void emitInstructionAnnot(const Instruction *I, + formatted_raw_ostream &OS) override { + const char *Name; + const Value *Value; + if (const CallInst *CI = dyn_cast(I)) { + Name = "call"; + Value = CI->getCalledOperand(); + } else if (const InvokeInst *II = dyn_cast(I)) { + Name = "invoke"; + Value = II->getCalledOperand(); + } else { + // Could demangle more operations, e. g. + // `store %place, @function`. + return; + } + + if (!Value->hasName()) { + return; + } + + StringRef Demangled = CallDemangle(Value->getName()); + if (Demangled.empty()) { + return; + } + + OS << "; " << Name << " " << Demangled << "\n"; + } +}; + +} // namespace + +extern "C" LLVMRustResult LLVMRustPrintModule(LLVMModuleRef M, const char *Path, + DemangleFn Demangle) { + std::string ErrorInfo; + std::error_code EC; + auto OS = raw_fd_ostream(Path, EC, sys::fs::OF_None); + if (EC) + ErrorInfo = EC.message(); + if (ErrorInfo != "") { + LLVMRustSetLastError(ErrorInfo.c_str()); + return LLVMRustResult::Failure; + } + + // Verify the module before printing + /*Module *Mod = unwrap(M); + std::string VerifyError; + raw_string_ostream VerifyOS(VerifyError); + if (verifyModule(*Mod, &VerifyOS)) { + // Module verification failed, print error and skip printing + ErrorInfo = "Module verification failed: " + VerifyOS.str(); + LLVMRustSetLastError(ErrorInfo.c_str()); + return LLVMRustResult::Failure; + }*/ + + auto AAW = RustAssemblyAnnotationWriter(Demangle); + auto FOS = formatted_raw_ostream(OS); + unwrap(M)->print(FOS, &AAW); + + return LLVMRustResult::Success; +} + +extern "C" void LLVMRustPrintPasses() { + PassBuilder PB; + PB.printPassNames(outs()); +} + +extern "C" void LLVMRustRunRestrictionPass(LLVMModuleRef M, char **Symbols, + size_t Len) { + auto PreserveFunctions = [=](const GlobalValue &GV) { + // Preserve LLVM-injected, ASAN-related symbols. + // See also https://github.com/rust-lang/rust/issues/113404. + if (GV.getName() == "___asan_globals_registered") { + return true; + } + + // Preserve symbols exported from Rust modules. + for (size_t I = 0; I < Len; I++) { + if (GV.getName() == Symbols[I]) { + return true; + } + } + return false; + }; + + internalizeModule(*unwrap(M), PreserveFunctions); +} + +extern "C" void +LLVMRustSetDataLayoutFromTargetMachine(LLVMModuleRef Module, + LLVMTargetMachineRef TMR) { + TargetMachine *Target = unwrap(TMR); + unwrap(Module)->setDataLayout(Target->createDataLayout()); +} + +extern "C" void LLVMRustSetModulePICLevel(LLVMModuleRef M) { + unwrap(M)->setPICLevel(PICLevel::Level::BigPIC); +} + +extern "C" void LLVMRustSetModulePIELevel(LLVMModuleRef M) { + unwrap(M)->setPIELevel(PIELevel::Level::Large); +} + +extern "C" void LLVMRustSetModuleCodeModel(LLVMModuleRef M, + LLVMRustCodeModel Model) { + auto CM = fromRust(Model); + if (!CM) + return; + unwrap(M)->setCodeModel(*CM); +} + +// Here you'll find an implementation of ThinLTO as used by the Rust compiler +// right now. This ThinLTO support is only enabled on "recent ish" versions of +// LLVM, and otherwise it's just blanket rejected from other compilers. +// +// Most of this implementation is straight copied from LLVM. At the time of +// this writing it wasn't *quite* suitable to reuse more code from upstream +// for our purposes, but we should strive to upstream this support once it's +// ready to go! I figure we may want a bit of testing locally first before +// sending this upstream to LLVM. I hear though they're quite eager to receive +// feedback like this! +// +// If you're reading this code and wondering "what in the world" or you're +// working "good lord by LLVM upgrade is *still* failing due to these bindings" +// then fear not! (ok maybe fear a little). All code here is mostly based +// on `lib/LTO/ThinLTOCodeGenerator.cpp` in LLVM. +// +// You'll find that the general layout here roughly corresponds to the `run` +// method in that file as well as `ProcessThinLTOModule`. Functions are +// specifically commented below as well, but if you're updating this code +// or otherwise trying to understand it, the LLVM source will be useful in +// interpreting the mysteries within. +// +// Otherwise I'll apologize in advance, it probably requires a relatively +// significant investment on your part to "truly understand" what's going on +// here. Not saying I do myself, but it took me awhile staring at LLVM's source +// and various online resources about ThinLTO to make heads or tails of all +// this. + +// This is a shared data structure which *must* be threadsafe to share +// read-only amongst threads. This also corresponds basically to the arguments +// of the `ProcessThinLTOModule` function in the LLVM source. +struct LLVMRustThinLTOData { + // The combined index that is the global analysis over all modules we're + // performing ThinLTO for. This is mostly managed by LLVM. + ModuleSummaryIndex Index; + + // All modules we may look at, stored as in-memory serialized versions. This + // is later used when inlining to ensure we can extract any module to inline + // from. + StringMap ModuleMap; + + // A set that we manage of everything we *don't* want internalized. Note that + // this includes all transitive references right now as well, but it may not + // always! + DenseSet GUIDPreservedSymbols; + + // Not 100% sure what these are, but they impact what's internalized and + // what's inlined across modules, I believe. +#if LLVM_VERSION_GE(20, 0) + FunctionImporter::ImportListsTy ImportLists; +#else + DenseMap ImportLists; +#endif + DenseMap ExportLists; + DenseMap ModuleToDefinedGVSummaries; + StringMap> ResolvedODR; + + LLVMRustThinLTOData() : Index(/* HaveGVs = */ false) {} +}; + +// Just an argument to the `LLVMRustCreateThinLTOData` function below. +struct LLVMRustThinLTOModule { + const char *identifier; + const char *data; + size_t len; +}; + +// This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp`, not sure what it +// does. +static const GlobalValueSummary * +getFirstDefinitionForLinker(const GlobalValueSummaryList &GVSummaryList) { + auto StrongDefForLinker = llvm::find_if( + GVSummaryList, [](const std::unique_ptr &Summary) { + auto Linkage = Summary->linkage(); + return !GlobalValue::isAvailableExternallyLinkage(Linkage) && + !GlobalValue::isWeakForLinker(Linkage); + }); + if (StrongDefForLinker != GVSummaryList.end()) + return StrongDefForLinker->get(); + + auto FirstDefForLinker = llvm::find_if( + GVSummaryList, [](const std::unique_ptr &Summary) { + auto Linkage = Summary->linkage(); + return !GlobalValue::isAvailableExternallyLinkage(Linkage); + }); + if (FirstDefForLinker == GVSummaryList.end()) + return nullptr; + return FirstDefForLinker->get(); +} + +// The main entry point for creating the global ThinLTO analysis. The structure +// here is basically the same as before threads are spawned in the `run` +// function of `lib/LTO/ThinLTOCodeGenerator.cpp`. +extern "C" LLVMRustThinLTOData * +LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules, size_t num_modules, + const char **preserved_symbols, size_t num_symbols) { + auto Ret = std::make_unique(); + + // Load each module's summary and merge it into one combined index + for (size_t i = 0; i < num_modules; i++) { + auto module = &modules[i]; + auto buffer = StringRef(module->data, module->len); + auto mem_buffer = MemoryBufferRef(buffer, module->identifier); + + Ret->ModuleMap[module->identifier] = mem_buffer; + + if (Error Err = readModuleSummaryIndex(mem_buffer, Ret->Index)) { + LLVMRustSetLastError(toString(std::move(Err)).c_str()); + return nullptr; + } + } + + // Collect for each module the list of function it defines (GUID -> Summary) + Ret->Index.collectDefinedGVSummariesPerModule( + Ret->ModuleToDefinedGVSummaries); + + // Convert the preserved symbols set from string to GUID, this is then needed + // for internalization. + for (size_t i = 0; i < num_symbols; i++) { + auto GUID = GlobalValue::getGUID(preserved_symbols[i]); + Ret->GUIDPreservedSymbols.insert(GUID); + } + + // Collect the import/export lists for all modules from the call-graph in the + // combined index + // + // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` + auto deadIsPrevailing = [&](GlobalValue::GUID G) { + return PrevailingType::Unknown; + }; + // We don't have a complete picture in our use of ThinLTO, just our immediate + // crate, so we need `ImportEnabled = false` to limit internalization. + // Otherwise, we sometimes lose `static` values -- see #60184. + computeDeadSymbolsWithConstProp(Ret->Index, Ret->GUIDPreservedSymbols, + deadIsPrevailing, + /* ImportEnabled = */ false); + // Resolve LinkOnce/Weak symbols, this has to be computed early be cause it + // impacts the caching. + // + // This is copied from `lib/LTO/ThinLTOCodeGenerator.cpp` with some of this + // being lifted from `lib/LTO/LTO.cpp` as well + DenseMap PrevailingCopy; + for (auto &I : Ret->Index) { + if (I.second.SummaryList.size() > 1) + PrevailingCopy[I.first] = + getFirstDefinitionForLinker(I.second.SummaryList); + } + auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { + const auto &Prevailing = PrevailingCopy.find(GUID); + if (Prevailing == PrevailingCopy.end()) + return true; + return Prevailing->second == S; + }; + ComputeCrossModuleImport(Ret->Index, Ret->ModuleToDefinedGVSummaries, + isPrevailing, Ret->ImportLists, Ret->ExportLists); + + auto recordNewLinkage = [&](StringRef ModuleIdentifier, + GlobalValue::GUID GUID, + GlobalValue::LinkageTypes NewLinkage) { + Ret->ResolvedODR[ModuleIdentifier][GUID] = NewLinkage; + }; + + // Uses FromPrevailing visibility scheme which works for many binary + // formats. We probably could and should use ELF visibility scheme for many of + // our targets, however. + lto::Config conf; + thinLTOResolvePrevailingInIndex(conf, Ret->Index, isPrevailing, + recordNewLinkage, Ret->GUIDPreservedSymbols); + + // Here we calculate an `ExportedGUIDs` set for use in the `isExported` + // callback below. This callback below will dictate the linkage for all + // summaries in the index, and we basically just only want to ensure that dead + // symbols are internalized. Otherwise everything that's already external + // linkage will stay as external, and internal will stay as internal. + std::set ExportedGUIDs; + for (auto &List : Ret->Index) { + for (auto &GVS : List.second.SummaryList) { + if (GlobalValue::isLocalLinkage(GVS->linkage())) + continue; + auto GUID = GVS->getOriginalName(); + if (GVS->flags().Live) + ExportedGUIDs.insert(GUID); + } + } + auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) { + const auto &ExportList = Ret->ExportLists.find(ModuleIdentifier); + return (ExportList != Ret->ExportLists.end() && + ExportList->second.count(VI)) || + ExportedGUIDs.count(VI.getGUID()); + }; + thinLTOInternalizeAndPromoteInIndex(Ret->Index, isExported, isPrevailing); + + return Ret.release(); +} + +extern "C" void LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) { + delete Data; +} + +// Below are the various passes that happen *per module* when doing ThinLTO. +// +// In other words, these are the functions that are all run concurrently +// with one another, one per module. The passes here correspond to the analysis +// passes in `lib/LTO/ThinLTOCodeGenerator.cpp`, currently found in the +// `ProcessThinLTOModule` function. Here they're split up into separate steps +// so rustc can save off the intermediate bytecode between each step. + +static bool clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) { + // When linking an ELF shared object, dso_local should be dropped. We + // conservatively do this for -fpic. + bool ClearDSOLocalOnDeclarations = TM.getTargetTriple().isOSBinFormatELF() && + TM.getRelocationModel() != Reloc::Static && + Mod.getPIELevel() == PIELevel::Default; + return ClearDSOLocalOnDeclarations; +} + +extern "C" void LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, + LLVMModuleRef M, + LLVMTargetMachineRef TM) { + Module &Mod = *unwrap(M); + TargetMachine &Target = *unwrap(TM); + + bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); + renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal); +} + +extern "C" bool +LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, + LLVMModuleRef M) { + Module &Mod = *unwrap(M); + const auto &DefinedGlobals = + Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); + thinLTOFinalizeInModule(Mod, DefinedGlobals, /*PropagateAttrs=*/true); + return true; +} + +extern "C" bool +LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, + LLVMModuleRef M) { + Module &Mod = *unwrap(M); + const auto &DefinedGlobals = + Data->ModuleToDefinedGVSummaries.lookup(Mod.getModuleIdentifier()); + thinLTOInternalizeModule(Mod, DefinedGlobals); + return true; +} + +extern "C" bool LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, + LLVMModuleRef M, + LLVMTargetMachineRef TM) { + Module &Mod = *unwrap(M); + TargetMachine &Target = *unwrap(TM); + + const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier()); + auto Loader = [&](StringRef Identifier) { + const auto &Memory = Data->ModuleMap.lookup(Identifier); + auto &Context = Mod.getContext(); + auto MOrErr = getLazyBitcodeModule(Memory, Context, true, true); + + if (!MOrErr) + return MOrErr; + + // The rest of this closure is a workaround for + // https://bugs.llvm.org/show_bug.cgi?id=38184 where during ThinLTO imports + // we accidentally import wasm custom sections into different modules, + // duplicating them by in the final output artifact. + // + // The issue is worked around here by manually removing the + // `wasm.custom_sections` named metadata node from any imported module. This + // we know isn't used by any optimization pass so there's no need for it to + // be imported. + // + // Note that the metadata is currently lazily loaded, so we materialize it + // here before looking up if there's metadata inside. The `FunctionImporter` + // will immediately materialize metadata anyway after an import, so this + // shouldn't be a perf hit. + if (Error Err = (*MOrErr)->materializeMetadata()) { + Expected> Ret(std::move(Err)); + return Ret; + } + + auto *WasmCustomSections = + (*MOrErr)->getNamedMetadata("wasm.custom_sections"); + if (WasmCustomSections) + WasmCustomSections->eraseFromParent(); + + // `llvm.ident` named metadata also gets duplicated. + auto *llvmIdent = (*MOrErr)->getNamedMetadata("llvm.ident"); + if (llvmIdent) + llvmIdent->eraseFromParent(); + + return MOrErr; + }; + bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target); + auto Importer = FunctionImporter(Data->Index, Loader, ClearDSOLocal); + Expected Result = Importer.importFunctions(Mod, ImportList); + if (!Result) { + LLVMRustSetLastError(toString(Result.takeError()).c_str()); + return false; + } + return true; +} + +extern "C" LLVMRustThinLTOBuffer * +LLVMRustThinLTOBufferCreate(LLVMModuleRef M, bool is_thin, bool emit_summary) { + auto Ret = std::make_unique(); + { + auto OS = raw_string_ostream(Ret->data); + auto ThinLinkOS = raw_string_ostream(Ret->thin_link_data); + { + if (is_thin) { + PassBuilder PB; + LoopAnalysisManager LAM; + FunctionAnalysisManager FAM; + CGSCCAnalysisManager CGAM; + ModuleAnalysisManager MAM; + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + ModulePassManager MPM; + // We only pass ThinLinkOS to be filled in if we want the summary, + // because otherwise LLVM does extra work and may double-emit some + // errors or warnings. + MPM.addPass( + ThinLTOBitcodeWriterPass(OS, emit_summary ? &ThinLinkOS : nullptr)); + MPM.run(*unwrap(M), MAM); // TODO: run this + } else { + WriteBitcodeToFile(*unwrap(M), OS); + } + } + } + return Ret.release(); +} + +extern "C" void LLVMRustThinLTOBufferFree(LLVMRustThinLTOBuffer *Buffer) { + delete Buffer; +} + +extern "C" const void * +LLVMRustThinLTOBufferPtr(const LLVMRustThinLTOBuffer *Buffer) { + return Buffer->data.data(); +} + +extern "C" size_t +LLVMRustThinLTOBufferLen(const LLVMRustThinLTOBuffer *Buffer) { + return Buffer->data.length(); +} + +extern "C" const void * +LLVMRustThinLTOBufferThinLinkDataPtr(const LLVMRustThinLTOBuffer *Buffer) { + return Buffer->thin_link_data.data(); +} + +extern "C" size_t +LLVMRustThinLTOBufferThinLinkDataLen(const LLVMRustThinLTOBuffer *Buffer) { + return Buffer->thin_link_data.length(); +} + +// This is what we used to parse upstream bitcode for actual ThinLTO +// processing. We'll call this once per module optimized through ThinLTO, and +// it'll be called concurrently on many threads. +extern "C" LLVMModuleRef LLVMRustParseBitcodeForLTO(LLVMContextRef Context, + const char *data, + size_t len, + const char *identifier) { + auto Data = StringRef(data, len); + auto Buffer = MemoryBufferRef(Data, identifier); + unwrap(Context)->enableDebugTypeODRUniquing(); + Expected> SrcOrError = + parseBitcodeFile(Buffer, *unwrap(Context)); + if (!SrcOrError) { + LLVMRustSetLastError(toString(SrcOrError.takeError()).c_str()); + return nullptr; + } + return wrap(std::move(*SrcOrError).release()); +} + +// Find a section of an object file by name. Fail if the section is missing or +// empty. +extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data, + size_t len, + const char *name, + size_t name_len, + size_t *out_len) { + *out_len = 0; + auto Name = StringRef(name, name_len); + auto Data = StringRef(data, len); + auto Buffer = MemoryBufferRef(Data, ""); // The id is unused. + file_magic Type = identify_magic(Buffer.getBuffer()); + Expected> ObjFileOrError = + object::ObjectFile::createObjectFile(Buffer, Type); + if (!ObjFileOrError) { + LLVMRustSetLastError(toString(ObjFileOrError.takeError()).c_str()); + return nullptr; + } + for (const object::SectionRef &Sec : (*ObjFileOrError)->sections()) { + Expected SecName = Sec.getName(); + if (SecName && *SecName == Name) { + Expected SectionOrError = Sec.getContents(); + if (!SectionOrError) { + LLVMRustSetLastError(toString(SectionOrError.takeError()).c_str()); + return nullptr; + } + *out_len = SectionOrError->size(); + return SectionOrError->data(); + } + } + LLVMRustSetLastError("could not find requested section"); + return nullptr; +} + +// Computes the LTO cache key for the provided 'ModId' in the given 'Data', +// storing the result in 'KeyOut'. +// Currently, this cache key is a SHA-1 hash of anything that could affect +// the result of optimizing this module (e.g. module imports, exports, liveness +// of access globals, etc). +// The precise details are determined by LLVM in `computeLTOCacheKey`, which is +// used during the normal linker-plugin incremental thin-LTO process. +extern "C" void LLVMRustComputeLTOCacheKey(RustStringRef KeyOut, + const char *ModId, + LLVMRustThinLTOData *Data) { + SmallString<40> Key; + llvm::lto::Config conf; + const auto &ImportList = Data->ImportLists.lookup(ModId); + const auto &ExportList = Data->ExportLists.lookup(ModId); + const auto &ResolvedODR = Data->ResolvedODR.lookup(ModId); + const auto &DefinedGlobals = Data->ModuleToDefinedGVSummaries.lookup(ModId); +#if LLVM_VERSION_GE(20, 0) + DenseSet CfiFunctionDefs; + DenseSet CfiFunctionDecls; +#else + std::set CfiFunctionDefs; + std::set CfiFunctionDecls; +#endif + + // Based on the 'InProcessThinBackend' constructor in LLVM + for (auto &Name : Data->Index.cfiFunctionDefs()) + CfiFunctionDefs.insert( + GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); + for (auto &Name : Data->Index.cfiFunctionDecls()) + CfiFunctionDecls.insert( + GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name))); + +#if LLVM_VERSION_GE(20, 0) + Key = llvm::computeLTOCacheKey(conf, Data->Index, ModId, ImportList, + ExportList, ResolvedODR, DefinedGlobals, + CfiFunctionDefs, CfiFunctionDecls); +#else + llvm::computeLTOCacheKey(Key, conf, Data->Index, ModId, ImportList, + ExportList, ResolvedODR, DefinedGlobals, + CfiFunctionDefs, CfiFunctionDecls); +#endif + + auto OS = RawRustStringOstream(KeyOut); + OS << Key.str(); +} + +// TODO: non-standard +extern "C" void +LLVMRustThinLTOGetDICompileUnit(LLVMModuleRef Mod, + DICompileUnit **A, + DICompileUnit **B) +{ + Module *M = unwrap(Mod); + DICompileUnit **Cur = A; + DICompileUnit **Next = B; + for (DICompileUnit *CU : M->debug_compile_units()) + { + *Cur = CU; + Cur = Next; + Next = nullptr; + if (Cur == nullptr) + break; + } +} + +// TODO: non-standard +extern "C" void +LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) +{ + Module *M = unwrap(Mod); + // If the original source module didn't have a `DICompileUnit` then try to + // merge all the existing compile units. If there aren't actually any though + // then there's not much for us to do so return. + if (Unit == nullptr) + { + for (DICompileUnit *CU : M->debug_compile_units()) + { + Unit = CU; + break; + } + if (Unit == nullptr) + return; + } + // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and + // process it recursively. Note that we specifically iterate over instructions + // to ensure we feed everything into it. + DebugInfoFinder Finder; + Finder.processModule(*M); + for (Function &F : M->functions()) + { + for (auto &FI : F) + { + for (Instruction &BI : FI) + { + if (auto Loc = BI.getDebugLoc()) + Finder.processLocation(*M, Loc); + // Updated for LLVM v19: DbgVariableIntrinsic is the base class + if (auto DVI = dyn_cast(&BI)) + if (auto Var = DVI->getVariable()) + Finder.processVariable(*M, Var); + } + } + } + // After we've found all our debuginfo, rewrite all subprograms to point to + // the same `DICompileUnit`. + for (auto *F : Finder.subprograms()) + { + F->replaceUnit(Unit); + } + // Erase any other references to other `DICompileUnit` instances, the verifier + // will later ensure that we don't actually have any other stale references to + // worry about. + auto *MD = M->getNamedMetadata("llvm.dbg.cu"); + if (MD) { + MD->clearOperands(); + MD->addOperand(Unit); + } +} diff --git a/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/README.md b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/README.md new file mode 100644 index 00000000..c639063f --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/README.md @@ -0,0 +1,5 @@ +# rustc_llvm_wrapper + +This is basically 99.99% from https://github.com/rust-lang/rust/tree/8c392966a013fd8a09e6b78b3c8d6e442bc278e1/compiler/rustc_llvm/llvm-wrapper with some added legacy functions convert LLVM v7 -> LLVM v19. + +This to workaround the fact that `rustc_codegen_nvvm` was written on LLVM v7 but we want to look towards the future (Blackwell+) which is based on LLVM v19. diff --git a/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/RustWrapper.cpp b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/RustWrapper.cpp new file mode 100644 index 00000000..29cb8fbf --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/RustWrapper.cpp @@ -0,0 +1,2616 @@ +#include "LLVMWrapper.h" + +#include "llvm-c/Analysis.h" +#include "llvm-c/Core.h" +#include "llvm-c/DebugInfo.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/IR/DIBuilder.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DiagnosticHandler.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/DiagnosticPrinter.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InlineAsm.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicsARM.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/LLVMRemarkStreamer.h" +#include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Value.h" +#include "llvm/Object/COFFImportFile.h" +#include "llvm/Remarks/RemarkFormat.h" +#include "llvm/Remarks/RemarkSerializer.h" +#include "llvm/Remarks/RemarkStreamer.h" +#include "llvm/Support/Compression.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/ModRef.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/ToolOutputFile.h" +#include + +// for raw `write` in the bad-alloc handler +#ifdef _MSC_VER +#include +#else +#include +#endif + +// version check +#if LLVM_VERSION_MAJOR != 19 +#error "This code requires LLVM major version 19" +#endif + +//===----------------------------------------------------------------------=== +// +// This file defines alternate interfaces to core functions that are more +// readily callable by Rust's FFI. +// +//===----------------------------------------------------------------------=== + +using namespace llvm; +using namespace llvm::sys; +using namespace llvm::object; + +// This opcode is an LLVM detail that could hypothetically change (?), so +// verify that the hard-coded value in `dwarf_const.rs` still agrees with LLVM. +static_assert(dwarf::DW_OP_LLVM_fragment == 0x1000); + +// LLVMAtomicOrdering is already an enum - don't create another +// one. +static AtomicOrdering fromRust(LLVMAtomicOrdering Ordering) { + switch (Ordering) { + case LLVMAtomicOrderingNotAtomic: + return AtomicOrdering::NotAtomic; + case LLVMAtomicOrderingUnordered: + return AtomicOrdering::Unordered; + case LLVMAtomicOrderingMonotonic: + return AtomicOrdering::Monotonic; + case LLVMAtomicOrderingAcquire: + return AtomicOrdering::Acquire; + case LLVMAtomicOrderingRelease: + return AtomicOrdering::Release; + case LLVMAtomicOrderingAcquireRelease: + return AtomicOrdering::AcquireRelease; + case LLVMAtomicOrderingSequentiallyConsistent: + return AtomicOrdering::SequentiallyConsistent; + } + + report_fatal_error("Invalid LLVMAtomicOrdering value!"); +} + +static LLVM_THREAD_LOCAL char *LastError; + +// Custom error handler for fatal LLVM errors. +// +// Notably it exits the process with code 101, unlike LLVM's default of 1. +static void FatalErrorHandler(void *UserData, const char *Reason, + bool GenCrashDiag) { + // Once upon a time we emitted "LLVM ERROR:" specifically to mimic LLVM. Then, + // we developed crater and other tools which only expose logs, not error + // codes. Use a more greppable prefix that will still match the "LLVM ERROR:" + // prefix. + std::cerr << "rustc-LLVM ERROR: " << Reason << std::endl; + + // Since this error handler exits the process, we have to run any cleanup that + // LLVM would run after handling the error. This might change with an LLVM + // upgrade. + // + // In practice, this will do nothing, because the only cleanup LLVM does is + // to remove all files that were registered with it via a frontend calling + // one of the `createOutputFile` family of functions in LLVM and passing true + // to RemoveFileOnSignal, something that rustc does not do. However, it would + // be... inadvisable to suddenly stop running these handlers, if LLVM gets + // "interesting" ideas in the future about what cleanup should be done. + // We might even find it useful for generating less artifacts. + sys::RunInterruptHandlers(); + + exit(101); +} + +// Custom error handler for bad-alloc LLVM errors. +// +// It aborts the process without any further allocations, similar to LLVM's +// default except that may be configured to `throw std::bad_alloc()` instead. +static void BadAllocErrorHandler(void *UserData, const char *Reason, + bool GenCrashDiag) { + const char *OOM = "rustc-LLVM ERROR: out of memory\n"; + (void)!::write(2, OOM, strlen(OOM)); + (void)!::write(2, Reason, strlen(Reason)); + (void)!::write(2, "\n", 1); + abort(); +} + +extern "C" void LLVMRustInstallErrorHandlers() { + install_bad_alloc_error_handler(BadAllocErrorHandler); + install_fatal_error_handler(FatalErrorHandler); + install_out_of_memory_new_handler(); +} + +extern "C" void LLVMRustDisableSystemDialogsOnCrash() { + sys::DisableSystemDialogsOnCrash(); +} + +extern "C" char *LLVMRustGetLastError(void) { + char *Ret = LastError; + LastError = nullptr; + return Ret; +} + +extern "C" void LLVMRustSetLastError(const char *Err) { + free((void *)LastError); + LastError = strdup(Err); +} + +extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames) { + auto ctx = new LLVMContext(); + ctx->setDiscardValueNames(shouldDiscardNames); + return wrap(ctx); +} + +extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, + const char *Triple) { + unwrap(M)->setTargetTriple(Triple::normalize(Triple)); +} + +extern "C" void LLVMRustPrintPassTimings(RustStringRef OutBuf) { + auto OS = RawRustStringOstream(OutBuf); + TimerGroup::printAll(OS); +} + +extern "C" void LLVMRustPrintStatistics(RustStringRef OutBuf) { + auto OS = RawRustStringOstream(OutBuf); + llvm::PrintStatistics(OS); +} + +// TODO: non-standard +extern "C" void LLVMRustSetOldDebugFormat(LLVMModuleRef M) { + unwrap(M)->setIsNewDbgInfoFormat(false); +} + +extern "C" LLVMValueRef LLVMRustGetNamedValue(LLVMModuleRef M, const char *Name, + size_t NameLen) { + return wrap(unwrap(M)->getNamedValue(StringRef(Name, NameLen))); +} + +enum class LLVMRustVerifierFailureAction { + AbortProcessAction = 0, + PrintMessageAction = 1, + ReturnStatusAction = 2, +}; + +static LLVMVerifierFailureAction +fromRust(LLVMRustVerifierFailureAction Action) { + switch (Action) { + case LLVMRustVerifierFailureAction::AbortProcessAction: + return LLVMAbortProcessAction; + case LLVMRustVerifierFailureAction::PrintMessageAction: + return LLVMPrintMessageAction; + case LLVMRustVerifierFailureAction::ReturnStatusAction: + return LLVMReturnStatusAction; + } + report_fatal_error("Invalid LLVMVerifierFailureAction value!"); +} + +extern "C" LLVMBool +LLVMRustVerifyFunction(LLVMValueRef Fn, LLVMRustVerifierFailureAction Action) { + return LLVMVerifyFunction(Fn, fromRust(Action)); +} + +extern "C" LLVMValueRef LLVMRustGetOrInsertFunction(LLVMModuleRef M, + const char *Name, + size_t NameLen, + LLVMTypeRef FunctionTy) { + return wrap(unwrap(M) + ->getOrInsertFunction(StringRef(Name, NameLen), + unwrap(FunctionTy)) + .getCallee()); +} + +extern "C" LLVMValueRef LLVMRustGetOrInsertGlobal(LLVMModuleRef M, + const char *Name, + size_t NameLen, + LLVMTypeRef Ty) { + Module *Mod = unwrap(M); + auto NameRef = StringRef(Name, NameLen); + + // We don't use Module::getOrInsertGlobal because that returns a Constant*, + // which may either be the real GlobalVariable*, or a constant bitcast of it + // if our type doesn't match the original declaration. We always want the + // GlobalVariable* so we can access linkage, visibility, etc. + GlobalVariable *GV = Mod->getGlobalVariable(NameRef, true); + if (!GV) + GV = new GlobalVariable(*Mod, unwrap(Ty), false, + GlobalValue::ExternalLinkage, nullptr, NameRef); + return wrap(GV); +} + +extern "C" LLVMValueRef LLVMRustInsertPrivateGlobal(LLVMModuleRef M, + LLVMTypeRef Ty) { + return wrap(new GlobalVariable(*unwrap(M), unwrap(Ty), false, + GlobalValue::PrivateLinkage, nullptr)); +} + +// Must match the layout of `rustc_codegen_llvm::llvm::ffi::AttributeKind`. +enum class LLVMRustAttributeKind { + AlwaysInline = 0, + ByVal = 1, + Cold = 2, + InlineHint = 3, + MinSize = 4, + Naked = 5, + NoAlias = 6, + NoCapture = 7, + NoInline = 8, + NonNull = 9, + NoRedZone = 10, + NoReturn = 11, + NoUnwind = 12, + OptimizeForSize = 13, + ReadOnly = 14, + SExt = 15, + StructRet = 16, + UWTable = 17, + ZExt = 18, + InReg = 19, + SanitizeThread = 20, + SanitizeAddress = 21, + SanitizeMemory = 22, + NonLazyBind = 23, + OptimizeNone = 24, + ReadNone = 26, + SanitizeHWAddress = 28, + WillReturn = 29, + StackProtectReq = 30, + StackProtectStrong = 31, + StackProtect = 32, + NoUndef = 33, + SanitizeMemTag = 34, + NoCfCheck = 35, + ShadowCallStack = 36, + AllocSize = 37, + AllocatedPointer = 38, + AllocAlign = 39, + SanitizeSafeStack = 40, + FnRetThunkExtern = 41, + Writable = 42, + DeadOnUnwind = 43, +}; + +static Attribute::AttrKind fromRust(LLVMRustAttributeKind Kind) { + switch (Kind) { + case LLVMRustAttributeKind::AlwaysInline: + return Attribute::AlwaysInline; + case LLVMRustAttributeKind::ByVal: + return Attribute::ByVal; + case LLVMRustAttributeKind::Cold: + return Attribute::Cold; + case LLVMRustAttributeKind::InlineHint: + return Attribute::InlineHint; + case LLVMRustAttributeKind::MinSize: + return Attribute::MinSize; + case LLVMRustAttributeKind::Naked: + return Attribute::Naked; + case LLVMRustAttributeKind::NoAlias: + return Attribute::NoAlias; + case LLVMRustAttributeKind::NoCapture: +#if LLVM_VERSION_GE(21, 0) + report_fatal_error("NoCapture doesn't exist in LLVM 21"); +#else + return Attribute::NoCapture; +#endif + case LLVMRustAttributeKind::NoCfCheck: + return Attribute::NoCfCheck; + case LLVMRustAttributeKind::NoInline: + return Attribute::NoInline; + case LLVMRustAttributeKind::NonNull: + return Attribute::NonNull; + case LLVMRustAttributeKind::NoRedZone: + return Attribute::NoRedZone; + case LLVMRustAttributeKind::NoReturn: + return Attribute::NoReturn; + case LLVMRustAttributeKind::NoUnwind: + return Attribute::NoUnwind; + case LLVMRustAttributeKind::OptimizeForSize: + return Attribute::OptimizeForSize; + case LLVMRustAttributeKind::ReadOnly: + return Attribute::ReadOnly; + case LLVMRustAttributeKind::SExt: + return Attribute::SExt; + case LLVMRustAttributeKind::StructRet: + return Attribute::StructRet; + case LLVMRustAttributeKind::UWTable: + return Attribute::UWTable; + case LLVMRustAttributeKind::ZExt: + return Attribute::ZExt; + case LLVMRustAttributeKind::InReg: + return Attribute::InReg; + case LLVMRustAttributeKind::SanitizeThread: + return Attribute::SanitizeThread; + case LLVMRustAttributeKind::SanitizeAddress: + return Attribute::SanitizeAddress; + case LLVMRustAttributeKind::SanitizeMemory: + return Attribute::SanitizeMemory; + case LLVMRustAttributeKind::NonLazyBind: + return Attribute::NonLazyBind; + case LLVMRustAttributeKind::OptimizeNone: + return Attribute::OptimizeNone; + case LLVMRustAttributeKind::ReadNone: + return Attribute::ReadNone; + case LLVMRustAttributeKind::SanitizeHWAddress: + return Attribute::SanitizeHWAddress; + case LLVMRustAttributeKind::WillReturn: + return Attribute::WillReturn; + case LLVMRustAttributeKind::StackProtectReq: + return Attribute::StackProtectReq; + case LLVMRustAttributeKind::StackProtectStrong: + return Attribute::StackProtectStrong; + case LLVMRustAttributeKind::StackProtect: + return Attribute::StackProtect; + case LLVMRustAttributeKind::NoUndef: + return Attribute::NoUndef; + case LLVMRustAttributeKind::SanitizeMemTag: + return Attribute::SanitizeMemTag; + case LLVMRustAttributeKind::ShadowCallStack: + return Attribute::ShadowCallStack; + case LLVMRustAttributeKind::AllocSize: + return Attribute::AllocSize; + case LLVMRustAttributeKind::AllocatedPointer: + return Attribute::AllocatedPointer; + case LLVMRustAttributeKind::AllocAlign: + return Attribute::AllocAlign; + case LLVMRustAttributeKind::SanitizeSafeStack: + return Attribute::SafeStack; + case LLVMRustAttributeKind::FnRetThunkExtern: + return Attribute::FnRetThunkExtern; + case LLVMRustAttributeKind::Writable: + return Attribute::Writable; + case LLVMRustAttributeKind::DeadOnUnwind: + return Attribute::DeadOnUnwind; + } + report_fatal_error("bad LLVMRustAttributeKind"); +} + +template +static inline void AddAttributes(T *t, unsigned Index, LLVMAttributeRef *Attrs, + size_t AttrsLen) { + AttributeList PAL = t->getAttributes(); + auto B = AttrBuilder(t->getContext()); + for (LLVMAttributeRef Attr : ArrayRef(Attrs, AttrsLen)) + B.addAttribute(unwrap(Attr)); + AttributeList PALNew = PAL.addAttributesAtIndex(t->getContext(), Index, B); + t->setAttributes(PALNew); +} + +extern "C" void LLVMRustAddFunctionAttributes(LLVMValueRef Fn, unsigned Index, + LLVMAttributeRef *Attrs, + size_t AttrsLen) { + Function *F = unwrap(Fn); + AddAttributes(F, Index, Attrs, AttrsLen); +} + +extern "C" void LLVMRustAddCallSiteAttributes(LLVMValueRef Instr, + unsigned Index, + LLVMAttributeRef *Attrs, + size_t AttrsLen) { + CallBase *Call = unwrap(Instr); + AddAttributes(Call, Index, Attrs, AttrsLen); +} + +extern "C" LLVMValueRef LLVMRustGetTerminator(LLVMBasicBlockRef BB) { + Instruction *ret = unwrap(BB)->getTerminator(); + return wrap(ret); +} + +extern "C" void LLVMRustEraseInstFromParent(LLVMValueRef Instr) { + if (auto I = dyn_cast(unwrap(Instr))) { + I->eraseFromParent(); + } +} + +extern "C" LLVMAttributeRef +LLVMRustCreateAttrNoValue(LLVMContextRef C, LLVMRustAttributeKind RustAttr) { +#if LLVM_VERSION_GE(21, 0) + // LLVM 21 replaced the NoCapture attribute with Captures(none). + if (RustAttr == LLVMRustAttributeKind::NoCapture) { + return wrap(Attribute::getWithCaptureInfo(*unwrap(C), CaptureInfo::none())); + } +#endif + return wrap(Attribute::get(*unwrap(C), fromRust(RustAttr))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateAlignmentAttr(LLVMContextRef C, + uint64_t Bytes) { + return wrap(Attribute::getWithAlignment(*unwrap(C), llvm::Align(Bytes))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateDereferenceableAttr(LLVMContextRef C, + uint64_t Bytes) { + return wrap(Attribute::getWithDereferenceableBytes(*unwrap(C), Bytes)); +} + +extern "C" LLVMAttributeRef +LLVMRustCreateDereferenceableOrNullAttr(LLVMContextRef C, uint64_t Bytes) { + return wrap(Attribute::getWithDereferenceableOrNullBytes(*unwrap(C), Bytes)); +} + +extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C, + LLVMTypeRef Ty) { + return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C, + LLVMTypeRef Ty) { + return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateElementTypeAttr(LLVMContextRef C, + LLVMTypeRef Ty) { + return wrap(Attribute::get(*unwrap(C), Attribute::ElementType, unwrap(Ty))); +} + +extern "C" LLVMAttributeRef LLVMRustCreateUWTableAttr(LLVMContextRef C, + bool Async) { + return wrap(Attribute::getWithUWTableKind( + *unwrap(C), Async ? UWTableKind::Async : UWTableKind::Sync)); +} + +extern "C" LLVMAttributeRef +LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) { + return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, + std::nullopt)); +} + +extern "C" LLVMAttributeRef +LLVMRustCreateRangeAttribute(LLVMContextRef C, unsigned NumBits, + const uint64_t LowerWords[], + const uint64_t UpperWords[]) { +#if LLVM_VERSION_GE(19, 0) + return LLVMCreateConstantRangeAttribute(C, Attribute::Range, NumBits, + LowerWords, UpperWords); +#else + report_fatal_error("LLVM 19.0 is required for Range Attribute"); +#endif +} + +// These values **must** match ffi::AllocKindFlags. +// It _happens_ to match the LLVM values of llvm::AllocFnKind, +// but that's happenstance and we do explicit conversions before +// passing them to LLVM. +enum class LLVMRustAllocKindFlags : uint64_t { + Unknown = 0, + Alloc = 1, + Realloc = 1 << 1, + Free = 1 << 2, + Uninitialized = 1 << 3, + Zeroed = 1 << 4, + Aligned = 1 << 5, +}; + +static LLVMRustAllocKindFlags operator&(LLVMRustAllocKindFlags A, + LLVMRustAllocKindFlags B) { + return static_cast(static_cast(A) & + static_cast(B)); +} + +static bool isSet(LLVMRustAllocKindFlags F) { + return F != LLVMRustAllocKindFlags::Unknown; +} + +static llvm::AllocFnKind allocKindFromRust(LLVMRustAllocKindFlags F) { + llvm::AllocFnKind AFK = llvm::AllocFnKind::Unknown; + if (isSet(F & LLVMRustAllocKindFlags::Alloc)) { + AFK |= llvm::AllocFnKind::Alloc; + } + if (isSet(F & LLVMRustAllocKindFlags::Realloc)) { + AFK |= llvm::AllocFnKind::Realloc; + } + if (isSet(F & LLVMRustAllocKindFlags::Free)) { + AFK |= llvm::AllocFnKind::Free; + } + if (isSet(F & LLVMRustAllocKindFlags::Uninitialized)) { + AFK |= llvm::AllocFnKind::Uninitialized; + } + if (isSet(F & LLVMRustAllocKindFlags::Zeroed)) { + AFK |= llvm::AllocFnKind::Zeroed; + } + if (isSet(F & LLVMRustAllocKindFlags::Aligned)) { + AFK |= llvm::AllocFnKind::Aligned; + } + return AFK; +} + +extern "C" LLVMAttributeRef LLVMRustCreateAllocKindAttr(LLVMContextRef C, + uint64_t AllocKindArg) { + return wrap( + Attribute::get(*unwrap(C), Attribute::AllocKind, + static_cast(allocKindFromRust( + static_cast(AllocKindArg))))); +} + +// Simplified representation of `MemoryEffects` across the FFI boundary. +// +// Each variant corresponds to one of the static factory methods on +// `MemoryEffects`. +enum class LLVMRustMemoryEffects { + None, + ReadOnly, + InaccessibleMemOnly, +}; + +extern "C" LLVMAttributeRef +LLVMRustCreateMemoryEffectsAttr(LLVMContextRef C, + LLVMRustMemoryEffects Effects) { + switch (Effects) { + case LLVMRustMemoryEffects::None: + return wrap( + Attribute::getWithMemoryEffects(*unwrap(C), MemoryEffects::none())); + case LLVMRustMemoryEffects::ReadOnly: + return wrap( + Attribute::getWithMemoryEffects(*unwrap(C), MemoryEffects::readOnly())); + case LLVMRustMemoryEffects::InaccessibleMemOnly: + return wrap(Attribute::getWithMemoryEffects( + *unwrap(C), MemoryEffects::inaccessibleMemOnly())); + default: + report_fatal_error("bad MemoryEffects."); + } +} + +// Enable all fast-math flags, including those which will cause floating-point +// operations to return poison for some well-defined inputs. This function can +// only be used to build unsafe Rust intrinsics. That unsafety does permit +// additional optimizations, but at the time of writing, their value is not +// well-understood relative to those enabled by LLVMRustSetAlgebraicMath. +// +// https://llvm.org/docs/LangRef.html#fast-math-flags +extern "C" void LLVMRustSetFastMath(LLVMValueRef V) { + if (auto I = dyn_cast(unwrap(V))) { + I->setFast(true); + } +} + +// Enable fast-math flags which permit algebraic transformations that are not +// allowed by IEEE floating point. For example: a + (b + c) = (a + b) + c and a +// / b = a * (1 / b) Note that this does NOT enable any flags which can cause a +// floating-point operation on well-defined inputs to return poison, and +// therefore this function can be used to build safe Rust intrinsics (such as +// fadd_algebraic). +// +// https://llvm.org/docs/LangRef.html#fast-math-flags +extern "C" void LLVMRustSetAlgebraicMath(LLVMValueRef V) { + if (auto I = dyn_cast(unwrap(V))) { + I->setHasAllowReassoc(true); + I->setHasAllowContract(true); + I->setHasAllowReciprocal(true); + I->setHasNoSignedZeros(true); + } +} + +// Enable the reassoc fast-math flag, allowing transformations that pretend +// floating-point addition and multiplication are associative. +// +// Note that this does NOT enable any flags which can cause a floating-point +// operation on well-defined inputs to return poison, and therefore this +// function can be used to build safe Rust intrinsics (such as fadd_algebraic). +// +// https://llvm.org/docs/LangRef.html#fast-math-flags +extern "C" void LLVMRustSetAllowReassoc(LLVMValueRef V) { + if (auto I = dyn_cast(unwrap(V))) { + I->setHasAllowReassoc(true); + } +} + +extern "C" LLVMValueRef +LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source, + const char *Name, LLVMAtomicOrdering Order) { + Value *Ptr = unwrap(Source); + LoadInst *LI = unwrap(B)->CreateLoad(unwrap(Ty), Ptr, Name); + LI->setAtomic(fromRust(Order)); + return wrap(LI); +} + +extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B, + LLVMValueRef V, + LLVMValueRef Target, + LLVMAtomicOrdering Order) { + StoreInst *SI = unwrap(B)->CreateStore(unwrap(V), unwrap(Target)); + SI->setAtomic(fromRust(Order)); + return wrap(SI); +} + +enum class LLVMRustAsmDialect { + Att, + Intel, +}; + +static InlineAsm::AsmDialect fromRust(LLVMRustAsmDialect Dialect) { + switch (Dialect) { + case LLVMRustAsmDialect::Att: + return InlineAsm::AD_ATT; + case LLVMRustAsmDialect::Intel: + return InlineAsm::AD_Intel; + default: + report_fatal_error("bad AsmDialect."); + } +} + +extern "C" LLVMValueRef +LLVMRustInlineAsm(LLVMTypeRef Ty, char *AsmString, size_t AsmStringLen, + char *Constraints, size_t ConstraintsLen, + LLVMBool HasSideEffects, LLVMBool IsAlignStack, + LLVMRustAsmDialect Dialect, LLVMBool CanThrow) { + return wrap(InlineAsm::get( + unwrap(Ty), StringRef(AsmString, AsmStringLen), + StringRef(Constraints, ConstraintsLen), HasSideEffects, IsAlignStack, + fromRust(Dialect), CanThrow)); +} + +extern "C" bool LLVMRustInlineAsmVerify(LLVMTypeRef Ty, char *Constraints, + size_t ConstraintsLen) { + // llvm::Error converts to true if it is an error. + return !llvm::errorToBool(InlineAsm::verify( + unwrap(Ty), StringRef(Constraints, ConstraintsLen))); +} + +template DIT *unwrapDIPtr(LLVMMetadataRef Ref) { + return (DIT *)(Ref ? unwrap(Ref) : nullptr); +} + +#define DIDescriptor DIScope +#define DIArray DINodeArray +#define unwrapDI unwrapDIPtr + +// Statically assert that `LLVMDIFlags` (C) and `DIFlags` (C++) have the same +// layout, at least for the flags we know about. This isn't guaranteed, but is +// likely to remain true, and as long as it is true it makes conversions easy. +#define ASSERT_DIFLAG_VALUE(FLAG, VALUE) \ + static_assert((LLVMDI##FLAG == (VALUE)) && (DINode::DIFlags::FLAG == (VALUE))) +ASSERT_DIFLAG_VALUE(FlagZero, 0); +ASSERT_DIFLAG_VALUE(FlagPrivate, 1); +ASSERT_DIFLAG_VALUE(FlagProtected, 2); +ASSERT_DIFLAG_VALUE(FlagPublic, 3); +// Bit (1 << 1) is part of the private/protected/public values above. +ASSERT_DIFLAG_VALUE(FlagFwdDecl, 1 << 2); +ASSERT_DIFLAG_VALUE(FlagAppleBlock, 1 << 3); +ASSERT_DIFLAG_VALUE(FlagReservedBit4, 1 << 4); +ASSERT_DIFLAG_VALUE(FlagVirtual, 1 << 5); +ASSERT_DIFLAG_VALUE(FlagArtificial, 1 << 6); +ASSERT_DIFLAG_VALUE(FlagExplicit, 1 << 7); +ASSERT_DIFLAG_VALUE(FlagPrototyped, 1 << 8); +ASSERT_DIFLAG_VALUE(FlagObjcClassComplete, 1 << 9); +ASSERT_DIFLAG_VALUE(FlagObjectPointer, 1 << 10); +ASSERT_DIFLAG_VALUE(FlagVector, 1 << 11); +ASSERT_DIFLAG_VALUE(FlagStaticMember, 1 << 12); +ASSERT_DIFLAG_VALUE(FlagLValueReference, 1 << 13); +ASSERT_DIFLAG_VALUE(FlagRValueReference, 1 << 14); +// Bit (1 << 15) has been recycled, but the C API value hasn't been renamed. +static_assert((LLVMDIFlagReserved == (1 << 15)) && + (DINode::DIFlags::FlagExportSymbols == (1 << 15))); +ASSERT_DIFLAG_VALUE(FlagSingleInheritance, 1 << 16); +ASSERT_DIFLAG_VALUE(FlagMultipleInheritance, 2 << 16); +ASSERT_DIFLAG_VALUE(FlagVirtualInheritance, 3 << 16); +// Bit (1 << 17) is part of the inheritance values above. +ASSERT_DIFLAG_VALUE(FlagIntroducedVirtual, 1 << 18); +ASSERT_DIFLAG_VALUE(FlagBitField, 1 << 19); +ASSERT_DIFLAG_VALUE(FlagNoReturn, 1 << 20); +// Bit (1 << 21) is unused, but was `LLVMDIFlagMainSubprogram`. +ASSERT_DIFLAG_VALUE(FlagTypePassByValue, 1 << 22); +ASSERT_DIFLAG_VALUE(FlagTypePassByReference, 1 << 23); +ASSERT_DIFLAG_VALUE(FlagEnumClass, 1 << 24); +ASSERT_DIFLAG_VALUE(FlagThunk, 1 << 25); +ASSERT_DIFLAG_VALUE(FlagNonTrivial, 1 << 26); +ASSERT_DIFLAG_VALUE(FlagBigEndian, 1 << 27); +ASSERT_DIFLAG_VALUE(FlagLittleEndian, 1 << 28); +ASSERT_DIFLAG_VALUE(FlagIndirectVirtualBase, (1 << 2) | (1 << 5)); +#undef ASSERT_DIFLAG_VALUE + +// There are two potential ways to convert `LLVMDIFlags` to `DIFlags`: +// - Check and copy every individual bit/subvalue from input to output. +// - Statically assert that both have the same layout, and cast. +// As long as the static assertions succeed, a cast is easier and faster. +// In the (hopefully) unlikely event that the assertions do fail someday, and +// LLVM doesn't expose its own conversion function, we'll have to switch over +// to copying each bit/subvalue. +static DINode::DIFlags fromRust(LLVMDIFlags Flags) { + // Check that all set bits are covered by the static assertions above. + const unsigned UNKNOWN_BITS = (1 << 31) | (1 << 30) | (1 << 29) | (1 << 21); + if (Flags & UNKNOWN_BITS) { + report_fatal_error("bad LLVMDIFlags"); + } + + // As long as the static assertions are satisfied and no unknown bits are + // present, we can convert from `LLVMDIFlags` to `DIFlags` with a cast. + return static_cast(Flags); +} + +// These values **must** match debuginfo::DISPFlags! They also *happen* +// to match LLVM, but that isn't required as we do giant sets of +// matching below. The value shouldn't be directly passed to LLVM. +enum class LLVMRustDISPFlags : uint32_t { + SPFlagZero = 0, + SPFlagVirtual = 1, + SPFlagPureVirtual = 2, + SPFlagLocalToUnit = (1 << 2), + SPFlagDefinition = (1 << 3), + SPFlagOptimized = (1 << 4), + SPFlagMainSubprogram = (1 << 5), + // Do not add values that are not supported by the minimum LLVM + // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def + // (In LLVM < 8, createFunction supported these as separate bool arguments.) +}; + +inline LLVMRustDISPFlags operator&(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { + return static_cast(static_cast(A) & + static_cast(B)); +} + +inline LLVMRustDISPFlags operator|(LLVMRustDISPFlags A, LLVMRustDISPFlags B) { + return static_cast(static_cast(A) | + static_cast(B)); +} + +inline LLVMRustDISPFlags &operator|=(LLVMRustDISPFlags &A, + LLVMRustDISPFlags B) { + return A = A | B; +} + +inline bool isSet(LLVMRustDISPFlags F) { + return F != LLVMRustDISPFlags::SPFlagZero; +} + +inline LLVMRustDISPFlags virtuality(LLVMRustDISPFlags F) { + return static_cast(static_cast(F) & 0x3); +} + +static DISubprogram::DISPFlags fromRust(LLVMRustDISPFlags SPFlags) { + DISubprogram::DISPFlags Result = DISubprogram::DISPFlags::SPFlagZero; + + switch (virtuality(SPFlags)) { + case LLVMRustDISPFlags::SPFlagVirtual: + Result |= DISubprogram::DISPFlags::SPFlagVirtual; + break; + case LLVMRustDISPFlags::SPFlagPureVirtual: + Result |= DISubprogram::DISPFlags::SPFlagPureVirtual; + break; + default: + // The rest are handled below + break; + } + + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagLocalToUnit)) { + Result |= DISubprogram::DISPFlags::SPFlagLocalToUnit; + } + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagDefinition)) { + Result |= DISubprogram::DISPFlags::SPFlagDefinition; + } + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagOptimized)) { + Result |= DISubprogram::DISPFlags::SPFlagOptimized; + } + if (isSet(SPFlags & LLVMRustDISPFlags::SPFlagMainSubprogram)) { + Result |= DISubprogram::DISPFlags::SPFlagMainSubprogram; + } + + return Result; +} + +enum class LLVMRustDebugEmissionKind { + NoDebug, + FullDebug, + LineTablesOnly, + DebugDirectivesOnly, +}; + +static DICompileUnit::DebugEmissionKind +fromRust(LLVMRustDebugEmissionKind Kind) { + switch (Kind) { + case LLVMRustDebugEmissionKind::NoDebug: + return DICompileUnit::DebugEmissionKind::NoDebug; + case LLVMRustDebugEmissionKind::FullDebug: + return DICompileUnit::DebugEmissionKind::FullDebug; + case LLVMRustDebugEmissionKind::LineTablesOnly: + return DICompileUnit::DebugEmissionKind::LineTablesOnly; + case LLVMRustDebugEmissionKind::DebugDirectivesOnly: + return DICompileUnit::DebugEmissionKind::DebugDirectivesOnly; + default: + report_fatal_error("bad DebugEmissionKind."); + } +} + +enum class LLVMRustDebugNameTableKind { + Default, + GNU, + None, + Apple, +}; + +static DICompileUnit::DebugNameTableKind +fromRust(LLVMRustDebugNameTableKind Kind) { + switch (Kind) { + case LLVMRustDebugNameTableKind::Default: + return DICompileUnit::DebugNameTableKind::Default; + case LLVMRustDebugNameTableKind::GNU: + return DICompileUnit::DebugNameTableKind::GNU; + case LLVMRustDebugNameTableKind::None: + return DICompileUnit::DebugNameTableKind::None; + case LLVMRustDebugNameTableKind::Apple: + return DICompileUnit::DebugNameTableKind::Apple; + default: + report_fatal_error("bad DebugNameTableKind."); + } +} + +enum class LLVMRustChecksumKind { + None, + MD5, + SHA1, + SHA256, +}; + +static std::optional fromRust(LLVMRustChecksumKind Kind) { + switch (Kind) { + case LLVMRustChecksumKind::None: + return std::nullopt; + case LLVMRustChecksumKind::MD5: + return DIFile::ChecksumKind::CSK_MD5; + case LLVMRustChecksumKind::SHA1: + return DIFile::ChecksumKind::CSK_SHA1; + case LLVMRustChecksumKind::SHA256: + return DIFile::ChecksumKind::CSK_SHA256; + default: + report_fatal_error("bad ChecksumKind."); + } +} + +extern "C" uint32_t LLVMRustDebugMetadataVersion() { + return DEBUG_METADATA_VERSION; +} + +extern "C" uint32_t LLVMRustVersionPatch() { return LLVM_VERSION_PATCH; } + +extern "C" uint32_t LLVMRustVersionMinor() { return LLVM_VERSION_MINOR; } + +extern "C" uint32_t LLVMRustVersionMajor() { return LLVM_VERSION_MAJOR; } + +// FFI equivalent of LLVM's `llvm::Module::ModFlagBehavior`. +// Must match the layout of +// `rustc_codegen_llvm::llvm::ffi::ModuleFlagMergeBehavior`. +// +// There is a stable LLVM-C version of this enum (`LLVMModuleFlagBehavior`), +// but as of LLVM 19 it does not support all of the enum values in the unstable +// C++ API. +enum class LLVMRustModuleFlagMergeBehavior { + Error = 1, + Warning = 2, + Require = 3, + Override = 4, + Append = 5, + AppendUnique = 6, + Max = 7, + Min = 8, +}; + +static Module::ModFlagBehavior +fromRust(LLVMRustModuleFlagMergeBehavior Behavior) { + switch (Behavior) { + case LLVMRustModuleFlagMergeBehavior::Error: + return Module::ModFlagBehavior::Error; + case LLVMRustModuleFlagMergeBehavior::Warning: + return Module::ModFlagBehavior::Warning; + case LLVMRustModuleFlagMergeBehavior::Require: + return Module::ModFlagBehavior::Require; + case LLVMRustModuleFlagMergeBehavior::Override: + return Module::ModFlagBehavior::Override; + case LLVMRustModuleFlagMergeBehavior::Append: + return Module::ModFlagBehavior::Append; + case LLVMRustModuleFlagMergeBehavior::AppendUnique: + return Module::ModFlagBehavior::AppendUnique; + case LLVMRustModuleFlagMergeBehavior::Max: + return Module::ModFlagBehavior::Max; + case LLVMRustModuleFlagMergeBehavior::Min: + return Module::ModFlagBehavior::Min; + } + report_fatal_error("bad LLVMRustModuleFlagMergeBehavior"); +} + +extern "C" void +LLVMRustAddModuleFlagU32(LLVMModuleRef M, + LLVMRustModuleFlagMergeBehavior MergeBehavior, + const char *Name, size_t NameLen, uint32_t Value) { + unwrap(M)->addModuleFlag(fromRust(MergeBehavior), StringRef(Name, NameLen), + Value); +} + +extern "C" void LLVMRustAddModuleFlagString( + LLVMModuleRef M, LLVMRustModuleFlagMergeBehavior MergeBehavior, + const char *Name, size_t NameLen, const char *Value, size_t ValueLen) { + unwrap(M)->addModuleFlag( + fromRust(MergeBehavior), StringRef(Name, NameLen), + MDString::get(unwrap(M)->getContext(), StringRef(Value, ValueLen))); +} + +extern "C" LLVMValueRef LLVMRustGetLastInstruction(LLVMBasicBlockRef BB) { + auto Point = unwrap(BB)->rbegin(); + if (Point != unwrap(BB)->rend()) + return wrap(&*Point); + return nullptr; +} + +extern "C" void LLVMRustEraseInstUntilInclusive(LLVMBasicBlockRef bb, + LLVMValueRef I) { + auto &BB = *unwrap(bb); + auto &Inst = *unwrap(I); + auto It = BB.begin(); + while (&*It != &Inst) + ++It; + // Make sure we found the Instruction. + assert(It != BB.end()); + // Delete in rev order to ensure no dangling references. + while (It != BB.begin()) { + auto Prev = std::prev(It); + It->eraseFromParent(); + It = Prev; + } + It->eraseFromParent(); +} + +extern "C" bool LLVMRustHasMetadata(LLVMValueRef inst, unsigned kindID) { + if (auto *I = dyn_cast(unwrap(inst))) { + return I->hasMetadata(kindID); + } + return false; +} + +extern "C" LLVMMetadataRef LLVMRustDIGetInstMetadata(LLVMValueRef x) { + if (auto *I = dyn_cast(unwrap(x))) { + auto *MD = I->getDebugLoc().getAsMDNode(); + return wrap(MD); + } + return nullptr; +} + +extern "C" void LLVMRustGlobalAddMetadata(LLVMValueRef Global, unsigned Kind, + LLVMMetadataRef MD) { + unwrap(Global)->addMetadata(Kind, *unwrap(MD)); +} + +extern "C" LLVMDIBuilderRef LLVMRustDIBuilderCreate(LLVMModuleRef M) { + return wrap(new DIBuilder(*unwrap(M))); +} + +extern "C" void LLVMRustDIBuilderDispose(LLVMDIBuilderRef Builder) { + delete unwrap(Builder); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateCompileUnit( + LLVMDIBuilderRef Builder, unsigned Lang, LLVMMetadataRef FileRef, + const char *Producer, size_t ProducerLen, bool isOptimized, + const char *Flags, unsigned RuntimeVer, const char *SplitName, + size_t SplitNameLen, LLVMRustDebugEmissionKind Kind, uint64_t DWOId, + bool SplitDebugInlining, LLVMRustDebugNameTableKind TableKind) { + auto *File = unwrapDI(FileRef); + + return wrap(unwrap(Builder)->createCompileUnit( + Lang, File, StringRef(Producer, ProducerLen), isOptimized, Flags, + RuntimeVer, StringRef(SplitName, SplitNameLen), fromRust(Kind), DWOId, + SplitDebugInlining, false, fromRust(TableKind))); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateFile(LLVMDIBuilderRef Builder, const char *Filename, + size_t FilenameLen, const char *Directory, + size_t DirectoryLen, LLVMRustChecksumKind CSKind, + const char *Checksum, size_t ChecksumLen, + const char *Source, size_t SourceLen) { + + std::optional llvmCSKind = fromRust(CSKind); + std::optional> CSInfo{}; + if (llvmCSKind) + CSInfo.emplace(*llvmCSKind, StringRef{Checksum, ChecksumLen}); + std::optional oSource{}; + if (Source) + oSource = StringRef(Source, SourceLen); + return wrap(unwrap(Builder)->createFile(StringRef(Filename, FilenameLen), + StringRef(Directory, DirectoryLen), + CSInfo, oSource)); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateSubroutineType(LLVMDIBuilderRef Builder, + LLVMMetadataRef ParameterTypes) { + return wrap(unwrap(Builder)->createSubroutineType( + DITypeRefArray(unwrap(ParameterTypes)))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFunction( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, const char *LinkageName, size_t LinkageNameLen, + LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty, + unsigned ScopeLine, LLVMDIFlags Flags, LLVMRustDISPFlags SPFlags, + LLVMValueRef MaybeFn, LLVMMetadataRef TParam, LLVMMetadataRef Decl) { + DITemplateParameterArray TParams = + DITemplateParameterArray(unwrap(TParam)); + DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); + DINode::DIFlags llvmFlags = fromRust(Flags); + DISubprogram *Sub = unwrap(Builder)->createFunction( + unwrapDI(Scope), StringRef(Name, NameLen), + StringRef(LinkageName, LinkageNameLen), unwrapDI(File), LineNo, + unwrapDI(Ty), ScopeLine, llvmFlags, llvmSPFlags, + TParams, unwrapDIPtr(Decl)); + if (MaybeFn) + unwrap(MaybeFn)->setSubprogram(Sub); + return wrap(Sub); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMethod( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, const char *LinkageName, size_t LinkageNameLen, + LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty, + LLVMDIFlags Flags, LLVMRustDISPFlags SPFlags, LLVMMetadataRef TParam) { + DITemplateParameterArray TParams = + DITemplateParameterArray(unwrap(TParam)); + DISubprogram::DISPFlags llvmSPFlags = fromRust(SPFlags); + DINode::DIFlags llvmFlags = fromRust(Flags); + DISubprogram *Sub = unwrap(Builder)->createMethod( + unwrapDI(Scope), StringRef(Name, NameLen), + StringRef(LinkageName, LinkageNameLen), unwrapDI(File), LineNo, + unwrapDI(Ty), 0, 0, + nullptr, // VTable params aren't used + llvmFlags, llvmSPFlags, TParams); + return wrap(Sub); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateBasicType(LLVMDIBuilderRef Builder, const char *Name, + size_t NameLen, uint64_t SizeInBits, + unsigned Encoding) { + return wrap(unwrap(Builder)->createBasicType(StringRef(Name, NameLen), + SizeInBits, Encoding)); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateTypedef(LLVMDIBuilderRef Builder, LLVMMetadataRef Type, + const char *Name, size_t NameLen, + LLVMMetadataRef File, unsigned LineNo, + LLVMMetadataRef Scope) { + return wrap(unwrap(Builder)->createTypedef( + unwrap(Type), StringRef(Name, NameLen), unwrap(File), + LineNo, unwrapDIPtr(Scope))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreatePointerType( + LLVMDIBuilderRef Builder, LLVMMetadataRef PointeeTy, uint64_t SizeInBits, + uint32_t AlignInBits, unsigned AddressSpace, const char *Name, + size_t NameLen) { + return wrap(unwrap(Builder)->createPointerType( + unwrapDI(PointeeTy), SizeInBits, AlignInBits, AddressSpace, + StringRef(Name, NameLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStructType( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, LLVMMetadataRef File, unsigned LineNumber, + uint64_t SizeInBits, uint32_t AlignInBits, LLVMDIFlags Flags, + LLVMMetadataRef DerivedFrom, LLVMMetadataRef Elements, unsigned RunTimeLang, + LLVMMetadataRef VTableHolder, const char *UniqueId, size_t UniqueIdLen) { + return wrap(unwrap(Builder)->createStructType( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNumber, SizeInBits, AlignInBits, + fromRust(Flags), unwrapDI(DerivedFrom), + DINodeArray(unwrapDI(Elements)), RunTimeLang, + unwrapDI(VTableHolder), StringRef(UniqueId, UniqueIdLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantPart( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, LLVMMetadataRef File, unsigned LineNumber, + uint64_t SizeInBits, uint32_t AlignInBits, LLVMDIFlags Flags, + LLVMMetadataRef Discriminator, LLVMMetadataRef Elements, + const char *UniqueId, size_t UniqueIdLen) { + return wrap(unwrap(Builder)->createVariantPart( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNumber, SizeInBits, AlignInBits, + fromRust(Flags), unwrapDI(Discriminator), + DINodeArray(unwrapDI(Elements)), + StringRef(UniqueId, UniqueIdLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateMemberType( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits, + uint32_t AlignInBits, uint64_t OffsetInBits, LLVMDIFlags Flags, + LLVMMetadataRef Ty) { + return wrap(unwrap(Builder)->createMemberType( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNo, SizeInBits, AlignInBits, OffsetInBits, + fromRust(Flags), unwrapDI(Ty))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariantMemberType( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, LLVMMetadataRef File, unsigned LineNo, uint64_t SizeInBits, + uint32_t AlignInBits, uint64_t OffsetInBits, LLVMValueRef Discriminant, + LLVMDIFlags Flags, LLVMMetadataRef Ty) { + llvm::ConstantInt *D = nullptr; + if (Discriminant) { + D = unwrap(Discriminant); + } + return wrap(unwrap(Builder)->createVariantMemberType( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNo, SizeInBits, AlignInBits, OffsetInBits, D, + fromRust(Flags), unwrapDI(Ty))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticMemberType( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty, + LLVMDIFlags Flags, LLVMValueRef val, uint32_t AlignInBits) { + return wrap(unwrap(Builder)->createStaticMemberType( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNo, unwrapDI(Ty), fromRust(Flags), + unwrap(val), llvm::dwarf::DW_TAG_member, AlignInBits)); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateQualifiedType(LLVMDIBuilderRef Builder, unsigned Tag, + LLVMMetadataRef Type) { + return wrap( + unwrap(Builder)->createQualifiedType(Tag, unwrapDI(Type))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticVariable( + LLVMDIBuilderRef Builder, LLVMMetadataRef Context, const char *Name, + size_t NameLen, const char *LinkageName, size_t LinkageNameLen, + LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty, + bool IsLocalToUnit, LLVMValueRef V, LLVMMetadataRef Decl = nullptr, + uint32_t AlignInBits = 0) { + llvm::GlobalVariable *InitVal = cast(unwrap(V)); + + llvm::DIExpression *InitExpr = nullptr; + if (llvm::ConstantInt *IntVal = llvm::dyn_cast(InitVal)) { + InitExpr = unwrap(Builder)->createConstantValueExpression( + IntVal->getValue().getSExtValue()); + } else if (llvm::ConstantFP *FPVal = + llvm::dyn_cast(InitVal)) { + InitExpr = unwrap(Builder)->createConstantValueExpression( + FPVal->getValueAPF().bitcastToAPInt().getZExtValue()); + } + + llvm::DIGlobalVariableExpression *VarExpr = + unwrap(Builder)->createGlobalVariableExpression( + unwrapDI(Context), StringRef(Name, NameLen), + StringRef(LinkageName, LinkageNameLen), unwrapDI(File), + LineNo, unwrapDI(Ty), IsLocalToUnit, + /* isDefined */ true, InitExpr, unwrapDIPtr(Decl), + /* templateParams */ nullptr, AlignInBits); + + InitVal->setMetadata("dbg", VarExpr); + + return wrap(VarExpr); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateVariable( + LLVMDIBuilderRef Builder, unsigned Tag, LLVMMetadataRef Scope, + const char *Name, size_t NameLen, LLVMMetadataRef File, unsigned LineNo, + LLVMMetadataRef Ty, bool AlwaysPreserve, LLVMDIFlags Flags, unsigned ArgNo, + uint32_t AlignInBits) { + if (Tag == 0x100) { // DW_TAG_auto_variable + return wrap(unwrap(Builder)->createAutoVariable( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNo, unwrapDI(Ty), AlwaysPreserve, + fromRust(Flags), AlignInBits)); + } else { + return wrap(unwrap(Builder)->createParameterVariable( + unwrapDI(Scope), StringRef(Name, NameLen), ArgNo, + unwrapDI(File), LineNo, unwrapDI(Ty), AlwaysPreserve, + fromRust(Flags))); + } +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateArrayType(LLVMDIBuilderRef Builder, uint64_t Size, + uint32_t AlignInBits, LLVMMetadataRef Ty, + LLVMMetadataRef Subscripts) { + return wrap(unwrap(Builder)->createArrayType( + Size, AlignInBits, unwrapDI(Ty), + DINodeArray(unwrapDI(Subscripts)))); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderGetOrCreateSubrange(LLVMDIBuilderRef Builder, int64_t Lo, + int64_t Count) { + return wrap(unwrap(Builder)->getOrCreateSubrange(Lo, Count)); +} + +extern "C" LLVMMetadataRef +LLVMRustDIBuilderGetOrCreateArray(LLVMDIBuilderRef Builder, + LLVMMetadataRef *Ptr, unsigned Count) { + Metadata **DataValue = unwrap(Ptr); + return wrap(unwrap(Builder) + ->getOrCreateArray(ArrayRef(DataValue, Count)) + .get()); +} + +extern "C" void +LLVMRustDIBuilderInsertDeclareAtEnd(LLVMDIBuilderRef Builder, LLVMValueRef V, + LLVMMetadataRef VarInfo, uint64_t *AddrOps, + unsigned AddrOpsCount, LLVMMetadataRef DL, + LLVMBasicBlockRef InsertAtEnd) { + + unwrap(Builder)->insertDeclare( + unwrap(V), unwrap(VarInfo), + unwrap(Builder)->createExpression( + llvm::ArrayRef(AddrOps, AddrOpsCount)), + DebugLoc(cast(unwrap(DL))), unwrap(InsertAtEnd)); +} + +/*extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateEnumerator(LLVMDIBuilderRef Builder, const char *Name, + size_t NameLen, const uint64_t Value[2], + unsigned SizeInBits, bool IsUnsigned) { + return wrap(unwrap(Builder)->createEnumerator( + StringRef(Name, NameLen), + APSInt(APInt(SizeInBits, ArrayRef(Value, 2)), IsUnsigned))); +}*/ + +// TODO: temporary workaround for uint64_t[2] array +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateEnumerator(LLVMDIBuilderRef Builder, const char *Name, + size_t NameLen, int64_t Value, // ⭐ Single i64 + bool IsUnsigned) { + // Determine bit width from the value + unsigned SizeInBits = 64; // Or calculate based on Value + + // Convert single i64 to uint64_t array + uint64_t ValueArray[2] = {(uint64_t)Value, Value < 0 ? UINT64_MAX : 0}; + + return wrap(unwrap(Builder)->createEnumerator( + StringRef(Name, NameLen), + APSInt(APInt(SizeInBits, ArrayRef(ValueArray, 2)), IsUnsigned))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, LLVMMetadataRef File, unsigned LineNumber, + uint64_t SizeInBits, uint32_t AlignInBits, LLVMMetadataRef Elements, + LLVMMetadataRef ClassTy, bool IsScoped) { + return wrap(unwrap(Builder)->createEnumerationType( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNumber, SizeInBits, AlignInBits, + DINodeArray(unwrapDI(Elements)), unwrapDI(ClassTy), + /* RunTimeLang */ 0, "", IsScoped)); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, LLVMMetadataRef File, unsigned LineNumber, + uint64_t SizeInBits, uint32_t AlignInBits, LLVMDIFlags Flags, + LLVMMetadataRef Elements, unsigned RunTimeLang, const char *UniqueId, + size_t UniqueIdLen) { + return wrap(unwrap(Builder)->createUnionType( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(File), LineNumber, SizeInBits, AlignInBits, + fromRust(Flags), DINodeArray(unwrapDI(Elements)), RunTimeLang, + StringRef(UniqueId, UniqueIdLen))); +} + +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateTemplateTypeParameter( + LLVMDIBuilderRef Builder, LLVMMetadataRef Scope, const char *Name, + size_t NameLen, LLVMMetadataRef Ty) { + bool IsDefault = false; // FIXME: should we ever set this true? + return wrap(unwrap(Builder)->createTemplateTypeParameter( + unwrapDI(Scope), StringRef(Name, NameLen), + unwrapDI(Ty), IsDefault)); +} + +extern "C" void LLVMRustDICompositeTypeReplaceArrays( + LLVMDIBuilderRef Builder, LLVMMetadataRef CompositeTy, + LLVMMetadataRef Elements, LLVMMetadataRef Params) { + DICompositeType *Tmp = unwrapDI(CompositeTy); + unwrap(Builder)->replaceArrays(Tmp, DINodeArray(unwrap(Elements)), + DINodeArray(unwrap(Params))); +} + +extern "C" LLVMMetadataRef +LLVMRustDILocationCloneWithBaseDiscriminator(LLVMMetadataRef Location, + unsigned BD) { + DILocation *Loc = unwrapDIPtr(Location); + auto NewLoc = Loc->cloneWithBaseDiscriminator(BD); + return wrap(NewLoc.has_value() ? NewLoc.value() : nullptr); +} + +extern "C" void LLVMRustWriteTypeToString(LLVMTypeRef Ty, RustStringRef Str) { + auto OS = RawRustStringOstream(Str); + unwrap(Ty)->print(OS); +} + +extern "C" void LLVMRustWriteValueToString(LLVMValueRef V, RustStringRef Str) { + auto OS = RawRustStringOstream(Str); + if (!V) { + OS << "(null)"; + } else { + OS << "("; + unwrap(V)->getType()->print(OS); + OS << ":"; + unwrap(V)->print(OS); + OS << ")"; + } +} + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Twine, LLVMTwineRef) + +extern "C" void LLVMRustWriteTwineToString(LLVMTwineRef T, RustStringRef Str) { + auto OS = RawRustStringOstream(Str); + unwrap(T)->print(OS); +} + +extern "C" void LLVMRustUnpackOptimizationDiagnostic( + LLVMDiagnosticInfoRef DI, RustStringRef PassNameOut, + LLVMValueRef *FunctionOut, unsigned *Line, unsigned *Column, + RustStringRef FilenameOut, RustStringRef MessageOut) { + // Undefined to call this not on an optimization diagnostic! + llvm::DiagnosticInfoOptimizationBase *Opt = + static_cast(unwrap(DI)); + + auto PassNameOS = RawRustStringOstream(PassNameOut); + PassNameOS << Opt->getPassName(); + *FunctionOut = wrap(&Opt->getFunction()); + + auto FilenameOS = RawRustStringOstream(FilenameOut); + DiagnosticLocation loc = Opt->getLocation(); + if (loc.isValid()) { + *Line = loc.getLine(); + *Column = loc.getColumn(); + FilenameOS << loc.getAbsolutePath(); + } + + auto MessageOS = RawRustStringOstream(MessageOut); + MessageOS << Opt->getMsg(); +} + +enum class LLVMRustDiagnosticLevel { + Error, + Warning, + Note, + Remark, +}; + +extern "C" void LLVMRustUnpackInlineAsmDiagnostic( + LLVMDiagnosticInfoRef DI, LLVMRustDiagnosticLevel *LevelOut, + uint64_t *CookieOut, LLVMTwineRef *MessageOut) { + // Undefined to call this not on an inline assembly diagnostic! + llvm::DiagnosticInfoInlineAsm *IA = + static_cast(unwrap(DI)); + + *CookieOut = IA->getLocCookie(); + *MessageOut = wrap(&IA->getMsgStr()); + + switch (IA->getSeverity()) { + case DS_Error: + *LevelOut = LLVMRustDiagnosticLevel::Error; + break; + case DS_Warning: + *LevelOut = LLVMRustDiagnosticLevel::Warning; + break; + case DS_Note: + *LevelOut = LLVMRustDiagnosticLevel::Note; + break; + case DS_Remark: + *LevelOut = LLVMRustDiagnosticLevel::Remark; + break; + default: + report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); + } +} + +extern "C" void LLVMRustWriteDiagnosticInfoToString(LLVMDiagnosticInfoRef DI, + RustStringRef Str) { + auto OS = RawRustStringOstream(Str); + auto DP = DiagnosticPrinterRawOStream(OS); + unwrap(DI)->print(DP); +} + +enum class LLVMRustDiagnosticKind { + Other, + InlineAsm, + StackSize, + DebugMetadataVersion, + SampleProfile, + OptimizationRemark, + OptimizationRemarkMissed, + OptimizationRemarkAnalysis, + OptimizationRemarkAnalysisFPCommute, + OptimizationRemarkAnalysisAliasing, + OptimizationRemarkOther, + OptimizationFailure, + PGOProfile, + Linker, + Unsupported, + SrcMgr, +}; + +static LLVMRustDiagnosticKind diagnosticKindToRust(DiagnosticKind Kind) { + switch (Kind) { + case DK_InlineAsm: + return LLVMRustDiagnosticKind::InlineAsm; + case DK_StackSize: + return LLVMRustDiagnosticKind::StackSize; + case DK_DebugMetadataVersion: + return LLVMRustDiagnosticKind::DebugMetadataVersion; + case DK_SampleProfile: + return LLVMRustDiagnosticKind::SampleProfile; + case DK_OptimizationRemark: + case DK_MachineOptimizationRemark: + return LLVMRustDiagnosticKind::OptimizationRemark; + case DK_OptimizationRemarkMissed: + case DK_MachineOptimizationRemarkMissed: + return LLVMRustDiagnosticKind::OptimizationRemarkMissed; + case DK_OptimizationRemarkAnalysis: + case DK_MachineOptimizationRemarkAnalysis: + return LLVMRustDiagnosticKind::OptimizationRemarkAnalysis; + case DK_OptimizationRemarkAnalysisFPCommute: + return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisFPCommute; + case DK_OptimizationRemarkAnalysisAliasing: + return LLVMRustDiagnosticKind::OptimizationRemarkAnalysisAliasing; + case DK_PGOProfile: + return LLVMRustDiagnosticKind::PGOProfile; + case DK_Linker: + return LLVMRustDiagnosticKind::Linker; + case DK_Unsupported: + return LLVMRustDiagnosticKind::Unsupported; + case DK_SrcMgr: + return LLVMRustDiagnosticKind::SrcMgr; + default: + return (Kind >= DK_FirstRemark && Kind <= DK_LastRemark) + ? LLVMRustDiagnosticKind::OptimizationRemarkOther + : LLVMRustDiagnosticKind::Other; + } +} + +extern "C" LLVMRustDiagnosticKind +LLVMRustGetDiagInfoKind(LLVMDiagnosticInfoRef DI) { + return diagnosticKindToRust((DiagnosticKind)unwrap(DI)->getKind()); +} + +// This is kept distinct from LLVMGetTypeKind, because when +// a new type kind is added, the Rust-side enum must be +// updated or UB will result. +extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty) { + switch (unwrap(Ty)->getTypeID()) { + case Type::VoidTyID: + return LLVMVoidTypeKind; + case Type::HalfTyID: + return LLVMHalfTypeKind; + case Type::FloatTyID: + return LLVMFloatTypeKind; + case Type::DoubleTyID: + return LLVMDoubleTypeKind; + case Type::X86_FP80TyID: + return LLVMX86_FP80TypeKind; + case Type::FP128TyID: + return LLVMFP128TypeKind; + case Type::PPC_FP128TyID: + return LLVMPPC_FP128TypeKind; + case Type::LabelTyID: + return LLVMLabelTypeKind; + case Type::MetadataTyID: + return LLVMMetadataTypeKind; + case Type::IntegerTyID: + return LLVMIntegerTypeKind; + case Type::FunctionTyID: + return LLVMFunctionTypeKind; + case Type::StructTyID: + return LLVMStructTypeKind; + case Type::ArrayTyID: + return LLVMArrayTypeKind; + case Type::PointerTyID: + return LLVMPointerTypeKind; + case Type::FixedVectorTyID: + return LLVMVectorTypeKind; + case Type::TokenTyID: + return LLVMTokenTypeKind; + case Type::ScalableVectorTyID: + return LLVMScalableVectorTypeKind; + case Type::BFloatTyID: + return LLVMBFloatTypeKind; + case Type::X86_AMXTyID: + return LLVMX86_AMXTypeKind; + default: { + std::string error; + auto stream = llvm::raw_string_ostream(error); + stream << "Rust does not support the TypeID: " << unwrap(Ty)->getTypeID() + << " for the type: " << *unwrap(Ty); + stream.flush(); + report_fatal_error(error.c_str()); + } + } +} + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SMDiagnostic, LLVMSMDiagnosticRef) + +extern "C" LLVMSMDiagnosticRef LLVMRustGetSMDiagnostic(LLVMDiagnosticInfoRef DI, + uint64_t *Cookie) { + llvm::DiagnosticInfoSrcMgr *SM = + static_cast(unwrap(DI)); + *Cookie = SM->getLocCookie(); + return wrap(&SM->getSMDiag()); +} + +extern "C" bool +LLVMRustUnpackSMDiagnostic(LLVMSMDiagnosticRef DRef, RustStringRef MessageOut, + RustStringRef BufferOut, + LLVMRustDiagnosticLevel *LevelOut, unsigned *LocOut, + unsigned *RangesOut, size_t *NumRanges) { + SMDiagnostic &D = *unwrap(DRef); + auto MessageOS = RawRustStringOstream(MessageOut); + MessageOS << D.getMessage(); + + switch (D.getKind()) { + case SourceMgr::DK_Error: + *LevelOut = LLVMRustDiagnosticLevel::Error; + break; + case SourceMgr::DK_Warning: + *LevelOut = LLVMRustDiagnosticLevel::Warning; + break; + case SourceMgr::DK_Note: + *LevelOut = LLVMRustDiagnosticLevel::Note; + break; + case SourceMgr::DK_Remark: + *LevelOut = LLVMRustDiagnosticLevel::Remark; + break; + default: + report_fatal_error("Invalid LLVMRustDiagnosticLevel value!"); + } + + if (D.getLoc() == SMLoc()) + return false; + + const SourceMgr &LSM = *D.getSourceMgr(); + const MemoryBuffer *LBuf = + LSM.getMemoryBuffer(LSM.FindBufferContainingLoc(D.getLoc())); + auto BufferOS = RawRustStringOstream(BufferOut); + BufferOS << LBuf->getBuffer(); + + *LocOut = D.getLoc().getPointer() - LBuf->getBufferStart(); + + *NumRanges = std::min(*NumRanges, D.getRanges().size()); + size_t LineStart = *LocOut - (size_t)D.getColumnNo(); + for (size_t i = 0; i < *NumRanges; i++) { + RangesOut[i * 2] = LineStart + D.getRanges()[i].first; + RangesOut[i * 2 + 1] = LineStart + D.getRanges()[i].second; + } + + return true; +} + +extern "C" LLVMValueRef LLVMRustBuildMemCpy(LLVMBuilderRef B, LLVMValueRef Dst, + unsigned DstAlign, LLVMValueRef Src, + unsigned SrcAlign, + LLVMValueRef Size, + bool IsVolatile) { + return wrap(unwrap(B)->CreateMemCpy(unwrap(Dst), MaybeAlign(DstAlign), + unwrap(Src), MaybeAlign(SrcAlign), + unwrap(Size), IsVolatile)); +} + +extern "C" LLVMValueRef +LLVMRustBuildMemMove(LLVMBuilderRef B, LLVMValueRef Dst, unsigned DstAlign, + LLVMValueRef Src, unsigned SrcAlign, LLVMValueRef Size, + bool IsVolatile) { + return wrap(unwrap(B)->CreateMemMove(unwrap(Dst), MaybeAlign(DstAlign), + unwrap(Src), MaybeAlign(SrcAlign), + unwrap(Size), IsVolatile)); +} + +extern "C" LLVMValueRef LLVMRustBuildMemSet(LLVMBuilderRef B, LLVMValueRef Dst, + unsigned DstAlign, LLVMValueRef Val, + LLVMValueRef Size, + bool IsVolatile) { + return wrap(unwrap(B)->CreateMemSet(unwrap(Dst), unwrap(Val), unwrap(Size), + MaybeAlign(DstAlign), IsVolatile)); +} + +// Polyfill for `LLVMBuildCallBr`, which was added in LLVM 19. +// +// FIXME: Remove when Rust's minimum supported LLVM version reaches 19. +#if LLVM_VERSION_LT(19, 0) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OperandBundleDef, LLVMOperandBundleRef) + +extern "C" LLVMValueRef +LLVMBuildCallBr(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, + LLVMBasicBlockRef DefaultDest, LLVMBasicBlockRef *IndirectDests, + unsigned NumIndirectDests, LLVMValueRef *Args, unsigned NumArgs, + LLVMOperandBundleRef *Bundles, unsigned NumBundles, + const char *Name) { + Value *Callee = unwrap(Fn); + FunctionType *FTy = unwrap(Ty); + + // FIXME: Is there a way around this? + std::vector IndirectDestsUnwrapped; + IndirectDestsUnwrapped.reserve(NumIndirectDests); + for (unsigned i = 0; i < NumIndirectDests; ++i) { + IndirectDestsUnwrapped.push_back(unwrap(IndirectDests[i])); + } + + // FIXME: Is there a way around this? + SmallVector OpBundles; + OpBundles.reserve(NumBundles); + for (unsigned i = 0; i < NumBundles; ++i) { + OpBundles.push_back(*unwrap(Bundles[i])); + } + + return wrap( + unwrap(B)->CreateCallBr(FTy, Callee, unwrap(DefaultDest), + ArrayRef(IndirectDestsUnwrapped), + ArrayRef(unwrap(Args), NumArgs), + ArrayRef(OpBundles), Name)); +} +#endif + +extern "C" void LLVMRustPositionBuilderAtStart(LLVMBuilderRef B, + LLVMBasicBlockRef BB) { + auto Point = unwrap(BB)->getFirstInsertionPt(); + unwrap(B)->SetInsertPoint(unwrap(BB), Point); +} + +extern "C" bool LLVMRustConstIntGetZExtValue(LLVMValueRef CV, uint64_t *value) { + auto C = unwrap(CV); + if (C->getBitWidth() > 64) + return false; + *value = C->getZExtValue(); + return true; +} + +// Returns true if both high and low were successfully set. Fails in case +// constant wasn’t any of the common sizes (1, 8, 16, 32, 64, 128 bits) +extern "C" bool LLVMRustConstInt128Get(LLVMValueRef CV, bool sext, + uint64_t *high, uint64_t *low) { + auto C = unwrap(CV); + if (C->getBitWidth() > 128) { + return false; + } + APInt AP; + if (sext) { + AP = C->getValue().sext(128); + } else { + AP = C->getValue().zext(128); + } + *low = AP.getLoBits(64).getZExtValue(); + *high = AP.getHiBits(64).getZExtValue(); + return true; +} + +extern "C" void LLVMRustSetDSOLocal(LLVMValueRef Global, bool is_dso_local) { + unwrap(Global)->setDSOLocal(is_dso_local); +} + +struct LLVMRustModuleBuffer { + std::string data; +}; + +extern "C" LLVMRustModuleBuffer *LLVMRustModuleBufferCreate(LLVMModuleRef M) { + auto Ret = std::make_unique(); + { + auto OS = raw_string_ostream(Ret->data); + WriteBitcodeToFile(*unwrap(M), OS); + } + return Ret.release(); +} + +extern "C" void LLVMRustModuleBufferFree(LLVMRustModuleBuffer *Buffer) { + delete Buffer; +} + +extern "C" const void * +LLVMRustModuleBufferPtr(const LLVMRustModuleBuffer *Buffer) { + return Buffer->data.data(); +} + +extern "C" size_t LLVMRustModuleBufferLen(const LLVMRustModuleBuffer *Buffer) { + return Buffer->data.length(); +} + +extern "C" uint64_t LLVMRustModuleCost(LLVMModuleRef M) { + auto f = unwrap(M)->functions(); + return std::distance(std::begin(f), std::end(f)); +} + +extern "C" void LLVMRustModuleInstructionStats(LLVMModuleRef M, + RustStringRef Str) { + auto OS = RawRustStringOstream(Str); + auto JOS = llvm::json::OStream(OS); + auto Module = unwrap(M); + + JOS.object([&] { + JOS.attribute("module", Module->getName()); + JOS.attribute("total", Module->getInstructionCount()); + }); +} + +// Vector reductions: +extern "C" LLVMValueRef LLVMRustBuildVectorReduceFAdd(LLVMBuilderRef B, + LLVMValueRef Acc, + LLVMValueRef Src) { + return wrap(unwrap(B)->CreateFAddReduce(unwrap(Acc), unwrap(Src))); +} +extern "C" LLVMValueRef LLVMRustBuildVectorReduceFMul(LLVMBuilderRef B, + LLVMValueRef Acc, + LLVMValueRef Src) { + return wrap(unwrap(B)->CreateFMulReduce(unwrap(Acc), unwrap(Src))); +} +extern "C" LLVMValueRef LLVMRustBuildVectorReduceAdd(LLVMBuilderRef B, + LLVMValueRef Src) { + return wrap(unwrap(B)->CreateAddReduce(unwrap(Src))); +} +extern "C" LLVMValueRef LLVMRustBuildVectorReduceMul(LLVMBuilderRef B, + LLVMValueRef Src) { + return wrap(unwrap(B)->CreateMulReduce(unwrap(Src))); +} +extern "C" LLVMValueRef LLVMRustBuildVectorReduceAnd(LLVMBuilderRef B, + LLVMValueRef Src) { + return wrap(unwrap(B)->CreateAndReduce(unwrap(Src))); +} +extern "C" LLVMValueRef LLVMRustBuildVectorReduceOr(LLVMBuilderRef B, + LLVMValueRef Src) { + return wrap(unwrap(B)->CreateOrReduce(unwrap(Src))); +} +extern "C" LLVMValueRef LLVMRustBuildVectorReduceXor(LLVMBuilderRef B, + LLVMValueRef Src) { + return wrap(unwrap(B)->CreateXorReduce(unwrap(Src))); +} +extern "C" LLVMValueRef LLVMRustBuildVectorReduceMin(LLVMBuilderRef B, + LLVMValueRef Src, + bool IsSigned) { + return wrap(unwrap(B)->CreateIntMinReduce(unwrap(Src), IsSigned)); +} +extern "C" LLVMValueRef LLVMRustBuildVectorReduceMax(LLVMBuilderRef B, + LLVMValueRef Src, + bool IsSigned) { + return wrap(unwrap(B)->CreateIntMaxReduce(unwrap(Src), IsSigned)); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceFMin(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { + Instruction *I = unwrap(B)->CreateFPMinReduce(unwrap(Src)); + I->setHasNoNaNs(NoNaN); + return wrap(I); +} +extern "C" LLVMValueRef +LLVMRustBuildVectorReduceFMax(LLVMBuilderRef B, LLVMValueRef Src, bool NoNaN) { + Instruction *I = unwrap(B)->CreateFPMaxReduce(unwrap(Src)); + I->setHasNoNaNs(NoNaN); + return wrap(I); +} + +extern "C" LLVMValueRef LLVMRustBuildMinNum(LLVMBuilderRef B, LLVMValueRef LHS, + LLVMValueRef RHS) { + return wrap(unwrap(B)->CreateMinNum(unwrap(LHS), unwrap(RHS))); +} +extern "C" LLVMValueRef LLVMRustBuildMaxNum(LLVMBuilderRef B, LLVMValueRef LHS, + LLVMValueRef RHS) { + return wrap(unwrap(B)->CreateMaxNum(unwrap(LHS), unwrap(RHS))); +} + +#if LLVM_VERSION_LT(19, 0) +enum { + LLVMGEPFlagInBounds = (1 << 0), + LLVMGEPFlagNUSW = (1 << 1), + LLVMGEPFlagNUW = (1 << 2), +}; +extern "C" LLVMValueRef +LLVMBuildGEPWithNoWrapFlags(LLVMBuilderRef B, LLVMTypeRef Ty, + LLVMValueRef Pointer, LLVMValueRef *Indices, + unsigned NumIndices, const char *Name, + unsigned NoWrapFlags) { + if (NoWrapFlags & LLVMGEPFlagInBounds) + return LLVMBuildInBoundsGEP2(B, Ty, Pointer, Indices, NumIndices, Name); + else + return LLVMBuildGEP2(B, Ty, Pointer, Indices, NumIndices, Name); +} +#endif + +// Transfers ownership of DiagnosticHandler unique_ptr to the caller. +extern "C" DiagnosticHandler * +LLVMRustContextGetDiagnosticHandler(LLVMContextRef C) { + std::unique_ptr DH = unwrap(C)->getDiagnosticHandler(); + return DH.release(); +} + +// Sets unique_ptr to object of DiagnosticHandler to provide custom diagnostic +// handling. Ownership of the handler is moved to the LLVMContext. +extern "C" void LLVMRustContextSetDiagnosticHandler(LLVMContextRef C, + DiagnosticHandler *DH) { + unwrap(C)->setDiagnosticHandler(std::unique_ptr(DH)); +} + +using LLVMDiagnosticHandlerTy = DiagnosticHandler::DiagnosticHandlerTy; + +// Configures a diagnostic handler that invokes provided callback when a +// backend needs to emit a diagnostic. +// +// When RemarkAllPasses is true, remarks are enabled for all passes. Otherwise +// the RemarkPasses array specifies individual passes for which remarks will be +// enabled. +// +// If RemarkFilePath is not NULL, optimization remarks will be streamed directly +// into this file, bypassing the diagnostics handler. +extern "C" void LLVMRustContextConfigureDiagnosticHandler( + LLVMContextRef C, LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, + void *DiagnosticHandlerContext, bool RemarkAllPasses, + const char *const *RemarkPasses, size_t RemarkPassesLen, + const char *RemarkFilePath, bool PGOAvailable) { + + class RustDiagnosticHandler final : public DiagnosticHandler { + public: + RustDiagnosticHandler( + LLVMDiagnosticHandlerTy DiagnosticHandlerCallback, + void *DiagnosticHandlerContext, bool RemarkAllPasses, + std::vector RemarkPasses, + std::unique_ptr RemarksFile, + std::unique_ptr RemarkStreamer, + std::unique_ptr LlvmRemarkStreamer) + : DiagnosticHandlerCallback(DiagnosticHandlerCallback), + DiagnosticHandlerContext(DiagnosticHandlerContext), + RemarkAllPasses(RemarkAllPasses), + RemarkPasses(std::move(RemarkPasses)), + RemarksFile(std::move(RemarksFile)), + RemarkStreamer(std::move(RemarkStreamer)), + LlvmRemarkStreamer(std::move(LlvmRemarkStreamer)) {} + + virtual bool handleDiagnostics(const DiagnosticInfo &DI) override { + // If this diagnostic is one of the optimization remark kinds, we can + // check if it's enabled before emitting it. This can avoid many + // short-lived allocations when unpacking the diagnostic and converting + // its various C++ strings into rust strings. + // FIXME: some diagnostic infos still allocate before we get here, and + // avoiding that would be good in the future. That will require changing a + // few call sites in LLVM. + if (auto *OptDiagBase = dyn_cast(&DI)) { + if (OptDiagBase->isEnabled()) { + if (this->LlvmRemarkStreamer) { + this->LlvmRemarkStreamer->emit(*OptDiagBase); + return true; + } + } else { + return true; + } + } + if (DiagnosticHandlerCallback) { +#if LLVM_VERSION_GE(19, 0) + DiagnosticHandlerCallback(&DI, DiagnosticHandlerContext); +#else + DiagnosticHandlerCallback(DI, DiagnosticHandlerContext); +#endif + return true; + } + return false; + } + + bool isAnalysisRemarkEnabled(StringRef PassName) const override { + return isRemarkEnabled(PassName); + } + + bool isMissedOptRemarkEnabled(StringRef PassName) const override { + return isRemarkEnabled(PassName); + } + + bool isPassedOptRemarkEnabled(StringRef PassName) const override { + return isRemarkEnabled(PassName); + } + + bool isAnyRemarkEnabled() const override { + return RemarkAllPasses || !RemarkPasses.empty(); + } + + private: + bool isRemarkEnabled(StringRef PassName) const { + if (RemarkAllPasses) + return true; + + for (auto &Pass : RemarkPasses) + if (Pass == PassName) + return true; + + return false; + } + + LLVMDiagnosticHandlerTy DiagnosticHandlerCallback = nullptr; + void *DiagnosticHandlerContext = nullptr; + + bool RemarkAllPasses = false; + std::vector RemarkPasses; + + // Since LlvmRemarkStreamer contains a pointer to RemarkStreamer, the + // ordering of the three members below is important. + std::unique_ptr RemarksFile; + std::unique_ptr RemarkStreamer; + std::unique_ptr LlvmRemarkStreamer; + }; + + std::vector Passes; + for (size_t I = 0; I != RemarkPassesLen; ++I) { + Passes.push_back(RemarkPasses[I]); + } + + // We need to hold onto both the streamers and the opened file + std::unique_ptr RemarkFile; + std::unique_ptr RemarkStreamer; + std::unique_ptr LlvmRemarkStreamer; + + if (RemarkFilePath != nullptr) { + if (PGOAvailable) { + // Enable PGO hotness data for remarks, if available + unwrap(C)->setDiagnosticsHotnessRequested(true); + } + + std::error_code EC; + RemarkFile = std::make_unique( + RemarkFilePath, EC, llvm::sys::fs::OF_TextWithCRLF); + if (EC) { + std::string Error = std::string("Cannot create remark file: ") + + toString(errorCodeToError(EC)); + report_fatal_error(Twine(Error)); + } + + // Do not delete the file after we gather remarks + RemarkFile->keep(); + + auto RemarkSerializer = remarks::createRemarkSerializer( + llvm::remarks::Format::YAML, remarks::SerializerMode::Separate, + RemarkFile->os()); + if (Error E = RemarkSerializer.takeError()) { + std::string Error = std::string("Cannot create remark serializer: ") + + toString(std::move(E)); + report_fatal_error(Twine(Error)); + } + RemarkStreamer = std::make_unique( + std::move(*RemarkSerializer)); + LlvmRemarkStreamer = std::make_unique(*RemarkStreamer); + } + + unwrap(C)->setDiagnosticHandler(std::make_unique( + DiagnosticHandlerCallback, DiagnosticHandlerContext, RemarkAllPasses, + Passes, std::move(RemarkFile), std::move(RemarkStreamer), + std::move(LlvmRemarkStreamer))); +} + +extern "C" void LLVMRustGetMangledName(LLVMValueRef V, RustStringRef Str) { + auto OS = RawRustStringOstream(Str); + GlobalValue *GV = unwrap(V); + Mangler().getNameWithPrefix(OS, GV, true); +} + +extern "C" int32_t LLVMRustGetElementTypeArgIndex(LLVMValueRef CallSite) { + auto *CB = unwrap(CallSite); + switch (CB->getIntrinsicID()) { + case Intrinsic::arm_ldrex: + return 0; + case Intrinsic::arm_strex: + return 1; + } + return -1; +} + +extern "C" bool LLVMRustIsNonGVFunctionPointerTy(LLVMValueRef V) { + if (unwrap(V)->getType()->isPointerTy()) { + if (auto *GV = dyn_cast(unwrap(V))) { + if (GV->getValueType()->isFunctionTy()) + return false; + } + return true; + } + return false; +} + +extern "C" bool LLVMRustLLVMHasZlibCompressionForDebugSymbols() { + return llvm::compression::zlib::isAvailable(); +} + +extern "C" bool LLVMRustLLVMHasZstdCompressionForDebugSymbols() { + return llvm::compression::zstd::isAvailable(); +} + +extern "C" void LLVMRustSetNoSanitizeAddress(LLVMValueRef Global) { + GlobalValue &GV = *unwrap(Global); + GlobalValue::SanitizerMetadata MD; + if (GV.hasSanitizerMetadata()) + MD = GV.getSanitizerMetadata(); + MD.NoAddress = true; + MD.IsDynInit = false; + GV.setSanitizerMetadata(MD); +} + +extern "C" void LLVMRustSetNoSanitizeHWAddress(LLVMValueRef Global) { + GlobalValue &GV = *unwrap(Global); + GlobalValue::SanitizerMetadata MD; + if (GV.hasSanitizerMetadata()) + MD = GV.getSanitizerMetadata(); + MD.NoHWAddress = true; + GV.setSanitizerMetadata(MD); +} + +// Operations on composite constants. +// These are clones of LLVM api functions that will become available in future +// releases. They can be removed once Rust's minimum supported LLVM version +// supports them. See https://github.com/rust-lang/rust/issues/121868 See +// https://llvm.org/doxygen/group__LLVMCCoreValueConstantComposite.html + +// FIXME: Remove when Rust's minimum supported LLVM version reaches 19. +// https://github.com/llvm/llvm-project/commit/e1405e4f71c899420ebf8262d5e9745598419df8 +#if LLVM_VERSION_LT(19, 0) +extern "C" LLVMValueRef LLVMConstStringInContext2(LLVMContextRef C, + const char *Str, + size_t Length, + bool DontNullTerminate) { + return wrap(ConstantDataArray::getString(*unwrap(C), StringRef(Str, Length), + !DontNullTerminate)); +} +#endif + +// TODO: not standard +extern "C" void LLVMRustAddDereferenceableOrNullAttr(LLVMValueRef Fn, + unsigned Index, + uint64_t Bytes) { + Function *A = unwrap(Fn); + LLVMContext &Ctx = A->getContext(); + + // In LLVM 19, AttrBuilder requires LLVMContext + AttrBuilder B(Ctx); + B.addDereferenceableOrNullAttr(Bytes); + + // In LLVM 19, use addFnAttr or addParamAttr instead of addAttributes + if (Index == AttributeList::FunctionIndex) { + A->addFnAttrs(B); + } else { + // For parameter attributes, use addParamAttrs + A->addParamAttrs(Index - 1, B); // Index is 1-based for parameters + } +} + +// TODO: not standard +extern "C" void LLVMRustAddDereferenceableAttr(LLVMValueRef Fn, unsigned Index, + uint64_t Bytes) { + Function *A = unwrap(Fn); + LLVMContext &Ctx = A->getContext(); + + // In LLVM 19, AttrBuilder requires LLVMContext + AttrBuilder B(Ctx); + B.addDereferenceableAttr(Bytes); + + // In LLVM 19, use addFnAttrs or addParamAttrs instead of addAttributes + if (Index == AttributeList::FunctionIndex) { + A->addFnAttrs(B); + } else { + // For parameter attributes, use addParamAttrs + A->addParamAttrs(Index - 1, B); // Index is 1-based for parameters + } +} + +// TODO: not standard +extern "C" void LLVMRustAddAlignmentAttr(LLVMValueRef Fn, + unsigned Index, + uint32_t Bytes) { + Function *A = unwrap(Fn); + LLVMContext &Ctx = A->getContext(); + + // In LLVM 19, AttrBuilder requires LLVMContext + AttrBuilder B(Ctx); + B.addAlignmentAttr(Bytes); + + // In LLVM 19, use addFnAttrs or addParamAttrs instead of addAttributes + if (Index == AttributeList::FunctionIndex) { + A->addFnAttrs(B); + } else { + // For parameter attributes, use addParamAttrs + A->addParamAttrs(Index - 1, B); // Index is 1-based for parameters + } +} + +// TODO: non standard +enum LLVMRustAttribute +{ + AlwaysInline = 0, + ByVal = 1, + Cold = 2, + InlineHint = 3, + MinSize = 4, + Naked = 5, + NoAlias = 6, + NoCapture = 7, + NoInline = 8, + NonNull = 9, + NoRedZone = 10, + NoReturn = 11, + NoUnwind = 12, + OptimizeForSize = 13, + OptimizeNone = 14, + ReadOnly = 15, + SExt = 16, + StructRet = 17, + UWTable = 18, + ZExt = 19, + InReg = 20, + SanitizeThread = 21, + SanitizeAddress = 22, + SanitizeMemory = 23, + ReadNone = 24 +}; + +// TODO: non-standard +static Attribute::AttrKind attrKindFromRust(LLVMRustAttribute Kind) +{ + switch (Kind) + { + case AlwaysInline: + return Attribute::AlwaysInline; + case ByVal: + report_fatal_error("ByVal not supported - needs type parameter in LLVM 19+"); + case Cold: + return Attribute::Cold; + case InlineHint: + return Attribute::InlineHint; + case MinSize: + return Attribute::MinSize; + case Naked: + return Attribute::Naked; + case NoAlias: + return Attribute::NoAlias; + case NoCapture: + return Attribute::NoCapture; + case NoInline: + return Attribute::NoInline; + case NonNull: + return Attribute::NonNull; + case NoRedZone: + return Attribute::NoRedZone; + case NoReturn: + return Attribute::NoReturn; + case NoUnwind: + return Attribute::NoUnwind; + case OptimizeForSize: + return Attribute::OptimizeForSize; + case OptimizeNone: + return Attribute::OptimizeNone; + case ReadOnly: + return Attribute::ReadOnly; + case SExt: + return Attribute::SExt; + case StructRet: + report_fatal_error("StructRet not supported - needs type parameter in LLVM 19+"); + case UWTable: + return Attribute::UWTable; + case ZExt: + return Attribute::ZExt; + case InReg: + return Attribute::InReg; + case SanitizeThread: + return Attribute::SanitizeThread; + case SanitizeAddress: + return Attribute::SanitizeAddress; + case SanitizeMemory: + return Attribute::SanitizeMemory; + case ReadNone: + return Attribute::ReadNone; + } + report_fatal_error("bad AttributeKind"); +} + +// TODO: non standard +extern "C" void LLVMRustAddFunctionAttribute(LLVMValueRef Fn, unsigned Index, + LLVMRustAttribute RustAttr) { + Function *F = unwrap(Fn); + LLVMContext &Ctx = F->getContext(); + + Attribute::AttrKind Kind = attrKindFromRust(RustAttr); + Attribute Attr = Attribute::get(Ctx, Kind); + LLVMAttributeRef AttrRef = wrap(Attr); + + // Now just call the existing function that handles a single attribute + AddAttributes(F, Index, &AttrRef, 1); +} + +// TODO: non-standard +extern "C" void LLVMRustAddFunctionAttributeWithType(LLVMValueRef Fn, unsigned Index, + LLVMRustAttribute RustAttr, + LLVMTypeRef Ty) { + Function *F = unwrap(Fn); + LLVMContext &Ctx = F->getContext(); + + Attribute Attr; + if (RustAttr == ByVal) { + Attr = Attribute::getWithByValType(Ctx, unwrap(Ty)); + } else if (RustAttr == StructRet) { + Attr = Attribute::getWithStructRetType(Ctx, unwrap(Ty)); + } else { + Attribute::AttrKind Kind = attrKindFromRust(RustAttr); + Attr = Attribute::get(Ctx, Kind); + } + LLVMAttributeRef AttrRef = wrap(Attr); + AddAttributes(F, Index, &AttrRef, 1); +} + +// TODO: non-standard +extern "C" void LLVMRustAddDereferenceableOrNullCallSiteAttr(LLVMValueRef Instr, + unsigned Index, + uint64_t Bytes) { + CallBase *Call = cast(unwrap(Instr)); + LLVMContext &Ctx = Call->getContext(); + + // In LLVM 19, AttrBuilder requires LLVMContext + AttrBuilder B(Ctx); + B.addDereferenceableOrNullAttr(Bytes); + + // In LLVM 19, CallSite is deprecated, use CallBase and addParamAttrs/addFnAttrs + AttributeList Attrs = Call->getAttributes(); + if (Index == AttributeList::FunctionIndex) { + Attrs = Attrs.addFnAttributes(Ctx, B); + } else { + Attrs = Attrs.addParamAttributes(Ctx, Index - 1, B); // Index is 1-based for parameters + } + Call->setAttributes(Attrs); +} + +// TODO: non-standard +extern "C" void LLVMRustAddDereferenceableCallSiteAttr(LLVMValueRef Instr, + unsigned Index, + uint64_t Bytes) { + CallBase *Call = cast(unwrap(Instr)); + LLVMContext &Ctx = Call->getContext(); + + // In LLVM 19, AttrBuilder requires LLVMContext + AttrBuilder B(Ctx); + B.addDereferenceableAttr(Bytes); + + // In LLVM 19, CallSite is deprecated, use CallBase and addParamAttrs/addFnAttrs + AttributeList Attrs = Call->getAttributes(); + if (Index == AttributeList::FunctionIndex) { + Attrs = Attrs.addFnAttributes(Ctx, B); + } else { + Attrs = Attrs.addParamAttributes(Ctx, Index - 1, B); // Index is 1-based for parameters + } + Call->setAttributes(Attrs); +} + +// TODO: non-standard +extern "C" void LLVMRustAddAlignmentCallSiteAttr(LLVMValueRef Instr, + unsigned Index, + uint32_t Bytes) { + CallBase *Call = cast(unwrap(Instr)); + LLVMContext &Ctx = Call->getContext(); + + // In LLVM 19, AttrBuilder requires LLVMContext + AttrBuilder B(Ctx); + B.addAlignmentAttr(Bytes); + + // In LLVM 19, CallSite is deprecated, use CallBase and addParamAttrs/addFnAttrs + AttributeList Attrs = Call->getAttributes(); + if (Index == AttributeList::FunctionIndex) { + Attrs = Attrs.addFnAttributes(Ctx, B); + } else { + Attrs = Attrs.addParamAttributes(Ctx, Index - 1, B); // Index is 1-based for parameters + } + Call->setAttributes(Attrs); +} + +// TODO: non-standard +extern "C" void LLVMRustAddCallSiteAttribute(LLVMValueRef Instr, unsigned Index, + LLVMRustAttribute RustAttr) { + CallBase *Call = cast(unwrap(Instr)); + LLVMContext &Ctx = Call->getContext(); + + Attribute Attr = Attribute::get(Ctx, attrKindFromRust(RustAttr)); + AttrBuilder B(Ctx); + B.addAttribute(Attr); + + // In LLVM 19, CallSite is deprecated, use CallBase and addParamAttrs/addFnAttrs + AttributeList Attrs = Call->getAttributes(); + if (Index == AttributeList::FunctionIndex) { + Attrs = Attrs.addFnAttributes(Ctx, B); + } else { + Attrs = Attrs.addParamAttributes(Ctx, Index - 1, B); // Index is 1-based for parameters + } + Call->setAttributes(Attrs); +} + +// TODO: non-standard +extern "C" LLVMTypeRef LLVMRustArrayType(LLVMTypeRef ElementTy, + uint64_t ElementCount) +{ + return wrap(ArrayType::get(unwrap(ElementTy), ElementCount)); +} + +// TODO: non-standard +extern "C" void LLVMRustRemoveFunctionAttributes(LLVMValueRef Fn, + unsigned Index, + LLVMRustAttribute RustAttr) { + Function *F = unwrap(Fn); + LLVMContext &Ctx = F->getContext(); + + Attribute Attr = Attribute::get(Ctx, attrKindFromRust(RustAttr)); + + // In LLVM 19, use removeAttributeAtIndex with the attribute kind enum + AttributeList PAL = F->getAttributes(); + AttributeList PALNew = PAL.removeAttributeAtIndex(Ctx, Index, Attr.getKindAsEnum()); + F->setAttributes(PALNew); +} + +// TODO: non-standard +extern "C" void LLVMRustAppendModuleInlineAsm(LLVMModuleRef M, const char *Asm, size_t AsmLen) +{ + unwrap(M)->appendModuleInlineAsm(StringRef(Asm, AsmLen)); +} + +// TODO: non-standard +enum class LLVMRustLinkage +{ + ExternalLinkage = 0, + AvailableExternallyLinkage = 1, + LinkOnceAnyLinkage = 2, + LinkOnceODRLinkage = 3, + WeakAnyLinkage = 4, + WeakODRLinkage = 5, + AppendingLinkage = 6, + InternalLinkage = 7, + PrivateLinkage = 8, + ExternalWeakLinkage = 9, + CommonLinkage = 10, +}; + +// TODO: non-standard +static LLVMLinkage linkageFromRust(LLVMRustLinkage Linkage) +{ + switch (Linkage) + { + case LLVMRustLinkage::ExternalLinkage: + return LLVMExternalLinkage; + case LLVMRustLinkage::AvailableExternallyLinkage: + return LLVMAvailableExternallyLinkage; + case LLVMRustLinkage::LinkOnceAnyLinkage: + return LLVMLinkOnceAnyLinkage; + case LLVMRustLinkage::LinkOnceODRLinkage: + return LLVMLinkOnceODRLinkage; + case LLVMRustLinkage::WeakAnyLinkage: + return LLVMWeakAnyLinkage; + case LLVMRustLinkage::WeakODRLinkage: + return LLVMWeakODRLinkage; + case LLVMRustLinkage::AppendingLinkage: + return LLVMAppendingLinkage; + case LLVMRustLinkage::InternalLinkage: + return LLVMInternalLinkage; + case LLVMRustLinkage::PrivateLinkage: + return LLVMPrivateLinkage; + case LLVMRustLinkage::ExternalWeakLinkage: + return LLVMExternalWeakLinkage; + case LLVMRustLinkage::CommonLinkage: + return LLVMCommonLinkage; + } + report_fatal_error("Invalid LLVMRustLinkage value!"); +} + +// TODO: non-standard +static LLVMRustLinkage linkageToRust(LLVMLinkage Linkage) { + switch (Linkage) { + case LLVMExternalLinkage: + return LLVMRustLinkage::ExternalLinkage; + case LLVMAvailableExternallyLinkage: + return LLVMRustLinkage::AvailableExternallyLinkage; + case LLVMLinkOnceAnyLinkage: + return LLVMRustLinkage::LinkOnceAnyLinkage; + case LLVMLinkOnceODRLinkage: + return LLVMRustLinkage::LinkOnceODRLinkage; + case LLVMWeakAnyLinkage: + return LLVMRustLinkage::WeakAnyLinkage; + case LLVMWeakODRLinkage: + return LLVMRustLinkage::WeakODRLinkage; + case LLVMAppendingLinkage: + return LLVMRustLinkage::AppendingLinkage; + case LLVMInternalLinkage: + return LLVMRustLinkage::InternalLinkage; + case LLVMPrivateLinkage: + return LLVMRustLinkage::PrivateLinkage; + case LLVMExternalWeakLinkage: + return LLVMRustLinkage::ExternalWeakLinkage; + case LLVMCommonLinkage: + return LLVMRustLinkage::CommonLinkage; + default: + report_fatal_error("Invalid LLVMRustLinkage value!"); + } +} + +// TODO: non-standard +extern "C" void LLVMRustSetLinkage(LLVMValueRef V, + LLVMRustLinkage RustLinkage) +{ + LLVMSetLinkage(V, linkageFromRust(RustLinkage)); +} + +// TODO: non-standard +extern "C" LLVMRustLinkage LLVMRustGetLinkage(LLVMValueRef V) { + return linkageToRust(LLVMGetLinkage(V)); +} + +// TODO: non-standard +enum class LLVMRustVisibility +{ + Default = 0, + Hidden = 1, + Protected = 2, +}; + +// TODO: non-standard +static LLVMRustVisibility visibilityToRust(LLVMVisibility Vis) { + switch (Vis) { + case LLVMDefaultVisibility: + return LLVMRustVisibility::Default; + case LLVMHiddenVisibility: + return LLVMRustVisibility::Hidden; + case LLVMProtectedVisibility: + return LLVMRustVisibility::Protected; + } + report_fatal_error("Invalid LLVMRustVisibility value!"); +} + + +// TODO: non-standard +static LLVMVisibility visibilityFromRust(LLVMRustVisibility Vis) +{ + switch (Vis) + { + case LLVMRustVisibility::Default: + return LLVMDefaultVisibility; + case LLVMRustVisibility::Hidden: + return LLVMHiddenVisibility; + case LLVMRustVisibility::Protected: + return LLVMProtectedVisibility; + } + report_fatal_error("Invalid LLVMRustVisibility value!"); +} + +// TODO: non-standard +extern "C" void LLVMRustSetVisibility(LLVMValueRef V, + LLVMRustVisibility RustVisibility) +{ + LLVMSetVisibility(V, visibilityFromRust(RustVisibility)); +} + +// TODO: non-standard +extern "C" LLVMRustVisibility LLVMRustGetVisibility(LLVMValueRef V) { + return visibilityToRust(LLVMGetVisibility(V)); +} + +// copied from https://github.com/rust-lang/rust/tree/d61f55d8b9d4703207a5980f27b6c28973ba27ee +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateDebugLocation(unsigned Line, unsigned Column, + LLVMMetadataRef ScopeRef, + LLVMMetadataRef InlinedAt) { + MDNode *Scope = unwrapDIPtr(ScopeRef); + DILocation *Loc = DILocation::get(Scope->getContext(), Line, Column, Scope, + unwrapDIPtr(InlinedAt)); + return wrap(Loc); +} + +// TODO: non-standard +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateLexicalBlockFile(LLVMDIBuilderRef Builder, + LLVMMetadataRef Scope, + LLVMMetadataRef File) { + return wrap(unwrap(Builder)->createLexicalBlockFile(unwrapDI(Scope), + unwrapDI(File))); +} + +// TODO: non-standard +extern "C" void LLVMRustDIBuilderFinalize(LLVMDIBuilderRef Builder) { + unwrap(Builder)->finalize(); +} + +// TODO: non-standard +extern "C" void LLVMRustAddModuleFlag(LLVMModuleRef M, const char *Name, + uint32_t Value) { + unwrap(M)->addModuleFlag(Module::ModFlagBehavior::Warning, Name, Value); +} + +// TODO: non-standard +extern "C" LLVMTypeRef LLVMRustMetadataTypeInContext(LLVMContextRef C) { + return wrap(Type::getMetadataTy(*unwrap(C))); +} + +// TODO: non-standard +extern "C" LLVMTypeRef LLVMRustGetFunctionType(LLVMValueRef V) +{ + if (auto *F = dyn_cast(unwrap(V))) { + return wrap(F->getFunctionType()); + } + return nullptr; // or handle error appropriately +} + +// TODO: non-standard +extern "C" LLVMValueRef LLVMRustBuildIntCast(LLVMBuilderRef B, LLVMValueRef Val, + LLVMTypeRef DestTy, bool isSigned) +{ + auto *Builder = unwrap(B); + auto *Value = unwrap(Val); + auto *Type = unwrap(DestTy); + + return wrap(Builder->CreateIntCast(Value, Type, isSigned, "")); +} + +// TODO: non-standard + +// OpBundlesIndirect is an array of pointers (*not* a pointer to an array). +extern "C" LLVMValueRef LLVMRustBuildCall(LLVMBuilderRef B, LLVMTypeRef Ty, + LLVMValueRef Fn, LLVMValueRef *Args, + unsigned NumArgs, + OperandBundleDef **OpBundlesIndirect, + unsigned NumOpBundles) { + Value *Callee = unwrap(Fn); + FunctionType *FTy = unwrap(Ty); + + // FIXME: Is there a way around this? + SmallVector OpBundles; + OpBundles.reserve(NumOpBundles); + for (unsigned i = 0; i < NumOpBundles; ++i) { + OpBundles.push_back(*OpBundlesIndirect[i]); + } + + return wrap(unwrap(B)->CreateCall(FTy, Callee, + ArrayRef(unwrap(Args), NumArgs), + ArrayRef(OpBundles))); +} + +// TODO: non-standard +extern "C" LLVMValueRef LLVMRustMetadataAsValue(LLVMContextRef C, LLVMMetadataRef MD) { + fprintf(stderr, "=== LLVMRustMetadataAsValue Debug ===\n"); + fprintf(stderr, "Context: %p\n", C); + fprintf(stderr, "Metadata: %p\n", MD); + + if (C) { + fprintf(stderr, "Context unwrapped: %p\n", unwrap(C)); + } else { + fprintf(stderr, "ERROR: Context is null!\n"); + } + + if (MD) { + fprintf(stderr, "Metadata unwrapped: %p\n", unwrap(MD)); + // Check if metadata is valid + auto metadata = unwrap(MD); + if (metadata) { + fprintf(stderr, "Metadata ID: %u\n", metadata->getMetadataID()); + } else { + fprintf(stderr, "ERROR: Metadata unwrap returned null!\n"); + } + } else { + fprintf(stderr, "ERROR: Metadata is null!\n"); + } + + fflush(stderr); + + return wrap(MetadataAsValue::get(*unwrap(C), unwrap(MD))); +} + +// TODO: non-standard +typedef DIBuilder *LLVMRustDIBuilderRef; +extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateLexicalBlock( + LLVMRustDIBuilderRef Builder, LLVMMetadataRef Scope, + LLVMMetadataRef File, unsigned Line, unsigned Col) +{ + return wrap(Builder->createLexicalBlock(unwrapDI(Scope), + unwrapDI(File), Line, Col)); +} + +// TODO: non-standard +extern "C" LLVMMetadataRef +LLVMRustDIBuilderCreateNameSpace(LLVMRustDIBuilderRef Builder, + LLVMMetadataRef Scope, const char *Name, size_t NameLen) +{ + return wrap(Builder->createNameSpace( + unwrapDI(Scope), // Changed from DIDescriptor to DIScope + StringRef(Name, NameLen), + false // ExportSymbols (always present in v19) + )); +} diff --git a/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/SuppressLLVMWarnings.h b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/SuppressLLVMWarnings.h new file mode 100644 index 00000000..a45d580f --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/rustc_llvm_wrapper/SuppressLLVMWarnings.h @@ -0,0 +1,17 @@ +#ifndef _rustc_llvm_SuppressLLVMWarnings_h +#define _rustc_llvm_SuppressLLVMWarnings_h + +// LLVM currently generates many warnings when compiled using MSVC. These +// warnings make it difficult to diagnose real problems when working on C++ +// code, so we suppress them. + +#ifdef _MSC_VER +#pragma warning(disable : 4530) // C++ exception handler used, but unwind + // semantics are not enabled. +#pragma warning( \ + disable : 4624) // 'xxx': destructor was implicitly defined as deleted +#pragma warning( \ + disable : 4244) // conversion from 'xxx' to 'yyy', possible loss of data +#endif + +#endif // _rustc_llvm_SuppressLLVMWarnings_h \ No newline at end of file diff --git a/crates/rustc_codegen_nvvm_v19/src/abi.rs b/crates/rustc_codegen_nvvm_v19/src/abi.rs new file mode 100644 index 00000000..ee9454ee --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/abi.rs @@ -0,0 +1,703 @@ +use std::cmp; + +use libc::c_uint; +use rustc_abi::BackendRepr::Scalar; +use rustc_abi::Size; +use rustc_abi::{HasDataLayout, Primitive, Reg, RegKind}; +use rustc_codegen_ssa::mir::operand::OperandRef; +use rustc_codegen_ssa::mir::operand::OperandValue; +use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; +use rustc_codegen_ssa::{MemFlags, traits::*}; +use rustc_middle::bug; +use rustc_middle::ty::layout::LayoutOf; +pub use rustc_middle::ty::layout::{WIDE_PTR_ADDR, WIDE_PTR_EXTRA}; +use rustc_middle::ty::{Ty, TyCtxt, TyKind}; +use rustc_target::callconv::{ + ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, Conv, FnAbi, PassMode, +}; +use tracing::trace; + +use crate::builder::Builder; +use crate::context::CodegenCx; +use crate::int_replace::{get_transformed_type, transmute_llval}; +use crate::llvm; +use crate::llvm::{AttributePlace, Type, Value}; +use crate::ty::LayoutLlvmExt; + +pub(crate) fn readjust_fn_abi<'tcx>( + tcx: TyCtxt<'tcx>, + fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, +) -> &'tcx FnAbi<'tcx, Ty<'tcx>> { + // dont override anything in the rust abi for now + if fn_abi.conv == Conv::Rust { + return fn_abi; + } + let readjust_arg_abi = |arg: &ArgAbi<'tcx, Ty<'tcx>>| { + let mut arg = ArgAbi { + layout: arg.layout, + mode: arg.mode.clone(), + }; + + // ignore zsts + if arg.layout.is_zst() { + arg.mode = PassMode::Ignore; + } + + if let TyKind::Ref(_, ty, _) = arg.layout.ty.kind() { + if matches!(ty.kind(), TyKind::Slice(_)) { + let mut ptr_attrs = ArgAttributes::new(); + if let PassMode::Indirect { attrs, .. } = arg.mode { + ptr_attrs.regular = attrs.regular; + } + arg.mode = PassMode::Pair(ptr_attrs, ArgAttributes::new()); + } + } + + if arg.layout.ty.is_array() && !matches!(arg.mode, PassMode::Direct { .. }) { + arg.mode = PassMode::Direct(ArgAttributes::new()); + } + + // pass all adts directly as values, ptx wants them to be passed all by value, but rustc's + // ptx-kernel abi seems to be wrong, and it's unstable. + if arg.layout.ty.is_adt() && !matches!(arg.mode, PassMode::Direct { .. }) { + arg.mode = PassMode::Direct(ArgAttributes::new()); + } + arg + }; + tcx.arena.alloc(FnAbi { + args: fn_abi.args.iter().map(readjust_arg_abi).collect(), + ret: readjust_arg_abi(&fn_abi.ret), + c_variadic: fn_abi.c_variadic, + fixed_count: fn_abi.fixed_count, + conv: fn_abi.conv, + can_unwind: fn_abi.can_unwind, + }) +} + +macro_rules! for_each_kind { + ($flags: ident, $f: ident, $($kind: ident),+) => ({ + $(if $flags.contains(ArgAttribute::$kind) { $f(llvm::Attribute::$kind) })+ + }) +} + +trait ArgAttributeExt { + fn for_each_kind(&self, f: F) + where + F: FnMut(llvm::Attribute); +} + +impl ArgAttributeExt for ArgAttribute { + fn for_each_kind(&self, mut f: F) + where + F: FnMut(llvm::Attribute), + { + for_each_kind!(self, f, NoAlias, NoCapture, NonNull, ReadOnly, InReg) + } +} + +pub(crate) trait ArgAttributesExt { + fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); + fn apply_attrs_to_callsite( + &self, + idx: AttributePlace, + cx: &CodegenCx<'_, '_>, + callsite: &Value, + ); +} + +impl ArgAttributesExt for ArgAttributes { + fn apply_attrs_to_llfn(&self, idx: AttributePlace, _cx: &CodegenCx<'_, '_>, llfn: &Value) { + let mut regular = self.regular; + unsafe { + let deref = self.pointee_size.bytes(); + if deref != 0 { + if regular.contains(ArgAttribute::NonNull) { + llvm::LLVMRustAddDereferenceableAttr(llfn, idx.as_uint(), deref); + } else { + llvm::LLVMRustAddDereferenceableOrNullAttr(llfn, idx.as_uint(), deref); + } + regular -= ArgAttribute::NonNull; + } + if let Some(align) = self.pointee_align { + llvm::LLVMRustAddAlignmentAttr(llfn, idx.as_uint(), align.bytes() as u32); + } + regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn)); + // TODO(RDambrosio016): Apply mutable noalias once we upgrade to LLVM 13 + match self.arg_ext { + ArgExtension::None => {} + ArgExtension::Zext => { + llvm::Attribute::ZExt.apply_llfn(idx, llfn); + } + ArgExtension::Sext => { + llvm::Attribute::SExt.apply_llfn(idx, llfn); + } + } + } + } + + fn apply_attrs_to_callsite( + &self, + idx: AttributePlace, + _cx: &CodegenCx<'_, '_>, + callsite: &Value, + ) { + let mut regular = self.regular; + unsafe { + let deref = self.pointee_size.bytes(); + if deref != 0 { + if regular.contains(ArgAttribute::NonNull) { + llvm::LLVMRustAddDereferenceableCallSiteAttr(callsite, idx.as_uint(), deref); + } else { + llvm::LLVMRustAddDereferenceableOrNullCallSiteAttr( + callsite, + idx.as_uint(), + deref, + ); + } + regular -= ArgAttribute::NonNull; + } + if let Some(align) = self.pointee_align { + llvm::LLVMRustAddAlignmentCallSiteAttr( + callsite, + idx.as_uint(), + align.bytes() as u32, + ); + } + regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite)); + match self.arg_ext { + ArgExtension::None => {} + ArgExtension::Zext => { + llvm::Attribute::ZExt.apply_callsite(idx, callsite); + } + ArgExtension::Sext => { + llvm::Attribute::SExt.apply_callsite(idx, callsite); + } + } + } + } +} + +pub(crate) trait LlvmType { + fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type; +} + +impl LlvmType for Reg { + fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { + match self.kind { + RegKind::Integer => cx.type_ix(self.size.bits()), + RegKind::Float => match self.size.bits() { + 16 => cx.type_f16(), + 32 => cx.type_f32(), + 64 => cx.type_f64(), + 128 => cx.type_f128(), + _ => bug!("unsupported float: {:?}", self), + }, + RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()), + } + } +} + +impl LlvmType for CastTarget { + fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { + let rest_ll_unit = self.rest.unit.llvm_type(cx); + let rest_count = if self.rest.total == Size::ZERO { + 0 + } else { + assert_ne!( + self.rest.unit.size, + Size::ZERO, + "total size {:?} cannot be divided into units of zero size", + self.rest.total + ); + if self.rest.total.bytes() % self.rest.unit.size.bytes() != 0 { + assert_eq!( + self.rest.unit.kind, + RegKind::Integer, + "only int regs can be split" + ); + } + self.rest + .total + .bytes() + .div_ceil(self.rest.unit.size.bytes()) + }; + + // Simplify to a single unit or an array if there's no prefix. + // This produces the same layout, but using a simpler type. + if self.prefix.iter().all(|x| x.is_none()) { + // We can't do this if is_consecutive is set and the unit would get + // split on the target. Currently, this is only relevant for i128 + // registers. + if rest_count == 1 && (!self.rest.is_consecutive || self.rest.unit != Reg::i128()) { + return rest_ll_unit; + } + + return cx.type_array(rest_ll_unit, rest_count); + } + + // Generate a struct type with the prefix and the "rest" arguments. + let prefix_args = self + .prefix + .iter() + .flat_map(|option_reg| option_reg.map(|reg| reg.llvm_type(cx))); + let rest_args = (0..rest_count).map(|_| rest_ll_unit); + let args: Vec<_> = prefix_args.chain(rest_args).collect(); + cx.type_struct(&args, false) + } +} + +impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { + fn store_fn_arg( + &mut self, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + idx: &mut usize, + dst: PlaceRef<'tcx, Self::Value>, + ) { + arg_abi.store_fn_arg(self, idx, dst) + } + fn store_arg( + &mut self, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + val: &'ll Value, + dst: PlaceRef<'tcx, &'ll Value>, + ) { + arg_abi.store(self, val, dst) + } + fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + arg_abi.memory_ty(self) + } +} + +pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> { + fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value); + fn apply_attrs_callsite<'a>(&self, bx: &mut Builder<'a, 'll, 'tcx>, callsite: &'ll Value); +} + +impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { + fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + // Ignore "extra" args when calling C variadic functions. + // Only the "fixed" args are part of the LLVM function signature. + let args = if self.c_variadic { + &self.args[..self.fixed_count as usize] + } else { + &self.args + }; + + // This capacity calculation is approximate. + let mut llargument_tys = Vec::with_capacity( + self.args.len() + + if let PassMode::Indirect { .. } = self.ret.mode { + 1 + } else { + 0 + }, + ); + + // the current index of each parameter in the function. Cant use enumerate on args because + // some pass modes pass args as multiple params, such as scalar pairs. + let mut idx = 0; + + let mut llreturn_ty = match &self.ret.mode { + PassMode::Ignore => cx.type_void(), + PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), + PassMode::Cast { cast, .. } => cast.llvm_type(cx), + PassMode::Indirect { .. } => { + idx += 1; + llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); + cx.type_void() + } + }; + + let mut transformed_types = Vec::new(); + let mut old_ret_ty = Some(llreturn_ty); + + let (new_ret, changed) = get_transformed_type(cx, llreturn_ty); + llreturn_ty = new_ret; + if !changed { + old_ret_ty = None; + } + + for arg in args { + let llarg_ty = match &arg.mode { + PassMode::Ignore => continue, + PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), + PassMode::Pair(..) => { + llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true)); + llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); + idx += 2; + continue; + } + PassMode::Indirect { + attrs: _, + meta_attrs: Some(_), + on_stack: _, + } => { + let ptr_ty = Ty::new_mut_ptr(cx.tcx, arg.layout.ty); + let ptr_layout = cx.layout_of(ptr_ty); + llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); + llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); + idx += 2; + continue; + } + PassMode::Cast { cast, pad_i32 } => { + // add padding + if *pad_i32 { + idx += 1; + llargument_tys.push(Reg::i32().llvm_type(cx)); + } + cast.llvm_type(cx) + } + PassMode::Indirect { + attrs: _, + meta_attrs: None, + on_stack: _, + } => cx.type_ptr_to(arg.memory_ty(cx)), + }; + let (new, changed) = get_transformed_type(cx, llarg_ty); + if changed { + transformed_types.push((idx, llarg_ty)); + } + llargument_tys.push(new); + idx += 1; + } + + let ty = if self.c_variadic { + cx.type_variadic_func(&llargument_tys, llreturn_ty) + } else { + cx.type_func(&llargument_tys, llreturn_ty) + }; + if !transformed_types.is_empty() || old_ret_ty.is_some() { + trace!("remapping args in {:?} to {:?}", ty, transformed_types); + cx.remapped_integer_args + .borrow_mut() + .insert(ty, (old_ret_ty, transformed_types)); + } + ty + } + + fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + unsafe { + llvm::LLVMPointerTypeInContext( + cx.llcx, + cx.data_layout().instruction_address_space.0 as c_uint, + ) + } + } + + fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { + if self.ret.layout.is_uninhabited() { + llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn); + } + + // TODO(RDambrosio016): should this always/never be applied? unwinding + // on the gpu doesnt exist. + if !self.can_unwind { + llvm::Attribute::NoUnwind.apply_llfn(llvm::AttributePlace::Function, llfn); + } + + let mut i = 0; + let mut apply = |attrs: &ArgAttributes| { + attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn); + i += 1; + i - 1 + }; + match &self.ret.mode { + PassMode::Direct(attrs) => { + attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); + } + PassMode::Indirect { + attrs, + meta_attrs: _, + on_stack, + } => { + assert!(!on_stack); + let i = apply(attrs); + llvm::Attribute::StructRet.apply_llfn(llvm::AttributePlace::Argument(i), llfn); + } + _ => {} + } + for arg in &self.args { + match &arg.mode { + PassMode::Ignore => {} + PassMode::Indirect { + attrs, + meta_attrs: None, + on_stack: true, + } => { + apply(attrs); + // TODO(RDambrosio016): we should technically apply byval here, + // llvm 7 seems to have it, but i could not find a way to apply it through the + // C++ API, so somebody more experienced in the C++ API should look at this. + // it shouldnt do anything bad since it seems to be only for optimization. + } + PassMode::Direct(attrs) + | PassMode::Indirect { + attrs, + meta_attrs: None, + on_stack: false, + } => { + apply(attrs); + } + PassMode::Indirect { + attrs, + meta_attrs: Some(extra_attrs), + on_stack, + } => { + assert!(!on_stack); + apply(attrs); + apply(extra_attrs); + } + PassMode::Pair(a, b) => { + apply(a); + apply(b); + } + PassMode::Cast { cast, pad_i32 } => { + if *pad_i32 { + apply(&ArgAttributes::new()); + } + apply(&cast.attrs); + } + } + } + } + + fn apply_attrs_callsite<'a>(&self, bx: &mut Builder<'a, 'll, 'tcx>, mut callsite: &'ll Value) { + // HACK(RDambrosio016): We sometimes lie to rustc with return values and give it a bitcast + // instead of a call. This is because we sometimes have to bitcast return types like <2 x i64> to i128. + // So we just check if the last call was remapped. + if let Some(old) = bx.cx.last_call_llfn.get() { + callsite = old; + bx.cx.last_call_llfn.set(None); + } + + let mut i = 0; + let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { + attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite); + i += 1; + i - 1 + }; + match self.ret.mode { + PassMode::Direct(attrs) => { + attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); + } + PassMode::Indirect { + attrs, + meta_attrs: _, + on_stack, + } => { + assert!(!on_stack); + apply(bx.cx, &attrs); + } + _ => {} + } + if let Scalar(scalar) = self.ret.layout.backend_repr { + // If the value is a boolean, the range is 0..2 and that ultimately + // become 0..0 when the type becomes i1, which would be rejected + // by the LLVM verifier. + if let Primitive::Int(..) = scalar.primitive() { + if !scalar.is_bool() && !scalar.is_always_valid(bx) { + trace!("apply_attrs_callsite -> range_metadata"); + bx.range_metadata(callsite, scalar.valid_range(bx)); + } + } + } + for arg in self.args.iter() { + match &arg.mode { + PassMode::Ignore => {} + PassMode::Indirect { + attrs, + meta_attrs: None, + on_stack: true, + } => { + apply(bx.cx, attrs); + } + PassMode::Direct(attrs) + | PassMode::Indirect { + attrs, + meta_attrs: None, + on_stack: false, + } => { + apply(bx.cx, attrs); + } + PassMode::Indirect { + attrs, + meta_attrs: Some(meta_attrs), + on_stack: _, + } => { + apply(bx.cx, attrs); + apply(bx.cx, meta_attrs); + } + PassMode::Pair(a, b) => { + apply(bx.cx, a); + apply(bx.cx, b); + } + PassMode::Cast { pad_i32, .. } => { + if *pad_i32 { + apply(bx.cx, &ArgAttributes::new()); + } + apply(bx.cx, &ArgAttributes::new()); + } + } + } + } +} + +impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + fn get_param(&mut self, index: usize) -> Self::Value { + let val = llvm::get_param(self.llfn(), index as c_uint); + // trace!("Get param `{:?}`", val); + let val = unsafe { + let llfnty: &Type = llvm::LLVMRustGetFunctionType(self.llfn()); + trace!("llfnty: {:?}", llfnty); + // destructure so rustc doesnt complain in the call to transmute_llval + let Self { cx, llbuilder } = self; + let map = cx.remapped_integer_args.borrow(); + if let Some((_, key)) = map.get(llfnty) { + if let Some((_, new_ty)) = key.iter().find(|t| t.0 == index) { + trace!("Casting irregular param {:?} to {:?}", val, new_ty); + return transmute_llval(llbuilder, cx, val, new_ty); + } + } + val + }; + trace!("Get param `{:?}`", val); + val + } +} + +pub(crate) trait ArgAbiExt<'ll, 'tcx> { + fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn store( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + val: &'ll Value, + dst: PlaceRef<'tcx, &'ll Value>, + ); + fn store_fn_arg( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + idx: &mut usize, + dst: PlaceRef<'tcx, &'ll Value>, + ); +} + +impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { + /// Gets the LLVM type for a place of the original Rust type of + /// this argument/return, i.e., the result of `type_of::type_of`. + fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + self.layout.llvm_type(cx) + } + + /// Stores a direct/indirect value described by this ArgAbi into a + /// place for the original Rust type of this argument/return. + /// Can be used for both storing formal arguments into Rust variables + /// or results of call/invoke instructions into their destinations. + fn store( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + val: &'ll Value, + dst: PlaceRef<'tcx, &'ll Value>, + ) { + match &self.mode { + PassMode::Ignore => {} + // Sized indirect arguments + PassMode::Indirect { + attrs, + meta_attrs: None, + on_stack: _, + } => { + let align = attrs.pointee_align.unwrap_or(self.layout.align.abi); + OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst); + } + // Unsized indirect arguments + PassMode::Indirect { + attrs: _, + meta_attrs: Some(_), + on_stack: _, + } => { + bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); + } + PassMode::Cast { pad_i32: _, cast } => { + trace!("store cast"); + // The ABI mandates that the value is passed as a different struct representation. + // Spill and reload it from the stack to convert from the ABI representation to + // the Rust representation. + let scratch_size = cast.size(bx); + let scratch_align = cast.align(bx); + // Note that the ABI type may be either larger or smaller than the Rust type, + // due to the presence or absence of trailing padding. For example: + // - On some ABIs, the Rust layout { f64, f32, } may omit padding + // when passed by value, making it smaller. + // - On some ABIs, the Rust layout { u16, u16, u16 } may be padded up to 8 bytes + // when passed by value, making it larger. + let copy_bytes = + cmp::min(cast.unaligned_size(bx).bytes(), self.layout.size.bytes()); + // Allocate some scratch space... + let llscratch = bx.alloca(scratch_size, scratch_align); + bx.lifetime_start(llscratch, scratch_size); + // ...store the value... + bx.store(val, llscratch, scratch_align); + // ... and then memcpy it to the intended destination. + bx.memcpy( + dst.val.llval, + self.layout.align.abi, + llscratch, + scratch_align, + bx.const_usize(copy_bytes), + MemFlags::empty(), + ); + + bx.lifetime_end(llscratch, scratch_size); + trace!("store cast end"); + } + _ => { + OperandRef::from_immediate_or_packed_pair(bx, val, self.layout) + .val + .store(bx, dst); + } + } + } + + fn store_fn_arg<'a>( + &self, + bx: &mut Builder<'a, 'll, 'tcx>, + idx: &mut usize, + dst: PlaceRef<'tcx, &'ll Value>, + ) { + let mut next = || { + let val = llvm::get_param(bx.llfn(), *idx as c_uint); + *idx += 1; + val + }; + match self.mode { + PassMode::Ignore => {} + PassMode::Pair(..) => { + OperandValue::Pair(next(), next()).store(bx, dst); + } + PassMode::Indirect { + attrs: _, + meta_attrs: Some(_), + on_stack: _, + } => { + let place_val = PlaceValue { + llval: next(), + llextra: Some(next()), + align: self.layout.align.abi, + }; + OperandValue::Ref(place_val).store(bx, dst); + } + PassMode::Direct(_) + | PassMode::Indirect { + attrs: _, + meta_attrs: None, + on_stack: _, + } + | PassMode::Cast { .. } => { + let next_arg = next(); + self.store(bx, next_arg, dst); + } + } + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/allocator.rs b/crates/rustc_codegen_nvvm_v19/src/allocator.rs new file mode 100644 index 00000000..de6ae978 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/allocator.rs @@ -0,0 +1,164 @@ +use crate::LlvmMod; +use crate::llvm; +use crate::llvm::{False, True}; +use crate::target; +use libc::c_uint; +use rustc_ast::expand::allocator::{ + ALLOCATOR_METHODS, AllocatorKind, AllocatorTy, alloc_error_handler_name, default_fn_name, +}; +use rustc_middle::ty::TyCtxt; + +// adapted from rustc_codegen_llvm +pub(crate) unsafe fn codegen( + _tcx: TyCtxt<'_>, + mods: &mut LlvmMod, + _module_name: &str, + kind: AllocatorKind, + alloc_error_handler_kind: AllocatorKind, +) { + let llcx = &*mods.llcx; + let llmod = unsafe { mods.llmod.as_ref().unwrap() }; + let usize = unsafe { target::usize_ty(llcx) }; + //let i8 = unsafe { llvm::LLVMInt8TypeInContext(llcx) }; + let i8p = unsafe { llvm::LLVMPointerTypeInContext(llcx, 0) }; + let void = unsafe { llvm::LLVMVoidTypeInContext(llcx) }; + + let mut used = Vec::new(); + + if kind == AllocatorKind::Default { + for method in ALLOCATOR_METHODS { + let mut args = Vec::with_capacity(method.inputs.len()); + for ty in method.inputs.iter() { + match ty.ty { + AllocatorTy::Layout => { + args.push(usize); // size + args.push(usize); // align + } + AllocatorTy::Ptr => args.push(i8p), + AllocatorTy::Usize => args.push(usize), + + AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + } + } + let output = match method.output { + AllocatorTy::ResultPtr => Some(i8p), + AllocatorTy::Unit => None, + + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; + + let ty = unsafe { + llvm::LLVMFunctionType( + output.unwrap_or(void), + args.as_ptr(), + args.len() as c_uint, + False, + ) + }; + let name = format!("__rust_{}", method.name); + let llfn = unsafe { + llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty) + }; + used.push(llfn); + // nvvm doesnt support uwtable so dont try to generate it + + let callee = default_fn_name(method.name); + let callee = unsafe { + llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty) + }; + unsafe { llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden) }; + + let llbb = unsafe { + llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()) + }; + + let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(llcx) }; + unsafe { llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb) }; + let args = args + .iter() + .enumerate() + .map(|(i, _)| unsafe { llvm::LLVMGetParam(llfn, i as c_uint) }) + .collect::>(); + let ret = unsafe { + llvm::LLVMRustBuildCall( + llbuilder, + ty, + callee, + args.as_ptr(), + args.len() as c_uint, + [].as_ptr(), + 0, + ) + }; + unsafe { llvm::LLVMSetTailCall(ret, True) }; + if output.is_some() { + unsafe { llvm::LLVMBuildRet(llbuilder, ret) }; + } else { + unsafe { llvm::LLVMBuildRetVoid(llbuilder) }; + } + unsafe { llvm::LLVMDisposeBuilder(llbuilder) }; + } + } + + // rust alloc error handler + let args = [usize, usize]; // size, align + + let ty = unsafe { llvm::LLVMFunctionType(void, args.as_ptr(), args.len() as c_uint, False) }; + let name = "__rust_alloc_error_handler".to_string(); + let llfn = unsafe { llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty) }; + used.push(llfn); + + // -> ! DIFlagNoReturn + llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn); + + let callee = alloc_error_handler_name(alloc_error_handler_kind); + let callee = unsafe { + llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty) + }; + used.push(callee); + + // -> ! DIFlagNoReturn + llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, callee); + unsafe { llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden) }; + + let llbb = unsafe { llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast()) }; + + let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(llcx) }; + unsafe { llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb) }; + let args = args + .iter() + .enumerate() + .map(|(i, _)| unsafe { llvm::LLVMGetParam(llfn, i as c_uint) }) + .collect::>(); + let ret = unsafe { + llvm::LLVMRustBuildCall( + llbuilder, + ty, + callee, + args.as_ptr(), + args.len() as c_uint, + [].as_ptr(), + 0 + ) + }; + unsafe { llvm::LLVMSetTailCall(ret, True) }; + unsafe { llvm::LLVMBuildRetVoid(llbuilder) }; + unsafe { llvm::LLVMDisposeBuilder(llbuilder) }; + + let ptr_ty = unsafe { llvm::LLVMPointerTypeInContext(llcx, 0) }; + + for used in &mut used { + *used = unsafe { llvm::LLVMConstBitCast(used, ptr_ty) }; + } + + let section = c"llvm.metadata"; + let array = unsafe { llvm::LLVMConstArray(ptr_ty, used.as_ptr(), used.len() as u32) }; + let g = unsafe { + llvm::LLVMAddGlobal(llmod, llvm::LLVMTypeOf(array), c"llvm.used".as_ptr().cast()) + }; + unsafe { llvm::LLVMSetInitializer(g, array) }; + unsafe { llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage) }; + unsafe { llvm::LLVMSetSection(g, section.as_ptr()) }; +} diff --git a/crates/rustc_codegen_nvvm_v19/src/asm.rs b/crates/rustc_codegen_nvvm_v19/src/asm.rs new file mode 100644 index 00000000..beb970a7 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/asm.rs @@ -0,0 +1,344 @@ +use std::os::raw::{c_char, c_uint}; + +use crate::common::AsCCharPtr; +use crate::llvm; +use crate::llvm::Value; +use crate::ty::LayoutLlvmExt; +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_codegen_ssa::{ + mir::operand::OperandValue, + traits::{ + AsmBuilderMethods, AsmCodegenMethods, BaseTypeCodegenMethods, BuilderMethods, + ConstCodegenMethods, GlobalAsmOperandRef, InlineAsmOperandRef, + }, +}; +use rustc_hash::FxHashMap; +use rustc_middle::{span_bug, ty::Instance}; +use rustc_span::{Pos, Span}; +use rustc_target::asm::{InlineAsmRegClass, InlineAsmRegOrRegClass, NvptxInlineAsmRegClass}; + +use crate::{builder::Builder, context::CodegenCx}; + +impl<'tcx> AsmBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + fn codegen_inline_asm( + &mut self, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperandRef<'tcx, Self>], + options: rustc_ast::InlineAsmOptions, + line_spans: &[Span], + _inst: Instance, + _dest: Option, + _catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>, + ) { + // Collect the types of output operands + let mut constraints = vec![]; + let mut output_types = vec![]; + let mut op_idx = FxHashMap::default(); + for (idx, op) in operands.iter().enumerate() { + match *op { + InlineAsmOperandRef::Out { reg, late, place } => { + let ty = if let Some(ref place) = place { + place.layout.llvm_type(self) + } else { + match reg.reg_class() { + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => { + self.type_i16() + } + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => { + self.type_i32() + } + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => { + self.type_i64() + } + _ => unreachable!(), + } + }; + output_types.push(ty); + op_idx.insert(idx, constraints.len()); + let prefix = if late { "=" } else { "=&" }; + constraints.push(format!("{}{}", prefix, reg_to_llvm(reg))); + } + InlineAsmOperandRef::InOut { + reg, + late, + in_value, + out_place, + } => { + let layout = if let Some(ref out_place) = out_place { + &out_place.layout + } else { + // LLVM required tied operands to have the same type, + // so we just use the type of the input. + &in_value.layout + }; + let ty = layout.llvm_type(self); + output_types.push(ty); + op_idx.insert(idx, constraints.len()); + let prefix = if late { "=" } else { "=&" }; + constraints.push(format!("{}{}", prefix, reg_to_llvm(reg))); + } + _ => {} + } + } + + // Collect input operands + let mut inputs = vec![]; + for (idx, op) in operands.iter().enumerate() { + match *op { + InlineAsmOperandRef::In { reg, value } => { + let llval = value.immediate(); + inputs.push(llval); + op_idx.insert(idx, constraints.len()); + constraints.push(reg_to_llvm(reg)); + } + InlineAsmOperandRef::InOut { + late: _, + in_value, + out_place: _, + .. + } => { + let value = in_value.immediate(); + inputs.push(value); + constraints.push(format!("{}", op_idx[&idx])); + } + InlineAsmOperandRef::SymFn { instance } => { + inputs.push(self.cx.get_fn(instance)); + op_idx.insert(idx, constraints.len()); + constraints.push("s".to_string()); + } + InlineAsmOperandRef::SymStatic { def_id } => { + inputs.push(self.cx.get_static(def_id)); + op_idx.insert(idx, constraints.len()); + constraints.push("s".to_string()); + } + _ => {} + } + } + + // Build the template string + let mut template_str = String::new(); + for piece in template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => { + if s.contains('$') { + for c in s.chars() { + if c == '$' { + template_str.push_str("$$"); + } else { + template_str.push(c); + } + } + } else { + template_str.push_str(s) + } + } + InlineAsmTemplatePiece::Placeholder { + operand_idx, span, .. + } => { + match operands[operand_idx] { + InlineAsmOperandRef::In { .. } + | InlineAsmOperandRef::Out { .. } + | InlineAsmOperandRef::InOut { .. } => { + template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx])); + } + InlineAsmOperandRef::Const { ref string } => { + // Const operands get injected directly into the template + template_str.push_str(string); + } + InlineAsmOperandRef::SymFn { .. } + | InlineAsmOperandRef::SymStatic { .. } => { + // Only emit the raw symbol name + template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); + } + InlineAsmOperandRef::Label { .. } => { + // template_str.push_str(&format!("${{{}:l}}", constraints.len())); + // constraints.push("!i".to_owned()); + // labels.push(label); + + self.tcx + .sess + .dcx() + .span_fatal(span, "Operands with label refs are unsupported"); + } + } + } + } + } + + if !options.contains(InlineAsmOptions::NOMEM) { + // This is actually ignored by LLVM, but it's probably best to keep + // it just in case. LLVM instead uses the ReadOnly/ReadNone + // attributes on the call instruction to optimize. + constraints.push("~{memory}".to_string()); + } + let volatile = !options.contains(InlineAsmOptions::PURE); + let alignstack = !options.contains(InlineAsmOptions::NOSTACK); + let output_type = match &output_types[..] { + [] => self.type_void(), + [ty] => ty, + tys => self.type_struct(tys, false), + }; + let dialect = llvm::AsmDialect::Att; + let result = inline_asm_call( + self, + &template_str, + &constraints.join(","), + &inputs, + output_type, + volatile, + alignstack, + dialect, + line_spans, + ) + .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); + + if options.contains(InlineAsmOptions::PURE) { + if options.contains(InlineAsmOptions::NOMEM) { + llvm::Attribute::ReadNone.apply_callsite(llvm::AttributePlace::Function, result); + } else if options.contains(InlineAsmOptions::READONLY) { + llvm::Attribute::ReadOnly.apply_callsite(llvm::AttributePlace::Function, result); + } + } + + // Write results to outputs + for (idx, op) in operands.iter().enumerate() { + if let InlineAsmOperandRef::Out { + place: Some(place), .. + } + | InlineAsmOperandRef::InOut { + out_place: Some(place), + .. + } = *op + { + let value = if output_types.len() == 1 { + result + } else { + self.extract_value(result, op_idx[&idx] as u64) + }; + OperandValue::Immediate(value).store(self, place); + } + } + } +} + +impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { + fn codegen_global_asm( + &self, + template: &[InlineAsmTemplatePiece], + operands: &[GlobalAsmOperandRef], + _options: InlineAsmOptions, + _line_spans: &[Span], + ) { + // Build the template string + let mut template_str = String::new(); + for piece in template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s), + InlineAsmTemplatePiece::Placeholder { + operand_idx, + modifier: _, + span: _, + } => { + match operands[operand_idx] { + GlobalAsmOperandRef::Const { ref string } => { + // Const operands get injected directly into the + // template. Note that we don't need to escape $ + // here unlike normal inline assembly. + template_str.push_str(string); + } + GlobalAsmOperandRef::SymFn { .. } => todo!(), + GlobalAsmOperandRef::SymStatic { .. } => todo!(), + } + } + } + } + + unsafe { + llvm::LLVMRustAppendModuleInlineAsm( + self.llmod, + template_str.as_c_char_ptr(), + template_str.len(), + ); + } + } + + fn mangled_name(&self, _instance: Instance<'tcx>) -> String { + todo!() + } +} + +fn reg_to_llvm(reg: InlineAsmRegOrRegClass) -> String { + match reg { + InlineAsmRegOrRegClass::Reg(reg) => { + format!("{{{}}}", reg.name()) + } + InlineAsmRegOrRegClass::RegClass(reg) => match reg { + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r", + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l", + _ => unreachable!(), + } + .to_string(), + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn inline_asm_call<'ll>( + bx: &mut Builder<'_, 'll, '_>, + asm: &str, + cons: &str, + inputs: &[&'ll Value], + output: &'ll llvm::Type, + volatile: bool, + alignstack: bool, + dia: llvm::AsmDialect, + line_spans: &[Span], +) -> Option<&'ll Value> { + let volatile = if volatile { llvm::True } else { llvm::False }; + let alignstack = if alignstack { llvm::True } else { llvm::False }; + + let argtys = inputs.iter().map(|v| bx.cx.val_ty(*v)).collect::>(); + + let fty = bx.cx.type_func(&argtys[..], output); + unsafe { + // Ask LLVM to verify that the constraints are well-formed. + let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); + if constraints_ok { + let v = llvm::LLVMRustInlineAsm( + fty, + asm.as_ptr().cast(), + asm.len(), + cons.as_ptr().cast(), + cons.len(), + volatile, + alignstack, + dia, + llvm::False, + ); + let call = bx.call(fty, None, None, v, inputs, None, None); + + // Store mark in a metadata node so we can map LLVM errors + // back to source locations. See #17552. + let key = "srcloc"; + let kind = llvm::LLVMGetMDKindIDInContext( + bx.llcx, + key.as_ptr() as *const c_char, + key.len() as c_uint, + ); + + let mut srcloc = vec![]; + srcloc.extend( + line_spans + .iter() + .map(|span| bx.const_i32(span.lo().to_u32() as i32)), + ); + let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32); + llvm::LLVMSetMetadata(call, kind, md); + + Some(call) + } else { + // LLVM has detected an issue with our constraints, bail out + None + } + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/attributes.rs b/crates/rustc_codegen_nvvm_v19/src/attributes.rs new file mode 100644 index 00000000..0f328c5f --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/attributes.rs @@ -0,0 +1,139 @@ +use crate::llvm; +use crate::llvm::{Value, AttributePlace::Function}; +use rustc_ast::{LitKind, MetaItemInner, MetaItemLit}; +use rustc_attr_parsing::{InlineAttr, OptimizeAttr}; +use rustc_hir::Attribute; +use rustc_middle::{bug, middle::codegen_fn_attrs::CodegenFnAttrFlags, ty}; +use rustc_session::{Session, config::OptLevel}; +use rustc_span::{Symbol, sym}; + +use crate::context::CodegenCx; + +#[inline] // so meta +fn inline(val: &'_ Value, inline: InlineAttr) { + use InlineAttr::*; + match inline { + Hint => llvm::Attribute::InlineHint.apply_llfn(Function, val), + Always => llvm::Attribute::AlwaysInline.apply_llfn(Function, val), + Never => llvm::Attribute::NoInline.apply_llfn(Function, val), + None => {} + Force { .. } => bug!("Force inline should have been inlined away by now"), // TODO: Verify this + } +} + +pub(crate) fn default_optimisation_attrs(sess: &Session, llfn: &'_ Value) { + match sess.opts.optimize { + OptLevel::Size => { + llvm::Attribute::MinSize.unapply_llfn(Function, llfn); + llvm::Attribute::OptimizeForSize.apply_llfn(Function, llfn); + llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn); + } + OptLevel::SizeMin => { + llvm::Attribute::MinSize.apply_llfn(Function, llfn); + llvm::Attribute::OptimizeForSize.apply_llfn(Function, llfn); + llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn); + } + OptLevel::No => { + llvm::Attribute::MinSize.unapply_llfn(Function, llfn); + llvm::Attribute::OptimizeForSize.unapply_llfn(Function, llfn); + llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn); + } + _ => {} + } +} + +/// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) +/// attributes. +pub(crate) fn from_fn_attrs<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + llfn: &'ll Value, + instance: ty::Instance<'tcx>, +) { + let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); + + match codegen_fn_attrs.optimize { + OptimizeAttr::Default => { + default_optimisation_attrs(cx.tcx.sess, llfn); + } + OptimizeAttr::Speed => { + llvm::Attribute::MinSize.unapply_llfn(Function, llfn); + llvm::Attribute::OptimizeForSize.unapply_llfn(Function, llfn); + llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn); + } + OptimizeAttr::Size => { + llvm::Attribute::MinSize.apply_llfn(Function, llfn); + llvm::Attribute::OptimizeForSize.apply_llfn(Function, llfn); + llvm::Attribute::OptimizeNone.unapply_llfn(Function, llfn); + } + OptimizeAttr::DoNotOptimize => { + llvm::Attribute::MinSize.unapply_llfn(Function, llfn); + llvm::Attribute::OptimizeForSize.unapply_llfn(Function, llfn); + llvm::Attribute::OptimizeNone.apply_llfn(Function, llfn); + } + } + + let inline_attr = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { + InlineAttr::Never + } else if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { + InlineAttr::Hint + } else { + codegen_fn_attrs.inline + }; + inline(llfn, inline_attr); + + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { + llvm::Attribute::Cold.apply_llfn(Function, llfn); + } + if codegen_fn_attrs + .flags + .contains(CodegenFnAttrFlags::FFI_PURE) + { + llvm::Attribute::ReadOnly.apply_llfn(Function, llfn); + } + if codegen_fn_attrs + .flags + .contains(CodegenFnAttrFlags::FFI_CONST) + { + llvm::Attribute::ReadNone.apply_llfn(Function, llfn); + } +} + +pub struct Symbols { + pub nvvm_internal: Symbol, + pub kernel: Symbol, + pub addrspace: Symbol, +} + +// inspired by rust-gpu's attribute handling +#[derive(Default, Clone, PartialEq)] +pub(crate) struct NvvmAttributes { + pub kernel: bool, + pub used: bool, + pub addrspace: Option, +} + +impl NvvmAttributes { + pub fn parse<'tcx>(cx: &CodegenCx<'_, 'tcx>, attrs: &'tcx [Attribute]) -> Self { + let mut nvvm_attrs = Self::default(); + + for attr in attrs { + if attr.path_matches(&[cx.symbols.nvvm_internal, cx.symbols.kernel]) { + nvvm_attrs.kernel = true; + } else if attr.path_matches(&[cx.symbols.nvvm_internal, sym::used]) { + nvvm_attrs.used = true; + } else if attr.path_matches(&[cx.symbols.nvvm_internal, cx.symbols.addrspace]) { + let args = attr.meta_item_list().unwrap_or_default(); + if let Some(MetaItemInner::Lit(MetaItemLit { + kind: LitKind::Int(val, _), + .. + })) = args.first() + { + nvvm_attrs.addrspace = Some(val.get() as u8); + } else { + panic!(); + } + } + } + nvvm_attrs + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/back.rs b/crates/rustc_codegen_nvvm_v19/src/back.rs new file mode 100644 index 00000000..1fcb6656 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/back.rs @@ -0,0 +1,546 @@ +use std::io::{self, Write}; +use std::slice; +use std::sync::Arc; +use std::ffi::CString; + +use libc::{c_char, size_t}; +use rustc_codegen_ssa::back::write::{TargetMachineFactoryConfig, TargetMachineFactoryFn}; +use rustc_codegen_ssa::traits::{DebugInfoCodegenMethods, MiscCodegenMethods}; +use rustc_codegen_ssa::{ + CompiledModule, ModuleCodegen, + back::write::{CodegenContext, ModuleConfig}, + base::maybe_create_entry_wrapper, + mono_item::MonoItemExt, + traits::{BaseTypeCodegenMethods, ThinBufferMethods}, +}; +use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_middle::mir::mono::{MonoItem, MonoItemData}; +use rustc_middle::{ + dep_graph, + ty::TyCtxt +}; +use rustc_session::Session; +use rustc_session::config::{self, DebugInfo, OutputType}; +use rustc_span::Symbol; +use rustc_target::spec::{CodeModel, RelocModel}; + +use crate::common::AsCCharPtr; +use crate::llvm; +use crate::override_fns::define_or_override_fn; +use crate::builder::Builder; +use crate::context::CodegenCx; +use crate::lto::ThinBuffer; +use crate::LlvmMod; +use crate::NvvmCodegenBackend; + +pub fn llvm_err(handle: DiagCtxtHandle, msg: &str) -> FatalError { + match llvm::last_error() { + Some(err) => handle.fatal(format!("{}: {}", msg, err)), + None => handle.fatal(msg.to_string()), + } +} + +pub fn to_llvm_opt_settings( + cfg: config::OptLevel, +) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { + use self::config::OptLevel::*; + match cfg { + No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone), + Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone), + More => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone), + Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone), + Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault), + SizeMin => ( + llvm::CodeGenOptLevel::Default, + llvm::CodeGenOptSizeAggressive, + ), + } +} + +fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocMode { + match relocation_model { + RelocModel::Static => llvm::RelocMode::Static, + RelocModel::Pic => llvm::RelocMode::PIC, + RelocModel::DynamicNoPic => llvm::RelocMode::DynamicNoPic, + RelocModel::Ropi => llvm::RelocMode::ROPI, + RelocModel::Rwpi => llvm::RelocMode::RWPI, + RelocModel::RopiRwpi => llvm::RelocMode::ROPI_RWPI, + RelocModel::Pie => panic!(), + } +} + +pub(crate) fn to_llvm_code_model(code_model: Option) -> llvm::CodeModel { + match code_model { + Some(CodeModel::Tiny) => llvm::CodeModel::Small, + Some(CodeModel::Small) => llvm::CodeModel::Small, + Some(CodeModel::Kernel) => llvm::CodeModel::Kernel, + Some(CodeModel::Medium) => llvm::CodeModel::Medium, + Some(CodeModel::Large) => llvm::CodeModel::Large, + None => llvm::CodeModel::None, + } +} + +pub fn target_machine_factory( + sess: &Session, + optlvl: config::OptLevel, +) -> TargetMachineFactoryFn { + let reloc_model = to_llvm_relocation_model(sess.relocation_model()); + + let (opt_level, _) = to_llvm_opt_settings(optlvl); + let use_softfp = sess.opts.cg.soft_float; + + let ffunction_sections = sess + .opts + .unstable_opts + .function_sections + .unwrap_or(sess.target.function_sections); + let fdata_sections = ffunction_sections; + + let code_model = to_llvm_code_model(sess.code_model()); + + let triple = sess.target.llvm_target.clone().to_string(); + let cpu_string = sess.opts.cg.target_cpu + .as_deref() + .unwrap_or("") // TODO: sm_120? + .to_string(); + let features_string = "".to_string(); + let trap_unreachable = sess + .opts + .unstable_opts + .trap_unreachable + .unwrap_or(sess.target.trap_unreachable); + + Arc::new(move |_config: TargetMachineFactoryConfig| { + let triple_cstr = CString::new(triple.as_str()) + .map_err(|e| format!("Invalid triple string: {}", e))?; + let cpu_cstr = CString::new(cpu_string.as_str()) + .map_err(|e| format!("Invalid CPU string: {}", e))?; + let features_cstr = CString::new(features_string.as_str()) + .map_err(|e| format!("Invalid features string: {}", e))?; + let abi_cstr = CString::new("").unwrap(); + let debug_compression_cstr = CString::new("none").unwrap(); + + let tm = unsafe { + llvm::LLVMRustCreateTargetMachine( + triple_cstr.as_ptr(), // triple + cpu_cstr.as_ptr(), // cpu + features_cstr.as_ptr(), // feature + abi_cstr.as_ptr(), // abistr + code_model, // RustCM + reloc_model, // reloc + opt_level, // opt + if use_softfp { llvm::FloatABIType::Soft } else { llvm::FloatABIType::Default }, // FloatABIType + ffunction_sections, // function sections + fdata_sections, + false, // unique section names + trap_unreachable, + false, // single thread + false, // VerboseAsm: bool, + false, // EmitStackSizeSection: bool, + false, // RelaxELFRelocations: bool, + false, // UseInitArray: bool, + std::ptr::null(), // SplitDwarfFile: *const c_char, + std::ptr::null(), // OutputObjFile: *const c_char, + debug_compression_cstr.as_ptr(), // DebugInfoCompression: *const c_char, + false, // UseEmulatedTls: bool, + std::ptr::null(), // ArgsCstrBuff: *const c_char, + 0, // ArgsCstrBuffLen: usize, + ) + }; + tm.ok_or_else(|| format!("Could not create LLVM TargetMachine for triple: {}", triple)) + }) +} + +pub extern "C" fn demangle_callback( + input_ptr: *const c_char, + input_len: size_t, + output_ptr: *mut c_char, + output_len: size_t, +) -> size_t { + let input = unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len) }; + + let input = match std::str::from_utf8(input) { + Ok(s) => s, + Err(_) => return 0, + }; + + let output = unsafe { slice::from_raw_parts_mut(output_ptr as *mut u8, output_len) }; + let mut cursor = io::Cursor::new(output); + + let demangled = match rustc_demangle::try_demangle(input) { + Ok(d) => d, + Err(_) => return 0, + }; + + if write!(cursor, "{:#}", demangled).is_err() { + // Possible only if provided buffer is not big enough + return 0; + } + + cursor.position() as size_t +} + +/// Compile a single module (in an nvvm context this means getting the llvm bitcode out of it) +pub(crate) unsafe fn codegen( + cgcx: &CodegenContext, + dcx: DiagCtxtHandle<'_>, + module: ModuleCodegen, + config: &ModuleConfig, +) -> Result { + // For NVVM, all the codegen we need to do is turn the llvm modules + // into llvm bitcode and write them to a tempdir. nvvm expects llvm + // bitcode as the modules to be added to the program. Then as the last step + // we gather all those tasty bitcode files, add them to the nvvm program + // and finally tell nvvm to compile it, which gives us a ptx file. + // + // we also implement emit_ir so we can dump the IR fed to nvvm in case we + // feed it anything it doesnt like + + let _timer = cgcx + .prof + .generic_activity_with_arg("NVVM_module_codegen", &module.name[..]); + + let llmod = unsafe { module.module_llvm.llmod.as_ref().unwrap() }; + let mod_name = module.name.clone(); + let module_name = Some(&mod_name[..]); + + let out = cgcx + .output_filenames + .temp_path(OutputType::Object, module_name); + + // nvvm ir *is* llvm ir so emit_ir fits the expectation of llvm ir which is why we + // implement this. this is copy and pasted straight from rustc_codegen_llvm + // because im too lazy to make it seem like i rewrote this when its the same logic + if config.emit_ir { + let _timer = cgcx + .prof + .generic_activity_with_arg("NVVM_module_codegen_emit_ir", &module.name[..]); + let out = cgcx + .output_filenames + .temp_path(OutputType::LlvmAssembly, module_name); + let out = out.to_str().unwrap(); + + let result = unsafe { + llvm::LLVMRustPrintModule(llmod, out.as_c_char_ptr(), out.len(), demangle_callback) + }; + + result.into_result().map_err(|()| { + let msg = format!("failed to write NVVM IR to {}", out); + llvm_err(dcx, &msg) + })?; + } + + let _bc_timer = cgcx + .prof + .generic_activity_with_arg("NVVM_module_codegen_make_bitcode", &module.name[..]); + + let thin = ThinBuffer::new(llmod); + + let data = thin.data(); + + let _bc_emit_timer = cgcx + .prof + .generic_activity_with_arg("NVVM_module_codegen_emit_bitcode", &module.name[..]); + + if let Err(e) = std::fs::write(&out, data) { + let msg = format!("failed to write bytecode to {}: {}", out.display(), e); + dcx.err(msg); + } + + Ok(CompiledModule { + name: mod_name, + kind: module.kind, + object: Some(out), + dwarf_object: None, + bytecode: None, + assembly: None, + llvm_ir: None, + }) +} + +/// compile a single codegen unit. +/// This involves getting its llvm module and doing some housekeeping such as +/// monomorphizing items and using RAUW on statics. This codegenned module is then +/// given to other functions to "compile it" (in our case not really because nvvm does +/// codegen on all the modules at once) and then link it (once again, nvvm does linking and codegen +/// in a single step) +pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen, u64) { + let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx); + let (module, _) = tcx.dep_graph.with_task( + dep_node, + tcx, + cgu_name, + module_codegen, + Some(dep_graph::hash_result), + ); + + fn module_codegen(tcx: TyCtxt<'_>, cgu_name: Symbol) -> ModuleCodegen { + let cgu = tcx.codegen_unit(cgu_name); + + // Instantiate monomorphizations without filling out definitions yet... + let llvm_module = LlvmMod::new(cgu_name.as_str()); + { + let cx = CodegenCx::new(tcx, cgu, &llvm_module); + + let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); + + for &( + mono_item, + MonoItemData { + linkage, + visibility, + .. + }, + ) in &mono_items + { + mono_item.predefine::>(&cx, linkage, visibility); + } + + // ... and now that we have everything pre-defined, fill out those definitions. + for &(mono_item, _) in &mono_items { + if let MonoItem::Fn(func) = mono_item { + define_or_override_fn(func, &cx); + } else { + mono_item.define::>(&cx); + } + } + + // a main function for gpu kernels really makes no sense but + // codegen it anyways. + // sanitize attrs are not allowed in nvvm so do nothing further. + maybe_create_entry_wrapper::>(&cx); + + // Run replace-all-uses-with for statics that need it + for &(old_g, new_g) in cx.statics_to_rauw.borrow().iter() { + unsafe { + let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); + llvm::LLVMReplaceAllUsesWith(old_g, bitcast); + llvm::LLVMDeleteGlobal(old_g); + } + } + + // Create the llvm.used and llvm.compiler.used variables. + if !cx.used_statics.borrow().is_empty() { + cx.create_used_variable_impl(c"llvm.used", &cx.used_statics.borrow()); + } + if !cx.compiler_used_statics.borrow().is_empty() { + cx.create_used_variable_impl( + c"llvm.compiler.used", + &cx.compiler_used_statics.borrow(), + ); + } + + // Finalize debuginfo + if cx.sess().opts.debuginfo != DebugInfo::None { + cx.debuginfo_finalize(); + } + } + + ModuleCodegen::new_regular(cgu_name.to_string(), llvm_module) + } + + // TODO(RDambrosio016): maybe the same cost as the llvm codegen works? + // nvvm does some exotic things and does linking too so it might be inaccurate + (module, 0) +} + +/*pub(crate) unsafe fn optimize( + _cgcx: &CodegenContext, + _diag_handler: DiagCtxtHandle<'_>, + _module: &ModuleCodegen, + _config: &ModuleConfig, +) -> Result<(), FatalError> { + // TODO: implement this + Ok(()) +}*/ + +// TODO: We use rustc's optimization approach from when it used llvm 7, because many things +// are incompatible with llvm 7 nowadays. Although we should probably consult a rustc dev on whether +// any big things were discovered in that timespan that we should modify. +pub(crate) unsafe fn optimize( + cgcx: &CodegenContext, + diag_handler: DiagCtxtHandle<'_>, + module: &ModuleCodegen, + config: &ModuleConfig, +) -> Result<(), FatalError> { + + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_module_optimize", &module.name[..]); + + let llmod = unsafe { &*module.module_llvm.llmod }; + + if config.emit_no_opt_bc { + let mod_name = module.name.clone(); + let module_name = Some(&mod_name[..]); + let out = cgcx + .output_filenames + .temp_path_ext("no-opt.bc", module_name); + let out = rustc_fs_util::path_to_c_string(&out); + unsafe { llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); } + } + + let tm_factory_config = TargetMachineFactoryConfig { + split_dwarf_file: None, + output_obj_file: None, + }; + + let tm = (cgcx.tm_factory)(tm_factory_config).expect("failed to create target machine"); + + // LLVM 19: Complete rewrite using new pass manager + if config.opt_level.is_some() { + unsafe { + let mut error_msg = std::ptr::null_mut(); + let verify_result = llvm::LLVMVerifyModule( + llmod, + llvm::LLVMVerifierFailureAction::LLVMPrintMessageAction, + &mut error_msg + ); + if verify_result != 0 { + llvm::LLVMDumpModule(llmod); + rustc_middle::bug!("Module verification failed!"); + } + + // Create pass builder options + let pass_options = llvm::LLVMCreatePassBuilderOptions(); + + // Configure pass builder options based on config + let opt_level = config + .opt_level + .map_or(llvm::CodeGenOptLevel::None, |x| to_llvm_opt_settings(x).0); + + // Set various options on the pass builder + // TODO: support these flags + /*if config.verify_each { + llvm::LLVMPassBuilderOptionsSetVerifyEach(pass_options, 1); + } + + // Enable debug logging if needed + if config.debug_pass_manager { + llvm::LLVMPassBuilderOptionsSetDebugLogging(pass_options, 1); + }*/ + + // Build the pass pipeline string based on optimization level and config + let mut pass_pipeline = String::new(); + + if !config.no_prepopulate_passes { + // Use default optimization pipeline based on level + match opt_level { + llvm::CodeGenOptLevel::None => { + pass_pipeline.push_str("default"); + }, + llvm::CodeGenOptLevel::Less => { + pass_pipeline.push_str("default"); + }, + llvm::CodeGenOptLevel::Default => { + pass_pipeline.push_str("default"); + }, + llvm::CodeGenOptLevel::Aggressive => { + pass_pipeline.push_str("default"); + }, + } + } + + // Add custom passes from config + for pass in &config.passes { + if !pass_pipeline.is_empty() { + pass_pipeline.push(','); + } + pass_pipeline.push_str(pass); + } + + + // Convert pass pipeline string to C string + let c_pass_pipeline = std::ffi::CString::new(pass_pipeline) + .expect("Pass pipeline string contains null byte"); + + // Run the passes using the new pass manager + + let result = llvm::LLVMRunPasses( + llmod, // Module + c_pass_pipeline.as_ptr(), // Pass pipeline string + tm, // TargetMachine + pass_options // PassBuilderOptions + ); + + + if result != 0 { + diag_handler.err("Failed to run optimization passes"); + } else { + } + + // Clean up + llvm::LLVMDisposePassBuilderOptions(pass_options); + } + } else { + } + + Ok(()) +} + +// TODO: remove this dead code? +/*unsafe fn with_llvm_pmb( + llmod: &llvm::Module, + config: &ModuleConfig, + opt_level: llvm::CodeGenOptLevel, + f: &mut impl FnMut(&llvm::PassManagerBuilder), +) { + unsafe { + use std::ptr; + + let builder = llvm::LLVMPassManagerBuilderCreate(); + let opt_size = config + .opt_size + .map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); + + llvm::LLVMRustConfigurePassManagerBuilder( + builder, + opt_level, + config.merge_functions, + config.vectorize_slp, + config.vectorize_loop, + false, + ptr::null(), + ptr::null(), + ); + + llvm::LLVMPassManagerBuilderSetSizeLevel(builder, opt_size as u32); + + if opt_size != llvm::CodeGenOptSizeNone { + llvm::LLVMPassManagerBuilderSetDisableUnrollLoops(builder, 1); + } + + llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins); + + // Here we match what clang does (kinda). For O0 we only inline + // always-inline functions (but don't add lifetime intrinsics), at O1 we + // inline with lifetime intrinsics, and O2+ we add an inliner with a + // thresholds copied from clang. + match (opt_level, opt_size) { + (llvm::CodeGenOptLevel::Aggressive, ..) => { + llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 275); + } + (_, llvm::CodeGenOptSizeDefault) => { + llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 75); + } + (_, llvm::CodeGenOptSizeAggressive) => { + llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 25); + } + (llvm::CodeGenOptLevel::None, ..) => { + llvm::LLVMRustAddAlwaysInlinePass(builder, false); + } + (llvm::CodeGenOptLevel::Less, ..) => { + llvm::LLVMRustAddAlwaysInlinePass(builder, true); + } + (llvm::CodeGenOptLevel::Default, ..) => { + llvm::LLVMPassManagerBuilderUseInlinerWithThreshold(builder, 225); + } + (llvm::CodeGenOptLevel::Other, ..) => { + bug!("CodeGenOptLevel::Other selected") + } + } + + f(builder); + llvm::LLVMPassManagerBuilderDispose(builder); + } +}*/ diff --git a/crates/rustc_codegen_nvvm_v19/src/builder.rs b/crates/rustc_codegen_nvvm_v19/src/builder.rs new file mode 100644 index 00000000..f135533a --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/builder.rs @@ -0,0 +1,1277 @@ +use std::borrow::Cow; +use std::ops::Deref; +use std::ptr; + +use libc::{c_char, c_uint}; +use rustc_abi as abi; +use rustc_abi::{Align, HasDataLayout, Size, TargetDataLayout, WrappingRange}; +use rustc_codegen_ssa::MemFlags; +use rustc_codegen_ssa::common::{AtomicOrdering, IntPredicate, RealPredicate, TypeKind}; +use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; +use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::def_id::DefId; +use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; +use rustc_middle::ty::layout::{ + FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers, + TyAndLayout, +}; +use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; +use rustc_session::config::OptLevel; +use rustc_span::Span; +use rustc_target::callconv::FnAbi; +use rustc_target::spec::{HasTargetSpec, Target}; +use tracing::{debug, info, trace}; + +use crate::abi::FnAbiLlvmExt; +use crate::context::CodegenCx; +use crate::int_replace::{get_transformed_type, transmute_llval}; +use crate::llvm; +use crate::llvm::{BasicBlock, Type, Value}; +use crate::ty::LayoutLlvmExt; + +// All Builders must have an llfn associated with them +#[must_use] +pub(crate) struct Builder<'a, 'll, 'tcx> { + pub llbuilder: &'ll mut llvm::Builder<'ll>, + pub cx: &'a CodegenCx<'ll, 'tcx>, +} + +impl Drop for Builder<'_, '_, '_> { + fn drop(&mut self) { + unsafe { + llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _)); + } + } +} + +const UNNAMED: *const c_char = c"".as_ptr(); + +/// Empty string, to be used where LLVM expects an instruction name, indicating +/// that the instruction is to be left unnamed (i.e. numbered, in textual IR). +pub(crate) fn unnamed() -> *const c_char { + UNNAMED +} + +impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> { + type Value = as BackendTypes>::Value; + type Function = as BackendTypes>::Function; + type BasicBlock = as BackendTypes>::BasicBlock; + type Type = as BackendTypes>::Type; + type Funclet = as BackendTypes>::Funclet; + + type DIScope = as BackendTypes>::DIScope; + type DILocation = as BackendTypes>::DILocation; + type DIVariable = as BackendTypes>::DIVariable; + + type Metadata = as BackendTypes>::Metadata; +} + +impl HasDataLayout for Builder<'_, '_, '_> { + fn data_layout(&self) -> &TargetDataLayout { + self.cx.data_layout() + } +} + +impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.cx.tcx + } +} + +impl<'tcx> HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> { + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.cx.typing_env() + } +} + +impl HasTargetSpec for Builder<'_, '_, '_> { + fn target_spec(&self) -> &Target { + self.cx.target_spec() + } +} + +impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { + type LayoutOfResult = TyAndLayout<'tcx>; + + #[inline] + fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { + self.cx.handle_layout_err(err, span, ty) + } +} + +impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { + type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; + + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + self.cx.handle_fn_abi_err(err, span, fn_abi_request) + } +} + +impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> { + type Target = CodegenCx<'ll, 'tcx>; + + #[inline] + fn deref(&self) -> &Self::Target { + self.cx + } +} + +macro_rules! math_builder_methods { + ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => { + $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value { + unsafe { + trace!("binary expr: {:?} with args {:?}", stringify!($name), [$($arg),*]); + llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED) + } + })+ + } +} + +macro_rules! set_math_builder_methods { + ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => { + $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value { + unsafe { + let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED); + llvm::$llvm_set_math(instr); + instr + } + })+ + } +} + +impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + fn add_coverage( + &mut self, + _instance: Instance<'tcx>, + _kind: &rustc_middle::mir::coverage::CoverageKind, + ) { + } +} + +impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { + type CodegenCx = CodegenCx<'ll, 'tcx>; + + fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self { + let bx = Builder::with_cx(cx); + unsafe { + llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb); + } + bx + } + + fn cx(&self) -> &CodegenCx<'ll, 'tcx> { + self.cx + } + + fn llbb(&self) -> &'ll BasicBlock { + unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) } + } + + fn set_span(&mut self, _span: Span) {} + + fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock { + unsafe { + let name = SmallCStr::new(name); + llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr()) + } + } + + fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock { + Self::append_block(self.cx, self.llfn(), name) + } + + fn switch_to_block(&mut self, llbb: Self::BasicBlock) { + *self = Self::build(self.cx, llbb) + } + + fn ret_void(&mut self) { + trace!("Ret void"); + unsafe { + llvm::LLVMBuildRetVoid(self.llbuilder); + } + } + + fn ret(&mut self, mut v: &'ll Value) { + trace!("Ret `{:?}`", v); + unsafe { + let ty = self.val_ty(v); + let (new_ty, changed) = get_transformed_type(self.cx, ty); + if changed { + v = transmute_llval(self.llbuilder, self.cx, v, new_ty); + } + llvm::LLVMBuildRet(self.llbuilder, v); + } + } + + fn br(&mut self, dest: &'ll BasicBlock) { + trace!("Br"); + unsafe { + llvm::LLVMBuildBr(self.llbuilder, dest); + } + } + + fn cond_br( + &mut self, + cond: &'ll Value, + then_llbb: &'ll BasicBlock, + else_llbb: &'ll BasicBlock, + ) { + trace!("Cond br `{:?}`", cond); + unsafe { + llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb); + } + } + + fn switch( + &mut self, + v: &'ll Value, + else_llbb: &'ll BasicBlock, + cases: impl ExactSizeIterator, + ) { + trace!("Switch `{:?}`", v); + let switch = + unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) }; + for (on_val, dest) in cases { + let on_val = self.const_uint_big(self.val_ty(v), on_val); + unsafe { llvm::LLVMAddCase(switch, on_val, dest) } + } + } + + fn invoke( + &mut self, + llty: &'ll Type, + fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + args: &[&'ll Value], + then: &'ll BasicBlock, + _catch: &'ll BasicBlock, + funclet: Option<&()>, + instance: Option>, + ) -> Self::Value { + trace!("invoke"); + let call = self.call(llty, fn_attrs, fn_abi, llfn, args, funclet, instance); + // exceptions arent a thing, go directly to the `then` block + unsafe { llvm::LLVMBuildBr(self.llbuilder, then) }; + call + } + + fn unreachable(&mut self) { + trace!("Unreachable"); + unsafe { + llvm::LLVMBuildUnreachable(self.llbuilder); + } + } + + math_builder_methods! { + add(a, b) => LLVMBuildAdd, + fadd(a, b) => LLVMBuildFAdd, + sub(a, b) => LLVMBuildSub, + fsub(a, b) => LLVMBuildFSub, + mul(a, b) => LLVMBuildMul, + fmul(a, b) => LLVMBuildFMul, + udiv(a, b) => LLVMBuildUDiv, + exactudiv(a, b) => LLVMBuildExactUDiv, + sdiv(a, b) => LLVMBuildSDiv, + exactsdiv(a, b) => LLVMBuildExactSDiv, + fdiv(a, b) => LLVMBuildFDiv, + urem(a, b) => LLVMBuildURem, + srem(a, b) => LLVMBuildSRem, + frem(a, b) => LLVMBuildFRem, + shl(a, b) => LLVMBuildShl, + lshr(a, b) => LLVMBuildLShr, + ashr(a, b) => LLVMBuildAShr, + and(a, b) => LLVMBuildAnd, + or(a, b) => LLVMBuildOr, + xor(a, b) => LLVMBuildXor, + neg(x) => LLVMBuildNeg, + fneg(x) => LLVMBuildFNeg, + not(x) => LLVMBuildNot, + unchecked_sadd(x, y) => LLVMBuildNSWAdd, + unchecked_uadd(x, y) => LLVMBuildNUWAdd, + unchecked_ssub(x, y) => LLVMBuildNSWSub, + unchecked_usub(x, y) => LLVMBuildNUWSub, + unchecked_smul(x, y) => LLVMBuildNSWMul, + unchecked_umul(x, y) => LLVMBuildNUWMul, + } + + set_math_builder_methods! { + fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath), + fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath), + fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath), + fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath), + frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath), + fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath), + fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath), + fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath), + fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath), + frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath), + } + + fn checked_binop( + &mut self, + oop: OverflowOp, + ty: Ty<'_>, + lhs: Self::Value, + rhs: Self::Value, + ) -> (Self::Value, Self::Value) { + trace!( + "Checked binop `{:?}`, lhs: `{:?}`, rhs: `{:?}`", + ty, lhs, rhs + ); + use rustc_middle::ty::IntTy::*; + use rustc_middle::ty::UintTy::*; + use rustc_middle::ty::{Int, Uint}; + + let new_kind = match ty.kind() { + Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)), + Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)), + t @ (Uint(_) | Int(_)) => *t, + _ => panic!("tried to get overflow intrinsic for op applied to non-int type"), + }; + + let name = match oop { + OverflowOp::Add => match new_kind { + Int(I8) => "__nvvm_i8_addo", + Int(I16) => "llvm.sadd.with.overflow.i16", + Int(I32) => "llvm.sadd.with.overflow.i32", + Int(I64) => "llvm.sadd.with.overflow.i64", + Int(I128) => "__nvvm_i128_addo", + + Uint(U8) => "__nvvm_u8_addo", + Uint(U16) => "llvm.uadd.with.overflow.i16", + Uint(U32) => "llvm.uadd.with.overflow.i32", + Uint(U64) => "llvm.uadd.with.overflow.i64", + Uint(U128) => "__nvvm_u128_addo", + _ => unreachable!(), + }, + OverflowOp::Sub => match new_kind { + Int(I8) => "__nvvm_i8_subo", + Int(I16) => "llvm.ssub.with.overflow.i16", + Int(I32) => "llvm.ssub.with.overflow.i32", + Int(I64) => "llvm.ssub.with.overflow.i64", + Int(I128) => "__nvvm_i128_subo", + + Uint(U8) => "__nvvm_u8_subo", + Uint(U16) => "llvm.usub.with.overflow.i16", + Uint(U32) => "llvm.usub.with.overflow.i32", + Uint(U64) => "llvm.usub.with.overflow.i64", + Uint(U128) => "__nvvm_u128_subo", + + _ => unreachable!(), + }, + OverflowOp::Mul => match new_kind { + Int(I8) => "__nvvm_i8_mulo", + Int(I16) => "llvm.smul.with.overflow.i16", + Int(I32) => "llvm.smul.with.overflow.i32", + Int(I64) => "llvm.smul.with.overflow.i64", + Int(I128) => "__nvvm_i128_mulo", + + Uint(U8) => "__nvvm_u8_mulo", + Uint(U16) => "llvm.umul.with.overflow.i16", + Uint(U32) => "llvm.umul.with.overflow.i32", + Uint(U64) => "llvm.umul.with.overflow.i64", + Uint(U128) => "__nvvm_u128_mulo", + + _ => unreachable!(), + }, + }; + + let (ty, f) = self.get_intrinsic(name); + let res = self.call(ty, None, None, f, &[lhs, rhs], None, None); + (self.extract_value(res, 0), self.extract_value(res, 1)) + } + + fn from_immediate(&mut self, val: Self::Value) -> Self::Value { + if self.cx().val_ty(val) == self.cx().type_i1() { + self.zext(val, self.cx().type_i8()) + } else { + val + } + } + fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value { + if scalar.is_bool() { + return self.trunc(val, self.cx().type_i1()); + } + val + } + + fn alloca(&mut self, size: Size, align: Align) -> &'ll Value { + trace!("Alloca `{:?}`", size); + let mut bx = Builder::with_cx(self.cx); + bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) }); + let ty = self.cx().type_array(self.cx().type_i8(), size.bytes()); + unsafe { + let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED); + llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); + // Cast to default addrspace if necessary + llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED) + } + } + + fn dynamic_alloca(&mut self, size: &'ll Value, align: Align) -> &'ll Value { + trace!("Dynamic Alloca `{:?}`", size); + unsafe { + let alloca = + llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED); + llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); + // Cast to default addrspace if necessary + llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED) + } + } + + fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value { + trace!("Load {ty:?} {:?}", ptr); + let ptr = self.pointercast(ptr, self.cx.type_ptr_to(ty)); + unsafe { + let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); + llvm::LLVMSetAlignment(load, align.bytes() as c_uint); + load + } + } + + fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { + trace!("Volatile load `{:?}`", ptr); + let ptr = self.pointercast(ptr, self.cx.type_ptr_to(ty)); + unsafe { + let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); + llvm::LLVMSetVolatile(load, llvm::True); + load + } + } + + fn atomic_load( + &mut self, + ty: &'ll Type, + _ptr: &'ll Value, + _order: AtomicOrdering, + _size: Size, + ) -> &'ll Value { + // core seems to think that nvptx has atomic loads, which is not true for NVVM IR, + // therefore our only option is to print that this is not supported then trap. + // i have heard of cursed things such as emulating this with __threadfence and volatile loads + // but that needs to be experimented with in terms of safety and behavior. + // NVVM has explicit intrinsics for adding and subtracting floats which we expose elsewhere + + // TODO(RDambrosio016): is there a way we can just generate a panic with a message instead + // of doing this ourselves? since all panics will be aborts, it should be equivalent + // let message = "Atomic Loads are not supported in CUDA.\0"; + + // let vprintf = self.get_intrinsic("vprintf"); + // let formatlist = self.const_str(Symbol::intern(message)).0; + // let valist = self.const_null(self.type_void()); + + // self.call(vprintf, &[formatlist, valist], None); + + let (trap_ty, f) = self.get_intrinsic("llvm.trap"); + self.call(trap_ty, None, None, f, &[], None, None); + //unsafe { llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, unnamed()) } + unsafe { llvm::LLVMGetUndef(ty) } + } + + fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> { + if place.layout.is_unsized() { + let tail = self + .tcx + .struct_tail_for_codegen(place.layout.ty, self.typing_env()); + if matches!(tail.kind(), ty::Foreign(..)) { + // Unsized locals and, at least conceptually, even unsized arguments must be copied + // around, which requires dynamically determining their size. Therefore, we cannot + // allow `extern` types here. Consult t-opsem before removing this check. + panic!("unsized locals must not be `extern` types"); + } + } + assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized()); + + if place.layout.is_zst() { + return OperandRef::zero_sized(place.layout); + } + + fn scalar_load_metadata<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + load: &'ll Value, + scalar: abi::Scalar, + layout: TyAndLayout<'tcx>, + offset: Size, + ) { + if bx.sess().opts.optimize == OptLevel::No { + // Don't emit metadata we're not going to use + return; + } + + if !scalar.is_uninit_valid() { + bx.noundef_metadata(load); + } + + match scalar.primitive() { + abi::Primitive::Int(..) => { + if !scalar.is_always_valid(bx) { + bx.range_metadata(load, scalar.valid_range(bx)); + } + } + abi::Primitive::Pointer(_) => { + if !scalar.valid_range(bx).contains(0) { + bx.nonnull_metadata(load); + } + + if let Some(pointee) = layout.pointee_info_at(bx, offset) { + if pointee.safe.is_some() { + bx.align_metadata(load, pointee.align); + } + } + } + abi::Primitive::Float(_) => {} + } + } + + let val = if place.val.llextra.is_some() { + // FIXME: Merge with the `else` below? + OperandValue::Ref(place.val) + } else if place.layout.is_llvm_immediate() { + let mut const_llval = None; + let llty = place.layout.llvm_type(self); + unsafe { + if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) { + if llvm::LLVMIsGlobalConstant(global) == llvm::True { + if let Some(init) = llvm::LLVMGetInitializer(global) { + if self.val_ty(init) == llty { + const_llval = Some(init); + } + } + } + } + } + + let llval = const_llval.unwrap_or_else(|| { + let load = self.load(llty, place.val.llval, place.val.align); + if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr { + scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO); + self.to_immediate_scalar(load, scalar) + } else { + load + } + }); + + OperandValue::Immediate(llval) + } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr { + let b_offset = a.size(self).align_to(b.align(self).abi); + + let mut load = |i, scalar: abi::Scalar, layout, align, offset| { + let llptr = if i == 0 { + place.val.llval + } else { + self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes())) + }; + let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); + let load = self.load(llty, llptr, align); + scalar_load_metadata(self, load, scalar, layout, offset); + self.to_immediate_scalar(load, scalar) + }; + + OperandValue::Pair( + load(0, a, place.layout, place.val.align, Size::ZERO), + load( + 1, + b, + place.layout, + place.val.align.restrict_for_offset(b_offset), + b_offset, + ), + ) + } else { + OperandValue::Ref(place.val) + }; + + OperandRef { + val, + layout: place.layout, + } + } + + fn write_operand_repeatedly( + &mut self, + cg_elem: OperandRef<'tcx, &'ll Value>, + count: u64, + dest: PlaceRef<'tcx, &'ll Value>, + ) { + trace!("write operand repeatedly"); + let zero = self.const_usize(0); + let count = self.const_usize(count); + + let header_bb = self.append_sibling_block("repeat_loop_header"); + let body_bb = self.append_sibling_block("repeat_loop_body"); + let next_bb = self.append_sibling_block("repeat_loop_next"); + + self.br(header_bb); + + let mut header_bx = Self::build(self.cx, header_bb); + let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]); + + let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count); + header_bx.cond_br(keep_going, body_bb, next_bb); + + let mut body_bx = Self::build(self.cx, body_bb); + let dest_elem = dest.project_index(&mut body_bx, i); + cg_elem.val.store(&mut body_bx, dest_elem); + + let next = body_bx.unchecked_uadd(i, self.const_usize(1)); + body_bx.br(header_bb); + header_bx.add_incoming_to_phi(i, next, body_bb); + + *self = Self::build(self.cx, next_bb); + } + + fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) { + trace!("range metadata on {load:?}: {range:?}"); + unsafe { + let llty = self.cx.val_ty(load); + let v = [ + self.cx.const_uint_big(llty, range.start), + self.cx.const_uint_big(llty, range.end.wrapping_add(1)), + ]; + + llvm::LLVMSetMetadata( + load, + llvm::MetadataType::MD_range as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint), + ); + } + } + + fn nonnull_metadata(&mut self, load: &'ll Value) { + unsafe { + llvm::LLVMSetMetadata( + load, + llvm::MetadataType::MD_nonnull as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), + ); + } + } + + fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { + trace!("Store val `{:?}` into ptr `{:?}`", val, ptr); + self.store_with_flags(val, ptr, align, MemFlags::empty()) + } + + fn store_with_flags( + &mut self, + val: &'ll Value, + ptr: &'ll Value, + align: Align, + flags: MemFlags, + ) -> &'ll Value { + trace!( + "store_with_flags: {:?} into {:?} with align {:?}", + val, + ptr, + align.bytes() + ); + assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer); + let ptr = self.check_store(val, ptr); + unsafe { + let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr); + let align = if flags.contains(MemFlags::UNALIGNED) { + 1 + } else { + align.bytes() as c_uint + }; + llvm::LLVMSetAlignment(store, align); + if flags.contains(MemFlags::VOLATILE) { + llvm::LLVMSetVolatile(store, llvm::True); + } + if flags.contains(MemFlags::NONTEMPORAL) { + // According to LLVM [1] building a nontemporal store must + // *always* point to a metadata value of the integer 1. + // + // [1]: http://llvm.org/docs/LangRef.html#store-instruction + let one = self.cx.const_i32(1); + let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1); + llvm::LLVMSetMetadata(store, llvm::MetadataType::MD_nontemporal as c_uint, node); + } + + store + } + } + + fn atomic_store( + &mut self, + _val: &'ll Value, + _ptr: &'ll Value, + _order: rustc_codegen_ssa::common::AtomicOrdering, + _size: Size, + ) { + // see comment in atomic_load + + // let message = "Atomic Stores are not supported in CUDA.\0"; + + // let vprintf = self.get_intrinsic("vprintf"); + // let formatlist = self.const_str(Symbol::intern(message)).0; + // let valist = self.const_null(self.type_void()); + + // self.call(vprintf, &[formatlist, valist], None); + let (ty, f) = self.get_intrinsic("llvm.trap"); + self.call(ty, None, None, f, &[], None, None); + //unsafe { llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, unnamed()); } + } + + fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value { + trace!("gep: {ty:?} {:?} with indices {:?}", ptr, indices); + let ptr = self.pointercast(ptr, self.cx().type_ptr_to(ty)); + unsafe { + llvm::LLVMBuildGEP2( + self.llbuilder, + ty, + ptr, + indices.as_ptr(), + indices.len() as c_uint, + UNNAMED, + ) + } + } + + fn inbounds_gep( + &mut self, + ty: &'ll Type, + ptr: &'ll Value, + indices: &[&'ll Value], + ) -> &'ll Value { + trace!("gep inbounds: {ty:?} {:?} with indices {:?}", ptr, indices); + let ptr = self.pointercast(ptr, self.cx().type_ptr_to(ty)); + unsafe { + llvm::LLVMBuildInBoundsGEP2( + self.llbuilder, + ty, + ptr, + indices.as_ptr(), + indices.len() as c_uint, + UNNAMED, + ) + } + } + + /* Casts */ + fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("trunc {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("sext {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + // NVVM does not have support for saturated conversion. Setting rustc flag + // `-Z saturating_float_casts=false` falls back to non-saturated, UB-prone + // conversion, and should prevent this codegen. Otherwise, fall back to UB + // prone conversion. + self.cx().sess().dcx() + .warn("Saturated float to int conversion is not supported on NVVM. Defaulting to UB prone conversion."); + self.fptoui(val, dest_ty) + } + + fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + // NVVM does not have support for saturated conversion. Setting rustc flag + // `-Z saturating_float_casts=false` falls back to non-saturated, UB-prone + // conversion, and should prevent this codegen. Otherwise, fall back to UB + // prone conversion. + self.cx().sess().dcx() + .warn("Saturated float to int conversion is not supported on NVVM. Defaulting to UB prone conversion."); + self.fptosi(val, dest_ty) + } + + fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("fptoui {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("fptosi {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("uitofp {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("sitofp {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("fptrunc {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("fpext {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("ptrtoint {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("inttoptr {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { + trace!("Intcast `{:?}` to ty `{:?}`", val, dest_ty); + unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) } + } + + fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("Pointercast `{:?}` to ty `{:?}`", val, dest_ty); + unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, unnamed()) } + } + + /* Comparisons */ + fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + trace!("Icmp lhs: `{:?}`, rhs: `{:?}`", lhs, rhs); + unsafe { + let op = llvm::IntPredicate::from_generic(op); + llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, unnamed()) + } + } + + fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + trace!("Fcmp lhs: `{:?}`, rhs: `{:?}`", lhs, rhs); + unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, unnamed()) } + } + + /* Miscellaneous instructions */ + fn memcpy( + &mut self, + dst: &'ll Value, + dst_align: Align, + src: &'ll Value, + src_align: Align, + size: &'ll Value, + flags: MemFlags, + ) { + assert!( + !flags.contains(MemFlags::NONTEMPORAL), + "non-temporal memcpy not supported" + ); + let size = self.intcast(size, self.type_isize(), false); + let is_volatile = flags.contains(MemFlags::VOLATILE); + unsafe { + llvm::LLVMRustBuildMemCpy( + self.llbuilder, + dst, + dst_align.bytes() as c_uint, + src, + src_align.bytes() as c_uint, + size, + is_volatile, + ); + } + } + + fn memmove( + &mut self, + dst: &'ll Value, + dst_align: Align, + src: &'ll Value, + src_align: Align, + size: &'ll Value, + flags: MemFlags, + ) { + assert!( + !flags.contains(MemFlags::NONTEMPORAL), + "non-temporal memmove not supported" + ); + let size = self.intcast(size, self.type_isize(), false); + let is_volatile = flags.contains(MemFlags::VOLATILE); + unsafe { + llvm::LLVMRustBuildMemMove( + self.llbuilder, + dst, + dst_align.bytes() as c_uint, + src, + src_align.bytes() as c_uint, + size, + is_volatile, + ); + } + } + + fn memset( + &mut self, + ptr: &'ll Value, + fill_byte: &'ll Value, + size: &'ll Value, + align: Align, + flags: MemFlags, + ) { + let is_volatile = flags.contains(MemFlags::VOLATILE); + unsafe { + llvm::LLVMRustBuildMemSet( + self.llbuilder, + ptr, + align.bytes() as c_uint, + fill_byte, + size, + is_volatile, + ); + } + } + + fn select( + &mut self, + mut cond: &'ll Value, + then_val: &'ll Value, + else_val: &'ll Value, + ) -> &'ll Value { + unsafe { + if self.val_ty(cond) == llvm::LLVMVectorType(self.type_i1(), 2) { + cond = self.const_bool(false); + } + llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, unnamed()) + } + } + + fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, unnamed()) } + } + + fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value { + trace!("extract element {:?}, {:?}", vec, idx); + unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, unnamed()) } + } + + fn vector_splat(&mut self, _num_elts: usize, _elt: &'ll Value) -> &'ll Value { + self.unsupported("vector splats"); + } + + fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value { + trace!("extract value {:?}, {:?}", agg_val, idx); + assert_eq!(idx as c_uint as u64, idx); + unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, unnamed()) } + } + + fn insert_value(&mut self, agg_val: &'ll Value, mut elt: &'ll Value, idx: u64) -> &'ll Value { + trace!("insert value {:?}, {:?}, {:?}", agg_val, elt, idx); + assert_eq!(idx as c_uint as u64, idx); + + let elt_ty = self.cx.val_ty(elt); + if self.cx.type_kind(elt_ty) == TypeKind::Pointer { + let agg_ty = self.cx.val_ty(agg_val); + let idx_ty = match self.cx.type_kind(agg_ty) { + TypeKind::Struct => unsafe { + llvm::LLVMStructGetTypeAtIndex(agg_ty, idx as c_uint) + }, + TypeKind::Array => unsafe { llvm::LLVMGetElementType(agg_ty) }, + _ => bug!( + "insert_value: expected struct or array type, found {:?}", + self.cx.type_kind(agg_ty) + ), + }; + assert_eq!(self.cx.type_kind(idx_ty), TypeKind::Pointer); + if idx_ty != elt_ty { + elt = self.pointercast(elt, idx_ty); + } + } + + unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) } + } + + fn cleanup_landing_pad(&mut self, _pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { + todo!() + } + + fn filter_landing_pad(&mut self, _pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { + todo!() + } + + fn resume(&mut self, _exn0: &'ll Value, _exn1: &'ll Value) { + self.unsupported("resumes"); + } + + fn cleanup_pad(&mut self, _parent: Option<&'ll Value>, _args: &[&'ll Value]) {} + + fn cleanup_ret(&mut self, _funclet: &(), _unwind: Option<&'ll BasicBlock>) {} + + fn catch_pad(&mut self, _parent: &'ll Value, _args: &[&'ll Value]) {} + + fn catch_switch( + &mut self, + _parent: Option<&'ll Value>, + _unwind: Option<&'ll BasicBlock>, + _handlers: &[&'ll BasicBlock], + ) -> &'ll Value { + self.unsupported("catch switches"); + } + + fn set_personality_fn(&mut self, _personality: &'ll Value) {} + + // Atomic Operations + fn atomic_cmpxchg( + &mut self, + _dst: &'ll Value, + _cmp: &'ll Value, + _src: &'ll Value, + _order: rustc_codegen_ssa::common::AtomicOrdering, + _failure_order: rustc_codegen_ssa::common::AtomicOrdering, + _weak: bool, + ) -> (&'ll Value, &'ll Value) { + // allowed but only for some things and with restrictions + // https://docs.nvidia.com/cuda/nvvm-ir-spec/index.html#cmpxchg-instruction + bug!("atomic cmpxchg is not supported") + } + fn atomic_rmw( + &mut self, + _op: rustc_codegen_ssa::common::AtomicRmwBinOp, + _dst: &'ll Value, + _src: &'ll Value, + _order: rustc_codegen_ssa::common::AtomicOrdering, + ) -> &'ll Value { + // see cmpxchg comment + bug!("atomic rmw is not supported") + } + + fn atomic_fence( + &mut self, + _order: rustc_codegen_ssa::common::AtomicOrdering, + _scope: rustc_codegen_ssa::common::SynchronizationScope, + ) { + bug!("atomic fence is not supported, use cuda_std intrinsics instead") + } + + fn set_invariant_load(&mut self, load: &'ll Value) { + unsafe { + llvm::LLVMSetMetadata( + load, + llvm::MetadataType::MD_invariant_load as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), + ); + } + } + + fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) { + self.call_lifetime_intrinsic("llvm.lifetime.start.p0", ptr, size); + } + + fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) { + self.call_lifetime_intrinsic("llvm.lifetime.end.p0", ptr, size); + } + + fn call( + &mut self, + llty: &'ll Type, + _fn_attrs: Option<&CodegenFnAttrs>, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + llfn: &'ll Value, + args: &[&'ll Value], + _funclet: Option<&Self::Funclet>, + _instance: Option>, + ) -> &'ll Value { + info!("builder::call Calling fn {:?} with args {:?}", llfn, args); + self.cx.last_call_llfn.set(None); + let args = self.check_call("call", llty, llfn, args); + let mut call = unsafe { + llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr(), + args.len() as c_uint, + [].as_ptr(), + 0 + ) + }; + if let Some(fn_abi) = fn_abi { + fn_abi.apply_attrs_callsite(self, call); + } + // bitcast return type if the type was remapped + let map = self.cx.remapped_integer_args.borrow(); + // Use the provided llty parameter instead of unwrapping pointers + let fn_ty = llty; + if let Some((Some(ret_ty), _)) = map.get(fn_ty) { + self.cx.last_call_llfn.set(Some(call)); + call = transmute_llval(self.llbuilder, self.cx, call, ret_ty); + } + call + } + + fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + trace!("Zext {:?} to {:?}", val, dest_ty); + unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, unnamed()) } + } + + fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) { + // Cleanup is always the cold path. + llvm::Attribute::Cold.apply_callsite(llvm::AttributePlace::Function, llret); + + // In LLVM versions with deferred inlining (currently, system LLVM < 14), + // inlining drop glue can lead to exponential size blowup. + // See rust_lang/rust #41696 and #92110. + llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret); + } + + fn assume_nonnull(&mut self, val: Self::Value) { + assert_eq!(self.cx.type_kind(self.cx.val_ty(val)), TypeKind::Pointer); + let val_ty = self.cx.val_ty(val); + let null = self.cx.const_null(val_ty); + let is_null = self.icmp(IntPredicate::IntNE, val, null); + self.assume(is_null); + } +} + +impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> { + fn get_static(&mut self, def_id: DefId) -> &'ll Value { + self.cx.get_static(def_id) + } +} + +impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { + // TODO(RDambrosio016): fix this when nvidia fixes i128 + pub(crate) fn abort_and_ret_i128(&mut self) -> &'ll Value { + self.abort(); + let first = self.const_u64(0); + let second = self.const_u64(0); + let vals = [first, second]; + unsafe { llvm::LLVMConstVector(vals.as_ptr(), 2) } + } + + fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self { + // Create a fresh builder from the crate context. + let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) }; + Builder { llbuilder, cx } + } + + pub fn llfn(&self) -> &'ll Value { + unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) } + } + + fn position_at_start(&mut self, llbb: &'ll BasicBlock) { + unsafe { + llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb); + } + } + + fn align_metadata(&mut self, _load: &'ll Value, _align: Align) {} + + fn noundef_metadata(&mut self, _load: &'ll Value) {} + + fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value { + let dest_ptr_ty = self.cx.val_ty(ptr); + let stored_ty = self.cx.val_ty(val); + let stored_ptr_ty = self.cx.type_ptr_to(stored_ty); + + assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer); + + if dest_ptr_ty == stored_ptr_ty { + ptr + } else { + self.bitcast(ptr, stored_ptr_ty) + } + } + + fn check_call<'b>( + &mut self, + typ: &str, + fn_ty: &'ll Type, + llfn: &'ll Value, + args: &'b [&'ll Value], + ) -> Cow<'b, [&'ll Value]> { + assert!( + self.cx.type_kind(fn_ty) == TypeKind::Function, + "builder::{} not passed a function, but {:?}", + typ, + fn_ty + ); + + let param_tys = self.cx.func_params_types(fn_ty); + + let all_args_match = param_tys + .iter() + .zip(args.iter().map(|&v| self.val_ty(v))) + .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty); + + if all_args_match { + return Cow::Borrowed(args); + } + + let casted_args: Vec<_> = param_tys + .into_iter() + .zip(args.iter()) + .enumerate() + .map(|(i, (expected_ty, &actual_val))| { + let actual_ty = self.val_ty(actual_val); + if expected_ty != actual_ty { + debug!( + "type mismatch in function call of {:?}. \ + Expected {:?} for param {}, got {:?}; injecting bitcast", + llfn, expected_ty, i, actual_ty + ); + self.bitcast(actual_val, expected_ty) + } else { + actual_val + } + }) + .collect(); + + Cow::Owned(casted_args) + } + + pub fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, unnamed()) } + } + + pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { + let (ty, f) = self.cx.get_intrinsic(intrinsic); + self.call(ty, None, None, f, args, None, None) + } + + fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) { + let size = size.bytes(); + if size == 0 { + return; + } + + if !self.cx().sess().emit_lifetime_markers() { + return; + } + + self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]); + } + + pub(crate) fn phi( + &mut self, + ty: &'ll Type, + vals: &[&'ll Value], + bbs: &[&'ll BasicBlock], + ) -> &'ll Value { + assert_eq!(vals.len(), bbs.len()); + let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, unnamed()) }; + unsafe { + llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint); + phi + } + } + + fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) { + unsafe { + llvm::LLVMAddIncoming(phi, &val, &bb, 1); + } + } +} diff --git a/crates/rustc_codegen_nvvm/src/common.rs b/crates/rustc_codegen_nvvm_v19/src/common.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/common.rs rename to crates/rustc_codegen_nvvm_v19/src/common.rs diff --git a/crates/rustc_codegen_nvvm_v19/src/const_ty.rs b/crates/rustc_codegen_nvvm_v19/src/const_ty.rs new file mode 100644 index 00000000..96d07b40 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/const_ty.rs @@ -0,0 +1,295 @@ +use crate::llvm; +use crate::llvm::{Bool, False, True, Type, Value}; +use crate::{consts::const_alloc_to_llvm, context::CodegenCx, ty::LayoutLlvmExt}; +use libc::c_uint; +use rustc_abi as abi; +use rustc_abi::Primitive::Pointer; +use rustc_abi::{AddressSpace, HasDataLayout}; +use rustc_ast::Mutability; +use rustc_codegen_ssa::common::TypeKind; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_hashes::Hash128; +use rustc_middle::bug; +use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; +use rustc_middle::ty::layout::LayoutOf; +use tracing::trace; + +impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn const_data_from_alloc(&self, alloc: ConstAllocation) -> &'ll Value { + const_alloc_to_llvm(self, alloc, /*static*/ false) + } + + fn const_null(&self, t: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMConstNull(t) } + } + + fn const_undef(&self, t: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMGetUndef(t) } + } + + fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value { + unsafe { llvm::LLVMConstInt(t, i as u64, True) } + } + + fn const_uint(&self, t: &'ll Type, i: u64) -> &'ll Value { + debug_assert!( + self.type_kind(t) == TypeKind::Integer, + "only allows integer types in const_int" + ); + unsafe { llvm::LLVMConstInt(t, i, False) } + } + + fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value { + debug_assert!( + self.type_kind(t) == TypeKind::Integer, + "only allows integer types in const_uint_big" + ); + unsafe { + let words = [u as u64, (u >> 64) as u64]; + llvm::LLVMConstIntOfArbitraryPrecision(t, 2, words.as_ptr()) + } + } + + fn const_bool(&self, val: bool) -> &'ll Value { + self.const_uint(self.type_i1(), val as u64) + } + + fn const_i32(&self, i: i32) -> &'ll Value { + self.const_int(self.type_i32(), i as i64) + } + + fn const_u32(&self, i: u32) -> &'ll Value { + self.const_uint(self.type_i32(), i as u64) + } + + fn const_u64(&self, i: u64) -> &'ll Value { + self.const_uint(self.type_i64(), i) + } + + fn const_usize(&self, i: u64) -> &'ll Value { + self.const_uint(self.isize_ty, i) + } + + fn const_u8(&self, i: u8) -> &'ll Value { + self.const_uint(self.type_i8(), i as u64) + } + + fn const_real(&self, t: &'ll Type, val: f64) -> &'ll Value { + unsafe { llvm::LLVMConstReal(t, val) } + } + + fn const_str(&self, s: &str) -> (&'ll Value, &'ll Value) { + let val = *self + .const_cstr_cache + .borrow_mut() + .raw_entry_mut() + .from_key(s) + .or_insert_with(|| { + let sc = self.const_bytes(s.as_bytes()); + let sym = self.generate_local_symbol_name("str"); + let g = self + .define_global(&sym[..], self.val_ty(sc), AddressSpace::DATA) + .unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + unsafe { + llvm::LLVMSetInitializer(g, sc); + llvm::LLVMSetGlobalConstant(g, True); + llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); + } + (s.to_owned(), g) + }) + .1; + let len = s.len(); + let ty = self.type_ptr_to(self.layout_of(self.tcx.types.str_).llvm_type(self)); + let cs = unsafe { llvm::LLVMConstPointerCast(val, ty) }; + (cs, self.const_usize(len as u64)) + } + + fn const_struct(&self, elts: &[&'ll Value], packed: bool) -> &'ll Value { + unsafe { + llvm::LLVMConstStructInContext( + self.llcx, + elts.as_ptr(), + elts.len() as c_uint, + packed as Bool, + ) + } + } + + fn const_to_opt_uint(&self, v: &'ll Value) -> Option { + unsafe { llvm::LLVMIsAConstantInt(v).map(|v| llvm::LLVMConstIntGetZExtValue(v)) } + } + + fn const_to_opt_u128(&self, v: &'ll Value, sign_ext: bool) -> Option { + unsafe { + llvm::LLVMIsAConstantInt(v).and_then(|v| { + let (mut lo, mut hi) = (0u64, 0u64); + let success = llvm::LLVMRustConstInt128Get(v, sign_ext, &mut hi, &mut lo); + success.then(|| hi_lo_to_u128(lo, hi)) + }) + } + } + + fn scalar_to_backend( + &self, + cv: Scalar, + layout: abi::Scalar, + mut llty: &'ll Type, + ) -> &'ll Value { + trace!("Scalar to backend `{:?}`, `{:?}`, `{:?}`", cv, layout, llty); + let bitsize = if layout.is_bool() { + 1 + } else { + layout.size(self).bits() + }; + let val = match cv { + Scalar::Int(int) => { + let data = int.to_bits(layout.size(self)); + let llval = self.const_uint_big(self.type_ix(bitsize), data); + if matches!(layout.primitive(), abi::Primitive::Pointer(_)) { + unsafe { llvm::LLVMConstIntToPtr(llval, llty) } + } else { + self.const_bitcast(llval, llty) + } + } + Scalar::Ptr(ptr, _) => { + let (prov, offset) = ptr.into_parts(); + let (base_addr, base_addr_space) = match self.tcx.global_alloc(prov.alloc_id()) { + GlobalAlloc::Memory(alloc) => { + // For ZSTs directly codegen an aligned pointer. + // This avoids generating a zero-sized constant value and actually needing a + // real address at runtime. + if alloc.inner().len() == 0 { + assert_eq!(offset.bytes(), 0); + let llval = self.const_usize(alloc.inner().align.bytes()); + return if matches!(layout.primitive(), abi::Primitive::Pointer(_)) { + unsafe { llvm::LLVMConstIntToPtr(llval, llty) } + } else { + self.const_bitcast(llval, llty) + }; + } else { + let init = const_alloc_to_llvm(self, alloc, /*static*/ false); + let alloc = alloc.inner(); + let value = match alloc.mutability { + Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), + _ => self.static_addr_of(init, alloc.align, None), + }; + if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty() + { + let hash = self.tcx.with_stable_hashing_context(|mut hcx| { + let mut hasher = StableHasher::new(); + alloc.hash_stable(&mut hcx, &mut hasher); + hasher.finish::() + }); + llvm::set_value_name( + value, + format!("alloc_{hash:032x}").as_bytes(), + ); + } + (value, AddressSpace::DATA) + } + } + GlobalAlloc::Function { instance, .. } => ( + self.get_fn_addr(instance), + self.data_layout().instruction_address_space, + ), + GlobalAlloc::VTable(ty, dyn_ty) => { + let alloc = self + .tcx + .global_alloc(self.tcx.vtable_allocation(( + ty, + dyn_ty.principal().map(|principal| { + self.tcx.instantiate_bound_regions_with_erased(principal) + }), + ))) + .unwrap_memory(); + let init = const_alloc_to_llvm(self, alloc, /*static*/ false); + let value = self.static_addr_of(init, alloc.inner().align, None); + (value, AddressSpace::DATA) + } + GlobalAlloc::Static(def_id) => { + assert!(self.tcx.is_static(def_id)); + assert!(!self.tcx.is_thread_local_static(def_id)); + let val = self.get_static(def_id); + let addrspace = + unsafe { llvm::LLVMGetPointerAddressSpace(self.val_ty(val)) }; + (val, AddressSpace(addrspace)) + } + }; + let llval = unsafe { + llvm::LLVMConstInBoundsGEP2( + self.type_i8(), + // Cast to the required address space if necessary + self.const_bitcast(base_addr, self.type_ptr_ext(base_addr_space)), + &self.const_usize(offset.bytes()), + 1, + ) + }; + + if !matches!(layout.primitive(), Pointer(_)) { + unsafe { llvm::LLVMConstPtrToInt(llval, llty) } + } else { + if base_addr_space != AddressSpace::DATA { + unsafe { + let element = llvm::LLVMGetElementType(llty); + llty = self.type_ptr_to_ext(element, base_addr_space); + } + } + self.const_bitcast(llval, llty) + } + } + }; + + trace!("...Scalar to backend: `{:?}`", val); + trace!("{:?}", std::backtrace::Backtrace::force_capture()); + + val + } + + fn is_undef(&self, v: Self::Value) -> bool { + unsafe { llvm::LLVMIsUndef(v) == True } + } + + fn const_poison(&self, t: Self::Type) -> Self::Value { + // FIXME: Use LLVMGetPoision when possible. + self.const_undef(t) + } + + fn const_i8(&self, i: i8) -> Self::Value { + self.const_int(self.type_i8(), i as i64) + } + + fn const_i16(&self, i: i16) -> Self::Value { + self.const_int(self.type_i16(), i as i64) + } + + fn const_u128(&self, i: u128) -> Self::Value { + trace!("const_u128 i = {i:?}"); + trace!("{}", std::backtrace::Backtrace::force_capture()); + self.const_uint_big(self.type_i128(), i) + } + + fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { + let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow"); + unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) } + } + + fn const_ptr_byte_offset(&self, mut base_addr: Self::Value, offset: abi::Size) -> Self::Value { + base_addr = self.const_ptrcast(base_addr, self.type_i8p()); + unsafe { + llvm::LLVMConstInBoundsGEP2( + self.type_i8(), + base_addr, + &self.const_usize(offset.bytes()), + 1, + ) + } + } +} + +#[inline] +fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 { + ((hi as u128) << 64) | (lo as u128) +} diff --git a/crates/rustc_codegen_nvvm_v19/src/consts.rs b/crates/rustc_codegen_nvvm_v19/src/consts.rs new file mode 100644 index 00000000..989ae244 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/consts.rs @@ -0,0 +1,448 @@ +use std::ops::Range; + +use crate::debug_info; +use crate::llvm; +use crate::llvm::{True, Type, Value}; +use libc::{c_char, c_uint}; +use rustc_abi::{AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; +use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::DefId; +use rustc_middle::mir::interpret::{ + Allocation, ConstAllocation, ErrorHandled, Pointer, read_target_uint, +}; +use rustc_middle::ty::layout::HasTypingEnv; +use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::{ + bug, + middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}, + mir::{ + interpret::{InitChunk, Scalar as InterpScalar}, + mono::{Linkage, MonoItem}, + }, + span_bug, +}; +use tracing::trace; + +use crate::{context::CodegenCx, ty::LayoutLlvmExt}; + +pub(crate) fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { + unsafe { + let ptr = bytes.as_ptr() as *const c_char; + llvm::LLVMConstStringInContext(llcx, ptr, bytes.len() as c_uint, True) + } +} + +impl<'ll> CodegenCx<'ll, '_> { + pub fn const_array(&self, ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { + unsafe { llvm::LLVMConstArray(ty, elts.as_ptr(), elts.len() as c_uint) } + } + + pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { + bytes_in_context(self.llcx, bytes) + } +} + +pub(crate) fn const_alloc_to_llvm<'ll>( + cx: &CodegenCx<'ll, '_>, + alloc: ConstAllocation<'_>, + is_static: bool, +) -> &'ll Value { + trace!("Const alloc to llvm"); + let alloc = alloc.inner(); + // We expect that callers of const_alloc_to_llvm will instead directly codegen a pointer or + // integer for any &ZST where the ZST is a constant (i.e. not a static). We should never be + // producing empty LLVM allocations as they're just adding noise to binaries and forcing less + // optimal codegen. + // + // Statics have a guaranteed meaningful address so it's less clear that we want to do + // something like this; it's also harder. + if !is_static { + assert!(alloc.len() != 0); + } + let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); + let dl = cx.data_layout(); + let pointer_size = dl.pointer_size.bytes() as usize; + + // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, + // so `range` must be within the bounds of `alloc` and not contain or overlap a relocation. + fn append_chunks_of_init_and_uninit_bytes<'ll, 'a>( + llvals: &mut Vec<&'ll Value>, + cx: &'a CodegenCx<'ll, '_>, + alloc: &'a Allocation, + range: Range, + ) { + let chunks = alloc.init_mask().range_as_init_chunks(range.clone().into()); + + let chunk_to_llval = move |chunk| match chunk { + InitChunk::Init(range) => { + let range = (range.start.bytes() as usize)..(range.end.bytes() as usize); + let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); + cx.const_bytes(bytes) + } + InitChunk::Uninit(range) => { + let len = range.end.bytes() - range.start.bytes(); + cx.const_undef(cx.type_array(cx.type_i8(), len)) + } + }; + + // Generating partially-uninit consts is limited to small numbers of chunks, + // to avoid the cost of generating large complex const expressions. + // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element, + // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`. + let max = cx.sess().opts.unstable_opts.uninit_const_chunk_threshold; + let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; + + if allow_uninit_chunks { + llvals.extend(chunks.map(chunk_to_llval)); + } else { + // If this allocation contains any uninit bytes, codegen as if it was initialized + // (using some arbitrary value for uninit bytes). + let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); + llvals.push(cx.const_bytes(bytes)); + } + } + + let mut next_offset = 0; + for &(offset, prov) in alloc.provenance().ptrs().iter() { + let offset = offset.bytes(); + assert_eq!(offset as usize as u64, offset); + let offset = offset as usize; + if offset > next_offset { + // This `inspect` is okay since we have checked that it is not within a relocation, it + // is within the bounds of the allocation, and it doesn't affect interpreter execution + // (we inspect the result after interpreter execution). + append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); + } + let ptr_offset = read_target_uint( + dl.endian, + // This `inspect` is okay since it is within the bounds of the allocation, it doesn't + // affect interpreter execution (we inspect the result after interpreter execution), + // and we properly interpret the relocation as a relocation pointer offset. + alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), + ) + .expect("const_alloc_to_llvm: could not read relocation pointer") + as u64; + + let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); + + llvals.push(cx.scalar_to_backend( + InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx), + Scalar::Initialized { + value: Primitive::Pointer(address_space), + valid_range: WrappingRange { start: 0, end: !0 }, + }, + cx.type_ptr_ext(address_space), + )); + next_offset = offset + pointer_size; + } + if alloc.len() >= next_offset { + let range = next_offset..alloc.len(); + // This `inspect` is okay since we have check that it is after all relocations, it is + // within the bounds of the allocation, and it doesn't affect interpreter execution (we + // inspect the result after interpreter execution). + append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); + } + + cx.const_struct(&llvals, true) +} + +pub(crate) fn codegen_static_initializer<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + def_id: DefId, +) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { + let alloc = cx.tcx.eval_static_initializer(def_id)?; + Ok((const_alloc_to_llvm(cx, alloc, /*static*/ true), alloc)) +} + +pub(crate) fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage { + match linkage { + Linkage::External => llvm::Linkage::ExternalLinkage, + Linkage::AvailableExternally => llvm::Linkage::AvailableExternallyLinkage, + Linkage::LinkOnceAny => llvm::Linkage::LinkOnceAnyLinkage, + Linkage::LinkOnceODR => llvm::Linkage::LinkOnceODRLinkage, + Linkage::WeakAny => llvm::Linkage::WeakAnyLinkage, + Linkage::WeakODR => llvm::Linkage::WeakODRLinkage, + Linkage::Internal => llvm::Linkage::InternalLinkage, + Linkage::ExternalWeak => llvm::Linkage::ExternalWeakLinkage, + Linkage::Common => llvm::Linkage::CommonLinkage, + } +} + +fn check_and_apply_linkage<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + attrs: &CodegenFnAttrs, + ty: Ty<'tcx>, + sym: &str, + span_def_id: DefId, + instance: Instance<'tcx>, +) -> &'ll Value { + let addrspace = cx.static_addrspace(instance); + let llty = cx.layout_of(ty).llvm_type(cx); + if let Some(linkage) = attrs.linkage { + // https://docs.nvidia.com/cuda/nvvm-ir-spec/index.html#linkage-types-nvvm + use Linkage::*; + match linkage { + External | Internal | Common | AvailableExternally | LinkOnceAny | LinkOnceODR + | WeakAny | WeakODR => {} + _ => cx + .sess() + .dcx() + .fatal(format!("Unsupported linkage kind: {:?}", linkage)), + } + + // If this is a static with a linkage specified, then we need to handle + // it a little specially. The typesystem prevents things like &T and + // extern "C" fn() from being non-null, so we can't just declare a + // static and call it a day. Some linkages (like weak) will make it such + // that the static actually has a null value. + let llty2 = if let ty::RawPtr(ty2, _) = ty.kind() { + cx.layout_of(*ty2).llvm_type(cx) + } else { + cx.sess().dcx().span_fatal( + cx.tcx.def_span(span_def_id), + "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", + ) + }; + unsafe { + // Declare a symbol `foo` with the desired linkage. + let g1 = cx.declare_global(sym, llty2, addrspace); + llvm::LLVMRustSetLinkage(g1, linkage_to_llvm(linkage)); + + // Declare an internal global `extern_with_linkage_foo` which + // is initialized with the address of `foo`. If `foo` is + // discarded during linking (for example, if `foo` has weak + // linkage and there are no definitions), then + // `extern_with_linkage_foo` will instead be initialized to + // zero. + let mut real_name = "_rust_extern_with_linkage_".to_string(); + real_name.push_str(sym); + let g2 = cx + .define_global(&real_name, llty, addrspace) + .unwrap_or_else(|| { + cx.sess().dcx().span_fatal( + cx.tcx.def_span(span_def_id), + format!("symbol `{}` is already defined", &sym), + ) + }); + llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage); + llvm::LLVMSetInitializer(g2, g1); + g2 + } + } else { + cx.declare_global(sym, llty, addrspace) + } +} + +impl<'ll> CodegenCx<'ll, '_> { + pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + trace!("Const bitcast: `{:?}` to `{:?}`", val, ty); + unsafe { llvm::LLVMConstBitCast(val, ty) } + } + + pub(crate) fn const_ptrcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + trace!("Const ptrcast: `{:?}` to `{:?}`", val, ty); + unsafe { llvm::LLVMConstPointerCast(val, ty) } + } + + pub(crate) fn static_addr_of_mut( + &self, + cv: &'ll Value, + align: Align, + kind: Option<&str>, + ) -> &'ll Value { + unsafe { + // TODO(RDambrosio016): replace this with latest rustc's handling when we use llvm 13 + let name = self.generate_local_symbol_name(kind.unwrap_or("private")); + let gv = self + .define_global(&name[..], self.val_ty(cv), AddressSpace::DATA) + .unwrap_or_else(|| bug!("symbol `{}` is already defined", name)); + llvm::LLVMRustSetLinkage(gv, llvm::Linkage::PrivateLinkage); + llvm::LLVMSetInitializer(gv, cv); + llvm::LLVMSetAlignment(gv, align.bytes() as c_uint); + llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); + gv + } + } + + pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { + let instance = Instance::mono(self.tcx, def_id); + if let Some(&g) = self.instances.borrow().get(&instance) { + return g; + } + + let defined_in_current_codegen_unit = self + .codegen_unit + .items() + .contains_key(&MonoItem::Static(def_id)); + assert!( + !defined_in_current_codegen_unit, + "consts::get_static() should always hit the cache for \ + statics defined in the same CGU, but did not for `{:?}`", + def_id + ); + + let ty = instance.ty(self.tcx, self.typing_env()); + let sym = self.tcx.symbol_name(instance).name; + let fn_attrs = self.tcx.codegen_fn_attrs(def_id); + + let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) { + let llty = self.layout_of(ty).llvm_type(self); + if let Some(g) = self.get_declared_value(sym) { + if self.val_ty(g) != self.type_ptr_to(llty) { + span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); + } + } + + let addrspace = self.static_addrspace(instance); + let g = self.declare_global(sym, llty, addrspace); + + if !self.tcx.is_reachable_non_generic(def_id) { + unsafe { + llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden); + } + } + + g + } else { + check_and_apply_linkage(self, fn_attrs, ty, sym, def_id, instance) + }; + + if fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { + self.unsupported("thread locals"); + } + + self.instances.borrow_mut().insert(instance, g); + g + } +} + +impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> { + fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { + if let Some(&gv) = self.const_globals.borrow().get(&cv) { + unsafe { + // Upgrade the alignment in cases where the same constant is used with different + // alignment requirements + let llalign = align.bytes() as u32; + if llalign > llvm::LLVMGetAlignment(gv) { + llvm::LLVMSetAlignment(gv, llalign); + } + } + return gv; + } + let gv = self.static_addr_of_mut(cv, align, kind); + unsafe { + llvm::LLVMSetGlobalConstant(gv, True); + } + self.const_globals.borrow_mut().insert(cv, gv); + gv + } + + fn codegen_static(&self, def_id: DefId) { + unsafe { + assert!( + llvm::LLVMGetInitializer( + self.instances + .borrow() + .get(&Instance::mono(self.tcx, def_id)) + .unwrap() + ) + .is_none() + ); + let attrs = self.tcx.codegen_fn_attrs(def_id); + + let (v, _) = match codegen_static_initializer(self, def_id) { + Ok(v) => v, + // Error has already been reported + Err(_) => return, + }; + + let g = self.get_static(def_id); + + let mut val_llty = self.val_ty(v); + let v: &Value = if val_llty == self.type_i1() { + val_llty = self.type_i8(); + // TODO: not sure about this + let const_int = v as *const llvm::Value as *const llvm::ConstantInt; + let const_val = llvm::LLVMConstIntGetZExtValue(&*const_int); + llvm::LLVMConstInt(val_llty, const_val, 0) + } else { + v + }; + + let instance = Instance::mono(self.tcx, def_id); + let ty = instance.ty(self.tcx, self.typing_env()); + let llty = self.layout_of(ty).llvm_type(self); + let g = if val_llty == llty { + g + } else { + trace!( + "Making new RAUW global: from ty `{:?}` to `{:?}`, initializer: `{:?}`", + llty, val_llty, v + ); + // If we created the global with the wrong type, + // correct the type. + let name = llvm::get_value_name(g).to_vec(); + + llvm::set_value_name(g, b""); + + let linkage = llvm::LLVMRustGetLinkage(g); + let visibility = llvm::LLVMRustGetVisibility(g); + + let addrspace = self.static_addrspace(instance); + let new_g = llvm::LLVMRustGetOrInsertGlobal( + self.llmod, + name.as_ptr().cast(), + name.len(), + val_llty, + addrspace.0, + ); + + llvm::LLVMRustSetLinkage(new_g, linkage); + llvm::LLVMRustSetVisibility(new_g, visibility); + + // To avoid breaking any invariants, we leave around the old + // global for the moment; we'll replace all references to it + // with the new global later. (See base::codegen_backend.) + self.statics_to_rauw.borrow_mut().push((g, new_g)); + new_g + }; + trace!("Codegen static `{:?}`", g); + llvm::LLVMSetAlignment(g, self.align_of(ty).bytes() as c_uint); + llvm::LLVMSetInitializer(g, v); + + debug_info::build_global_var_di_node(self, def_id, g); + + // As an optimization, all shared statics which do not have interior + // mutability are placed into read-only memory. + if self.type_is_freeze(ty) { + // TODO(RDambrosio016): is this the same as putting this in + // the __constant__ addrspace for nvvm? should we set this addrspace explicitly? + // llvm::LLVMSetGlobalConstant(g, llvm::True); + } + + debug_info::build_global_var_di_node(self, def_id, g); + + if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { + self.unsupported("thread locals"); + } + + if attrs.flags.contains(CodegenFnAttrFlags::USED) { + self.add_used_global(g); + } + trace!("Codegen static `{:?}`", g); + } + } + + /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*. + fn add_used_global(&self, global: &'ll Value) { + let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; + self.used_statics.borrow_mut().push(cast); + } + + /// Add a global value to a list to be stored in the `llvm.compiler.used` variable, + /// an array of i8*. + fn add_compiler_used_global(&self, global: &'ll Value) { + let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; + self.compiler_used_statics.borrow_mut().push(cast); + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/context.rs b/crates/rustc_codegen_nvvm_v19/src/context.rs new file mode 100644 index 00000000..88d4cc7b --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/context.rs @@ -0,0 +1,683 @@ +use std::cell::{Cell, RefCell}; +use std::ffi::CStr; +use std::path::PathBuf; +use std::ptr::null; +use std::str::FromStr; + +use crate::abi::FnAbiLlvmExt; +use crate::attributes::{self, NvvmAttributes, Symbols}; +use crate::debug_info::{self, CodegenUnitDebugContext}; +use crate::llvm; +use crate::llvm::{BasicBlock, Type, Value}; +use crate::{LlvmMod, target}; +use nvvm::NvvmOption; +use rustc_abi::AddressSpace; +use rustc_abi::{HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx}; +use rustc_codegen_ssa::errors as ssa_errors; +use rustc_codegen_ssa::traits::{ + BackendTypes, BaseTypeCodegenMethods, CoverageInfoBuilderMethods, DerivedTypeCodegenMethods, + MiscCodegenMethods, +}; +use rustc_data_structures::base_n::{ALPHANUMERIC_ONLY, ToBaseN}; +use rustc_errors::DiagMessage; +use rustc_hash::FxHashMap; +use rustc_middle::dep_graph::DepContext; +use rustc_middle::ty::layout::{ + FnAbiError, FnAbiOf, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOf, +}; +use rustc_middle::ty::layout::{FnAbiOfHelpers, LayoutOfHelpers}; +use rustc_middle::ty::{Ty, TypeVisitableExt}; +use rustc_middle::{bug, span_bug, ty}; +use rustc_middle::{ + mir::mono::CodegenUnit, + ty::{Instance, TyCtxt}, +}; +use rustc_session::Session; +use rustc_session::config::DebugInfo; +use rustc_span::source_map::Spanned; +use rustc_span::{Span, Symbol}; +use rustc_target::callconv::FnAbi; + +use rustc_target::spec::{HasTargetSpec, Target}; +use tracing::{debug, trace}; + +/// "There is a total of 64 KB constant memory on a device." +/// +const CONSTANT_MEMORY_SIZE_LIMIT_BYTES: u64 = 64 * 1024; + +pub(crate) struct CodegenCx<'ll, 'tcx> { + pub tcx: TyCtxt<'tcx>, + + pub llmod: &'ll llvm::Module, + pub llcx: &'ll llvm::Context, + pub codegen_unit: &'tcx CodegenUnit<'tcx>, + + /// Map of MIR functions to LLVM function values + pub instances: RefCell, &'ll Value>>, + /// A cache of the generated vtables for trait objects + pub vtables: RefCell, Option>), &'ll Value>>, + /// A cache of constant strings and their values + pub const_cstr_cache: RefCell>, + /// A map of functions which have parameters at specific indices replaced with an int-remapped type. + /// such as i128 --> <2 x i64> + #[allow(clippy::type_complexity)] + pub remapped_integer_args: + RefCell, Vec<(usize, &'ll Type)>)>>, + + /// Cache of emitted const globals (value -> global) + pub const_globals: RefCell>, + + /// List of globals for static variables which need to be passed to the + /// LLVM function ReplaceAllUsesWith (RAUW) when codegen is complete. + /// (We have to make sure we don't invalidate any Values referring + /// to constants.) + pub statics_to_rauw: RefCell>, + + /// Statics that will be placed in the llvm.used variable + /// See for details + pub used_statics: RefCell>, + + /// Statics that will be placed in the llvm.compiler.used variable + /// See for details + pub compiler_used_statics: RefCell>, + + pub lltypes: RefCell, Option), &'ll Type>>, + pub scalar_lltypes: RefCell, &'ll Type>>, + pub pointee_infos: RefCell, Size), Option>>, + pub isize_ty: &'ll Type, + + pub dbg_cx: Option>, + + /// A map of the intrinsics we actually declared for usage. + pub(crate) intrinsics: RefCell>, + /// A map of the intrinsics available but not yet declared. + pub(crate) intrinsics_map: RefCell, &'ll Type)>>, + + local_gen_sym_counter: Cell, + + nvptx_data_layout: TargetDataLayout, + nvptx_target: Target, + + /// empty eh_personality function + eh_personality: &'ll Value, + + pub symbols: Symbols, + pub codegen_args: CodegenArgs, + // the value of the last call instruction. Needed for return type remapping. + pub last_call_llfn: Cell>, +} + +impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { + pub(crate) fn new( + tcx: TyCtxt<'tcx>, + codegen_unit: &'tcx CodegenUnit<'tcx>, + llvm_module: &'ll LlvmMod, + ) -> Self { + debug!("Creating new CodegenCx"); + let (llcx, llmod) = (&*llvm_module.llcx, unsafe { + llvm_module.llmod.as_ref().unwrap() + }); + + let isize_ty = Type::ix_llcx(llcx, target::POINTER_WIDTH as u64); + // the eh_personality function doesnt make sense on the GPU, but we still need to give + // rustc something, so we just give it an empty function + let eh_personality = unsafe { + let void = llvm::LLVMVoidTypeInContext(llcx); + let llfnty = llvm::LLVMFunctionType(void, null(), 0, llvm::False); + let name = "__rust_eh_personality"; + llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), llfnty) + }; + + let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None { + let dctx = CodegenUnitDebugContext::new(llmod); + debug_info::build_compile_unit_di_node(tcx, codegen_unit.name().as_str(), &dctx); + Some(dctx) + } else { + None + }; + + let mut cx = CodegenCx { + tcx, + llmod, + llcx, + codegen_unit, + instances: Default::default(), + vtables: Default::default(), + const_cstr_cache: Default::default(), + remapped_integer_args: Default::default(), + const_globals: Default::default(), + statics_to_rauw: RefCell::new(Vec::new()), + used_statics: RefCell::new(Vec::new()), + compiler_used_statics: RefCell::new(Vec::new()), + lltypes: Default::default(), + scalar_lltypes: Default::default(), + pointee_infos: Default::default(), + isize_ty, + intrinsics: Default::default(), + intrinsics_map: RefCell::new(FxHashMap::with_capacity_and_hasher( + // ~319 libdevice intrinsics plus some headroom for llvm + 350, + Default::default(), + )), + local_gen_sym_counter: Cell::new(0), + nvptx_data_layout: TargetDataLayout::parse_from_llvm_datalayout_string( + &target::target().data_layout, + ) + .unwrap_or_else(|err| tcx.sess.dcx().emit_fatal(err)), + nvptx_target: target::target(), + eh_personality, + symbols: Symbols { + nvvm_internal: Symbol::intern("nvvm_internal"), + kernel: Symbol::intern("kernel"), + addrspace: Symbol::intern("addrspace"), + }, + dbg_cx, + codegen_args: CodegenArgs::from_session(tcx.sess()), + last_call_llfn: Cell::new(None), + }; + cx.build_intrinsics_map(); + cx + } + + pub(crate) fn fatal(&self, msg: impl Into) -> ! { + self.tcx.sess.dcx().fatal(msg) + } + + // im lazy i know + pub(crate) fn unsupported(&self, thing: &str) -> ! { + self.fatal(format!("{} is unsupported", thing)) + } + + pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { + let section = c"llvm.metadata"; + let array = self.const_array(self.type_ptr_to(self.type_i8()), values); + + unsafe { + trace!( + "Creating LLVM used variable with name `{}` and values:\n{:#?}", + name.to_str().unwrap(), + values + ); + let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); + llvm::LLVMSetInitializer(g, array); + llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); + llvm::LLVMSetSection(g, section.as_ptr()); + } + } +} + +fn sanitize_global_ident(name: &str) -> String { + name.replace(".", "$") +} + +impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn vtables( + &self, + ) -> &RefCell, Option>), &'ll Value>> { + &self.vtables + } + + fn get_fn(&self, instance: Instance<'tcx>) -> &'ll Value { + self.get_fn(instance) + } + + fn get_fn_addr(&self, instance: Instance<'tcx>) -> &'ll Value { + self.get_fn(instance) + } + + fn eh_personality(&self) -> &'ll Value { + self.eh_personality + } + + fn sess(&self) -> &Session { + self.tcx.sess + } + + fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx> { + self.codegen_unit + } + + fn declare_c_main( + &self, + _fn_type: as rustc_codegen_ssa::traits::BackendTypes>::Type, + ) -> Option< as rustc_codegen_ssa::traits::BackendTypes>::Function> { + // no point for gpu kernels + None + } + + fn apply_target_cpu_attr( + &self, + _llfn: as rustc_codegen_ssa::traits::BackendTypes>::Function, + ) { + // no point if we are running on the gpu ;) + } + + fn set_frame_pointer_type( + &self, + _llfn: as rustc_codegen_ssa::traits::BackendTypes>::Function, + ) { + } +} + +impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { + /// Computes the address space for a static. + pub fn static_addrspace(&self, instance: Instance<'tcx>) -> AddressSpace { + let ty = instance.ty(self.tcx, self.typing_env()); + let is_mutable = self.tcx().is_mutable_static(instance.def_id()); + let attrs = self.tcx.get_attrs_unchecked(instance.def_id()); // TODO: replace with get_attrs + let nvvm_attrs = NvvmAttributes::parse(self, attrs); + + if let Some(addr) = nvvm_attrs.addrspace { + return AddressSpace(addr as u32); + } + + if !is_mutable && self.type_is_freeze(ty) { + if !self.codegen_args.use_constant_memory_space { + // We aren't using constant memory, so put the instance in global memory. + AddressSpace(1) + } else { + // We are using constant memory, see if the instance will fit. + // + // FIXME(@LegNeato) ideally we keep track of what we have put into + // constant memory and when it is filled up spill instead of only + // spilling when a static is big. We'll probably want some packing + // strategy controlled by the user...for example, if you have one large + // static and many small ones, you might want the small ones to all be + // in constant memory or just the big one depending on your workload. + let layout = self.layout_of(ty); + if layout.size.bytes() > CONSTANT_MEMORY_SIZE_LIMIT_BYTES { + self.tcx.sess.dcx().warn(format!( + "static `{}` exceeds the constant memory limit; placing in global memory (performance may be reduced)", + instance + )); + // Place instance in global memory if it is too big for constant memory. + AddressSpace(1) + } else { + // Place instance in constant memory if it fits. + AddressSpace(4) + } + } + } else { + AddressSpace::DATA + } + } + + /// Declare a global value, returns the existing value if it was already declared. + pub fn declare_global( + &self, + name: &str, + ty: &'ll Type, + address_space: AddressSpace, + ) -> &'ll Value { + // NVVM doesnt allow `.` inside of globals, this should be sound, at worst it should result in an nvvm error if something goes wrong. + let name = sanitize_global_ident(name); + trace!("Declaring global `{}`", name); + unsafe { + llvm::LLVMRustGetOrInsertGlobal( + self.llmod, + name.as_ptr().cast(), + name.len(), + ty, + address_space.0, + ) + } + } + + /// Declare a function. All functions use the default ABI, NVVM ignores any calling convention markers. + /// All functions calls are generated according to the PTX calling convention. + /// + pub fn declare_fn( + &self, + name: &str, + ty: &'ll Type, + fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, + ) -> &'ll Value { + let llfn = unsafe { + llvm::LLVMRustGetOrInsertFunction(self.llmod, name.as_ptr().cast(), name.len(), ty) + }; + trace!("Declaring function `{}` with ty `{:?}`", name, ty); + + // TODO(RDambrosio016): we should probably still generate accurate calling conv for functions + // just to make it easier to debug IR and/or make it more compatible with compiling using llvm + llvm::SetUnnamedAddress(llfn, llvm::UnnamedAddr::Global); + if let Some(abi) = fn_abi { + abi.apply_attrs_llfn(self, llfn); + } + attributes::default_optimisation_attrs(self.tcx.sess, llfn); + llfn + } + + /// Declare a global with an intention to define it. + /// + /// Use this function when you intend to define a global. This function will + /// return `None` if the name already has a definition associated with it. In that + /// case an error should be reported to the user, because it usually happens due + /// to user’s fault (e.g., misuse of `#[no_mangle]` or `#[export_name]` attributes). + pub fn define_global( + &self, + name: &str, + ty: &'ll Type, + address_space: AddressSpace, + ) -> Option<&'ll Value> { + if self.get_defined_value(name).is_some() { + None + } else { + Some(self.declare_global(name, ty, address_space)) + } + } + + // /// Declare a private global + // /// + // /// Use this function when you intend to define a global without a name. + // pub fn define_private_global(&self, ty: &'ll Type) -> &'ll Value { + // println!("Declaring private global with ty `{:?}`", ty); + // unsafe { llvm::LLVMRustInsertPrivateGlobal(self.llmod, ty) } + // } + + /// Gets declared value by name. + pub fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { + // NVVM doesnt allow `.` inside of globals, this should be sound, at worst it should result in an llvm/nvvm error if something goes wrong. + let name = sanitize_global_ident(name); + trace!("Retrieving value with name `{}`...", name); + let res = + unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_ptr().cast(), name.len()) }; + trace!("...Retrieved value: `{:?}`", res); + res + } + + /// Gets defined or externally defined (AvailableExternally linkage) value by + /// name. + pub fn get_defined_value(&self, name: &str) -> Option<&'ll Value> { + self.get_declared_value(name).and_then(|val| { + let declaration = unsafe { llvm::LLVMIsDeclaration(val) != 0 }; + if !declaration { Some(val) } else { None } + }) + } + + pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { + if let Some(v) = self.intrinsics.borrow().get(key).cloned() { + return v; + } + let result = self.declare_intrinsic(key) + .unwrap_or_else(|| bug!("unknown intrinsic '{}'", key)); + result + } + + pub(crate) fn insert_intrinsic( + &self, + name: &str, + args: Option<&[&'ll Type]>, + ret: &'ll Type, + ) -> (&'ll Type, &'ll Value) { + let fn_ty = if let Some(args) = args { + self.type_func(args, ret) // This includes empty args: fn() -> ret + } else { + self.type_variadic_func(&[], ret) // This should rarely be used + }; + let f = self.declare_fn(name, fn_ty, None); + llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No); + self.intrinsics + .borrow_mut() + .insert(name.to_owned(), (fn_ty, f)); + (fn_ty, f) + } + + pub fn generate_local_symbol_name(&self, prefix: &str) -> String { + let idx = self.local_gen_sym_counter.get(); + self.local_gen_sym_counter.set(idx + 1); + // Include a '.' character, so there can be no accidental conflicts with + // user defined names + let mut name = String::with_capacity(prefix.len() + 6); + name.push_str(prefix); + name.push('.'); + name.push_str(&(idx as u64).to_base(ALPHANUMERIC_ONLY)); + name + } + + //// Codegens a reference to a function/method, monomorphizing and inlining as it goes. + pub fn get_fn(&self, instance: Instance<'tcx>) -> &'ll Value { + let tcx = self.tcx; + + assert!(!instance.args.has_infer()); + assert!(!instance.args.has_escaping_bound_vars()); + + if let Some(&llfn) = self.instances.borrow().get(&instance) { + return llfn; + } + + let sym = tcx.symbol_name(instance).name; + debug!( + "get_fn({:?}: {:?}) => {}", + instance, + instance.ty(self.tcx(), self.typing_env()), + sym + ); + + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + + let llfn = if let Some(llfn) = self.get_declared_value(sym) { + trace!("Returning existing llfn `{:?}`", llfn); + let llptrty = fn_abi.ptr_to_llvm_type(self); + + if self.val_ty(llfn) != llptrty { + trace!( + "ptrcasting llfn to different llptrty: `{:?}` --> `{:?}`", + llfn, llptrty + ); + self.const_ptrcast(llfn, llptrty) + } else { + llfn + } + } else { + let llfn = self.declare_fn(sym, fn_abi.llvm_type(self), Some(fn_abi)); + attributes::from_fn_attrs(self, llfn, instance); + let def_id = instance.def_id(); + + unsafe { + llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::ExternalLinkage); + + let is_generic = instance.args.non_erasable_generics().next().is_some(); + + // nvvm ignores visibility styles, but we still make them just in case it will do something + // with them in the future or we want to use that metadata + if is_generic { + if tcx.sess.opts.share_generics() { + if let Some(instance_def_id) = def_id.as_local() { + // This is a definition from the current crate. If the + // definition is unreachable for downstream crates or + // the current crate does not re-export generics, the + // definition of the instance will have been declared + // as `hidden`. + if tcx.is_unreachable_local_definition(instance_def_id) + || !tcx.local_crate_exports_generics() + { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } else { + // This is a monomorphization of a generic function + // defined in an upstream crate. + if instance.upstream_monomorphization(tcx).is_some() { + // This is instantiated in another crate. It cannot + // be `hidden`. + } else { + // This is a local instantiation of an upstream definition. + // If the current crate does not re-export it + // (because it is a C library or an executable), it + // will have been declared `hidden`. + if !tcx.local_crate_exports_generics() { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } + } + } else { + // When not sharing generics, all instances are in the same + // crate and have hidden visibility + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } else { + // This is a non-generic function + if tcx.is_codegened_item(def_id) { + // This is a function that is instantiated in the local crate + + if def_id.is_local() { + // This is function that is defined in the local crate. + // If it is not reachable, it is hidden. + if !tcx.is_reachable_non_generic(def_id) { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } else { + // This is a function from an upstream crate that has + // been instantiated here. These are always hidden. + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } + } + llfn + } + }; + + self.instances.borrow_mut().insert(instance, llfn); + + llfn + } +} + +#[derive(Default, Clone, Debug)] +pub struct CodegenArgs { + pub nvvm_options: Vec, + pub override_libm: bool, + pub use_constant_memory_space: bool, + pub final_module_path: Option, +} + +impl CodegenArgs { + pub fn from_session(sess: &Session) -> Self { + Self::parse(&sess.opts.cg.llvm_args) + } + + // we may want to use rustc's own option parsing facilities to have better errors in the future. + pub fn parse(args: &[String]) -> Self { + // TODO: replace this with a "proper" arg parser. + let mut cg_args = Self::default(); + + for (idx, arg) in args.iter().enumerate() { + if let Ok(flag) = NvvmOption::from_str(arg) { + cg_args.nvvm_options.push(flag); + } else if arg == "--override-libm" { + cg_args.override_libm = true; + } else if arg == "--use-constant-memory-space" { + cg_args.use_constant_memory_space = true; + } else if arg == "--final-module-path" { + cg_args.final_module_path = Some(PathBuf::from( + args.get(idx + 1).expect("No path for --final-module-path"), + )); + } + } + + cg_args + } +} + +impl<'ll> BackendTypes for CodegenCx<'ll, '_> { + type Value = &'ll Value; + type Function = &'ll Value; + + type BasicBlock = &'ll BasicBlock; + type Type = &'ll Type; + // not applicable to nvvm, unwinding/exception handling + // doesnt exist on the gpu + type Funclet = (); + + type DIScope = &'ll llvm::DIScope; + type DILocation = &'ll llvm::DILocation; + type DIVariable = &'ll llvm::DIVariable; + + type Metadata = &'ll llvm::Metadata; +} + +impl HasDataLayout for CodegenCx<'_, '_> { + fn data_layout(&self) -> &TargetDataLayout { + &self.nvptx_data_layout + } +} + +impl HasTargetSpec for CodegenCx<'_, '_> { + fn target_spec(&self) -> &Target { + &self.nvptx_target + } +} + +impl<'tcx> ty::layout::HasTyCtxt<'tcx> for CodegenCx<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } +} + +impl<'tcx> HasTypingEnv<'tcx> for CodegenCx<'_, 'tcx> { + fn typing_env<'a>(&'a self) -> ty::TypingEnv<'tcx> { + ty::TypingEnv::fully_monomorphized() + } +} + +impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { + #[inline] + fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { + if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { + self.tcx.dcx().emit_fatal(Spanned { + span, + node: err.into_diagnostic(), + }) + } else { + self.tcx + .dcx() + .emit_fatal(ssa_errors::FailedToGetLayout { span, ty, err }) + } + } +} + +impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + match err { + FnAbiError::Layout(LayoutError::SizeOverflow(_) | LayoutError::Cycle(_)) => { + self.tcx.dcx().emit_fatal(Spanned { span, node: err }); + } + _ => match fn_abi_request { + FnAbiRequest::OfFnPtr { sig, extra_args } => { + span_bug!( + span, + "`fn_abi_of_fn_ptr({sig}, {extra_args:?})` failed: {err:?}", + ); + } + FnAbiRequest::OfInstance { + instance, + extra_args, + } => { + span_bug!( + span, + "`fn_abi_of_instance({instance}, {extra_args:?})` failed: {err:?}", + ); + } + }, + } + } +} + +impl<'tcx> CoverageInfoBuilderMethods<'tcx> for CodegenCx<'_, 'tcx> { + fn init_coverage(&mut self, _instance: Instance<'tcx>) { + todo!() + } + + fn add_coverage( + &mut self, + _instance: Instance<'tcx>, + _kind: &rustc_middle::mir::coverage::CoverageKind, + ) { + todo!() + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/ctx_intrinsics.rs b/crates/rustc_codegen_nvvm_v19/src/ctx_intrinsics.rs new file mode 100644 index 00000000..6d7aac5c --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/ctx_intrinsics.rs @@ -0,0 +1,428 @@ +use crate::context::CodegenCx; +use crate::llvm::{Value, Type}; +use rustc_codegen_ssa::traits::BaseTypeCodegenMethods; +use rustc_session::config::DebugInfo; + +impl<'ll> CodegenCx<'ll, '_> { + pub(crate) fn declare_intrinsic(&self, key: &str) -> Option<(&'ll Type, &'ll Value)> { + let map = self.intrinsics_map.borrow(); + let (args, ret) = map.get(key)?; + let result = self.insert_intrinsic(key, Some(args), ret); + Some(result) + } + + #[rustfmt::skip] // stop rustfmt from making this 2k lines + pub(crate) fn build_intrinsics_map(&mut self) { + let mut map = self.intrinsics_map.borrow_mut(); + let mut remapped = self.remapped_integer_args.borrow_mut(); + + macro_rules! ifn { + ($map:expr, $($name:literal)|*, fn($($arg:expr),*) -> $ret:expr) => { + for name in [$($name),*] { + map.insert(name, (vec![$($arg),*], $ret)); + } + }; + } + + let real_t_i128 = self.type_i128(); + let real_t_i128_i1 = self.type_struct(&[real_t_i128, self.type_i1()], false); + + let i8p = self.type_i8p(); + let void = self.type_void(); + let i1 = self.type_i1(); + let t_i8 = self.type_i8(); + let t_i16 = self.type_i16(); + let t_i32 = self.type_i32(); + let t_i64 = self.type_i64(); + let t_i128 = self.type_vector(t_i64, 2); + let t_f32 = self.type_f32(); + let t_f64 = self.type_f64(); + let t_isize = self.type_isize(); + + let t_i8_i1 = self.type_struct(&[t_i8, i1], false); + let t_i16_i1 = self.type_struct(&[t_i16, i1], false); + let t_i32_i1 = self.type_struct(&[t_i32, i1], false); + let t_i64_i1 = self.type_struct(&[t_i64, i1], false); + let t_i128_i1 = self.type_struct(&[t_i128, i1], false); + + let voidp = self.voidp(); + + ifn!(map, "llvm.trap" | "llvm.sideeffect", fn() -> void); + ifn!(map, "llvm.assume", fn(i1) -> void); + ifn!(map, "llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void); + + ifn!(map, "llvm.sadd.with.overflow.i16", fn(t_i16, t_i16) -> t_i16_i1); + ifn!(map, "llvm.sadd.with.overflow.i32", fn(t_i32, t_i32) -> t_i32_i1); + ifn!(map, "llvm.sadd.with.overflow.i64", fn(t_i64, t_i64) -> t_i64_i1); + + ifn!(map, "llvm.uadd.with.overflow.i16", fn(t_i16, t_i16) -> t_i16_i1); + ifn!(map, "llvm.uadd.with.overflow.i32", fn(t_i32, t_i32) -> t_i32_i1); + ifn!(map, "llvm.uadd.with.overflow.i64", fn(t_i64, t_i64) -> t_i64_i1); + + ifn!(map, "llvm.ssub.with.overflow.i16", fn(t_i16, t_i16) -> t_i16_i1); + ifn!(map, "llvm.ssub.with.overflow.i32", fn(t_i32, t_i32) -> t_i32_i1); + ifn!(map, "llvm.ssub.with.overflow.i64", fn(t_i64, t_i64) -> t_i64_i1); + + ifn!(map, "llvm.usub.with.overflow.i16", fn(t_i16, t_i16) -> t_i16_i1); + ifn!(map, "llvm.usub.with.overflow.i32", fn(t_i32, t_i32) -> t_i32_i1); + ifn!(map, "llvm.usub.with.overflow.i64", fn(t_i64, t_i64) -> t_i64_i1); + + ifn!(map, "llvm.smul.with.overflow.i16", fn(t_i16, t_i16) -> t_i16_i1); + ifn!(map, "llvm.smul.with.overflow.i32", fn(t_i32, t_i32) -> t_i32_i1); + ifn!(map, "llvm.smul.with.overflow.i64", fn(t_i64, t_i64) -> t_i64_i1); + + ifn!(map, "llvm.umul.with.overflow.i16", fn(t_i16, t_i16) -> t_i16_i1); + ifn!(map, "llvm.umul.with.overflow.i32", fn(t_i32, t_i32) -> t_i32_i1); + ifn!(map, "llvm.umul.with.overflow.i64", fn(t_i64, t_i64) -> t_i64_i1); + + let i128_checked_binops = [ + "__nvvm_i128_addo", + "__nvvm_u128_addo", + "__nvvm_i128_subo", + "__nvvm_u128_subo", + "__nvvm_i128_mulo", + "__nvvm_u128_mulo" + ]; + + for binop in i128_checked_binops { + map.insert(binop, (vec![t_i128, t_i128], t_i128_i1)); + let llfn_ty = self.type_func(&[t_i128, t_i128], t_i128_i1); + remapped.insert(llfn_ty, (Some(real_t_i128_i1), vec![(0, real_t_i128), (1, real_t_i128)])); + } + + let i128_saturating_ops = [ + "llvm.sadd.sat.i128", + "llvm.uadd.sat.i128", + "llvm.ssub.sat.i128", + "llvm.usub.sat.i128", + ]; + + for binop in i128_saturating_ops { + map.insert(binop, (vec![t_i128, t_i128], t_i128)); + let llfn_ty = self.type_func(&[t_i128, t_i128], t_i128); + remapped.insert(llfn_ty, (Some(real_t_i128), vec![(0, real_t_i128), (1, real_t_i128)])); + } + + // for some very strange reason, they arent supported for i8 either, but that case + // is easy to handle and we declare our own functions for that which just + // zext to i16, use the i16 intrinsic, then trunc back to i8 + + // these are declared in libintrinsics, see libintrinsics.ll + ifn!(map, "__nvvm_i8_addo", fn(t_i8, t_i8) -> t_i8_i1); + ifn!(map, "__nvvm_u8_addo", fn(t_i8, t_i8) -> t_i8_i1); + ifn!(map, "__nvvm_i8_subo", fn(t_i8, t_i8) -> t_i8_i1); + ifn!(map, "__nvvm_u8_subo", fn(t_i8, t_i8) -> t_i8_i1); + ifn!(map, "__nvvm_i8_mulo", fn(t_i8, t_i8) -> t_i8_i1); + ifn!(map, "__nvvm_u8_mulo", fn(t_i8, t_i8) -> t_i8_i1); + + // see comment in libintrinsics.ll + // ifn!(map, "__nvvm_i128_trap", fn(t_i128, t_i128) -> t_i128); + + ifn!(map, "llvm.sadd.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!(map, "llvm.sadd.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!(map, "llvm.sadd.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!(map, "llvm.sadd.sat.i64", fn(t_i64, t_i64) -> t_i64); + + ifn!(map, "llvm.uadd.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!(map, "llvm.uadd.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!(map, "llvm.uadd.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!(map, "llvm.uadd.sat.i64", fn(t_i64, t_i64) -> t_i64); + + ifn!(map, "llvm.ssub.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!(map, "llvm.ssub.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!(map, "llvm.ssub.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!(map, "llvm.ssub.sat.i64", fn(t_i64, t_i64) -> t_i64); + + ifn!(map, "llvm.usub.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!(map, "llvm.usub.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!(map, "llvm.usub.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!(map, "llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64); + + ifn!(map, "llvm.fshl.i8", fn(t_i8, t_i8, t_i8) -> t_i8); + ifn!(map, "llvm.fshl.i16", fn(t_i16, t_i16, t_i16) -> t_i16); + ifn!(map, "llvm.fshl.i32", fn(t_i32, t_i32, t_i32) -> t_i32); + ifn!(map, "llvm.fshl.i64", fn(t_i64, t_i64, t_i64) -> t_i64); + + ifn!(map, "llvm.fshr.i8", fn(t_i8, t_i8, t_i8) -> t_i8); + ifn!(map, "llvm.fshr.i16", fn(t_i16, t_i16, t_i16) -> t_i16); + ifn!(map, "llvm.fshr.i32", fn(t_i32, t_i32, t_i32) -> t_i32); + ifn!(map, "llvm.fshr.i64", fn(t_i64, t_i64, t_i64) -> t_i64); + + ifn!(map, "llvm.ctpop.i8", fn(t_i8) -> t_i8); + ifn!(map, "llvm.ctpop.i16", fn(t_i16) -> t_i16); + ifn!(map, "llvm.ctpop.i32", fn(t_i32) -> t_i32); + ifn!(map, "llvm.ctpop.i64", fn(t_i64) -> t_i64); + + ifn!(map, "llvm.bitreverse.i8", fn(t_i8) -> t_i8); + ifn!(map, "llvm.bitreverse.i16", fn(t_i16) -> t_i16); + ifn!(map, "llvm.bitreverse.i32", fn(t_i32) -> t_i32); + ifn!(map, "llvm.bitreverse.i64", fn(t_i64) -> t_i64); + + ifn!(map, "llvm.bswap.i16", fn(t_i16) -> t_i16); + ifn!(map, "llvm.bswap.i32", fn(t_i32) -> t_i32); + ifn!(map, "llvm.bswap.i64", fn(t_i64) -> t_i64); + + ifn!(map, "llvm.ctlz.i8", fn(t_i8, i1) -> t_i8); + ifn!(map, "llvm.ctlz.i16", fn(t_i16, i1) -> t_i16); + ifn!(map, "llvm.ctlz.i32", fn(t_i32, i1) -> t_i32); + ifn!(map, "llvm.ctlz.i64", fn(t_i64, i1) -> t_i64); + + ifn!(map, "llvm.cttz.i8", fn(t_i8, i1) -> t_i8); + ifn!(map, "llvm.cttz.i16", fn(t_i16, i1) -> t_i16); + ifn!(map, "llvm.cttz.i32", fn(t_i32, i1) -> t_i32); + ifn!(map, "llvm.cttz.i64", fn(t_i64, i1) -> t_i64); + + ifn!(map, "llvm.lifetime.start.p0", fn(t_i64, i8p) -> void); + ifn!(map, "llvm.lifetime.end.p0", fn(t_i64, i8p) -> void); + + ifn!(map, "llvm.expect.i1", fn(i1, i1) -> i1); + ifn!(map, "llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void); + + // This isn't an "LLVM intrinsic", but LLVM's optimization passes + // recognize it like one and we assume it exists in `core::slice::cmp` + ifn!(map, "memcmp", fn(i8p, i8p, t_isize) -> t_i32); + + ifn!(map, "llvm.va_start", fn(i8p) -> void); + ifn!(map, "llvm.va_end", fn(i8p) -> void); + ifn!(map, "llvm.va_copy", fn(i8p, i8p) -> void); + + if self.tcx.sess.opts.debuginfo != DebugInfo::None { + ifn!(map, "llvm.dbg.declare", fn(self.type_metadata(), self.type_metadata()) -> void); + ifn!(map, "llvm.dbg.value", fn(self.type_metadata(), t_i64, self.type_metadata()) -> void); + } + + // misc syscalls, only the ones we use + + ifn!(map, "vprintf", fn(i8p, voidp) -> t_i32); + + // so, nvvm, instead of allowing llvm math intrinsics and resolving them + // to libdevice intrinsics, it forces us to explicitly use the libdevice + // intrinsics and add libdevice as a module. so we need to completely + // substitute the llvm intrinsics we would use, for the libdevice ones + // + // see https://docs.nvidia.com/cuda/pdf/libdevice-users-guide.pdf + // and https://docs.nvidia.com/cuda/libdevice-users-guide/index.html + // for docs on what these do + + // libdevice includes a lot of "exotic" intrinsics for common-ish formulas. + // we might want to do a pass in the future to substitute common ops for + // special libdevice intrinsics. We should also expose them as util traits + // for f32 and f64 in cuda_std. + + ifn!(map, "__nv_abs", fn(t_i32) -> t_i32); + + // f64 -> f64 intrinsics + ifn!( + map, + "__nv_acos" | + "__nv_acosh" | + "__nv_asin" | + "__nv_asinh" | + "__nv_atan" | + "__nv_atanh" | + "__nv_cbrt" | + "__nv_ceil" | + "__nv_cos" | + "__nv_cosh" | + "__nv_cospi" | + "__nv_drcp_rd" | + "__nv_drcp_rn" | + "__nv_drcp_ru" | + "__nv_drcp_rz" | + "__nv_dsqrt_rd" | + "__nv_dsqrt_rn" | + "__nv_dsqrt_ru" | + "__nv_dsqrt_rz" | + "__nv_erf" | + "__nv_erfc" | + "__nv_erfcinv" | + "__nv_erfcx" | + "__nv_erfinv" | + "__nv_exp" | + "__nv_exp10" | + "__nv_exp2" | + "__nv_expm1" | + "__nv_fabs" | + "__nv_floor" | + "__nv_j0" | + "__nv_j1" | + "__nv_lgamma" | + "__nv_log" | + "__nv_log10" | + "__nv_log1p" | + "__nv_log2" | + "__nv_logb" | + "__nv_nearbyint" | + "__nv_normcdf" | + "__nv_normcdfinv" | + "__nv_rcbrt" | + "__nv_rint" | + "__nv_round" | + "__nv_rsqrt" | + "__nv_sin" | + "__nv_sinh" | + "__nv_sinpi" | + "__nv_sqrt" | + "__nv_tan" | + "__nv_tanh" | + "__nv_tgamma" | + "__nv_trunc" | + "__nv_y0" | + "__nv_y1", + fn(t_f64) -> t_f64 + ); + + // f32 -> f32 intrinsics + ifn!( + map, + "__nv_acosf" | + "__nv_acoshf" | + "__nv_asinf" | + "__nv_asinhf" | + "__nv_atanf" | + "__nv_atanhf" | + "__nv_cbrtf" | + "__nv_ceilf" | + "__nv_cosf" | + "__nv_coshf" | + "__nv_cospif" | + "__nv_erff" | + "__nv_erfcf" | + "__nv_erfcinvf" | + "__nv_erfcxf" | + "__nv_erfinvf" | + "__nv_expf" | + "__nv_exp10f" | + "__nv_exp2f" | + "__nv_expm1f" | + "__nv_fabsf" | + "__nv_floorf" | + "__nv_j0f" | + "__nv_j1f" | + "__nv_lgammaf" | + "__nv_logf" | + "__nv_log10f" | + "__nv_log1pf" | + "__nv_log2f" | + "__nv_logbf" | + "__nv_nearbyintf" | + "__nv_normcdff" | + "__nv_normcdfinvf" | + "__nv_rcbrtf" | + "__nv_rintf" | + "__nv_roundf" | + "__nv_rsqrtf" | + "__nv_sinf" | + "__nv_sinhf" | + "__nv_sinpif" | + "__nv_sqrtf" | + "__nv_tanf" | + "__nv_tanhf" | + "__nv_tgammaf" | + "__nv_truncf" | + "__nv_y0f" | + "__nv_y1f", + fn(t_f32) -> t_f32 + ); + + // f64, f64 -> f64 intrinsics + ifn!( + map, + "__nv_atan2" | + "__nv_copysign" | + "__nv_dadd_rd" | + "__nv_dadd_rn" | + "__nv_dadd_ru" | + "__nv_dadd_rz" | + "__nv_ddiv_rd" | + "__nv_ddiv_rn" | + "__nv_ddiv_ru" | + "__nv_ddiv_rz" | + "__nv_dmul_rd" | + "__nv_dmul_rn" | + "__nv_dmul_ru" | + "__nv_dmul_rz" | + "__nv_fdim" | + "__nv_fmax" | + "__nv_fmin" | + "__nv_fmod" | + "__nv_hypot" | + "__nv_nextafter" | + "__nv_pow" | + "__nv_remainder", + fn(t_f64, t_f64) -> t_f64 + ); + + // f32, f32 -> f32 intrinsics + ifn!( + map, + "__nv_atan2f" | + "__nv_copysignf" | + "__nv_fadd_rd" | + "__nv_fadd_rn" | + "__nv_fadd_ru" | + "__nv_fadd_rz" | + "__nv_fast_fdividef" | + "__nv_fast_powf" | + "__nv_fdimf" | + "__nv_fdiv_rd" | + "__nv_fdiv_rn" | + "__nv_fdiv_ru" | + "__nv_fdiv_rz" | + "__nv_fmaxf" | + "__nv_fminf" | + "__nv_fmodf" | + "__nv_fmul_rd" | + "__nv_fmul_rn" | + "__nv_fmul_ru" | + "__nv_fmul_rz" | + "__nv_fsub_rd" | + "__nv_fsub_rn" | + "__nv_fsub_ru" | + "__nv_fsub_rz" | + "__nv_hypotf" | + "__nv_nextafterf" | + "__nv_powf" | + "__nv_remainderf", + fn(t_f32, t_f32) -> t_f32 + ); + + // other intrinsics + + ifn!( + map, + "__nv_powi", + fn(t_f64, t_i32) -> t_f64 + ); + + ifn!( + map, + "__nv_powif", + fn(t_f32, t_i32) -> t_f32 + ); + + ifn!( + map, + "__nv_fma", + fn(t_f64, t_f64, t_f64) -> t_f64 + ); + + ifn!( + map, + "__nv_fmaf", + fn(t_f32, t_f32, t_f32) -> t_f32 + ); + + ifn!( + map, + "__nv_yn", + fn(t_i32, t_f64) -> t_f64 + ); + + ifn!( + map, + "__nv_ynf", + fn(t_i32, t_f32) -> t_f32 + ); + } +} diff --git a/crates/rustc_codegen_nvvm/src/debug_info/create_scope_map.rs b/crates/rustc_codegen_nvvm_v19/src/debug_info/create_scope_map.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/debug_info/create_scope_map.rs rename to crates/rustc_codegen_nvvm_v19/src/debug_info/create_scope_map.rs diff --git a/crates/rustc_codegen_nvvm/src/debug_info/dwarf_const.rs b/crates/rustc_codegen_nvvm_v19/src/debug_info/dwarf_const.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/debug_info/dwarf_const.rs rename to crates/rustc_codegen_nvvm_v19/src/debug_info/dwarf_const.rs diff --git a/crates/rustc_codegen_nvvm_v19/src/debug_info/metadata.rs b/crates/rustc_codegen_nvvm_v19/src/debug_info/metadata.rs new file mode 100644 index 00000000..a512e383 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/debug_info/metadata.rs @@ -0,0 +1,1500 @@ +use std::borrow::Cow; +use std::ffi::CString; +use std::fmt::{self, Write}; +use std::hash::{Hash, Hasher}; +use std::path::PathBuf; +use std::{iter, ptr}; + +use libc::{c_longlong, c_uint}; +use rustc_abi::{Align, Size}; +use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; +use rustc_codegen_ssa::traits::*; +use rustc_hir::def::{CtorKind, DefKind}; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_middle::bug; +use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf, TyAndLayout}; +use rustc_middle::ty::{self, AdtKind, CoroutineArgsExt, Instance, Ty, TyCtxt, Visibility}; +use rustc_session::config::{self, DebugInfo}; +use rustc_span::symbol::Symbol; +use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, hygiene}; +use smallvec::smallvec; +use tracing::debug; + +pub(crate) use self::type_map::TypeMap; +use self::type_map::{DINodeCreationResult, Stub, UniqueTypeId}; +use super::CodegenUnitDebugContext; +use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name}; +use crate::common::AsCCharPtr; +use crate::context::CodegenCx; +use crate::debug_info::metadata::type_map::build_type_with_children; +use crate::debug_info::util::*; +use crate::debug_info::*; +use crate::llvm::{DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, Value}; +use crate::abi; + +// most of this code is taken from rustc_codegen_llvm, but adapted +// to use llvm 7 stuff. As well as removing some useless stuff to account for +// osx/wasm/msvc + +impl PartialEq for llvm::Metadata { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self, other) + } +} + +impl Eq for llvm::Metadata {} + +impl Hash for llvm::Metadata { + fn hash(&self, hasher: &mut H) { + (self as *const Self).hash(hasher); + } +} + +impl fmt::Debug for llvm::Metadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (self as *const Self).fmt(f) + } +} + +pub(super) const UNKNOWN_LINE_NUMBER: c_uint = 0; +pub(super) const UNKNOWN_COLUMN_NUMBER: c_uint = 0; + +const NO_SCOPE_METADATA: Option<&DIScope> = None; +/// A function that returns an empty list of generic parameter debuginfo nodes. +const NO_GENERICS: for<'ll> fn(&CodegenCx<'ll, '_>) -> SmallVec<&'ll DIType> = |_| SmallVec::new(); + +// SmallVec is used quite a bit in this module, so create a shorthand. +// The actual number of elements is not so important. +pub type SmallVec = smallvec::SmallVec<[T; 16]>; + +mod enums; +mod type_map; + +/// Returns from the enclosing function if the type debuginfo node with the given +/// unique ID can be found in the type map. +macro_rules! return_if_di_node_created_in_meantime { + ($cx: expr, $unique_type_id: expr) => { + if let Some(di_node) = debug_context($cx) + .type_map + .di_node_for_unique_id($unique_type_id) + { + return DINodeCreationResult::new(di_node, true); + } + }; +} + +/// Extract size and alignment from a TyAndLayout. +#[inline] +fn size_and_align_of(ty_and_layout: TyAndLayout<'_>) -> (Size, Align) { + (ty_and_layout.size, ty_and_layout.align.abi) +} + +/// Creates debuginfo for a fixed size array (e.g. `[u64; 123]`). +/// For slices (that is, "arrays" of unknown size) use [build_slice_type_di_node]. +fn build_fixed_size_array_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, + array_type: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { + let ty::Array(element_type, len) = array_type.kind() else { + bug!( + "build_fixed_size_array_di_node() called with non-ty::Array type `{:?}`", + array_type + ) + }; + + let element_type_di_node = type_di_node(cx, *element_type); + + return_if_di_node_created_in_meantime!(cx, unique_type_id); + + let (size, align) = cx.size_and_align_of(array_type); + + let upper_bound = len + .try_to_target_usize(cx.tcx) + .expect("expected monomorphic const in codegen") as c_longlong; + + let subrange = unsafe { + Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange( + DIB(cx), + 0, + upper_bound, + )) + }; + + let subscripts = create_DIArray(DIB(cx), &[subrange]); + let di_node = unsafe { + llvm::LLVMRustDIBuilderCreateArrayType( + DIB(cx), + size.bits(), + align.bits() as u32, + element_type_di_node, + subscripts, + ) + }; + + DINodeCreationResult::new(di_node, false) +} + +/// Creates debuginfo for built-in pointer-like things: +/// +/// - ty::Ref +/// - ty::RawPtr +/// - ty::Adt in the case it's Box +/// +/// At some point we might want to remove the special handling of Box +/// and treat it the same as other smart pointers (like Rc, Arc, ...). +fn build_pointer_or_reference_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ptr_type: Ty<'tcx>, + pointee_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + // The debuginfo generated by this function is only valid if `ptr_type` is really just + // a (wide) pointer. Make sure it is not called for e.g. `Box`. + assert_eq!( + cx.size_and_align_of(ptr_type), + cx.size_and_align_of(Ty::new_mut_ptr(cx.tcx, pointee_type)) + ); + + let pointee_type_di_node = type_di_node(cx, pointee_type); + + return_if_di_node_created_in_meantime!(cx, unique_type_id); + + let data_layout = &cx.tcx.data_layout; + let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); + + match wide_pointer_kind(cx, pointee_type) { + None => { + // This is a thin pointer. Create a regular pointer type and give it the correct name. + assert_eq!( + (data_layout.pointer_size, data_layout.pointer_align.abi), + cx.size_and_align_of(ptr_type), + "ptr_type={ptr_type}, pointee_type={pointee_type}", + ); + + let ptr_type_debuginfo_name_len = ptr_type_debuginfo_name.len(); + + let di_node = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + pointee_type_di_node, + data_layout.pointer_size.bits(), + data_layout.pointer_align.abi.bits() as u32, + 0, // TODO: guessing + CString::new(ptr_type_debuginfo_name).unwrap().as_ptr(), + ptr_type_debuginfo_name_len, + ) + }; + + DINodeCreationResult { + di_node, + already_stored_in_typemap: false, + } + } + Some(wide_pointer_kind) => { + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &ptr_type_debuginfo_name, + None, + cx.size_and_align_of(ptr_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |cx, owner| { + // FIXME: If this wide pointer is a `Box` then we don't want to use its + // type layout and instead use the layout of the raw pointer inside + // of it. + // The proper way to handle this is to not treat Box as a pointer + // at all and instead emit regular struct debuginfo for it. We just + // need to make sure that we don't break existing debuginfo consumers + // by doing that (at least not without a warning period). + let layout_type = if ptr_type.is_box() { + // The assertion at the start of this function ensures we have a ZST + // allocator. We'll make debuginfo "skip" all ZST allocators, not just the + // default allocator. + Ty::new_mut_ptr(cx.tcx, pointee_type) + } else { + ptr_type + }; + + let layout = cx.layout_of(layout_type); + let addr_field = layout.field(cx, abi::WIDE_PTR_ADDR); + let extra_field = layout.field(cx, abi::WIDE_PTR_EXTRA); + + let (addr_field_name, extra_field_name) = match wide_pointer_kind { + WidePtrKind::Dyn => ("pointer", "vtable"), + WidePtrKind::Slice => ("data_ptr", "length"), + }; + + assert_eq!(abi::WIDE_PTR_ADDR, 0); + assert_eq!(abi::WIDE_PTR_EXTRA, 1); + + // The data pointer type is a regular, thin pointer, regardless of whether this + // is a slice or a trait object. + let data_ptr_type_di_node = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + pointee_type_di_node, + addr_field.size.bits(), + addr_field.align.abi.bits() as u32, + 0, // TODO: guessing + c"".as_ptr(), + 0, + ) + }; + + smallvec![ + build_field_di_node( + cx, + owner, + addr_field_name, + (addr_field.size, addr_field.align.abi), + layout.fields.offset(abi::WIDE_PTR_ADDR), + DIFlags::FlagZero, + data_ptr_type_di_node, + None, + ), + build_field_di_node( + cx, + owner, + extra_field_name, + (extra_field.size, extra_field.align.abi), + layout.fields.offset(abi::WIDE_PTR_EXTRA), + DIFlags::FlagZero, + type_di_node(cx, extra_field.ty), + None, + ), + ] + }, + NO_GENERICS, + ) + } + } +} + +fn build_subroutine_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + // It's possible to create a self-referential type in Rust by using 'impl trait': + // + // fn foo() -> impl Copy { foo } + // + // Unfortunately LLVM's API does not allow us to create recursive subroutine types. + // In order to work around that restriction we place a marker type in the type map, + // before creating the actual type. If the actual type is recursive, it will hit the + // marker type. So we end up with a type that looks like + // + // fn foo() -> + // + // Once that is created, we replace the marker in the typemap with the actual type. + debug_context(cx) + .type_map + .unique_id_to_di_node + .borrow_mut() + .insert(unique_type_id, recursion_marker_type_di_node(cx)); + + let fn_ty = unique_type_id.expect_ty(); + let signature = cx + .tcx + .normalize_erasing_late_bound_regions(cx.typing_env(), fn_ty.fn_sig(cx.tcx)); + + let signature_di_nodes: SmallVec<_> = iter::once( + // return type + match signature.output().kind() { + ty::Tuple(tys) if tys.is_empty() => { + // this is a "void" function + None + } + _ => Some(type_di_node(cx, signature.output())), + }, + ) + .chain( + // regular arguments + signature + .inputs() + .iter() + .map(|&argument_type| Some(type_di_node(cx, argument_type))), + ) + .collect(); + + debug_context(cx) + .type_map + .unique_id_to_di_node + .borrow_mut() + .remove(&unique_type_id); + + let fn_di_node = unsafe { + llvm::LLVMRustDIBuilderCreateSubroutineType( + DIB(cx), + create_DIArray(DIB(cx), &signature_di_nodes[..]), + ) + }; + + // This is actually a function pointer, so wrap it in pointer DI. + let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); + let (size, align) = match fn_ty.kind() { + ty::FnDef(..) => (0, 1), + ty::FnPtr(..) => ( + cx.tcx.data_layout.pointer_size.bits(), + cx.tcx.data_layout.pointer_align.abi.bits() as u32, + ), + _ => unreachable!(), + }; + let name_len = name.len(); + let di_node = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + fn_di_node, + size, + align, + 0, // TODO: guessing + CString::new(name).unwrap().as_ptr(), + name_len, + ) + }; + + DINodeCreationResult::new(di_node, false) +} + +/// Create debuginfo for `dyn SomeTrait` types. Currently these are empty structs +/// we with the correct type name (e.g. "dyn SomeTrait + Sync"). +fn build_dyn_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + dyn_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + if let ty::Dynamic(..) = dyn_type.kind() { + let type_name = compute_debuginfo_type_name(cx.tcx, dyn_type, true); + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + None, + cx.size_and_align_of(dyn_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |_, _| smallvec![], + NO_GENERICS, + ) + } else { + bug!( + "Only ty::Dynamic is valid for build_dyn_type_di_node(). Found {:?} instead.", + dyn_type + ) + } +} + +/// Create debuginfo for `[T]` and `str`. These are unsized. +/// +/// NOTE: We currently emit just emit the debuginfo for the element type here +/// (i.e. `T` for slices and `u8` for `str`), so that we end up with +/// `*const T` for the `data_ptr` field of the corresponding wide-pointer +/// debuginfo of `&[T]`. +/// +/// It would be preferable and more accurate if we emitted a DIArray of T +/// without an upper bound instead. That is, LLVM already supports emitting +/// debuginfo of arrays of unknown size. But GDB currently seems to end up +/// in an infinite loop when confronted with such a type. +/// +/// As a side effect of the current encoding every instance of a type like +/// `struct Foo { unsized_field: [u8] }` will look like +/// `struct Foo { unsized_field: u8 }` in debuginfo. If the length of the +/// slice is zero, then accessing `unsized_field` in the debugger would +/// result in an out-of-bounds access. +fn build_slice_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + slice_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let element_type = match slice_type.kind() { + ty::Slice(element_type) => *element_type, + ty::Str => cx.tcx.types.u8, + _ => { + bug!( + "Only ty::Slice is valid for build_slice_type_di_node(). Found {:?} instead.", + slice_type + ) + } + }; + + let element_type_di_node = type_di_node(cx, element_type); + return_if_di_node_created_in_meantime!(cx, unique_type_id); + DINodeCreationResult { + di_node: element_type_di_node, + already_stored_in_typemap: false, + } +} + +pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { + // Get the unique type ID of this type. + let unique_type_id = UniqueTypeId::for_ty(cx.tcx, t); + + if let Some(existing_di_node) = debug_context(cx) + .type_map + .di_node_for_unique_id(unique_type_id) + { + return existing_di_node; + } + + //debug!("type_di_node: {:?} kind: {:?}", t, t.kind()); + + let DINodeCreationResult { + di_node, + already_stored_in_typemap, + } = match *t.kind() { + ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { + build_basic_type_di_node(cx, t) + } + ty::Tuple(elements) if elements.is_empty() => build_basic_type_di_node(cx, t), + ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t), + ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id), + ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id), + ty::Foreign(..) => build_foreign_type_di_node(cx, t, unique_type_id), + ty::RawPtr(pointee_type, _) | ty::Ref(_, pointee_type, _) => { + build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id) + } + // Some `Box` are newtyped pointers, make debuginfo aware of that. + // Only works if the allocator argument is a 1-ZST and hence irrelevant for layout + // (or if there is no allocator argument). + ty::Adt(def, args) + if def.is_box() + && args + .get(1) + .is_none_or(|arg| cx.layout_of(arg.expect_ty()).is_1zst()) => + { + build_pointer_or_reference_di_node(cx, t, t.expect_boxed_ty(), unique_type_id) + } + ty::FnDef(..) | ty::FnPtr(..) => build_subroutine_type_di_node(cx, unique_type_id), + ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), + ty::CoroutineClosure(..) => build_closure_env_di_node(cx, unique_type_id), + ty::Coroutine(..) => enums::build_coroutine_di_node(cx, unique_type_id), + ty::Adt(def, ..) => match def.adt_kind() { + AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), + AdtKind::Union => build_union_type_di_node(cx, unique_type_id), + AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id), + }, + ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), + _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t), + }; + + { + if already_stored_in_typemap { + // Make sure that we really do have a `TypeMap` entry for the unique type ID. + let di_node_for_uid = match debug_context(cx) + .type_map + .di_node_for_unique_id(unique_type_id) + { + Some(di_node) => di_node, + None => { + bug!( + "expected type debuginfo node for unique \ + type ID '{:?}' to already be in \ + the `debuginfo::TypeMap` but it \ + was not.", + unique_type_id, + ); + } + }; + + assert_eq!(di_node_for_uid as *const _, di_node as *const _); + } else { + debug_context(cx).type_map.insert(unique_type_id, di_node); + } + } + + di_node +} + +// FIXME(mw): Cache this via a regular UniqueTypeId instead of an extra field in the debug context. +fn recursion_marker_type_di_node<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIType { + debug_context(cx) + .recursion_marker_type + .get_or_init(move || { + unsafe { + // The choice of type here is pretty arbitrary - + // anything reading the debuginfo for a recursive + // type is going to see *something* weird - the only + // question is what exactly it will see. + // + // FIXME: the name `` does not fit the naming scheme + // of other types. + // + // FIXME: it might make sense to use an actual pointer type here + // so that debuggers can show the address. + let name = ""; + llvm::LLVMRustDIBuilderCreateBasicType( + DIB(cx), + name.as_c_char_ptr(), + name.len(), + cx.tcx.data_layout.pointer_size.bits(), + dwarf_const::DW_ATE_unsigned, + ) + } + }) +} + +fn hex_encode(data: &[u8]) -> String { + let mut hex_string = String::with_capacity(data.len() * 2); + for byte in data.iter() { + write!(&mut hex_string, "{byte:02x}").unwrap(); + } + hex_string +} + +pub(crate) fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { + let cache_key = Some((source_file.stable_id, source_file.src_hash)); + return debug_context(cx) + .created_files + .borrow_mut() + .entry(cache_key) + .or_insert_with(|| alloc_new_file_metadata(cx, source_file)); + + fn alloc_new_file_metadata<'ll>( + cx: &CodegenCx<'ll, '_>, + source_file: &SourceFile, + ) -> &'ll DIFile { + //debug!(?source_file.name); + + let filename_display_preference = cx + .sess() + .filename_display_preference(RemapPathScopeComponents::DEBUGINFO); + + use rustc_session::config::RemapPathScopeComponents; + let (directory, file_name) = match &source_file.name { + FileName::Real(filename) => { + let working_directory = &cx.sess().opts.working_dir; + //debug!(?working_directory); + + if filename_display_preference == FileNameDisplayPreference::Remapped { + let filename = cx + .sess() + .source_map() + .path_mapping() + .to_embeddable_absolute_path(filename.clone(), working_directory); + + // Construct the absolute path of the file + let abs_path = filename.remapped_path_if_available(); + //debug!(?abs_path); + + if let Ok(rel_path) = + abs_path.strip_prefix(working_directory.remapped_path_if_available()) + { + // If the compiler's working directory (which also is the DW_AT_comp_dir of + // the compilation unit) is a prefix of the path we are about to emit, then + // only emit the part relative to the working directory. Because of path + // remapping we sometimes see strange things here: `abs_path` might + // actually look like a relative path (e.g. + // `/src/lib.rs`), so if we emit it without taking + // the working directory into account, downstream tooling will interpret it + // as `//src/lib.rs`, which + // makes no sense. Usually in such cases the working directory will also be + // remapped to `` or some other prefix of the path + // we are remapping, so we end up with + // `//src/lib.rs`. + // By moving the working directory portion into the `directory` part of the + // DIFile, we allow LLVM to emit just the relative path for DWARF, while + // still emitting the correct absolute path for CodeView. + ( + working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), abs_path.to_string_lossy().into_owned()) + } + } else { + let working_directory = working_directory.local_path_if_available(); + let filename = filename.local_path_if_available(); + + //debug!(?working_directory, ?filename); + + let abs_path: Cow<'_, _> = if filename.is_absolute() { + filename.into() + } else { + let mut p = PathBuf::new(); + p.push(working_directory); + p.push(filename); + p.into() + }; + + if let Ok(rel_path) = abs_path.strip_prefix(working_directory) { + ( + working_directory.to_string_lossy(), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), abs_path.to_string_lossy().into_owned()) + } + } + } + other => { + //debug!(?other); + ( + "".into(), + other.display(filename_display_preference).to_string(), + ) + } + }; + + let hash_kind = match source_file.src_hash.kind { + rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5, + rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1, + // NVVM: Llvm 7 does not support either of these checksums + rustc_span::SourceFileHashAlgorithm::Sha256 + | rustc_span::SourceFileHashAlgorithm::Blake3 => llvm::ChecksumKind::None, + }; + let hash_value = hex_encode(source_file.src_hash.hash_bytes()); + + let source = cx + .sess() + .opts + .unstable_opts + .embed_source + .then_some(()) + .and(source_file.src.as_ref()); + + unsafe { + llvm::LLVMRustDIBuilderCreateFile( + DIB(cx), + file_name.as_c_char_ptr(), + file_name.len(), + directory.as_c_char_ptr(), + directory.len(), + hash_kind, + hash_value.as_c_char_ptr(), + hash_value.len(), + source.map_or(ptr::null(), |x| x.as_c_char_ptr()), + source.map_or(0, |x| x.len()), + ) + } + } +} + +pub(crate) fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { + debug_context(cx) + .created_files + .borrow_mut() + .entry(None) + .or_insert_with(|| unsafe { + let file_name = ""; + let directory = ""; + let hash_value = ""; + + llvm::LLVMRustDIBuilderCreateFile( + DIB(cx), + file_name.as_c_char_ptr(), + file_name.len(), + directory.as_c_char_ptr(), + directory.len(), + llvm::ChecksumKind::None, + hash_value.as_c_char_ptr(), + hash_value.len(), + ptr::null(), + 0, + ) + }) +} + +fn build_basic_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { + debug!("build_basic_type_di_node: {:?}", t); + + use dwarf_const::{DW_ATE_UTF, DW_ATE_boolean, DW_ATE_float, DW_ATE_signed, DW_ATE_unsigned}; + + let (name, encoding) = match t.kind() { + ty::Never => ("!", DW_ATE_unsigned), + ty::Tuple(elements) if elements.is_empty() => ("()", DW_ATE_unsigned), + ty::Bool => ("bool", DW_ATE_boolean), + ty::Char => ("char", DW_ATE_UTF), + ty::Int(int_ty) => (int_ty.name_str(), DW_ATE_signed), + ty::Uint(uint_ty) => (uint_ty.name_str(), DW_ATE_unsigned), + ty::Float(float_ty) => (float_ty.name_str(), DW_ATE_float), + _ => bug!("debuginfo::build_basic_type_di_node - `t` is invalid type"), + }; + + let ty_di_node = unsafe { + llvm::LLVMRustDIBuilderCreateBasicType( + DIB(cx), + name.as_c_char_ptr(), + name.len(), + cx.size_of(t).bits(), + encoding, + ) + }; + + DINodeCreationResult::new(ty_di_node, false) +} + +fn build_foreign_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + debug!("build_foreign_type_di_node: {:?}", t); + + let &ty::Foreign(def_id) = unique_type_id.expect_ty().kind() else { + bug!( + "build_foreign_type_di_node() called with unexpected type: {:?}", + unique_type_id.expect_ty() + ); + }; + + build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &compute_debuginfo_type_name(cx.tcx, t, false), + None, + cx.size_and_align_of(t), + Some(get_namespace_for_item(cx, def_id)), + DIFlags::FlagZero, + ), + |_, _| smallvec![], + NO_GENERICS, + ) +} + +pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>( + tcx: TyCtxt<'tcx>, + codegen_unit_name: &str, + debug_context: &CodegenUnitDebugContext<'ll, 'tcx>, +) -> &'ll DIDescriptor { + use rustc_session::RemapFileNameExt; + use rustc_session::config::RemapPathScopeComponents; + let mut name_in_debuginfo = tcx + .sess + .local_crate_source_file() + .map(|src| { + src.for_scope(tcx.sess, RemapPathScopeComponents::DEBUGINFO) + .to_path_buf() + }) + .unwrap_or_else(|| PathBuf::from(tcx.crate_name(LOCAL_CRATE).as_str())); + + // To avoid breaking split DWARF, we need to ensure that each codegen unit + // has a unique `DW_AT_name`. This is because there's a remote chance that + // different codegen units for the same module will have entirely + // identical DWARF entries for the purpose of the DWO ID, which would + // violate Appendix F ("Split Dwarf Object Files") of the DWARF 5 + // specification. LLVM uses the algorithm specified in section 7.32 "Type + // Signature Computation" to compute the DWO ID, which does not include + // any fields that would distinguish compilation units. So we must embed + // the codegen unit name into the `DW_AT_name`. (Issue #88521.) + // + // Additionally, the OSX linker has an idiosyncrasy where it will ignore + // some debuginfo if multiple object files with the same `DW_AT_name` are + // linked together. + // + // As a workaround for these two issues, we generate unique names for each + // object file. Those do not correspond to an actual source file but that + // is harmless. + name_in_debuginfo.push("@"); + name_in_debuginfo.push(codegen_unit_name); + + debug!("build_compile_unit_di_node: {:?}", name_in_debuginfo); + + let rustc_producer = "rustc_codegen_nvvm_v19".to_string(); + + // leave the clang LLVM in there just in case, although it shouldnt be needed because + // gpu stuff is different + let producer = format!("clang LLVM ({})", rustc_producer); + + let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); + let work_dir = tcx + .sess + .opts + .working_dir + .for_scope(tcx.sess, RemapPathScopeComponents::DEBUGINFO) + .to_string_lossy(); + let output_filenames = tcx.output_filenames(()); + let split_name = if tcx.sess.target_can_use_split_dwarf() + && let Some(f) = output_filenames.split_dwarf_path( + tcx.sess.split_debuginfo(), + tcx.sess.opts.unstable_opts.split_dwarf_kind, + Some(codegen_unit_name), + ) { + // We get a path relative to the working directory from split_dwarf_path + Some(tcx.sess.source_map().path_mapping().to_real_filename(f)) + } else { + None + }; + let split_name = split_name + .as_ref() + .map(|f| { + f.for_scope(tcx.sess, RemapPathScopeComponents::DEBUGINFO) + .to_string_lossy() + }) + .unwrap_or_default(); + let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); + + unsafe { + let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( + debug_context.builder, + name_in_debuginfo.as_c_char_ptr(), + name_in_debuginfo.len(), + work_dir.as_c_char_ptr(), + work_dir.len(), + llvm::ChecksumKind::None, + ptr::null(), + 0, + ptr::null(), + 0, + ); + + let unit_metadata = llvm::LLVMRustDIBuilderCreateCompileUnit( + debug_context.builder, + dwarf_const::DW_LANG_Rust, + compile_unit_file, + producer.as_c_char_ptr(), + producer.len(), + tcx.sess.opts.optimize != config::OptLevel::No, + c"".as_ptr(), + 0, + // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead + // put the path supplied to `MCSplitDwarfFile` into the debug info of the final + // output(s). + split_name.as_c_char_ptr(), + split_name.len(), + kind, + 0, + tcx.sess.opts.unstable_opts.split_dwarf_inlining, + llvm::DebugNameTableKind::Default, + ); + + unit_metadata + } +} + +/// Creates a `DW_TAG_member` entry inside the DIE represented by the given `type_di_node`. +#[allow(clippy::too_many_arguments)] +fn build_field_di_node<'ll>( + cx: &CodegenCx<'ll, '_>, + owner: &'ll DIScope, + name: &str, + size_and_align: (Size, Align), + offset: Size, + flags: DIFlags, + type_di_node: &'ll DIType, + def_id: Option, +) -> &'ll DIType { + let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers + { + file_metadata_from_def_id(cx, def_id) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + unsafe { + llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + owner, + name.as_c_char_ptr(), + name.len(), + file_metadata, + line_number, + size_and_align.0.bits(), + size_and_align.1.bits() as u32, + offset.bits(), + flags, + type_di_node, + ) + } +} + +/// Returns the `DIFlags` corresponding to the visibility of the item identified by `did`. +/// +/// `DIFlags::Flag{Public,Protected,Private}` correspond to `DW_AT_accessibility` +/// (public/protected/private) aren't exactly right for Rust, but neither is `DW_AT_visibility` +/// (local/exported/qualified), and there's no way to set `DW_AT_visibility` in LLVM's API. +fn visibility_di_flags(cx: &CodegenCx<'_, '_>, did: DefId, type_did: DefId) -> DIFlags { + let parent_did = cx.tcx.parent(type_did); + let visibility = cx.tcx.visibility(did); + match visibility { + Visibility::Public => DIFlags::FlagPublic, + // Private fields have a restricted visibility of the module containing the type. + Visibility::Restricted(did) if did == parent_did => DIFlags::FlagPrivate, + // `pub(crate)`/`pub(super)` visibilities are any other restricted visibility. + Visibility::Restricted(..) => DIFlags::FlagProtected, + } +} + +/// Creates the debuginfo node for a Rust struct type. Maybe be a regular struct or a tuple-struct. +fn build_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let struct_type = unique_type_id.expect_ty(); + let ty::Adt(adt_def, _) = struct_type.kind() else { + bug!( + "build_struct_type_di_node() called with non-struct-type: {:?}", + struct_type + ); + }; + assert!(adt_def.is_struct()); + let containing_scope = get_namespace_for_item(cx, adt_def.did()); + let struct_type_and_layout = cx.layout_of(struct_type); + let variant_def = adt_def.non_enum_variant(); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(adt_def.did()))) + } else { + None + }; + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &compute_debuginfo_type_name(cx.tcx, struct_type, false), + def_location, + size_and_align_of(struct_type_and_layout), + Some(containing_scope), + visibility_di_flags(cx, adt_def.did(), adt_def.did()), + ), + // Fields: + |cx, owner| { + variant_def + .fields + .iter() + .enumerate() + .map(|(i, f)| { + let field_name = if variant_def.ctor_kind() == Some(CtorKind::Fn) { + // This is a tuple struct + tuple_field_name(i) + } else { + // This is struct with named fields + Cow::Borrowed(f.name.as_str()) + }; + let field_layout = struct_type_and_layout.field(cx, i); + let def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(f.did) + } else { + None + }; + build_field_di_node( + cx, + owner, + &field_name[..], + (field_layout.size, field_layout.align.abi), + struct_type_and_layout.fields.offset(i), + visibility_di_flags(cx, f.did, adt_def.did()), + type_di_node(cx, field_layout.ty), + def_id, + ) + }) + .collect() + }, + |cx| build_generic_type_param_di_nodes(cx, struct_type), + ) +} + +//=----------------------------------------------------------------------------- +// Tuples +//=----------------------------------------------------------------------------- + +/// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or coroutine. +/// For a coroutine, this will handle upvars shared by all states. +fn build_upvar_field_di_nodes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + closure_or_coroutine_ty: Ty<'tcx>, + closure_or_coroutine_di_node: &'ll DIType, +) -> SmallVec<&'ll DIType> { + let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { + ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()), + ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), + ty::CoroutineClosure(def_id, args) => (def_id, args.as_coroutine_closure().upvar_tys()), + _ => { + bug!( + "build_upvar_field_di_nodes() called with non-closure-or-coroutine-type: {:?}", + closure_or_coroutine_ty + ) + } + }; + + assert!( + up_var_tys + .iter() + .all(|t| t == cx.tcx.normalize_erasing_regions(cx.typing_env(), t)) + ); + + let capture_names = cx.tcx.closure_saved_names_of_captured_variables(def_id); + let layout = cx.layout_of(closure_or_coroutine_ty); + + up_var_tys + .into_iter() + .zip(capture_names.iter()) + .enumerate() + .map(|(index, (up_var_ty, capture_name))| { + build_field_di_node( + cx, + closure_or_coroutine_di_node, + capture_name.as_str(), + cx.size_and_align_of(up_var_ty), + layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, up_var_ty), + None, + ) + }) + .collect() +} + +/// Builds the DW_TAG_structure_type debuginfo node for a Rust tuple type. +fn build_tuple_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let tuple_type = unique_type_id.expect_ty(); + let &ty::Tuple(component_types) = tuple_type.kind() else { + bug!( + "build_tuple_type_di_node() called with non-tuple-type: {:?}", + tuple_type + ) + }; + + let tuple_type_and_layout = cx.layout_of(tuple_type); + let type_name = compute_debuginfo_type_name(cx.tcx, tuple_type, false); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + None, + size_and_align_of(tuple_type_and_layout), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + // Fields: + |cx, tuple_di_node| { + component_types + .into_iter() + .enumerate() + .map(|(index, component_type)| { + build_field_di_node( + cx, + tuple_di_node, + &tuple_field_name(index), + cx.size_and_align_of(component_type), + tuple_type_and_layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, component_type), + None, + ) + }) + .collect() + }, + NO_GENERICS, + ) +} + +/// Builds the debuginfo node for a closure environment. +fn build_closure_env_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let closure_env_type = unique_type_id.expect_ty(); + let &(ty::Closure(def_id, _) | ty::CoroutineClosure(def_id, _)) = closure_env_type.kind() + else { + bug!( + "build_closure_env_di_node() called with non-closure-type: {:?}", + closure_env_type + ) + }; + let containing_scope = get_namespace_for_item(cx, def_id); + let type_name = compute_debuginfo_type_name(cx.tcx, closure_env_type, false); + + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(def_id))) + } else { + None + }; + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + def_location, + cx.size_and_align_of(closure_env_type), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| build_upvar_field_di_nodes(cx, closure_env_type, owner), + NO_GENERICS, + ) +} + +/// Build the debuginfo node for a Rust `union` type. +fn build_union_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let union_type = unique_type_id.expect_ty(); + let (union_def_id, variant_def) = match union_type.kind() { + ty::Adt(def, _) => (def.did(), def.non_enum_variant()), + _ => bug!("build_union_type_di_node on a non-ADT"), + }; + let containing_scope = get_namespace_for_item(cx, union_def_id); + let union_ty_and_layout = cx.layout_of(union_type); + let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(union_def_id))) + } else { + None + }; + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Union, + unique_type_id, + &type_name, + def_location, + size_and_align_of(union_ty_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| { + variant_def + .fields + .iter() + .enumerate() + .map(|(i, f)| { + let field_layout = union_ty_and_layout.field(cx, i); + let def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(f.did) + } else { + None + }; + build_field_di_node( + cx, + owner, + f.name.as_str(), + size_and_align_of(field_layout), + Size::ZERO, + DIFlags::FlagZero, + type_di_node(cx, field_layout.ty), + def_id, + ) + }) + .collect() + }, + // Generics: + |cx| build_generic_type_param_di_nodes(cx, union_type), + ) +} + +/// Computes the type parameters for a type, if any, for the given metadata. +fn build_generic_type_param_di_nodes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, +) -> SmallVec<&'ll DIType> { + if let ty::Adt(def, args) = *ty.kind() { + if args.types().next().is_some() { + let generics = cx.tcx.generics_of(def.did()); + let names = get_parameter_names(cx, generics); + let template_params: SmallVec<_> = iter::zip(args, names) + .filter_map(|(kind, name)| { + kind.as_type().map(|ty| { + let actual_type = cx.tcx.normalize_erasing_regions(cx.typing_env(), ty); + let actual_type_di_node = type_di_node(cx, actual_type); + let name = name.as_str(); + unsafe { + llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( + DIB(cx), + None, + name.as_c_char_ptr(), + name.len(), + actual_type_di_node, + ) + } + }) + }) + .collect(); + + return template_params; + } + } + + return smallvec![]; + + fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec { + let mut names = generics.parent.map_or_else(Vec::new, |def_id| { + get_parameter_names(cx, cx.tcx.generics_of(def_id)) + }); + names.extend(generics.own_params.iter().map(|param| param.name)); + names + } +} + +/// Creates debug information for the given global variable. +/// +/// Adds the created metadata nodes directly to the crate's IR. +pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, global: &'ll Value) { + if cx.dbg_cx.is_none() { + return; + } + + // Only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo != DebugInfo::Full { + return; + } + + let tcx = cx.tcx; + + // We may want to remove the namespace scope if we're in an extern block (see + // https://github.com/rust-lang/rust/pull/46457#issuecomment-351750952). + let var_scope = get_namespace_for_item(cx, def_id); + let (file_metadata, line_number) = file_metadata_from_def_id(cx, Some(def_id)); + + let is_local_to_unit = is_node_local_to_unit(cx, def_id); + + let DefKind::Static { nested, .. } = cx.tcx.def_kind(def_id) else { + bug!() + }; + if nested { + return; + } + + let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, cx.typing_env()); + let type_metadata = type_di_node(cx, variable_type); + let var_name = tcx.item_name(def_id); + let var_name = var_name.as_str(); + let linkage_name = mangled_name_of_instance(cx, Instance::mono(tcx, def_id)).name; + // When empty, linkage_name field is omitted, + // which is what we want for no_mangle statics + let linkage_name = if var_name == linkage_name { + "" + } else { + linkage_name + }; + + let global_align = cx.align_of(variable_type); + + unsafe { + llvm::LLVMRustDIBuilderCreateStaticVariable( + DIB(cx), + Some(var_scope), + var_name.as_c_char_ptr(), + var_name.len(), + linkage_name.as_c_char_ptr(), + linkage_name.len(), + file_metadata, + line_number, + type_metadata, + is_local_to_unit, + global, + None, + global_align.bits() as u32, + ); + } +} + +/// Generates LLVM debuginfo for a vtable. +/// +/// The vtable type looks like a struct with a field for each function pointer and super-trait +/// pointer it contains (plus the `size` and `align` fields). +/// +/// Except for `size`, `align`, and `drop_in_place`, the field names don't try to mirror +/// the name of the method they implement. This can be implemented in the future once there +/// is a proper disambiguation scheme for dealing with methods from different traits that have +/// the same name. +fn build_vtable_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + poly_trait_ref: Option>, +) -> &'ll DIType { + let tcx = cx.tcx; + + let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref { + let trait_ref = poly_trait_ref.with_self_ty(tcx, ty); + let trait_ref = tcx.erase_regions(trait_ref); + + tcx.vtable_entries(trait_ref) + } else { + TyCtxt::COMMON_VTABLE_ENTRIES + }; + + // All function pointers are described as opaque pointers. This could be improved in the future + // by describing them as actual function pointers. + let void_pointer_ty = Ty::new_imm_ptr(tcx, tcx.types.unit); + let void_pointer_type_di_node = type_di_node(cx, void_pointer_ty); + let usize_di_node = type_di_node(cx, tcx.types.usize); + let (pointer_size, pointer_align) = cx.size_and_align_of(void_pointer_ty); + // If `usize` is not pointer-sized and -aligned then the size and alignment computations + // for the vtable as a whole would be wrong. Let's make sure this holds even on weird + // platforms. + assert_eq!( + cx.size_and_align_of(tcx.types.usize), + (pointer_size, pointer_align) + ); + + let vtable_type_name = + compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::Type); + let unique_type_id = UniqueTypeId::for_vtable_ty(tcx, ty, poly_trait_ref); + let size = pointer_size * vtable_entries.len() as u64; + + // This gets mapped to a DW_AT_containing_type attribute which allows GDB to correlate + // the vtable to the type it is for. + let vtable_holder = type_di_node(cx, ty); + + build_type_with_children( + cx, + type_map::stub( + cx, + Stub::VTableTy { vtable_holder }, + unique_type_id, + &vtable_type_name, + None, + (size, pointer_align), + NO_SCOPE_METADATA, + DIFlags::FlagArtificial, + ), + |cx, vtable_type_di_node| { + vtable_entries + .iter() + .enumerate() + .filter_map(|(index, vtable_entry)| { + let (field_name, field_type_di_node) = match vtable_entry { + ty::VtblEntry::MetadataDropInPlace => { + ("drop_in_place".to_string(), void_pointer_type_di_node) + } + ty::VtblEntry::Method(_) => { + // Note: This code does not try to give a proper name to each method + // because their might be multiple methods with the same name + // (coming from different traits). + (format!("__method{index}"), void_pointer_type_di_node) + } + ty::VtblEntry::TraitVPtr(_) => ( + format!("__super_trait_ptr{index}"), + void_pointer_type_di_node, + ), + ty::VtblEntry::MetadataAlign => ("align".to_string(), usize_di_node), + ty::VtblEntry::MetadataSize => ("size".to_string(), usize_di_node), + ty::VtblEntry::Vacant => return None, + }; + + let field_offset = pointer_size * index as u64; + + Some(build_field_di_node( + cx, + vtable_type_di_node, + &field_name, + (pointer_size, pointer_align), + field_offset, + DIFlags::FlagZero, + field_type_di_node, + None, + )) + }) + .collect() + }, + NO_GENERICS, + ) + .di_node +} + +/// Creates debug information for the given vtable, which is for the +/// given type. +/// +/// Adds the created metadata nodes directly to the crate's IR. +pub(crate) fn create_vtable_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + poly_trait_ref: Option>, + vtable: &'ll Value, +) { + if cx.dbg_cx.is_none() { + return; + } + + // Only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo != DebugInfo::Full { + return; + } + + // When full debuginfo is enabled, we want to try and prevent vtables from being + // merged. Otherwise debuggers will have a hard time mapping from dyn pointer + // to concrete type. + llvm::SetUnnamedAddress(vtable, llvm::UnnamedAddr::No); + + let vtable_name = + compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable); + let vtable_type_di_node = build_vtable_type_di_node(cx, ty, poly_trait_ref); + let linkage_name = ""; + + unsafe { + llvm::LLVMRustDIBuilderCreateStaticVariable( + DIB(cx), + NO_SCOPE_METADATA, + vtable_name.as_c_char_ptr(), + vtable_name.len(), + linkage_name.as_c_char_ptr(), + linkage_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + vtable_type_di_node, + true, + vtable, + None, + 0, + ); + } +} + +/// Creates an "extension" of an existing `DIScope` into another file. +pub(crate) fn extend_scope_to_file<'ll>( + cx: &CodegenCx<'ll, '_>, + scope_metadata: &'ll DIScope, + file: &SourceFile, +) -> &'ll DILexicalBlock { + let file_metadata = file_metadata(cx, file); + unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlockFile(DIB(cx), scope_metadata, file_metadata) } +} + +fn tuple_field_name(field_index: usize) -> Cow<'static, str> { + const TUPLE_FIELD_NAMES: [&str; 16] = [ + "__0", "__1", "__2", "__3", "__4", "__5", "__6", "__7", "__8", "__9", "__10", "__11", + "__12", "__13", "__14", "__15", + ]; + TUPLE_FIELD_NAMES + .get(field_index) + .map(|s| Cow::from(*s)) + .unwrap_or_else(|| Cow::from(format!("__{field_index}"))) +} + +pub(crate) type DefinitionLocation<'ll> = (&'ll DIFile, c_uint); + +pub(crate) fn file_metadata_from_def_id<'ll>( + cx: &CodegenCx<'ll, '_>, + def_id: Option, +) -> DefinitionLocation<'ll> { + if let Some(def_id) = def_id + && let span = hygiene::walk_chain_collapsed(cx.tcx.def_span(def_id), DUMMY_SP) + && !span.is_dummy() + { + let loc = cx.lookup_debug_loc(span.lo()); + (file_metadata(cx, &loc.file), loc.line) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/debug_info/metadata/enums.rs b/crates/rustc_codegen_nvvm_v19/src/debug_info/metadata/enums.rs new file mode 100644 index 00000000..a5a22edc --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/debug_info/metadata/enums.rs @@ -0,0 +1,878 @@ +use std::borrow::Cow; + +use libc::c_uint; +use rustc_abi::{FieldIdx, Size, TagEncoding, VariantIdx, Variants}; +use rustc_codegen_ssa::debuginfo::{ + tag_base_type, type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo, +}; +use rustc_codegen_ssa::traits::{ConstCodegenMethods, MiscCodegenMethods}; +use rustc_hir::def::CtorKind; +use rustc_index::IndexSlice; +use rustc_middle::bug; +use rustc_middle::mir::CoroutineLayout; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; +use rustc_span::Symbol; +use smallvec::smallvec; + +use super::type_map::{DINodeCreationResult, UniqueTypeId}; +use super::{SmallVec, size_and_align_of}; +use crate::common::AsCCharPtr; +use crate::context::CodegenCx; +use crate::debug_info::file_metadata_from_def_id; +use crate::debug_info::metadata::type_map::{self, Stub, StubInfo}; +use crate::debug_info::metadata::{ + NO_GENERICS, UNKNOWN_LINE_NUMBER, build_field_di_node, build_generic_type_param_di_nodes, + file_metadata, type_di_node, unknown_file_metadata, visibility_di_flags, +}; +use crate::debug_info::util::{DIB, create_DIArray, get_namespace_for_item}; +use crate::llvm::debuginfo::{DIFile, DIFlags, DIType}; +use crate::llvm; + +/// Build the debuginfo node for an enum type. The listing below shows how such a +/// type looks like at the LLVM IR/DWARF level. It is a `DW_TAG_structure_type` +/// with a single `DW_TAG_variant_part` that in turn contains a `DW_TAG_variant` +/// for each variant of the enum. The variant-part also contains a single member +/// describing the discriminant, and a nested struct type for each of the variants. +/// +/// ```txt +/// ---> DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +pub(super) fn build_enum_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let enum_type = unique_type_id.expect_ty(); + let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { + bug!( + "build_enum_type_di_node() called with non-enum type: `{:?}`", + enum_type + ) + }; + + let enum_type_and_layout = cx.layout_of(enum_type); + + if wants_c_like_enum_debuginfo(cx.tcx, enum_type_and_layout) { + return build_c_style_enum_di_node(cx, enum_adt_def, enum_type_and_layout); + } + + let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); + let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); + + let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()); + + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(enum_adt_def.did()))) + } else { + None + }; + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &enum_type_name, + def_location, + size_and_align_of(enum_type_and_layout), + Some(containing_scope), + visibility_flags, + ), + |cx, enum_type_di_node| { + // Build the struct type for each variant. These will be referenced by the + // DW_TAG_variant DIEs inside of the DW_TAG_variant_part DIE. + // We also called the names for the corresponding DW_TAG_variant DIEs here. + let variant_member_infos: SmallVec<_> = enum_adt_def + .variant_range() + .map(|variant_index| VariantMemberInfo { + variant_index, + variant_name: Cow::from(enum_adt_def.variant(variant_index).name.as_str()), + variant_struct_type_di_node: build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + variant_index, + enum_adt_def.variant(variant_index), + enum_type_and_layout.for_variant(cx, variant_index), + visibility_flags, + ), + source_info: None, + }) + .collect(); + + let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(enum_adt_def.did()) + } else { + None + }; + smallvec![build_enum_variant_part_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + enum_adt_def_id, + &variant_member_infos[..], + )] + }, + // We don't seem to be emitting generic args on the enum type, it seems. Rather + // they get attached to the struct type of each variant. + NO_GENERICS, + ) +} + +/// Build the debuginfo node for a coroutine environment. It looks the same as the debuginfo for +/// an enum. See [build_enum_type_di_node] for more information. +/// +/// ```txt +/// +/// ---> DW_TAG_structure_type (top-level type for the coroutine) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// +/// ``` +pub(super) fn build_coroutine_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let coroutine_type = unique_type_id.expect_ty(); + let &ty::Coroutine(coroutine_def_id, coroutine_args) = coroutine_type.kind() else { + bug!( + "build_coroutine_di_node() called with non-coroutine type: `{:?}`", + coroutine_type + ) + }; + + let containing_scope = get_namespace_for_item(cx, coroutine_def_id); + let coroutine_type_and_layout = cx.layout_of(coroutine_type); + + assert!(!wants_c_like_enum_debuginfo( + cx.tcx, + coroutine_type_and_layout + )); + + let coroutine_type_name = compute_debuginfo_type_name(cx.tcx, coroutine_type, false); + + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(coroutine_def_id))) + } else { + None + }; + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &coroutine_type_name, + def_location, + size_and_align_of(coroutine_type_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + |cx, coroutine_type_di_node| { + let coroutine_layout = cx + .tcx + .coroutine_layout(coroutine_def_id, coroutine_args.as_coroutine().kind_ty()) + .unwrap(); + + let Variants::Multiple { + tag_encoding: TagEncoding::Direct, + ref variants, + .. + } = coroutine_type_and_layout.variants + else { + bug!( + "Encountered coroutine with non-direct-tag layout: {:?}", + coroutine_type_and_layout + ) + }; + + let common_upvar_names = cx + .tcx + .closure_saved_names_of_captured_variables(coroutine_def_id); + + // Build variant struct types + let variant_struct_type_di_nodes: SmallVec<_> = variants + .indices() + .map(|variant_index| { + // FIXME: This is problematic because just a number is not a valid identifier. + // CoroutineArgs::variant_name(variant_index), would be consistent + // with enums? + let variant_name = format!("{}", variant_index.as_usize()).into(); + + let span = coroutine_layout.variant_source_info[variant_index].span; + let source_info = if !span.is_dummy() { + let loc = cx.lookup_debug_loc(span.lo()); + Some((file_metadata(cx, &loc.file), loc.line)) + } else { + None + }; + + VariantMemberInfo { + variant_index, + variant_name, + variant_struct_type_di_node: build_coroutine_variant_struct_type_di_node( + cx, + variant_index, + coroutine_type_and_layout, + coroutine_type_di_node, + coroutine_layout, + common_upvar_names, + ), + source_info, + } + }) + .collect(); + + let coroutine_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(coroutine_def_id) + } else { + None + }; + smallvec![build_enum_variant_part_di_node( + cx, + coroutine_type_and_layout, + coroutine_type_di_node, + coroutine_def_id, + &variant_struct_type_di_nodes[..], + )] + }, + // We don't seem to be emitting generic args on the coroutine type, it seems. Rather + // they get attached to the struct type of each variant. + NO_GENERICS, + ) +} + +/// Builds the DW_TAG_variant_part of an enum or coroutine debuginfo node: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// ---> DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +fn build_enum_variant_part_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + enum_type_def_id: Option, + variant_member_infos: &[VariantMemberInfo<'_, 'll>], +) -> &'ll DIType { + let tag_member_di_node = + build_discr_member_di_node(cx, enum_type_and_layout, enum_type_di_node); + + let variant_part_unique_type_id = + UniqueTypeId::for_enum_variant_part(cx.tcx, enum_type_and_layout.ty); + + let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers + { + file_metadata_from_def_id(cx, enum_type_def_id) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + + let stub = StubInfo::new( + cx, + variant_part_unique_type_id, + |cx, variant_part_unique_type_id_str| unsafe { + let variant_part_name = ""; + llvm::LLVMRustDIBuilderCreateVariantPart( + DIB(cx), + enum_type_di_node, + variant_part_name.as_c_char_ptr(), + variant_part_name.len(), + file_metadata, + line_number, + enum_type_and_layout.size.bits(), + enum_type_and_layout.align.abi.bits() as u32, + DIFlags::FlagZero, + tag_member_di_node, + create_DIArray(DIB(cx), &[]), + variant_part_unique_type_id_str.as_c_char_ptr(), + variant_part_unique_type_id_str.len(), + ) + }, + ); + + type_map::build_type_with_children( + cx, + stub, + |cx, variant_part_di_node| { + variant_member_infos + .iter() + .map(|variant_member_info| { + build_enum_variant_member_di_node( + cx, + enum_type_and_layout, + variant_part_di_node, + variant_member_info, + ) + }) + .collect() + }, + NO_GENERICS, + ) + .di_node +} + +/// Builds the DW_TAG_member describing where we can find the tag of an enum. +/// Returns `None` if the enum does not have a tag. +/// +/// ```txt +/// +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// ---> DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// +/// ``` +fn build_discr_member_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_or_coroutine_type_and_layout: TyAndLayout<'tcx>, + enum_or_coroutine_type_di_node: &'ll DIType, +) -> Option<&'ll DIType> { + let tag_name = match enum_or_coroutine_type_and_layout.ty.kind() { + ty::Coroutine(..) => "__state", + _ => "", + }; + + // NOTE: This is actually wrong. This will become a member of + // of the DW_TAG_variant_part. But, due to LLVM's API, that + // can only be constructed with this DW_TAG_member already in created. + // In LLVM IR the wrong scope will be listed but when DWARF is + // generated from it, the DW_TAG_member will be a child the + // DW_TAG_variant_part. + let containing_scope = enum_or_coroutine_type_di_node; + + match enum_or_coroutine_type_and_layout.layout.variants() { + // A single-variant or no-variant enum has no discriminant. + &Variants::Single { .. } | &Variants::Empty => None, + + &Variants::Multiple { tag_field, .. } => { + let tag_base_type = tag_base_type(cx.tcx, enum_or_coroutine_type_and_layout); + let (size, align) = cx.size_and_align_of(tag_base_type); + + unsafe { + Some(llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + containing_scope, + tag_name.as_c_char_ptr(), + tag_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + enum_or_coroutine_type_and_layout + .fields + .offset(tag_field) + .bits(), + DIFlags::FlagArtificial, + type_di_node(cx, tag_base_type), + )) + } + } + } +} + +/// Build the debuginfo node for `DW_TAG_variant`: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// ---> DW_TAG_variant (variant 1) +/// ---> DW_TAG_variant (variant 2) +/// ---> DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +/// +/// This node looks like: +/// +/// ```txt +/// DW_TAG_variant +/// DW_AT_discr_value 0 +/// DW_TAG_member +/// DW_AT_name None +/// DW_AT_type <0x000002a1> +/// DW_AT_alignment 0x00000002 +/// DW_AT_data_member_location 0 +/// ``` +/// +/// The DW_AT_discr_value is optional, and is omitted if +/// - This is the only variant of a univariant enum (i.e. their is no discriminant) +/// - This is the "untagged" variant of a niche-layout enum +/// (where only the other variants are identified by a single value) +/// +/// There is only ever a single member, the type of which is a struct that describes the +/// fields of the variant (excluding the discriminant). The name of the member is the name +/// of the variant as given in the source code. The DW_AT_data_member_location is always +/// zero. +/// +/// Note that the LLVM DIBuilder API is a bit unintuitive here. The DW_TAG_variant subtree +/// (including the DW_TAG_member) is built by a single call to +/// `LLVMRustDIBuilderCreateVariantMemberType()`. +fn build_enum_variant_member_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + variant_part_di_node: &'ll DIType, + variant_member_info: &VariantMemberInfo<'_, 'll>, +) -> &'ll DIType { + let variant_index = variant_member_info.variant_index; + let discr_value = compute_discriminant_value(cx, enum_type_and_layout, variant_index); + + let (file_di_node, line_number) = variant_member_info + .source_info + .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); + + let discr = discr_value.opt_single_val().map(|value| { + let tag_base_type = tag_base_type(cx.tcx, enum_type_and_layout); + let size = cx.size_of(tag_base_type); + cx.const_uint_big(cx.type_ix(size.bits()), value) + }); + + unsafe { + llvm::LLVMRustDIBuilderCreateVariantMemberType( + DIB(cx), + variant_part_di_node, + variant_member_info.variant_name.as_c_char_ptr(), + variant_member_info.variant_name.len(), + file_di_node, + line_number, + enum_type_and_layout.size.bits(), + enum_type_and_layout.align.abi.bits() as u32, + Size::ZERO.bits(), + discr, + DIFlags::FlagZero, + variant_member_info.variant_struct_type_di_node, + ) + } +} + +/// Information needed for building a `DW_TAG_variant`: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// ---> DW_TAG_variant (variant 1) +/// ---> DW_TAG_variant (variant 2) +/// ---> DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +struct VariantMemberInfo<'a, 'll> { + variant_index: VariantIdx, + variant_name: Cow<'a, str>, + variant_struct_type_di_node: &'ll DIType, + source_info: Option<(&'ll DIFile, c_uint)>, +} + +/// Build the debuginfo node for a C-style enum, i.e. an enum the variants of which have no fields. +/// +/// The resulting debuginfo will be a DW_TAG_enumeration_type. +fn build_c_style_enum_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, +) -> DINodeCreationResult<'ll> { + let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); + let enum_adt_def_id = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(enum_adt_def.did()) + } else { + None + }; + DINodeCreationResult { + di_node: build_enumeration_type_di_node( + cx, + &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false), + tag_base_type(cx.tcx, enum_type_and_layout), + enum_adt_def + .discriminants(cx.tcx) + .map(|(variant_index, discr)| { + let name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); + (name, discr.val) + }), + enum_adt_def_id, + containing_scope, + ), + already_stored_in_typemap: false, + } +} + +/// Build a DW_TAG_enumeration_type debuginfo node, with the given base type and variants. +/// This is a helper function and does not register anything in the type map by itself. +/// +/// `variants` is an iterator of (discr-value, variant-name). +/// +/// NVVM: Discrimant values are mapped from u128 to i64 to conform with LLVM 7's API. +fn build_enumeration_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + type_name: &str, + base_type: Ty<'tcx>, + enumerators: impl Iterator, u128)>, + def_id: Option, + containing_scope: &'ll DIType, +) -> &'ll DIType { + let is_unsigned = match base_type.kind() { + ty::Int(_) => false, + ty::Uint(_) => true, + _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."), + }; + let (size, align) = cx.size_and_align_of(base_type); + + let enumerator_di_nodes: SmallVec> = enumerators + .map(|(name, value)| unsafe { + // FIXME: pass all 128 bits with newer LLVM API. + Some(llvm::LLVMRustDIBuilderCreateEnumerator( + DIB(cx), + name.as_c_char_ptr(), + name.len(), + value as i64, + is_unsigned, + )) + }) + .collect(); + + let (file_metadata, line_number) = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers + { + file_metadata_from_def_id(cx, def_id) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + + unsafe { + llvm::LLVMRustDIBuilderCreateEnumerationType( + DIB(cx), + containing_scope, + type_name.as_c_char_ptr(), + type_name.len(), + file_metadata, + line_number, + size.bits(), + align.bits() as u32, + create_DIArray(DIB(cx), &enumerator_di_nodes[..]), + type_di_node(cx, base_type), + ) + } +} + +/// Build the debuginfo node for the struct type describing a single variant of an enum. +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// ``` +/// +/// The node looks like: +/// +/// ```txt +/// DW_TAG_structure_type +/// DW_AT_name +/// DW_AT_byte_size 0x00000010 +/// DW_AT_alignment 0x00000008 +/// DW_TAG_member +/// DW_AT_name +/// DW_AT_type <0x0000018e> +/// DW_AT_alignment 0x00000004 +/// DW_AT_data_member_location 4 +/// DW_TAG_member +/// DW_AT_name +/// DW_AT_type <0x00000195> +/// DW_AT_alignment 0x00000008 +/// DW_AT_data_member_location 8 +/// ... +/// ``` +/// +/// The type of a variant is always a struct type with the name of the variant +/// and a DW_TAG_member for each field (but not the discriminant). +fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_index: VariantIdx, + variant_def: &VariantDef, + variant_layout: TyAndLayout<'tcx>, + di_flags: DIFlags, +) -> &'ll DIType { + assert_eq!(variant_layout.ty, enum_type_and_layout.ty); + + let def_location = if cx.sess().opts.unstable_opts.debug_info_type_line_numbers { + Some(file_metadata_from_def_id(cx, Some(variant_def.def_id))) + } else { + None + }; + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + UniqueTypeId::for_enum_variant_struct_type( + cx.tcx, + enum_type_and_layout.ty, + variant_index, + ), + variant_def.name.as_str(), + def_location, + // NOTE: We use size and align of enum_type, not from variant_layout: + size_and_align_of(enum_type_and_layout), + Some(enum_type_di_node), + di_flags, + ), + |cx, struct_type_di_node| { + (0..variant_layout.fields.count()) + .map(|field_index| { + let field_name = if variant_def.ctor_kind() != Some(CtorKind::Fn) { + // Fields have names + let field = &variant_def.fields[FieldIdx::from_usize(field_index)]; + Cow::from(field.name.as_str()) + } else { + // Tuple-like + super::tuple_field_name(field_index) + }; + + let field_layout = variant_layout.field(cx, field_index); + + build_field_di_node( + cx, + struct_type_di_node, + &field_name, + (field_layout.size, field_layout.align.abi), + variant_layout.fields.offset(field_index), + di_flags, + type_di_node(cx, field_layout.ty), + None, + ) + }) + .collect::>() + }, + |cx| build_generic_type_param_di_nodes(cx, enum_type_and_layout.ty), + ) + .di_node +} + +/// Build the struct type for describing a single coroutine state. +/// See [build_coroutine_variant_struct_type_di_node]. +/// +/// ```txt +/// +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// +/// ``` +fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + variant_index: VariantIdx, + coroutine_type_and_layout: TyAndLayout<'tcx>, + coroutine_type_di_node: &'ll DIType, + coroutine_layout: &CoroutineLayout<'tcx>, + common_upvar_names: &IndexSlice, +) -> &'ll DIType { + let variant_name = CoroutineArgs::variant_name(variant_index); + let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( + cx.tcx, + coroutine_type_and_layout.ty, + variant_index, + ); + + let variant_layout = coroutine_type_and_layout.for_variant(cx, variant_index); + + let coroutine_args = match coroutine_type_and_layout.ty.kind() { + ty::Coroutine(_, args) => args.as_coroutine(), + _ => unreachable!(), + }; + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &variant_name, + None, + size_and_align_of(coroutine_type_and_layout), + Some(coroutine_type_di_node), + DIFlags::FlagZero, + ), + |cx, variant_struct_type_di_node| { + // Fields that just belong to this variant/state + let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) + .map(|field_index| { + let coroutine_saved_local = coroutine_layout.variant_fields[variant_index] + [FieldIdx::from_usize(field_index)]; + let field_name_maybe = coroutine_layout.field_names[coroutine_saved_local]; + let field_name = field_name_maybe + .as_ref() + .map(|s| Cow::from(s.as_str())) + .unwrap_or_else(|| super::tuple_field_name(field_index)); + + let field_type = variant_layout.field(cx, field_index).ty; + + build_field_di_node( + cx, + variant_struct_type_di_node, + &field_name, + cx.size_and_align_of(field_type), + variant_layout.fields.offset(field_index), + DIFlags::FlagZero, + type_di_node(cx, field_type), + None, + ) + }) + .collect(); + + // Fields that are common to all states + let common_fields: SmallVec<_> = coroutine_args + .prefix_tys() + .iter() + .zip(common_upvar_names) + .enumerate() + .map(|(index, (upvar_ty, upvar_name))| { + build_field_di_node( + cx, + variant_struct_type_di_node, + upvar_name.as_str(), + cx.size_and_align_of(upvar_ty), + coroutine_type_and_layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, upvar_ty), + None, + ) + }) + .collect(); + + state_specific_fields + .into_iter() + .chain(common_fields) + .collect() + }, + |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty), + ) + .di_node +} + +#[derive(Copy, Clone)] +enum DiscrResult { + NoDiscriminant, + Value(u128), + #[allow(dead_code)] + Range(u128, u128), +} + +impl DiscrResult { + fn opt_single_val(&self) -> Option { + if let Self::Value(d) = *self { + Some(d) + } else { + None + } + } +} + +/// Returns the discriminant value corresponding to the variant index. +/// +/// Will return `None` if there is less than two variants (because then the enum won't have) +/// a tag, and if this is the untagged variant of a niche-layout enum (because then there is no +/// single discriminant value). +fn compute_discriminant_value<'tcx>( + cx: &CodegenCx<'_, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + variant_index: VariantIdx, +) -> DiscrResult { + match enum_type_and_layout.layout.variants() { + &Variants::Single { .. } | &Variants::Empty => DiscrResult::NoDiscriminant, + &Variants::Multiple { + tag_encoding: TagEncoding::Direct, + .. + } => DiscrResult::Value( + enum_type_and_layout + .ty + .discriminant_for_variant(cx.tcx, variant_index) + .unwrap() + .val, + ), + &Variants::Multiple { + tag_encoding: + TagEncoding::Niche { + ref niche_variants, + niche_start, + untagged_variant, + }, + tag, + .. + } => { + if variant_index == untagged_variant { + let valid_range = enum_type_and_layout + .for_variant(cx, variant_index) + .largest_niche + .as_ref() + .unwrap() + .valid_range; + + let min = valid_range.start.min(valid_range.end); + let min = tag.size(cx).truncate(min); + + let max = valid_range.start.max(valid_range.end); + let max = tag.size(cx).truncate(max); + + DiscrResult::Range(min, max) + } else { + let value = (variant_index.as_u32() as u128) + .wrapping_sub(niche_variants.start().as_u32() as u128) + .wrapping_add(niche_start); + let value = tag.size(cx).truncate(value); + DiscrResult::Value(value) + } + } + } +} diff --git a/crates/rustc_codegen_nvvm/src/debug_info/metadata/type_map.rs b/crates/rustc_codegen_nvvm_v19/src/debug_info/metadata/type_map.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/debug_info/metadata/type_map.rs rename to crates/rustc_codegen_nvvm_v19/src/debug_info/metadata/type_map.rs diff --git a/crates/rustc_codegen_nvvm_v19/src/debug_info/mod.rs b/crates/rustc_codegen_nvvm_v19/src/debug_info/mod.rs new file mode 100644 index 00000000..9a9f1d86 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/debug_info/mod.rs @@ -0,0 +1,552 @@ +use std::cell::OnceCell; +use std::cell::RefCell; +use std::ffi::CString; +use std::iter; +use std::ops::Range; +use std::sync::Arc; + +use libc::c_uint; +use rustc_abi::Size; +use rustc_codegen_ssa::debuginfo::type_names; +use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; +use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::unord::UnordMap; +use rustc_hir::def_id::{DefId, DefIdMap}; +use rustc_index::IndexVec; +use rustc_middle::mir; +use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv}; +use rustc_middle::ty::{self, GenericArgKind, GenericArgsRef, Instance, Ty, TypeVisitableExt}; +use rustc_session::config::{self, DebugInfo}; +use rustc_span::symbol::Symbol; +use rustc_span::{ + self, BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span, StableSourceFileId, +}; +use rustc_target::callconv::FnAbi; +use smallvec::SmallVec; + +use crate::builder::Builder; +use crate::common::AsCCharPtr; +use crate::context::CodegenCx; +use crate::debug_info::util::{create_DIArray, is_node_local_to_unit}; +use crate::llvm; +use crate::llvm::LLVMRustDISPFlags; +use crate::llvm::{ + Context, DIArray, DIBuilder, DIFile, DIFlags, DILocation, DIScope, DIType, DIVariable, Module, + Value, +}; + +use self::namespace::*; +use self::util::DIB; +use create_scope_map::compute_mir_scopes; + +mod create_scope_map; +mod dwarf_const; +pub(crate) mod metadata; +mod namespace; +mod util; + +pub(crate) use metadata::*; + +#[allow(non_upper_case_globals)] +const DW_TAG_auto_variable: c_uint = 0x100; +#[allow(non_upper_case_globals)] +const DW_TAG_arg_variable: c_uint = 0x101; + +pub struct CodegenUnitDebugContext<'ll, 'tcx> { + #[allow(dead_code)] + llcontext: &'ll Context, + llmod: &'ll Module, + builder: &'ll mut DIBuilder<'ll>, + created_files: RefCell, &'ll DIFile>>, + + type_map: metadata::TypeMap<'ll, 'tcx>, + namespace_map: RefCell>, + recursion_marker_type: OnceCell<&'ll DIType>, +} + +impl Drop for CodegenUnitDebugContext<'_, '_> { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustDIBuilderDispose(&mut *(self.builder as *mut _)); + } + } +} + +impl<'a> CodegenUnitDebugContext<'a, '_> { + pub(crate) fn new(llmod: &'a llvm::Module) -> Self { + let builder = unsafe { llvm::LLVMRustDIBuilderCreate(llmod) }; + // DIBuilder inherits context from the module, so we'd better use the same one + let llcontext = unsafe { llvm::LLVMGetModuleContext(llmod) }; + // TODO: remove this legacy old thing + unsafe { llvm::LLVMRustSetOldDebugFormat(llmod) }; + CodegenUnitDebugContext { + llcontext, + llmod, + builder, + created_files: Default::default(), + type_map: Default::default(), + namespace_map: RefCell::new(Default::default()), + recursion_marker_type: OnceCell::new(), + } + } + + pub(crate) fn finalize(&self) { + unsafe { + llvm::LLVMRustDIBuilderFinalize(self.builder); + + // Prevent bitcode readers from deleting the debug info. + llvm::LLVMRustAddModuleFlag( + self.llmod, + c"Debug Info Version".as_ptr(), + llvm::LLVMRustDebugMetadataVersion(), + ); + } + } +} + +/// Creates any deferred debug metadata nodes +pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) { + if let Some(dbg_cx) = &cx.dbg_cx { + dbg_cx.finalize(); + } +} + +impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { + fn dbg_var_addr( + &mut self, + dbg_var: &'ll DIVariable, + dbg_loc: &'ll DILocation, + variable_alloca: &'ll Value, + direct_offset: Size, + indirect_offsets: &[Size], + fragment: Option>, + ) { + use dwarf_const::{DW_OP_LLVM_fragment, DW_OP_deref, DW_OP_plus_uconst}; + + let mut addr_ops = SmallVec::<[u64; 8]>::new(); + + if direct_offset.bytes() > 0 { + addr_ops.push(DW_OP_plus_uconst as u64); + addr_ops.push(direct_offset.bytes()); + } + for &offset in indirect_offsets { + addr_ops.push(DW_OP_deref as u64); + if offset.bytes() > 0 { + addr_ops.push(DW_OP_plus_uconst as u64); + addr_ops.push(offset.bytes()); + } + } + + if let Some(fragment) = fragment { + // `DW_OP_LLVM_fragment` takes as arguments the fragment's + // offset and size, both of them in bits. + addr_ops.push(DW_OP_LLVM_fragment as u64); + addr_ops.push(fragment.start.bits()); + addr_ops.push((fragment.end - fragment.start).bits()); + } + + unsafe { + llvm::LLVMRustDIBuilderInsertDeclareAtEnd( + DIB(self.cx()), + variable_alloca, + dbg_var, + addr_ops.as_ptr(), + addr_ops.len() as c_uint, + dbg_loc, + self.llbb(), + ); + } + } + + fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { + unsafe { + llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, Some(dbg_loc)); + } + } + + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { + // do nothing + } + + fn set_var_name(&mut self, value: &'ll Value, name: &str) { + if self.sess().fewer_names() { + return; + } + + // Only function parameters and instructions are local to a function, + // don't change the name of anything else (e.g. globals). + let param_or_inst = unsafe { + llvm::LLVMIsAArgument(value).is_some() || llvm::LLVMIsAInstruction(value).is_some() + }; + if !param_or_inst { + return; + } + + // Avoid replacing the name if it already exists. + // While we could combine the names somehow, it'd + // get noisy quick, and the usefulness is dubious. + if llvm::get_value_name(value).is_empty() { + llvm::set_value_name(value, name.as_bytes()); + } + } + + fn clear_dbg_loc(&mut self) { + unsafe { + llvm::LLVMSetCurrentDebugLocation2(self.llbuilder, None); + } + } + + fn get_dbg_loc(&self) -> Option { + None // TODO: implement this + } +} + +/// A source code location used to generate debug information. +pub struct DebugLoc { + /// Information about the original source file. + pub file: Arc, + /// The (1-based) line number. + pub line: u32, + /// The (1-based) column number. + pub col: u32, +} + +impl CodegenCx<'_, '_> { + /// Looks up debug source information about a `BytePos`. + pub fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { + let (file, line, col) = match self.sess().source_map().lookup_line(pos) { + Ok(SourceFileAndLine { sf: file, line }) => { + let line_pos = file.line_begin_pos(pos); + + // Use 1-based indexing. + let line = (line + 1) as u32; + let col = (pos - line_pos).to_u32() + 1; + + (file, line, col) + } + Err(file) => (file, UNKNOWN_LINE_NUMBER, UNKNOWN_COLUMN_NUMBER), + }; + + DebugLoc { file, line, col } + } +} + +impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn create_function_debug_context( + &self, + instance: Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + llfn: Self::Function, + mir: &mir::Body<'tcx>, + ) -> Option> { + if self.sess().opts.debuginfo == DebugInfo::None { + return None; + } + + // Initialize fn debug context (including scopes). + let empty_scope = DebugScope { + dbg_scope: self.dbg_scope_fn(instance, fn_abi, Some(llfn)), + inlined_at: None, + file_start_pos: BytePos(0), + file_end_pos: BytePos(0), + }; + let mut fn_debug_context = FunctionDebugContext { + scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes), + inlined_function_scopes: Default::default(), + }; + + // Fill in all the scopes, with the information from the MIR body. + compute_mir_scopes(self, instance, mir, &mut fn_debug_context); + + Some(fn_debug_context) + } + + fn dbg_scope_fn( + &self, + instance: Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + maybe_definition_llfn: Option<&'ll Value>, + ) -> &'ll DIScope { + let tcx = self.tcx(); + + let def_id = instance.def_id(); + let containing_scope = get_containing_scope(self, instance); + let span = tcx.def_span(def_id); + let loc = self.lookup_debug_loc(span.lo()); + let file_metadata = file_metadata(self, &loc.file); + + let function_type_metadata = unsafe { + let fn_signature = get_function_signature(self, fn_abi); + llvm::LLVMRustDIBuilderCreateSubroutineType(DIB(self), fn_signature) + }; + + let mut name = String::new(); + type_names::push_item_name(tcx, def_id, false, &mut name); + + // Find the enclosing function, in case this is a closure. + let enclosing_fn_def_id = tcx.typeck_root_def_id(def_id); + + // Get_template_parameters() will append a `<...>` clause to the function + // name if necessary. + let generics = tcx.generics_of(enclosing_fn_def_id); + let args = instance.args.truncate_to(tcx, generics); + let template_parameters = get_template_parameters(self, generics, args); + + let linkage_name = &mangled_name_of_instance(self, instance).name; + // Omit the linkage_name if it is the same as subprogram name. + let linkage_name = if &name == linkage_name { + "" + } else { + linkage_name + }; + let name_len = name.len(); + let linkage_name_len = linkage_name.len(); + let name = CString::new(name).unwrap(); + let linkage_name = CString::new(linkage_name).unwrap(); + + let scope_line = loc.line; + + let mut flags = DIFlags::FlagPrototyped; + + if fn_abi.ret.layout.is_uninhabited() { + flags |= DIFlags::FlagNoReturn; + } + + let is_local_to_unit = is_node_local_to_unit(self, def_id); + let is_definition = true; // <- This was hardcoded as `true` in your call + let is_optimized = self.sess().opts.optimize != config::OptLevel::No; // <- This was the boolean expression + + let mut sp_flags = LLVMRustDISPFlags::SPFlagZero; + if is_local_to_unit { + sp_flags |= LLVMRustDISPFlags::SPFlagLocalToUnit; + } + if is_definition { + sp_flags |= LLVMRustDISPFlags::SPFlagDefinition; + } + if is_optimized { + sp_flags |= LLVMRustDISPFlags::SPFlagOptimized; + } + + unsafe { + return llvm::LLVMRustDIBuilderCreateFunction( + DIB(self), + containing_scope.0, + name.as_ptr(), + name_len, + linkage_name.as_ptr(), + linkage_name_len, + file_metadata, + loc.line, + function_type_metadata, + scope_line, + flags, + sp_flags, + maybe_definition_llfn, + template_parameters, + None, + ); + } + + fn get_function_signature<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + ) -> &'ll DIArray { + if cx.sess().opts.debuginfo == DebugInfo::Limited { + return create_DIArray(DIB(cx), &[]); + } + + let mut signature = Vec::with_capacity(fn_abi.args.len() + 1); + + // Return type -- llvm::DIBuilder wants this at index 0 + signature.push(if fn_abi.ret.is_ignore() { + None + } else { + Some(type_di_node(cx, fn_abi.ret.layout.ty)) + }); + + signature.extend( + fn_abi + .args + .iter() + .map(|arg| Some(type_di_node(cx, arg.layout.ty))), + ); + + create_DIArray(DIB(cx), &signature[..]) + } + + fn get_template_parameters<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + generics: &ty::Generics, + args: GenericArgsRef<'tcx>, + ) -> &'ll DIArray { + if args.types().next().is_none() { + return create_DIArray(DIB(cx), &[]); + } + + // Again, only create type information if full debuginfo is enabled + let template_params: Vec<_> = if cx.sess().opts.debuginfo == DebugInfo::Full { + let names = get_parameter_names(cx, generics); + iter::zip(args, names) + .filter_map(|(kind, name)| { + if let GenericArgKind::Type(ty) = kind.unpack() { + let actual_type = cx.tcx.normalize_erasing_regions(cx.typing_env(), ty); + let actual_type_metadata = type_di_node(cx, actual_type); + let name = name.as_str(); + Some(unsafe { + Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( + DIB(cx), + None, + name.as_c_char_ptr(), + name.len(), + actual_type_metadata, + )) + }) + } else { + None + } + }) + .collect() + } else { + vec![] + }; + + create_DIArray(DIB(cx), &template_params) + } + + fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec { + let mut names = generics.parent.map_or_else(Vec::new, |def_id| { + get_parameter_names(cx, cx.tcx.generics_of(def_id)) + }); + names.extend(generics.own_params.iter().map(|param| param.name)); + names + } + + fn get_containing_scope<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + ) -> (&'ll DIScope, bool) { + // First, let's see if this is a method within an inherent impl. Because + // if yes, we want to make the result subroutine DIE a child of the + // subroutine's self-type. + if let Some(impl_def_id) = cx.tcx.impl_of_method(instance.def_id()) { + // If the method does *not* belong to a trait, proceed + if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { + let impl_self_ty = cx.tcx.instantiate_and_normalize_erasing_regions( + instance.args, + cx.typing_env(), + cx.tcx.type_of(impl_def_id), + ); + + // Only "class" methods are generally understood by LLVM, + // so avoid methods on other types (e.g., `<*mut T>::null`). + if let ty::Adt(def, ..) = impl_self_ty.kind() + && !def.is_box() + { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full && !impl_self_ty.has_param() + { + return (type_di_node(cx, impl_self_ty), true); + } else { + return (namespace::item_namespace(cx, def.did()), false); + } + } + } else { + // For trait method impls we still use the "parallel namespace" + // strategy + } + } + let scope = namespace::item_namespace( + cx, + DefId { + krate: instance.def_id().krate, + index: cx + .tcx + .def_key(instance.def_id()) + .parent + .expect("get_containing_scope: missing parent?"), + }, + ); + + (scope, false) + } + } + + fn dbg_loc( + &self, + scope: Self::DIScope, + inlined_at: Option, + span: Span, + ) -> Self::DILocation { + let DebugLoc { line, col, .. } = self.lookup_debug_loc(span.lo()); + + unsafe { + llvm::LLVMRustDIBuilderCreateDebugLocation( + line, + col, + scope, + inlined_at, + ) + } + } + + fn create_vtable_debuginfo( + &self, + ty: Ty<'tcx>, + trait_ref: Option>, + vtable: Self::Value, + ) { + metadata::create_vtable_di_node(self, ty, trait_ref, vtable) + } + + fn extend_scope_to_file( + &self, + scope_metadata: Self::DIScope, + file: &SourceFile, + ) -> Self::DIScope { + metadata::extend_scope_to_file(self, scope_metadata, file) + } + + fn debuginfo_finalize(&self) { + finalize(self) + } + + fn create_dbg_var( + &self, + variable_name: Symbol, + variable_type: Ty<'tcx>, + scope_metadata: Self::DIScope, + variable_kind: VariableKind, + span: Span, + ) -> Self::DIVariable { + let loc = self.lookup_debug_loc(span.lo()); + let file_metadata = file_metadata(self, &loc.file); + + let type_metadata = type_di_node(self, variable_type); + + let (argument_index, dwarf_tag) = match variable_kind { + ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), + LocalVariable => (0, DW_TAG_auto_variable), + }; + let align = self.align_of(variable_type); + + let name_len = variable_name.as_str().len(); + let name = CString::new(variable_name.as_str()).unwrap(); + unsafe { + llvm::LLVMRustDIBuilderCreateVariable( + DIB(self), + dwarf_tag, + scope_metadata, + name.as_ptr().cast(), + name_len, + file_metadata, + loc.line, + type_metadata, + true, + DIFlags::FlagZero, + argument_index, + align.bytes() as u32, + ) + } + } +} diff --git a/crates/rustc_codegen_nvvm/src/debug_info/namespace.rs b/crates/rustc_codegen_nvvm_v19/src/debug_info/namespace.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/debug_info/namespace.rs rename to crates/rustc_codegen_nvvm_v19/src/debug_info/namespace.rs diff --git a/crates/rustc_codegen_nvvm/src/debug_info/util.rs b/crates/rustc_codegen_nvvm_v19/src/debug_info/util.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/debug_info/util.rs rename to crates/rustc_codegen_nvvm_v19/src/debug_info/util.rs diff --git a/crates/rustc_codegen_nvvm_v19/src/init.rs b/crates/rustc_codegen_nvvm_v19/src/init.rs new file mode 100644 index 00000000..555811a8 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/init.rs @@ -0,0 +1,136 @@ +use std::ffi::CString; +use std::mem; +use std::path::Path; +use std::str; +use std::sync::Once; +use std::sync::atomic::{AtomicBool, Ordering}; + +use libc::c_int; +use rustc_middle::bug; +use rustc_session::Session; +use rustc_target::spec::MergeFunctions; + +use crate::llvm; + +static POISONED: AtomicBool = AtomicBool::new(false); +static INIT: Once = Once::new(); + +pub(crate) fn init(sess: &Session) { + unsafe { + // Before we touch LLVM, make sure that multithreading is enabled. + INIT.call_once(|| { + if llvm::LLVMStartMultithreaded() != 1 { + // use an extra bool to make sure that all future usage of LLVM + // cannot proceed despite the Once not running more than once. + POISONED.store(true, Ordering::SeqCst); + } + + configure_llvm(sess); + }); + + if POISONED.load(Ordering::SeqCst) { + bug!("couldn't enable multi-threaded LLVM"); + } + } +} + +unsafe fn configure_llvm(sess: &Session) { + // TODO(RDambrosio016): We override the meaning of llvm-args to pass our own nvvm args, + // but we should probably retain a way to pass args to LLVM. + let n_args = sess.opts.cg.llvm_args.len() + sess.target.llvm_args.len(); + let mut llvm_c_strs = Vec::with_capacity(n_args + 1); + let mut llvm_args = Vec::with_capacity(n_args + 1); + + // fn llvm_arg_to_arg_name(full_arg: &str) -> &str { + // full_arg + // .trim() + // .split(|c: char| c == '=' || c.is_whitespace()) + // .next() + // .unwrap_or("") + // } + + // let cg_opts = sess.opts.cg.llvm_args.iter(); + // let tg_opts = sess.target.llvm_args.iter(); + // let sess_args = cg_opts.chain(tg_opts); + + // dont print anything in here or it will interfere with cargo trying to print stuff which will + // cause the compilation to fail in mysterious ways. + + // let user_specified_args: FxHashSet<_> = sess_args + // .clone() + // .map(|s| llvm_arg_to_arg_name(s)) + // .filter(|s| !s.is_empty()) + // .collect(); + + { + // This adds the given argument to LLVM. Unless `force` is true + // user specified arguments are *not* overridden. + let mut add = |arg: &str, _force: bool| { + // if force || !user_specified_args.contains(llvm_arg_to_arg_name(arg)) { + let s = CString::new(arg).unwrap(); + llvm_args.push(s.as_ptr()); + llvm_c_strs.push(s); + // } + }; + // Set the llvm "program name" to make usage and invalid argument messages more clear. + // add("rustc -Cllvm-args=\"...\" with", true); + + if sess.opts.unstable_opts.time_llvm_passes { + add("-time-passes", false); + } + if sess.opts.unstable_opts.print_llvm_passes { + add("-debug-pass=Structure", false); + } + if !sess.opts.unstable_opts.no_generate_arange_section { + add("-generate-arange-section", false); + } + + match sess + .opts + .unstable_opts + .merge_functions + .unwrap_or(sess.target.merge_functions) + { + MergeFunctions::Disabled | MergeFunctions::Trampolines => {} + MergeFunctions::Aliases => { + add("-mergefunc-use-aliases", false); + } + } + + // HACK(eddyb) LLVM inserts `llvm.assume` calls to preserve align attributes + // during inlining. Unfortunately these may block other optimizations. + add( + "-preserve-alignment-assumptions-during-inlining=false", + false, + ); + + // Use non-zero `import-instr-limit` multiplier for cold callsites. + add("-import-cold-multiplier=0.1", false); + + // for arg in sess_args { + // add(&(*arg), true); + // } + } + + // TODO: removed as part of llvm v19? + //unsafe { llvm::LLVMInitializePasses() }; + + for plugin in &sess.opts.unstable_opts.llvm_plugins { + let path = Path::new(plugin); + let res = unsafe { libloading::Library::new(path) }; + match res { + Ok(_) => {} + Err(e) => bug!("couldn't load plugin: {}", e), + } + mem::forget(res); + } + + unsafe { + llvm::LLVMInitializeNVPTXTarget(); + llvm::LLVMInitializeNVPTXTargetInfo(); + llvm::LLVMInitializeNVPTXTargetMC(); + llvm::LLVMInitializeNVPTXAsmPrinter(); + + llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()); + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/int_replace.rs b/crates/rustc_codegen_nvvm_v19/src/int_replace.rs new file mode 100644 index 00000000..e54f1122 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/int_replace.rs @@ -0,0 +1,113 @@ +use tracing::trace; + +use crate::builder::unnamed; +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm::{True, TypeKind, Type}; + +// for some reason nvvm doesnt accept "irregular" types like i4 or i48, so we need to resort to using "regular" +// types and falling back to 1 if excrement hits the rotating fan. +const WIDTH_CANDIDATES: &[u32] = &[64, 32, 16, 8, 1]; + +/// returns `true` if the type is or contains irregular integers. +pub(crate) fn type_needs_transformation(ty: &Type) -> bool { + unsafe { + let kind = llvm::LLVMRustGetTypeKind(ty); + match kind { + TypeKind::Integer => { + let width = llvm::LLVMGetIntTypeWidth(ty); + !WIDTH_CANDIDATES.contains(&width) + } + TypeKind::Struct => struct_type_fields(ty) + .into_iter() + .any(type_needs_transformation), + _ => false, + } + } +} + +fn struct_type_fields(ty: &Type) -> Vec<&Type> { + unsafe { + let count = llvm::LLVMCountStructElementTypes(ty); + let mut fields = Vec::with_capacity(count as usize); + llvm::LLVMGetStructElementTypes(ty, fields.as_mut_ptr()); + fields.set_len(count as usize); + fields + } +} + +/// Transforms a type to an nvvm-friendly vector type if the type is an int over i64. +/// returns a bool on whether the type was transformed. +pub(crate) fn get_transformed_type<'ll>( + cx: &CodegenCx<'ll, '_>, + ty: &'ll Type, +) -> (&'ll Type, bool) { + unsafe { + if type_needs_transformation(ty) { + let kind = llvm::LLVMRustGetTypeKind(ty); + match kind { + TypeKind::Integer => { + let width = llvm::LLVMGetIntTypeWidth(ty); + let (width, count) = target_vector_width_and_count(width); + let int_ty = llvm::LLVMIntTypeInContext(cx.llcx, width); + trace!( + "Transforming irregular int type `{:?}` to vector ty `{:?}` with length {}", + ty, int_ty, count + ); + (llvm::LLVMVectorType(int_ty, count), true) + } + TypeKind::Struct => { + let fields = struct_type_fields(ty); + let transformed = fields + .into_iter() + .map(|field| get_transformed_type(cx, field).0) + .collect::>(); + + let packed = llvm::LLVMIsPackedStruct(ty); + (cx.type_struct(&transformed, packed == True), true) + } + _ => unreachable!(), + } + } else { + (ty, false) + } + } +} + +// try to find the largest possible int type to use for the target vector type. +// going from i64 down. +pub(crate) fn target_vector_width_and_count(int_width: u32) -> (u32, u32) { + for &i in WIDTH_CANDIDATES { + if int_width % i == 0 { + return (i, int_width / i); + } + } + unreachable!() +} + +/// transmutes a value to a certain type, accounting for structs. +pub(crate) fn transmute_llval<'ll>( + bx: &mut llvm::Builder<'ll>, + _cx: &CodegenCx<'ll, '_>, + a_val: &'ll llvm::Value, + ty: &'ll llvm::Type, +) -> &'ll llvm::Value { + trace!("transmute_llval: transmuting `{:?}` to `{:?}`", a_val, ty); + unsafe { + let kind = llvm::LLVMRustGetTypeKind(ty); + match kind { + // structs cannot be bitcasted, so we need to do it using a bunch of extract/insert values. + TypeKind::Struct => { + let new_struct = llvm::LLVMGetUndef(ty); + let mut last_val = new_struct; + for (idx, field) in struct_type_fields(ty).into_iter().enumerate() { + let field_val = llvm::LLVMBuildExtractValue(bx, a_val, idx as u32, unnamed()); + let new_val = transmute_llval(bx, _cx, field_val, field); + last_val = llvm::LLVMBuildInsertValue(bx, last_val, new_val, idx as u32, unnamed()); + } + last_val + } + _ => llvm::LLVMBuildBitCast(bx, a_val, ty, unnamed()), + } + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/intrinsic.rs b/crates/rustc_codegen_nvvm_v19/src/intrinsic.rs new file mode 100644 index 00000000..3fe8e668 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/intrinsic.rs @@ -0,0 +1,623 @@ +use rustc_abi as abi; +use rustc_abi::{self, Float, HasDataLayout, Primitive}; +use rustc_codegen_ssa::errors::InvalidMonomorphization; +use rustc_codegen_ssa::mir::operand::OperandValue; +use rustc_codegen_ssa::mir::place::PlaceValue; +use rustc_codegen_ssa::mir::{operand::OperandRef, place::PlaceRef}; +use rustc_codegen_ssa::traits::{ + BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, IntrinsicCallBuilderMethods, + OverflowOp, +}; +use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; +use rustc_middle::ty::{self, Ty}; +use rustc_middle::{bug, span_bug}; +use rustc_span::symbol::kw; +use rustc_span::{Span, Symbol, sym}; +use rustc_target::callconv::{FnAbi, PassMode}; +use tracing::trace; + +use crate::abi::LlvmType; +use crate::builder::Builder; +use crate::context::CodegenCx; +use crate::llvm::{self, Metadata, Type, Value}; +use crate::ty::LayoutLlvmExt; + +// libnvvm does not support some advanced intrinsics for i128 so we just abort on them for now. In the future +// we should emulate them in software. +fn handle_128_bit_intrinsic<'ll>(b: &mut Builder<'_, 'll, '_>) -> &'ll Value { + b.abort_and_ret_i128() +} + +// llvm 7 does not have saturating intrinsics, so we reimplement them right here. +// This is derived from what rustc used to do before the intrinsics. It should map to the same assembly. +fn saturating_intrinsic_impl<'ll, 'tcx>( + b: &mut Builder<'_, 'll, 'tcx>, + width: u32, + signed: bool, + is_add: bool, + args: &[OperandRef<'tcx, &'ll Value>], +) -> &'ll Value { + use rustc_middle::ty::IntTy::*; + use rustc_middle::ty::UintTy::*; + + let tcx = b.tcx; + let ty = match (signed, width) { + (true, 8) => Ty::new_int(tcx, I8), + (true, 16) => Ty::new_int(tcx, I16), + (true, 32) => Ty::new_int(tcx, I32), + (true, 64) => Ty::new_int(tcx, I64), + (true, 128) => Ty::new_int(tcx, I128), + (false, 8) => Ty::new_uint(tcx, U8), + (false, 16) => Ty::new_uint(tcx, U16), + (false, 32) => Ty::new_uint(tcx, U32), + (false, 64) => Ty::new_uint(tcx, U64), + (false, 128) => Ty::new_uint(tcx, U128), + _ => unreachable!(), + }; + + let unsigned_max_value = match width { + 8 => u8::MAX as i64, + 16 => u16::MAX as i64, + 32 => u32::MAX as i64, + 64 => u64::MAX as i64, + _ => unreachable!(), + }; + + let (min_value, max_value) = if signed { + (-((unsigned_max_value / 2) + 1), (unsigned_max_value / 2)) + } else { + (0, unsigned_max_value) + }; + + let overflow_op = if is_add { + OverflowOp::Add + } else { + OverflowOp::Sub + }; + let llty = b.type_ix(width as u64); + let lhs = args[0].immediate(); + let rhs = args[1].immediate(); + + let (val, overflowed) = b.checked_binop(overflow_op, ty, lhs, rhs); + + if !signed { + let select_val = if is_add { + b.const_int(llty, -1) + } else { + b.const_int(llty, 0) + }; + b.select(overflowed, select_val, val) + } else { + let const_val = b.const_int(llty, (width - 1) as i64); + let first_val = if is_add { + b.ashr(rhs, const_val) + } else { + b.lshr(rhs, const_val) + }; + let second_val = if is_add { + b.unchecked_uadd(first_val, b.const_int(llty, max_value)) + } else { + b.xor(first_val, b.const_int(llty, min_value)) + }; + b.select(overflowed, second_val, val) + } +} + +fn get_simple_intrinsic<'ll>( + cx: &CodegenCx<'ll, '_>, + name: Symbol, +) -> Option<(&'ll Type, &'ll Value)> { + #[rustfmt::skip] + let llvm_name = match name { + sym::sqrtf32 => "__nv_sqrtf", + sym::sqrtf64 => "__nv_sqrt", + sym::powif32 => "__nv_powif", + sym::powif64 => "__nv_powi", + sym::sinf32 => "__nv_sinf", + sym::sinf64 => "__nv_sin", + sym::cosf32 => "__nv_cosf", + sym::cosf64 => "__nv_cos", + sym::powf32 => "__nv_powf", + sym::powf64 => "__nv_pow", + sym::expf32 => "__nv_expf", + sym::expf64 => "__nv_exp", + sym::exp2f32 => "__nv_exp2f", + sym::exp2f64 => "__nv_exp2", + sym::logf32 => "__nv_logf", + sym::logf64 => "__nv_log", + sym::log10f32 => "__nv_log10f", + sym::log10f64 => "__nv_log10", + sym::log2f32 => "__nv_log2f", + sym::log2f64 => "__nv_log2", + sym::fmaf32 => "__nv_fmaf", + sym::fmaf64 => "__nv_fma", + sym::fabsf32 => "__nv_fabsf", + sym::fabsf64 => "__nv_fabs", + sym::minnumf32 => "__nv_fminf", + sym::minnumf64 => "__nv_fmin", + sym::maxnumf32 => "__nv_fmaxf", + sym::maxnumf64 => "__nv_fmax", + sym::copysignf32 => "__nv_copysignf", + sym::copysignf64 => "__nv_copysign", + sym::floorf32 => "__nv_floorf", + sym::floorf64 => "__nv_floor", + sym::ceilf32 => "__nv_ceilf", + sym::ceilf64 => "__nv_ceil", + sym::truncf32 => "__nv_truncf", + sym::truncf64 => "__nv_trunc", + sym::roundf32 => "__nv_roundf", + sym::roundf64 => "__nv_round", + sym::round_ties_even_f32 => "__nv_rintf", + sym::round_ties_even_f64 => "__nv_rint", + _ => return None, + }; + trace!("Retrieving nv intrinsic `{:?}`", llvm_name); + Some(cx.get_intrinsic(llvm_name)) +} + +impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { + fn codegen_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + args: &[OperandRef<'tcx, &'ll Value>], + llresult: &'ll Value, + span: Span, + ) -> Result<(), ty::Instance<'tcx>> { + let tcx = self.tcx; + let callee_ty = instance.ty(tcx, self.typing_env()); + + let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else { + bug!("expected fn item type, found {}", callee_ty); + }; + + let sig = callee_ty.fn_sig(tcx); + let sig = tcx.normalize_erasing_late_bound_regions(self.typing_env(), sig); + let arg_tys = sig.inputs(); + let ret_ty = sig.output(); + let name = tcx.item_name(def_id); + let name_str = name.as_str(); + + trace!( + "Beginning intrinsic call: `{:?}`, args: `{:?}`, ret: `{:?}`", + name, arg_tys, ret_ty + ); + + let llret_ty = self.layout_of(ret_ty).llvm_type(self); + let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); + + let simple = get_simple_intrinsic(self, name); + let llval = match name { + _ if simple.is_some() => { + let (simple_ty, simple_fn) = simple.unwrap(); + self.call( + simple_ty, + None, + None, + simple_fn, + &args.iter().map(|arg| arg.immediate()).collect::>(), + None, + Some(instance), + ) + } + sym::is_val_statically_known => { + // LLVM 7 does not support this intrinsic, so always assume false. + self.const_bool(false) + } + sym::select_unpredictable => { + // This should set MD_unpredictable on the select instruction, but + // nvvm ignores it, so just use a normal select. + let cond = args[0].immediate(); + assert_eq!(args[1].layout, args[2].layout); + match (args[1].val, args[2].val) { + (OperandValue::Ref(true_val), OperandValue::Ref(false_val)) => { + assert!(true_val.llextra.is_none()); + assert!(false_val.llextra.is_none()); + assert_eq!(true_val.align, false_val.align); + let ptr = self.select(cond, true_val.llval, false_val.llval); + let selected = + OperandValue::Ref(PlaceValue::new_sized(ptr, true_val.align)); + selected.store(self, result); + return Ok(()); + } + (OperandValue::Immediate(_), OperandValue::Immediate(_)) + | (OperandValue::Pair(_, _), OperandValue::Pair(_, _)) => { + let true_val = args[1].immediate_or_packed_pair(self); + let false_val = args[2].immediate_or_packed_pair(self); + self.select(cond, true_val, false_val) + } + (OperandValue::ZeroSized, OperandValue::ZeroSized) => return Ok(()), + _ => span_bug!(span, "Incompatible OperandValue for select_unpredictable"), + } + } + sym::likely => self.call_intrinsic( + "llvm.expect.i1", + &[args[0].immediate(), self.const_bool(true)], + ), + sym::unlikely => self.call_intrinsic( + "llvm.expect.i1", + &[args[0].immediate(), self.const_bool(false)], + ), + kw::Try => { + let try_func = args[0].immediate(); + let data = args[1].immediate(); + + self.call(self.type_i1(), None, None, try_func, &[data], None, None); + let ret_align = self.data_layout().i32_align.abi; + self.store(self.const_i32(0), llresult, ret_align) + } + sym::breakpoint => { + // debugtrap is not supported + return Ok(()); + } + sym::va_copy => { + self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()]) + } + sym::va_arg => { + match fn_abi.ret.layout.backend_repr { + abi::BackendRepr::Scalar(scalar) => { + match scalar.primitive() { + Primitive::Int(..) => { + if self.cx().size_of(ret_ty).bytes() < 4 { + // `va_arg` should not be called on a integer type + // less than 4 bytes in length. If it is, promote + // the integer to a `i32` and truncate the result + // back to the smaller type. + let promoted_result = self.va_arg( + args[0].immediate(), + self.cx.layout_of(tcx.types.i32).llvm_type(self.cx), + ); + self.trunc(promoted_result, llret_ty) + } else { + self.va_arg( + args[0].immediate(), + self.cx.layout_of(ret_ty).llvm_type(self.cx), + ) + } + } + Primitive::Float(Float::F16) => { + bug!("the va_arg intrinsic does not work with `f16`") + } + Primitive::Float(Float::F64) | Primitive::Pointer(_) => self.va_arg( + args[0].immediate(), + self.cx.layout_of(ret_ty).llvm_type(self.cx), + ), + // `va_arg` should never be used with the return type f32. + Primitive::Float(Float::F32) => { + bug!("the va_arg intrinsic does not work with `f32`") + } + Primitive::Float(Float::F128) => { + bug!("the va_arg intrinsic does not work with `f128`") + } + } + } + _ => bug!("the va_arg intrinsic does not work with non-scalar types"), + } + } + sym::volatile_load | sym::unaligned_volatile_load => { + // The `*const T` or `*mut T` operand. + let ptr_operand = &args[0]; + let src_ptr_llval = ptr_operand.immediate(); + + // Determine the type T (the pointee type) and its LLVM representation. + // `ptr_operand.layout.ty` is the Rust type `*const T` (or `*mut T`). We + // need the layout of `T`. + let pointee_rust_ty = + ptr_operand + .layout + .ty + .builtin_deref(true) + .unwrap_or_else(|| { + span_bug!(span, "volatile_load input pointer is not a pointer type") + }); + let layout_of_pointee = self.layout_of(pointee_rust_ty); + let llvm_ty_of_pointee = layout_of_pointee.llvm_type(self); + + // Call volatile_load with the correct LLVM type of T. The + // `volatile_load` does a pointercast so we do not need to do it here. + let loaded_llval = self.volatile_load(llvm_ty_of_pointee, src_ptr_llval); + + // Set alignment for the LLVM load instruction based on the alignment of + // `T`. + let align = if name == sym::unaligned_volatile_load { + 1 + } else { + layout_of_pointee.align.abi.bytes() as u32 + }; + unsafe { + llvm::LLVMSetAlignment(loaded_llval, align); + } + + if !result.layout.is_zst() { + self.store_to_place(loaded_llval, result.val); + } + return Ok(()); + } + sym::volatile_store => { + let dst = args[0].deref(self.cx()); + args[1].val.volatile_store(self, dst); + return Ok(()); + } + sym::unaligned_volatile_store => { + let dst = args[0].deref(self.cx()); + args[1].val.unaligned_volatile_store(self, dst); + return Ok(()); + } + sym::prefetch_read_data + | sym::prefetch_write_data + | sym::prefetch_read_instruction + | sym::prefetch_write_instruction => { + let (rw, cache_type) = match name { + sym::prefetch_read_data => (0, 1), + sym::prefetch_write_data => (1, 1), + sym::prefetch_read_instruction => (0, 0), + sym::prefetch_write_instruction => (1, 0), + _ => bug!(), + }; + self.call_intrinsic( + "llvm.prefetch", + &[ + args[0].immediate(), + self.const_i32(rw), + args[1].immediate(), + self.const_i32(cache_type), + ], + ) + } + sym::carrying_mul_add => { + let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx); + + let wide_llty = self.type_ix(size.bits() * 2); + let args = args.as_array().unwrap(); + let [a, b, c, d] = args.map(|a| self.intcast(a.immediate(), wide_llty, signed)); + + let wide = if signed { + let prod = self.unchecked_smul(a, b); + let acc = self.unchecked_sadd(prod, c); + self.unchecked_sadd(acc, d) + } else { + let prod = self.unchecked_umul(a, b); + let acc = self.unchecked_uadd(prod, c); + self.unchecked_uadd(acc, d) + }; + + let narrow_llty = self.type_ix(size.bits()); + let low = self.trunc(wide, narrow_llty); + let bits_const = self.const_uint(wide_llty, size.bits()); + // No need for ashr when signed; LLVM changes it to lshr anyway. + let high = self.lshr(wide, bits_const); + // FIXME: could be `trunc nuw`, even for signed. + let high = self.trunc(high, narrow_llty); + + let pair_llty = self.type_struct(&[narrow_llty, narrow_llty], false); + let pair = self.const_poison(pair_llty); + let pair = self.insert_value(pair, low, 0); + self.insert_value(pair, high, 1) + } + sym::ctlz + | sym::ctlz_nonzero + | sym::cttz + | sym::cttz_nonzero + | sym::ctpop + | sym::bswap + | sym::bitreverse + | sym::rotate_left + | sym::rotate_right + | sym::saturating_add + | sym::saturating_sub => { + let ty = arg_tys[0]; + if !ty.is_integral() { + tcx.dcx() + .emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty }); + return Ok(()); + } + let (size, signed) = ty.int_size_and_signed(self.tcx); + let width = size.bits(); + if name == sym::saturating_add || name == sym::saturating_sub { + saturating_intrinsic_impl( + self, + width as u32, + signed, + name == sym::saturating_add, + args, + ) + } else if width == 128 { + handle_128_bit_intrinsic(self) + } else { + match name { + sym::ctlz | sym::cttz => { + let y = self.const_bool(false); + let llvm_name = format!("llvm.{}.i{}", name, width); + self.call_intrinsic(&llvm_name, &[args[0].immediate(), y]) + } + sym::ctlz_nonzero | sym::cttz_nonzero => { + let y = self.const_bool(true); + let llvm_name = format!("llvm.{}.i{}", &name_str[..4], width); + self.call_intrinsic(&llvm_name, &[args[0].immediate(), y]) + } + sym::ctpop => self.call_intrinsic( + &format!("llvm.ctpop.i{}", width), + &[args[0].immediate()], + ), + sym::bswap => { + if width == 8 { + args[0].immediate() // byte swap a u8/i8 is just a no-op + } else { + self.call_intrinsic( + &format!("llvm.bswap.i{}", width), + &[args[0].immediate()], + ) + } + } + sym::bitreverse => self.call_intrinsic( + &format!("llvm.bitreverse.i{}", width), + &[args[0].immediate()], + ), + sym::rotate_left | sym::rotate_right => { + let is_left = name == sym::rotate_left; + let val = args[0].immediate(); + let raw_shift = args[1].immediate(); + // rotate = funnel shift with first two args the same + let llvm_name = + &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); + + // llvm expects shift to be the same type as the values, but rust + // always uses `u32`. + let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); + + self.call_intrinsic(llvm_name, &[val, val, raw_shift]) + } + sym::saturating_add | sym::saturating_sub => { + let is_add = name == sym::saturating_add; + let lhs = args[0].immediate(); + let rhs = args[1].immediate(); + let llvm_name = &format!( + "llvm.{}{}.sat.i{}", + if signed { 's' } else { 'u' }, + if is_add { "add" } else { "sub" }, + width + ); + self.call_intrinsic(llvm_name, &[lhs, rhs]) + } + _ => unreachable!(), + } + } + } + sym::raw_eq => { + use abi::BackendRepr::*; + use rustc_codegen_ssa::common::IntPredicate; + let tp_ty = fn_args.type_at(0); + let layout = self.layout_of(tp_ty).layout; + let use_integer_compare = match layout.backend_repr() { + Scalar(_) | ScalarPair(_, _) => true, + Vector { .. } => false, + Memory { .. } => { + // For rusty ABIs, small aggregates are actually passed + // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), + // so we re-use that same threshold here. + layout.size <= self.data_layout().pointer_size * 2 + } + }; + + let a = args[0].immediate(); + let b = args[1].immediate(); + if layout.size.bytes() == 0 { + self.const_bool(true) + } else if use_integer_compare { + let integer_ty = self.type_ix(layout.size.bits()); + let ptr_ty = self.type_ptr_to(integer_ty); + let a_ptr = self.bitcast(a, ptr_ty); + let a_val = self.load(integer_ty, a_ptr, layout.align.abi); + let b_ptr = self.bitcast(b, ptr_ty); + let b_val = self.load(integer_ty, b_ptr, layout.align.abi); + self.icmp(IntPredicate::IntEQ, a_val, b_val) + } else { + let i8p_ty = self.type_i8p(); + let a_ptr = self.bitcast(a, i8p_ty); + let b_ptr = self.bitcast(b, i8p_ty); + let n = self.const_usize(layout.size.bytes()); + let cmp = self.call_intrinsic("memcmp", &[a_ptr, b_ptr, n]); + self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)) + } + } + sym::compare_bytes => self.call_intrinsic( + "memcmp", + &[ + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + ], + ), + + sym::black_box => { + args[0].val.store(self, result); + let result_val_span = [result.val.llval]; + // We need to "use" the argument in some way LLVM can't introspect, and on + // targets that support it we can typically leverage inline assembly to do + // this. LLVM's interpretation of inline assembly is that it's, well, a black + // box. This isn't the greatest implementation since it probably deoptimizes + // more than we want, but it's so far good enough. + // + // For zero-sized types, the location pointed to by the result may be + // uninitialized. Do not "use" the result in this case; instead just clobber + // the memory. + let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() { + ("~{memory}", &[]) + } else { + ("r,~{memory}", &result_val_span) + }; + crate::asm::inline_asm_call( + self, + "", + constraint, + inputs, + self.type_void(), + true, + false, + llvm::AsmDialect::Att, + &[span], + ) + .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); + + // We have copied the value to `result` already. + return Ok(()); + } + // is this even supported by nvvm? i did not find a definitive answer + _ if name_str.starts_with("simd_") => todo!("simd intrinsics"), + _ => bug!("unknown intrinsic '{}'", name), + }; + trace!("Finish intrinsic call: `{:?}`", llval); + if !fn_abi.ret.is_ignore() { + if let PassMode::Cast { cast, .. } = &fn_abi.ret.mode { + let ptr_llty = self.type_ptr_to(cast.llvm_type(self)); + let ptr = self.pointercast(result.val.llval, ptr_llty); + self.store(llval, ptr, result.val.align); + } else { + OperandRef::from_immediate_or_packed_pair(self, llval, result.layout) + .val + .store(self, result); + } + } + Ok(()) + } + + fn abort(&mut self) { + let (trap_ty, f) = self.get_intrinsic("llvm.trap"); + self.call(trap_ty, None, None, f, &[], None, None); + } + + fn assume(&mut self, val: &'ll Value) { + trace!("Generate assume call with `{:?}`", val); + self.call_intrinsic("llvm.assume", &[val]); + } + + fn expect(&mut self, cond: &'ll Value, expected: bool) -> &'ll Value { + trace!("Generate expect call with `{:?}`, {}", cond, expected); + self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) + } + + fn type_test(&mut self, _pointer: &'ll Value, _typeid: &'ll Metadata) -> &'ll Value { + // LLVM CFI doesnt make sense on the GPU + self.const_i32(0) + } + + fn type_checked_load( + &mut self, + _llvtable: Self::Value, + _vtable_byte_offset: u64, + _typeid: Self::Metadata, + ) -> Self::Value { + // LLVM CFI doesnt make sense on the GPU + self.const_i32(0) + } + + fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { + trace!("Generate va_start `{:?}`", va_list); + self.call_intrinsic("llvm.va.start", &[va_list]) + } + + fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value { + trace!("Generate va_end call `{:?}`", va_list); + self.call_intrinsic("llvm.va_end", &[va_list]) + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/lib.rs b/crates/rustc_codegen_nvvm_v19/src/lib.rs new file mode 100644 index 00000000..071d9318 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/lib.rs @@ -0,0 +1,397 @@ +#![feature(rustc_private)] +// crate is perma-unstable because of rustc_private so might as well +// make our lives a lot easier for llvm ffi with this. And since rustc's core infra +// relies on it its almost guaranteed to not be removed/broken +#![feature(extern_types)] +#![feature(hash_raw_entry)] +#![feature(let_chains)] +#![feature(slice_as_array)] + +extern crate rustc_abi; +extern crate rustc_arena; +extern crate rustc_ast; +extern crate rustc_attr_parsing; +extern crate rustc_codegen_ssa; +extern crate rustc_data_structures; +extern crate rustc_driver; +extern crate rustc_errors; +extern crate rustc_fs_util; +extern crate rustc_hash; +extern crate rustc_hashes; +extern crate rustc_hir; +extern crate rustc_index; +extern crate rustc_interface; +extern crate rustc_macros; +extern crate rustc_metadata; +extern crate rustc_middle; +extern crate rustc_query_system; +extern crate rustc_session; +extern crate rustc_span; +extern crate rustc_symbol_mangling; +extern crate rustc_target; +extern crate rustc_type_ir; + +mod abi; +mod allocator; +mod asm; +mod attributes; +mod back; +mod builder; +mod common; +mod const_ty; +mod consts; +mod context; +mod ctx_intrinsics; +mod debug_info; +mod init; +mod int_replace; +mod intrinsic; +mod link; +mod llvm; +mod lto; +mod mono_item; +mod nvvm; +mod override_fns; +mod target; +mod ty; + +use abi::readjust_fn_abi; +use back::target_machine_factory; +use lto::ThinBuffer; +use rustc_ast::expand::allocator::AllocatorKind; +use rustc_ast::expand::autodiff_attrs::AutoDiffItem; +use rustc_codegen_ssa::{ + CodegenResults, CompiledModule, ModuleCodegen, + back::{ + lto::{LtoModuleCodegen, SerializedModule, ThinModule}, + write::{CodegenContext, FatLtoInput, ModuleConfig, OngoingCodegen}, + }, + traits::{CodegenBackend, ExtraBackendMethods, WriteBackendMethods}, +}; +use rustc_data_structures::fx::FxIndexMap; +use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_metadata::EncodedMetadata; +use rustc_metadata::creader::MetadataLoaderDyn; +use rustc_middle::util::Providers; +use rustc_middle::{ + dep_graph::{WorkProduct, WorkProductId}, + ty::TyCtxt, +}; +use rustc_session::{ + Session, + config::{self, OutputFilenames}, +}; +use tracing::debug; + +use std::ffi::CString; + +// codegen dylib entrypoint +#[unsafe(no_mangle)] +pub fn __rustc_codegen_backend() -> Box { + rustc_driver::install_ice_hook( + "https://github.com/Rust-GPU/Rust-CUDA/issues/new", + |handler| { + handler.handle().note(concat!( + "`rust-cuda` version `", + env!("CARGO_PKG_VERSION"), + "`" + )); + }, + ); + Box::new(NvvmCodegenBackend) +} + +#[derive(Clone)] +pub struct NvvmCodegenBackend; + +unsafe impl Send for NvvmCodegenBackend {} +unsafe impl Sync for NvvmCodegenBackend {} + +impl CodegenBackend for NvvmCodegenBackend { + fn locale_resource(&self) -> &'static str { + "" + } + + fn init(&self, sess: &Session) { + let filter = tracing_subscriber::EnvFilter::from_env("NVVM_LOG"); + let subscriber = tracing_subscriber::fmt() + .with_env_filter(filter) + .without_time() + .with_ansi(false) + .compact() + .finish(); + tracing::subscriber::set_global_default(subscriber).expect("no default subscriber"); + init::init(sess); + } + + // FIXME If we can use the default metadata loader in the LLVM backend + // we can remove this and use the default provided impl instead. + fn metadata_loader(&self) -> Box { + Box::new(link::NvvmMetadataLoader) + } + + fn provide(&self, providers: &mut Providers) { + // FIXME(eddyb) this is currently only passed back to us, specifically + // into `target_machine_factory` (which is a noop), but it might make + // sense to move some of the target feature parsing into here. + providers.global_backend_features = |_tcx, ()| vec![]; + + providers.fn_abi_of_fn_ptr = |tcx, key| { + let result = (rustc_interface::DEFAULT_QUERY_PROVIDERS.fn_abi_of_fn_ptr)(tcx, key); + Ok(readjust_fn_abi(tcx, result?)) + }; + providers.fn_abi_of_instance = |tcx, key| { + let result = (rustc_interface::DEFAULT_QUERY_PROVIDERS.fn_abi_of_instance)(tcx, key); + Ok(readjust_fn_abi(tcx, result?)) + }; + } + + fn codegen_crate( + &self, + tcx: TyCtxt<'_>, + metadata: EncodedMetadata, + need_metadata_module: bool, + ) -> Box { + debug!("Codegen crate"); + Box::new(rustc_codegen_ssa::base::codegen_crate( + Self, + tcx, + tcx.sess + .opts + .cg + .target_cpu + .clone() + .unwrap_or_else(|| tcx.sess.target.cpu.to_string()), + metadata, + need_metadata_module, + )) + } + + fn join_codegen( + &self, + ongoing_codegen: Box, + sess: &Session, + _outputs: &OutputFilenames, + ) -> (CodegenResults, FxIndexMap) { + debug!("Join codegen"); + let (codegen_results, work_products) = ongoing_codegen + .downcast::>() + .expect("Expected OngoingCodegen, found Box") + .join(sess); + + // sess.compile_status(); + + (codegen_results, work_products) + } + + fn link( + &self, + sess: &rustc_session::Session, + codegen_results: rustc_codegen_ssa::CodegenResults, + outputs: &config::OutputFilenames, + ) { + link::link( + sess, + &codegen_results, + outputs, + codegen_results.crate_info.local_crate_name.as_str(), + ); + } +} + +impl WriteBackendMethods for NvvmCodegenBackend { + type Module = LlvmMod; + type ModuleBuffer = lto::ModuleBuffer; + type TargetMachine = &'static mut llvm::TargetMachine; + type TargetMachineError = String; + type ThinData = (); + type ThinBuffer = ThinBuffer; + + fn run_link( + _cgcx: &CodegenContext, + _diag_handler: DiagCtxtHandle<'_>, + _modules: Vec>, + ) -> Result, FatalError> { + // TODO(Rdambrosio016): + // we can probably call the llvm codegen to do this, but cgcx + // is a codegen context of NvvmCodegenBackend not LlvmCodegenBackend + // and to make a new cgcx we need to make a new LlvmCodegenBackend which + // cannot be done through the API currently + todo!(); + } + + fn run_fat_lto( + _: &CodegenContext, + _: Vec>, + _: Vec<(SerializedModule, WorkProduct)>, + ) -> Result, FatalError> { + todo!() + } + + fn run_thin_lto( + cgcx: &CodegenContext, + modules: Vec<(String, Self::ThinBuffer)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + ) -> Result<(Vec>, Vec), FatalError> { + lto::run_thin(cgcx, modules, cached_modules) + } + + fn print_pass_timings(&self) { + // Not applicable, nvvm doesnt expose pass timing info, maybe we could print llvm pass stuff here. + } + + fn print_statistics(&self) { + // Not applicable, nvvm doesnt expose pass timing info, maybe we could print llvm pass stuff here. + } + + unsafe fn optimize( + cgcx: &CodegenContext, + diag_handler: DiagCtxtHandle<'_>, + module: &mut ModuleCodegen, + config: &ModuleConfig, + ) -> Result<(), FatalError> { + unsafe { back::optimize(cgcx, diag_handler, module, config) } + } + + unsafe fn optimize_thin( + cgcx: &CodegenContext, + thin_module: ThinModule, + ) -> Result, FatalError> { + unsafe { lto::optimize_thin(cgcx, thin_module) } + } + + unsafe fn codegen( + cgcx: &CodegenContext, + diag_handler: DiagCtxtHandle<'_>, + module: ModuleCodegen, + config: &ModuleConfig, + ) -> Result { + unsafe { back::codegen(cgcx, diag_handler, module, config) } + } + + fn prepare_thin( + module: ModuleCodegen, + _want_summary: bool, + ) -> (String, Self::ThinBuffer) { + debug!("Prepare thin"); + unsafe { + ( + module.name, + lto::ThinBuffer::new(module.module_llvm.llmod.as_ref().unwrap()), + ) + } + } + + fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer) { + debug!("Serializing module"); + unsafe { + ( + module.name, + lto::ModuleBuffer::new(module.module_llvm.llmod.as_ref().unwrap()), + ) + } + } + + fn optimize_fat( + _cgcx: &CodegenContext, + _llmod: &mut ModuleCodegen, + ) -> Result<(), FatalError> { + todo!() + } + + fn autodiff( + _cgcx: &CodegenContext, + _module: &ModuleCodegen, + _diff_fncs: Vec, + _config: &ModuleConfig, + ) -> Result<(), FatalError> { + todo!() + } +} + +impl ExtraBackendMethods for NvvmCodegenBackend { + fn codegen_allocator( + &self, + tcx: TyCtxt<'_>, + module_name: &str, + kind: AllocatorKind, + alloc_error_handler_kind: AllocatorKind, + ) -> LlvmMod { + let mut module_llvm = LlvmMod::new(module_name); + unsafe { + allocator::codegen( + tcx, + &mut module_llvm, + module_name, + kind, + alloc_error_handler_kind, + ); + } + module_llvm + } + + fn compile_codegen_unit( + &self, + tcx: TyCtxt<'_>, + cgu_name: rustc_span::Symbol, + ) -> (rustc_codegen_ssa::ModuleCodegen, u64) { + back::compile_codegen_unit(tcx, cgu_name) + } + + fn target_machine_factory( + &self, + sess: &Session, + opt_level: config::OptLevel, + _target_features: &[String], + ) -> rustc_codegen_ssa::back::write::TargetMachineFactoryFn { + target_machine_factory(sess, opt_level) + } +} + +/// Create the LLVM module for the rest of the compilation, this houses +/// the LLVM bitcode we then add to the NVVM program and feed to libnvvm. +/// LLVM's codegen is never actually called. +pub(crate) unsafe fn create_module<'ll>( + llcx: &'ll llvm::Context, + mod_name: &str, +) -> &'ll llvm::Module { + debug!("Creating llvm module with name `{}`", mod_name); + let mod_name = CString::new(mod_name).expect("nul in module name"); + let llmod = unsafe { llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx) }; + + let data_layout = CString::new(target::DATA_LAYOUT).unwrap(); + unsafe { llvm::LLVMSetDataLayout(llmod, data_layout.as_ptr()) }; + + let target = CString::new(target::TARGET_TRIPLE).unwrap(); + unsafe { llvm::LLVMSetTarget(llmod, target.as_ptr()) }; + + llmod +} + +/// Wrapper over raw llvm structures +pub struct LlvmMod { + llcx: &'static mut llvm::Context, + llmod: *const llvm::Module, +} + +unsafe impl Send for LlvmMod {} +unsafe impl Sync for LlvmMod {} + +impl LlvmMod { + pub fn new(name: &str) -> Self { + unsafe { + // TODO(RDambrosio016): does shouldDiscardNames affect NVVM at all? + let llcx = llvm::LLVMRustContextCreate(false); + let llmod = create_module(llcx, name) as *const _; + LlvmMod { llcx, llmod } + } + } +} + +impl Drop for LlvmMod { + fn drop(&mut self) { + unsafe { + llvm::LLVMContextDispose(&mut *(self.llcx as *mut _)); + } + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/link.rs b/crates/rustc_codegen_nvvm_v19/src/link.rs new file mode 100644 index 00000000..53a59866 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/link.rs @@ -0,0 +1,410 @@ +use object::{Object, ObjectSection}; +use rustc_ast::CRATE_NODE_ID; +use rustc_codegen_ssa::CodegenResults; +use rustc_codegen_ssa::CompiledModule; +use rustc_codegen_ssa::NativeLib; +use rustc_data_structures::memmap::Mmap; +use rustc_data_structures::owned_slice::{OwnedSlice, slice_owned, try_slice_owned}; +use rustc_hash::FxHashSet; +use rustc_metadata::creader::MetadataLoader; +use rustc_middle::bug; +use rustc_middle::middle::dependency_format::Linkage; +use rustc_session::output::out_filename; +use rustc_session::{ + Session, + config::{CrateType, OutputFilenames, OutputType}, + output::check_file_is_writeable, + utils::NativeLibKind, +}; +use rustc_span::Symbol; +use rustc_target::spec::Target; +use std::ops::Deref; +use std::{ + ffi::OsStr, + fs::File, + io::{self, Read}, + path::{Path, PathBuf}, +}; +use tar::{Archive, Builder, Header}; +use tracing::{debug, trace}; + +use crate::LlvmMod; +use crate::context::CodegenArgs; + +pub(crate) struct NvvmMetadataLoader; + +fn load_metadata_with( + path: &Path, + f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>, +) -> Result { + let file = + File::open(path).map_err(|e| format!("failed to open file '{}': {}", path.display(), e))?; + + unsafe { Mmap::map(file) } + .map_err(|e| format!("failed to mmap file '{}': {}", path.display(), e)) + .and_then(|mmap| try_slice_owned(mmap, |mmap| f(mmap))) +} + +impl MetadataLoader for NvvmMetadataLoader { + fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result { + trace!("Retrieving rlib metadata for `{:?}`", path); + read_metadata(path) + } + + fn get_dylib_metadata(&self, target: &Target, path: &Path) -> Result { + debug!("getting rlib metadata for {}", path.display()); + // This is required for loading metadata from proc macro crates compiled as dylibs for the host target. + if target.is_like_aix { + bug!("aix dynlibs unsupported"); + } else { + load_metadata_with(path, |data| search_for_section(path, data, ".rustc")) + } + } +} + +fn read_metadata(rlib: &Path) -> Result { + let read_meta = || -> Result, io::Error> { + for entry in Archive::new(File::open(rlib)?).entries()? { + let mut entry = entry?; + if entry.path()? == Path::new(".metadata") { + let mut bytes = Vec::new(); + entry.read_to_end(&mut bytes)?; + return Ok(Some(slice_owned(bytes, Deref::deref))); + } + } + Ok(None) + }; + + match read_meta() { + Ok(Some(m)) => Ok(m), + Ok(None) => Err(format!("No .metadata file in rlib: {:?}", rlib)), + Err(io) => Err(format!("Failed to read rlib at {:?}: {}", rlib, io)), + } +} + +fn search_for_section<'a>(path: &Path, bytes: &'a [u8], section: &str) -> Result<&'a [u8], String> { + let Ok(file) = object::File::parse(bytes) else { + // The parse above could fail for odd reasons like corruption, but for + // now we just interpret it as this target doesn't support metadata + // emission in object files so the entire byte slice itself is probably + // a metadata file. Ideally though if necessary we could at least check + // the prefix of bytes to see if it's an actual metadata object and if + // not forward the error along here. + return Ok(bytes); + }; + file.section_by_name(section) + .ok_or_else(|| format!("no `{}` section in '{}'", section, path.display()))? + .data() + .map_err(|e| { + format!( + "failed to read {} section in '{}': {}", + section, + path.display(), + e + ) + }) +} + +pub fn link( + sess: &Session, + codegen_results: &CodegenResults, + outputs: &OutputFilenames, + crate_name: &str, +) { + debug!("Linking crate `{}`", crate_name); + // largely inspired by rust-gpu + let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata); + for &crate_type in sess.opts.crate_types.iter() { + if (sess.opts.unstable_opts.no_codegen || !sess.opts.output_types.should_codegen()) + && !output_metadata + && crate_type == CrateType::Executable + { + continue; + } + + for obj in codegen_results + .modules + .iter() + .filter_map(|m| m.object.as_ref()) + { + check_file_is_writeable(obj, sess); + } + + if outputs.outputs.should_codegen() { + let out_filename = out_filename(sess, crate_type, outputs, Symbol::intern(crate_name)); + let out_filename_file_for_writing = + out_filename.file_for_writing(outputs, OutputType::Exe, None); + match crate_type { + CrateType::Rlib => { + link_rlib(sess, codegen_results, &out_filename_file_for_writing); + } + CrateType::Executable | CrateType::Cdylib | CrateType::Dylib => { + let _ = link_exe( + &codegen_results.allocator_module, + sess, + crate_type, + &out_filename_file_for_writing, + codegen_results, + ); + } + other => sess.dcx().fatal(format!("Invalid crate type: {:?}", other)), + } + } + } +} + +fn link_rlib(sess: &Session, codegen_results: &CodegenResults, out_filename: &Path) { + debug!("Linking rlib `{:?}`", out_filename); + let mut file_list = Vec::<&Path>::new(); + + for obj in codegen_results + .modules + .iter() + .filter_map(|m| m.object.as_ref()) + { + file_list.push(obj); + } + + for lib in codegen_results.crate_info.used_libraries.iter() { + // native libraries in cuda doesnt make much sense, extern functions + // do exist in nvvm for stuff like cuda syscalls and cuda provided functions + // but including libraries doesnt make sense because nvvm would have to translate + // the binary directly to ptx. We might want to add some way of linking in + // ptx files or custom bitcode modules as "libraries" perhaps in the future. + if let NativeLibKind::Static { + bundle: None | Some(true), + .. + } = lib.kind + { + sess.dcx().err(format!( + "Adding native libraries to rlib is not supported in CUDA: {}", + lib.name + )); + } + } + trace!("Files linked in rlib:\n{:#?}", file_list); + + create_archive( + sess, + &file_list, + codegen_results.metadata.raw_data(), + out_filename, + ); +} + +fn link_exe( + allocator: &Option, + sess: &Session, + crate_type: CrateType, + out_filename: &Path, + codegen_results: &CodegenResults, +) -> io::Result<()> { + let mut objects = Vec::new(); + let mut rlibs = Vec::new(); + for obj in codegen_results + .modules + .iter() + .filter_map(|m| m.object.as_ref()) + { + objects.push(obj.clone()); + } + + link_local_crate_native_libs_and_dependent_crate_libs( + &mut rlibs, + sess, + crate_type, + codegen_results, + ); + + let mut root_file_name = out_filename.file_name().unwrap().to_owned(); + root_file_name.push(".dir"); + let out_dir = out_filename.with_file_name(root_file_name); + if !out_dir.is_dir() { + std::fs::create_dir_all(&out_dir)?; + } + + codegen_into_ptx_file(allocator, sess, &objects, &rlibs, out_filename) +} + +/// This is the meat of the codegen, taking all of the llvm bitcode modules we have, and giving them to +/// nvvm to make into a final +fn codegen_into_ptx_file( + allocator: &Option, + sess: &Session, + objects: &[PathBuf], + rlibs: &[PathBuf], + out_filename: &Path, +) -> io::Result<()> { + debug!( + "Codegenning crate into PTX, allocator: {}, objects:\n{:#?}, rlibs:\n{:#?}, out_filename:\n{:#?}", + allocator.is_some(), + objects, + rlibs, + out_filename + ); + + // we need to make a new llvm context because we need it for linking together modules, + // but we dont have our original one because rustc drops tyctxt and codegencx before linking. + let cx = LlvmMod::new("link_tmp"); + + let mut modules = Vec::with_capacity(objects.len() + rlibs.len()); + + // object files (theyre not object files, they are impostors ඞ) are the bitcode modules produced by this codegen session + // they *should* be the final crate. + for obj in objects { + let bitcode = std::fs::read(obj)?; + modules.push(bitcode); + } + + // rlibs are archives that we made previously, they are usually made for crates that are referenced + // in this crate. We must unpack them and devour their bitcode to link in. + for rlib in rlibs { + let mut cgus = Vec::with_capacity(16); + for entry in Archive::new(File::open(rlib)?).entries()? { + let mut entry = entry?; + // metadata is where rustc puts rlib metadata, so its not a cgu we are interested in. + if entry.path().unwrap() != Path::new(".metadata") { + // std::fs::read adds 1 to the size, so do the same here - see comment: + // https://github.com/rust-lang/rust/blob/72868e017bdade60603a25889e253f556305f996/library/std/src/fs.rs#L200-L202 + let mut bitcode = Vec::with_capacity(entry.size() as usize + 1); + entry.read_to_end(&mut bitcode).unwrap(); + cgus.push(bitcode); + } + } + + modules.extend(cgus); + } + + if let Some(alloc) = allocator { + let bc = std::fs::read( + alloc + .object + .clone() + .expect("expected obj path for allocator module"), + )?; + modules.push(bc); + } + + // now that we have our nice bitcode modules, we just need to find libdevice and give our + // modules to nvvm to make a final ptx file + + // we need to actually parse the codegen args again, because codegencx is not available at link time. + let codegen_args = CodegenArgs::from_session(sess); + + let ptx_bytes = match crate::nvvm::codegen_bitcode_modules(&codegen_args, sess, modules, cx.llcx) { + Ok(bytes) => bytes, + Err(err) => { + // TODO(RDambrosio016): maybe include the nvvm log with this fatal error + sess.dcx().fatal(err.to_string()) + } + }; + + std::fs::write(out_filename, ptx_bytes) +} + +fn create_archive(sess: &Session, files: &[&Path], metadata: &[u8], out_filename: &Path) { + if let Err(err) = try_create_archive(files, metadata, out_filename) { + sess.dcx() + .fatal(format!("Failed to create archive: {}", err)); + } +} + +fn try_create_archive(files: &[&Path], metadata: &[u8], out_filename: &Path) -> io::Result<()> { + let file = File::create(out_filename)?; + let mut builder = Builder::new(file); + { + let mut header = Header::new_gnu(); + header.set_path(".metadata")?; + header.set_size(metadata.len() as u64); + header.set_cksum(); + builder.append(&header, metadata)?; + } + let mut filenames = FxHashSet::default(); + filenames.insert(OsStr::new(".metadata")); + for file in files { + assert!( + filenames.insert(file.file_name().unwrap()), + "Duplicate filename in archive: {:?}", + file.file_name().unwrap() + ); + builder.append_path_with_name(file, file.file_name().unwrap())?; + } + builder.into_inner()?; + Ok(()) +} + +// most of the code from here is derived from rust-gpu + +fn link_local_crate_native_libs_and_dependent_crate_libs( + rlibs: &mut Vec, + sess: &Session, + crate_type: CrateType, + codegen_results: &CodegenResults, +) { + if sess.opts.unstable_opts.link_native_libraries { + add_local_native_libraries(sess, codegen_results); + } + add_upstream_rust_crates(sess, rlibs, codegen_results, crate_type); + if sess.opts.unstable_opts.link_native_libraries { + add_upstream_native_libraries(sess, codegen_results, crate_type); + } +} + +fn add_local_native_libraries(sess: &Session, codegen_results: &CodegenResults) { + let relevant_libs = codegen_results + .crate_info + .used_libraries + .iter() + .filter(|l| relevant_lib(sess, l)); + assert_eq!(relevant_libs.count(), 0); +} + +fn add_upstream_rust_crates( + sess: &Session, + rlibs: &mut Vec, + codegen_results: &CodegenResults, + crate_type: CrateType, +) { + let (_, data) = codegen_results + .crate_info + .dependency_formats + .iter() + .find(|(ty, _)| **ty == crate_type) + .expect("failed to find crate type in dependency format list"); + let deps = &codegen_results.crate_info.used_crates; + for cnum in deps.iter() { + let src = &codegen_results.crate_info.used_crate_source[cnum]; + match data[*cnum] { + Linkage::NotLinked => {} + Linkage::Static => rlibs.push(src.rlib.as_ref().unwrap().0.clone()), + // should we just ignore includedFromDylib? + Linkage::Dynamic | Linkage::IncludedFromDylib => { + sess.dcx().fatal("Dynamic Linking is not supported in CUDA") + } + } + } +} + +fn add_upstream_native_libraries( + sess: &Session, + codegen_results: &CodegenResults, + _crate_type: CrateType, +) { + let crates = &codegen_results.crate_info.used_crates; + for cnum in crates { + for lib in codegen_results.crate_info.native_libraries[cnum].iter() { + if !relevant_lib(sess, lib) { + continue; + } + sess.dcx() + .fatal("Native libraries are not supported in CUDA"); + } + } +} + +fn relevant_lib(sess: &Session, lib: &NativeLib) -> bool { + match &lib.cfg { + Some(cfg) => rustc_attr_parsing::cfg_matches(cfg, sess, CRATE_NODE_ID, None), + None => true, + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/llvm.rs b/crates/rustc_codegen_nvvm_v19/src/llvm.rs new file mode 100644 index 00000000..39528bfc --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/llvm.rs @@ -0,0 +1,2077 @@ +//! LLVM FFI functions. +//! +//! The reason we don't use rustc's llvm FFI is because rustc uses llvm 13 (at the time of writing). +//! While NVVM expects llvm 7 bitcode/suppported things. And we don't use llvm-sys because this allows us +//! to only include what we need, as well as use safe references instead of pointers. +//! +//! Most of this code was taken from rustc_codegen_llvm with many things removed. + +#![allow( + non_camel_case_types, + non_snake_case, + non_upper_case_globals, + clippy::enum_variant_names +)] +// we have a lot of functions we linked to from cg_llvm that we don't use +// but likely will use in the future, so we ignore any unused functions +// in case we need them in the future for things like debug info or LTO. +#![allow(dead_code)] + +use libc::{c_char, c_uint, c_void, size_t}; +use libc::{c_int, c_ulonglong}; +use std::ffi::{CStr, CString}; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; +use std::ptr::{self}; + +pub use debuginfo::*; + +impl PartialEq for Value { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self, other) + } +} + +impl Eq for Value {} + +impl Hash for Value { + fn hash(&self, hasher: &mut H) { + (self as *const Self).hash(hasher); + } +} + +impl fmt::Debug for Value { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + unsafe { + // ideally we'd print the type but the llvm 7 C api doesnt have a way to do this :( + f.write_str("(")?; + let ptr = LLVMPrintValueToString(self); + let cstr = CString::from_raw(ptr); + let string = cstr.to_string_lossy(); + f.write_str(&string)?; + f.write_str(")") + } + } +} + +impl LLVMRustResult { + pub fn into_result(self) -> Result<(), ()> { + match self { + LLVMRustResult::Success => Ok(()), + LLVMRustResult::Failure => Err(()), + } + } +} + +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum CodeGenOptSize { + CodeGenOptSizeNone = 0, + CodeGenOptSizeDefault = 1, + CodeGenOptSizeAggressive = 2, +} + +pub use self::CodeGenOptSize::*; + +#[derive(Copy, Clone)] +pub enum AttributePlace { + ReturnValue, + Argument(u32), + Function, +} + +impl AttributePlace { + pub fn as_uint(self) -> c_uint { + match self { + AttributePlace::ReturnValue => 0, + AttributePlace::Argument(i) => 1 + i, + AttributePlace::Function => !0, + } + } +} + +impl Attribute { + pub fn apply_llfn(&self, idx: AttributePlace, llfn: &Value) { + unsafe { + match *self { + Attribute::StructRet => { + // These attributes need type information in LLVM 19+ + // We need to infer the type from the function signature + let fn_type = LLVMRustGetFunctionType(llfn); + + // Get the parameter type based on the index + let param_type = match idx { + AttributePlace::Argument(arg_idx) => { + let n_args = LLVMCountParamTypes(fn_type) as usize; + let mut args = Vec::with_capacity(n_args); + LLVMGetParamTypes(fn_type, args.as_mut_ptr()); + args.set_len(n_args); + args[arg_idx as usize] + } + AttributePlace::ReturnValue => { + LLVMGetReturnType(fn_type) + } + _ => { + panic!("StructRet attributes only valid on parameters and return values"); + } + }; + + LLVMRustAddFunctionAttributeWithType(llfn, idx.as_uint(), *self, param_type); + } + _ => { + // Regular attributes that don't need type info + LLVMRustAddFunctionAttribute(llfn, idx.as_uint(), *self); + } + } + } + } + + pub fn apply_callsite(&self, idx: AttributePlace, callsite: &Value) { + unsafe { LLVMRustAddCallSiteAttribute(callsite, idx.as_uint(), *self) } + } + + pub fn unapply_llfn(&self, idx: AttributePlace, llfn: &Value) { + unsafe { LLVMRustRemoveFunctionAttributes(llfn, idx.as_uint(), *self) } + } +} + +/// Safe wrapper around `LLVMGetParam`, because segfaults are no fun. +pub(crate) fn get_param(llfn: &Value, index: c_uint) -> &Value { + unsafe { + assert!( + index < LLVMCountParams(llfn), + "out of bounds argument access: {} out of {} arguments", + index, + LLVMCountParams(llfn) + ); + LLVMGetParam(llfn, index) + } +} + +/// Safe wrapper around `LLVMGetParams`. +pub(crate) fn get_params(llfn: &Value) -> Vec<&Value> { + unsafe { + let count = LLVMCountParams(llfn) as usize; + let mut params = Vec::with_capacity(count); + LLVMGetParams(llfn, params.as_mut_ptr()); + params.set_len(count); + params + } +} + +/// Safe wrapper for `LLVMGetValueName2` into a byte slice +pub(crate) fn get_value_name(value: &Value) -> &[u8] { + unsafe { + let mut len = 0; + let data = LLVMGetValueName2(value, &mut len); + std::slice::from_raw_parts(data.cast(), len) + } +} + +/// Safe wrapper for `LLVMSetValueName2` from a byte slice +pub(crate) fn set_value_name(value: &Value, name: &[u8]) { + unsafe { + let data = name.as_ptr().cast(); + LLVMSetValueName2(value, data, name.len()); + } +} + +pub fn last_error() -> Option { + unsafe { + let cstr = LLVMRustGetLastError(); + if cstr.is_null() { + None + } else { + let err = CStr::from_ptr(cstr).to_bytes(); + let err = String::from_utf8_lossy(err).to_string(); + libc::free(cstr as *mut _); + Some(err) + } + } +} + +pub(crate) fn SetUnnamedAddress(global: &'_ Value, unnamed: UnnamedAddr) { + unsafe { + LLVMSetUnnamedAddress(global, unnamed); + } +} + +pub(crate) type Bool = c_uint; + +pub const True: Bool = 1; +pub const False: Bool = 0; + +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +#[allow(dead_code)] // Variants constructed by C++. +pub(crate) enum LLVMRustResult { + Success, + Failure, +} + +/// LLVMRustLinkage +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub(crate) enum Linkage { + ExternalLinkage = 0, + AvailableExternallyLinkage = 1, + LinkOnceAnyLinkage = 2, + LinkOnceODRLinkage = 3, + WeakAnyLinkage = 4, + WeakODRLinkage = 5, + AppendingLinkage = 6, + InternalLinkage = 7, + PrivateLinkage = 8, + ExternalWeakLinkage = 9, + CommonLinkage = 10, +} + +// LLVMRustVisibility +#[repr(C)] +#[derive(Copy, Clone, PartialEq)] +pub(crate) enum Visibility { + Default = 0, + Hidden = 1, + Protected = 2, +} + +/// LLVMUnnamedAddr +#[repr(C)] +pub(crate) enum UnnamedAddr { + No, + Local, + Global, +} + +/// LLVMDLLStorageClass +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) enum DLLStorageClass { + #[allow(dead_code)] + Default = 0, + #[allow(dead_code)] + DllExport = 2, // Function to be accessible from DLL. +} + +/// Matches LLVMRustAttribute in LLVMWrapper.h +/// Semantically a subset of the C++ enum llvm::Attribute::AttrKind, +/// though it is not ABI compatible (since it's a C++ enum) +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub(crate) enum Attribute { + AlwaysInline = 0, + Cold = 2, + InlineHint = 3, + MinSize = 4, + NoAlias = 6, + NoCapture = 7, + NoInline = 8, + NonNull = 9, + NoReturn = 11, + NoUnwind = 12, + OptimizeForSize = 13, + OptimizeNone = 14, + ReadOnly = 15, + SExt = 16, + StructRet = 17, + ZExt = 19, + InReg = 20, + ReadNone = 24, +} + +/// LLVMIntPredicate +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) enum IntPredicate { + IntEQ = 32, + IntNE = 33, + IntUGT = 34, + IntUGE = 35, + IntULT = 36, + IntULE = 37, + IntSGT = 38, + IntSGE = 39, + IntSLT = 40, + IntSLE = 41, +} + +impl IntPredicate { + pub fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self { + match intpre { + rustc_codegen_ssa::common::IntPredicate::IntEQ => IntPredicate::IntEQ, + rustc_codegen_ssa::common::IntPredicate::IntNE => IntPredicate::IntNE, + rustc_codegen_ssa::common::IntPredicate::IntUGT => IntPredicate::IntUGT, + rustc_codegen_ssa::common::IntPredicate::IntUGE => IntPredicate::IntUGE, + rustc_codegen_ssa::common::IntPredicate::IntULT => IntPredicate::IntULT, + rustc_codegen_ssa::common::IntPredicate::IntULE => IntPredicate::IntULE, + rustc_codegen_ssa::common::IntPredicate::IntSGT => IntPredicate::IntSGT, + rustc_codegen_ssa::common::IntPredicate::IntSGE => IntPredicate::IntSGE, + rustc_codegen_ssa::common::IntPredicate::IntSLT => IntPredicate::IntSLT, + rustc_codegen_ssa::common::IntPredicate::IntSLE => IntPredicate::IntSLE, + } + } +} + +/// LLVMTypeKind +#[allow(dead_code)] +#[derive(Copy, Clone, PartialEq, Debug)] +#[repr(C)] +pub(crate) enum TypeKind { + Void = 0, + Half = 1, + Float = 2, + Double = 3, + X86_FP80 = 4, + FP128 = 5, + PPC_FP128 = 6, + Label = 7, + Integer = 8, + Function = 9, + Struct = 10, + Array = 11, + Pointer = 12, + Vector = 13, + Metadata = 14, + Token = 16, + ScalableVector = 17, + BFloat = 18, + X86_AMX = 19, + TargetExt = 20, +} + +impl TypeKind { + pub fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind { + match self { + TypeKind::Void => rustc_codegen_ssa::common::TypeKind::Void, + TypeKind::Half => rustc_codegen_ssa::common::TypeKind::Half, + TypeKind::Float => rustc_codegen_ssa::common::TypeKind::Float, + TypeKind::Double => rustc_codegen_ssa::common::TypeKind::Double, + TypeKind::X86_FP80 => rustc_codegen_ssa::common::TypeKind::X86_FP80, + TypeKind::FP128 => rustc_codegen_ssa::common::TypeKind::FP128, + TypeKind::PPC_FP128 => rustc_codegen_ssa::common::TypeKind::PPC_FP128, + TypeKind::Label => rustc_codegen_ssa::common::TypeKind::Label, + TypeKind::Metadata => rustc_codegen_ssa::common::TypeKind::Metadata, + TypeKind::Integer => rustc_codegen_ssa::common::TypeKind::Integer, + TypeKind::Function => rustc_codegen_ssa::common::TypeKind::Function, + TypeKind::Struct => rustc_codegen_ssa::common::TypeKind::Struct, + TypeKind::Array => rustc_codegen_ssa::common::TypeKind::Array, + TypeKind::Pointer => rustc_codegen_ssa::common::TypeKind::Pointer, + TypeKind::Vector => rustc_codegen_ssa::common::TypeKind::Vector, + TypeKind::Token => rustc_codegen_ssa::common::TypeKind::Token, + TypeKind::ScalableVector => rustc_codegen_ssa::common::TypeKind::ScalableVector, + TypeKind::BFloat => rustc_codegen_ssa::common::TypeKind::BFloat, + TypeKind::X86_AMX => rustc_codegen_ssa::common::TypeKind::X86_AMX, + TypeKind::TargetExt => unimplemented!() + } + } +} + +/// LLVMMetadataType +#[derive(Copy, Clone)] +#[repr(C)] +pub(crate) enum MetadataType { + MD_range = 4, + MD_invariant_load = 6, + MD_nontemporal = 9, + MD_nonnull = 11, +} + +/// LLVMRustAsmDialect +#[derive(Copy, Clone)] +#[repr(C)] +pub enum AsmDialect { + Att, + Intel, +} + +// impl AsmDialect { +// pub fn from_generic(asm: rustc_ast::LlvmAsmDialect) -> Self { +// match asm { +// rustc_ast::LlvmAsmDialect::Att => AsmDialect::Att, +// rustc_ast::LlvmAsmDialect::Intel => AsmDialect::Intel, +// } +// } +// } + +/// LLVMRustDiagnosticKind +#[derive(Copy, Clone)] +#[repr(C)] +#[allow(dead_code)] // Variants constructed by C++. +pub(crate) enum DiagnosticKind { + Other, + InlineAsm, + StackSize, + DebugMetadataVersion, + SampleProfile, + OptimizationRemark, + OptimizationRemarkMissed, + OptimizationRemarkAnalysis, + OptimizationRemarkAnalysisFPCommute, + OptimizationRemarkAnalysisAliasing, + OptimizationRemarkOther, + OptimizationFailure, + PGOProfile, + Linker, + Unsupported, +} + +/// LLVMRustDiagnosticLevel +#[derive(Copy, Clone)] +#[repr(C)] +#[allow(dead_code)] // Variants constructed by C++. +pub(crate) enum DiagnosticLevel { + Error, + Warning, + Note, + Remark, +} + +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum LLVMVerifierFailureAction { + /// Print to stderr and abort the process. + LLVMAbortProcessAction = 0, + /// Print to stderr and return 1. + LLVMPrintMessageAction = 1, + /// Return 1 and print nothing. + LLVMReturnStatusAction = 2, +} + +/// LLVMRustPassKind +#[derive(Copy, Clone, PartialEq, Debug)] +#[repr(C)] +#[allow(dead_code)] // Variants constructed by C++. +pub(crate) enum PassKind { + Other, + Function, + Module, +} + +// LLVMRustThinLTOData +unsafe extern "C" { + pub(crate) type ThinLTOData; +} + +unsafe impl Send for ThinLTOData {} + +// LLVMRustThinLTOBuffer +unsafe extern "C" { + pub(crate) type ThinLTOBuffer; +} + +unsafe impl Send for ThinLTOBuffer {} + +/// LLVMRustThinLTOModule +#[repr(C)] +pub(crate) struct ThinLTOModule { + pub identifier: *const c_char, + pub data: *const u8, + pub len: usize, +} + +unsafe extern "C" { + type Opaque; +} +#[repr(C)] +struct InvariantOpaque<'a> { + _marker: PhantomData<&'a mut &'a ()>, + _opaque: Opaque, +} + +// Opaque pointer types +unsafe extern "C" { + pub(crate) type Module; +} +unsafe extern "C" { + pub type Context; +} + +unsafe impl Send for Context {} + +unsafe extern "C" { + pub(crate) type Type; +} +unsafe extern "C" { + pub(crate) type Value; +} +unsafe extern "C" { + pub(crate) type ConstantInt; +} +unsafe extern "C" { + pub type Metadata; +} +unsafe extern "C" { + pub(crate) type BasicBlock; +} +#[repr(C)] +pub(crate) struct Builder<'a> { + _inv: InvariantOpaque<'a>, +} +#[repr(C)] +pub(crate) struct OperandBundleDef<'a>(InvariantOpaque<'a>); + +unsafe extern "C" { + pub(crate) type ModuleBuffer; +} + +unsafe impl Send for ModuleBuffer {} + +#[repr(C)] +pub struct PassManager<'a>(InvariantOpaque<'a>); +unsafe extern "C" { + pub type PassManagerBuilder; +} +unsafe extern "C" { + pub type Pass; +} +unsafe extern "C" { + pub type TargetMachine; +} +unsafe extern "C" { + pub(crate) type MemoryBuffer; +} + +// TODO: remove this llvm v19 type +unsafe extern "C" { + pub type PassBuilderOptions; +} + +/// LLVMRustChecksumKind +#[derive(Copy, Clone)] +#[repr(C)] +pub enum ChecksumKind { + None, + MD5, + SHA1, +} + +pub mod debuginfo { + use super::{InvariantOpaque, Metadata}; + use bitflags::bitflags; + + #[repr(C)] + pub(crate) struct DIBuilder<'a>(InvariantOpaque<'a>); + + pub type DIDescriptor = Metadata; + pub type DIScope = DIDescriptor; + pub type DILocation = DIDescriptor; + pub type DIFile = DIScope; + pub type DILexicalBlock = DIScope; + pub type DISubprogram = DIScope; + pub type DINameSpace = DIScope; + pub type DIType = DIDescriptor; + pub type DIBasicType = DIType; + pub type DIDerivedType = DIType; + pub type DICompositeType = DIDerivedType; + pub type DIVariable = DIDescriptor; + pub type DIGlobalVariable = DIDescriptor; + pub type DIArray = DIDescriptor; + pub type DISubrange = DIDescriptor; + pub type DIEnumerator = DIDescriptor; + pub type DITemplateTypeParameter = DIDescriptor; + + bitflags! { + /// Must match the layout of `LLVMDIFlags` in the LLVM-C API. + /// + /// Each value declared here must also be covered by the static + /// assertions in `RustWrapper.cpp` used by `fromRust(LLVMDIFlags)`. + #[repr(C)] + #[derive(Clone, Copy, Default)] + pub struct DIFlags: u32 { + const FlagZero = 0; + const FlagPrivate = 1; + const FlagProtected = 2; + const FlagPublic = 3; + const FlagFwdDecl = (1 << 2); + const FlagAppleBlock = (1 << 3); + const FlagBlockByrefStruct = (1 << 4); + const FlagVirtual = (1 << 5); + const FlagArtificial = (1 << 6); + const FlagExplicit = (1 << 7); + const FlagPrototyped = (1 << 8); + const FlagObjcClassComplete = (1 << 9); + const FlagObjectPointer = (1 << 10); + const FlagVector = (1 << 11); + const FlagStaticMember = (1 << 12); + const FlagLValueReference = (1 << 13); + const FlagRValueReference = (1 << 14); + const FlagExternalTypeRef = (1 << 15); + const FlagIntroducedVirtual = (1 << 18); + const FlagBitField = (1 << 19); + const FlagNoReturn = (1 << 20); + const FlagMainSubprogram = (1 << 21); + } + } + + bitflags! { + /// These values **must** match debuginfo::DISPFlags! They also *happen* + /// to match LLVM, but that isn't required as we do giant sets of + /// matching below. The value shouldn't be directly passed to LLVM. + /// + /// Each value declared here must also be covered by the static + /// assertions in `RustWrapper.cpp` used by `fromRust(LLVMRustDISPFlags)`. + #[repr(C)] + #[derive(Clone, Copy, Default)] + pub struct LLVMRustDISPFlags: u32 { + const SPFlagZero = 0; + const SPFlagVirtual = 1; + const SPFlagPureVirtual = 2; + const SPFlagLocalToUnit = (1 << 2); + const SPFlagDefinition = (1 << 3); + const SPFlagOptimized = (1 << 4); + const SPFlagMainSubprogram = (1 << 5); + // Do not add values that are not supported by the minimum LLVM + // version we support! see llvm/include/llvm/IR/DebugInfoFlags.def + // (In LLVM < 8, createFunction supported these as separate bool arguments.) + } + } + + /// LLVMRustDebugEmissionKind + #[derive(Copy, Clone)] + #[repr(C)] + pub enum DebugEmissionKind { + NoDebug, + FullDebug, + LineTablesOnly, + } + + impl DebugEmissionKind { + pub(crate) fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { + // We should be setting LLVM's emission kind to `LineTablesOnly` if + // we are compiling with "limited" debuginfo. However, some of the + // existing tools relied on slightly more debuginfo being generated than + // would be the case with `LineTablesOnly`, and we did not want to break + // these tools in a "drive-by fix", without a good idea or plan about + // what limited debuginfo should exactly look like. So for now we are + // instead adding a new debuginfo option "line-tables-only" so as to + // not break anything and to allow users to have 'limited' debug info. + // + // See https://github.com/rust-lang/rust/issues/60020 for details. + use rustc_session::config::DebugInfo; + match kind { + // NVVM: Llvm 7 is missing LineDirectivesOnly, so don't emit anything. + DebugInfo::None | DebugInfo::LineDirectivesOnly => DebugEmissionKind::NoDebug, + DebugInfo::LineTablesOnly => DebugEmissionKind::LineTablesOnly, + DebugInfo::Limited | DebugInfo::Full => DebugEmissionKind::FullDebug, + } + } + } +} + +/// LLVMRustCodeGenOptLevel +#[derive(Copy, Clone, PartialEq, Debug)] +#[repr(C)] +pub enum CodeGenOptLevel { + None, + Less, + Default, + Aggressive, +} + +/// LLVMRelocMode +#[derive(Copy, Clone, PartialEq)] +#[allow(clippy::upper_case_acronyms)] +#[repr(C)] +pub enum RelocMode { + Default, + Static, + PIC, + DynamicNoPic, + ROPI, + RWPI, + ROPI_RWPI, +} + +/// LLVMRustFloatABI +#[derive(Copy, Clone)] +#[repr(C)] +pub enum FloatABIType { + Default, + Soft, + Hard +} + +/// LLVMRustCodeModel +#[derive(Copy, Clone)] +#[repr(C)] +pub enum CodeModel { + Other, + Small, + Kernel, + Medium, + Large, + None, +} + +// LLVMRustDebugNameTableKind +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(C)] +pub enum DebugNameTableKind { + Default = 0, + GNU = 1, + None = 2, + Apple = 3, +} + + +unsafe extern "C" { + pub fn LLVMRustBuildCall<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + OpBundles: *const &OperandBundleDef<'a>, + NumOpBundles: c_uint, + ) -> &'a Value; + + pub(crate) fn LLVMRustGetOrInsertFunction<'a>( + M: &'a Module, + Name: *const c_char, + NameLen: size_t, // TODO: llvm19 added this + FunctionTy: &'a Type, + ) -> &'a Value; + + // dont trace these functions or cargo will error, see init.rs + pub(crate) fn LLVMStartMultithreaded() -> Bool; + pub(crate) fn LLVMInitializeNVPTXTargetInfo(); + pub(crate) fn LLVMInitializeNVPTXTarget(); + pub(crate) fn LLVMInitializeNVPTXTargetMC(); + pub(crate) fn LLVMInitializeNVPTXAsmPrinter(); + // TODO: removed? + //pub(crate) fn LLVMInitializePasses(); + pub(crate) fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); +} + +// use rustc_codegen_nvvm_macros::trace_ffi_calls; +// #[trace_ffi_calls] +unsafe extern "C" { + pub(crate) fn LLVMGetPointerAddressSpace(PointerTy: &Type) -> c_uint; + pub(crate) fn LLVMBuildAddrSpaceCast<'a>( + arg1: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMRustGetOrInsertGlobal<'a>( + M: &'a Module, + Name: *const c_char, + NameLen: usize, + T: &'a Type, + AddressSpace: c_uint, + ) -> &'a Value; + // TODO: removed + //pub(crate) fn LLVMAddGlobalDCEPass(PM: &mut PassManager); + pub(crate) fn LLVMGetNamedMetadataOperands(M: &Module, name: *const c_char, Dest: *mut &Value); + pub(crate) fn LLVMGetNamedMetadataNumOperands(M: &Module, name: *const c_char) -> c_uint; + pub(crate) fn LLVMGetMDNodeOperands(V: &Value, Dest: *mut &Value); + pub(crate) fn LLVMGetMDNodeNumOperands(V: &Value) -> c_uint; + pub(crate) fn LLVMGetFirstFunction(M: &Module) -> Option<&Value>; + pub(crate) fn LLVMGetNextFunction(Fn: &Value) -> Option<&Value>; + pub(crate) fn LLVMAddGlobalInAddressSpace<'a>( + M: &'a Module, + Ty: &'a Type, + Name: *const c_char, + AddressSpace: c_uint, + ) -> &'a Value; + pub(crate) fn LLVMGetOperand(Val: &Value, Index: c_uint) -> &Value; + pub(crate) fn LLVMIsABitCastInst(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMIsASelectInst(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMRustGetFunctionType(V: &Value) -> &Type; + pub(crate) fn LLVMLinkModules2(Dest: &Module, Src: &Module) -> Bool; + pub(crate) fn LLVMParseIRInContext<'ll, 'a, 'b>( + ContextRef: &'ll Context, + MemBuf: &'a MemoryBuffer, + OutM: *mut &'b Module, + OutMessage: *mut *mut c_char, + ) -> Bool; + pub(crate) fn LLVMCreateMemoryBufferWithMemoryRange<'a>( + InputData: *const c_char, + InputDataLength: usize, + BufferName: *const c_char, + RequiresNullTerminator: Bool, + ) -> &'a mut MemoryBuffer; + pub(crate) fn LLVMDisposeMemoryBuffer<'a>(MemBuf: &'a mut MemoryBuffer); + + pub(crate) fn LLVMGetCurrentDebugLocation<'a>(Builder: &Builder<'a>) -> Option<&'a Value>; + //pub(crate) fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: Option<&'a Value>); + pub(crate) fn LLVMSetCurrentDebugLocation2<'a>( + Builder: &Builder<'a>, + Loc: Option<&'a Metadata>, + ); + + pub(crate) fn LLVMGetModuleContext(M: &Module) -> &Context; + pub(crate) fn LLVMGetMDKindIDInContext( + C: &Context, + Name: *const c_char, + SLen: c_uint, + ) -> c_uint; + + pub(crate) fn LLVMRustDebugMetadataVersion() -> u32; + pub(crate) fn LLVMRustVersionMajor() -> u32; + pub(crate) fn LLVMRustVersionMinor() -> u32; + + pub(crate) fn LLVMRustAddModuleFlag(M: &Module, name: *const c_char, value: u32); + + pub(crate) fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; + + pub(crate) fn LLVMRustDIBuilderCreate<'a>(M: &'a Module) -> &'a mut DIBuilder<'a>; + + pub(crate) fn LLVMRustDIBuilderDispose<'a>(Builder: &'a mut DIBuilder<'a>); + + pub(crate) fn LLVMRustDIBuilderFinalize<'a>(Builder: &DIBuilder<'a>); + + pub fn LLVMRustDIBuilderCreateCompileUnit<'a>( + Builder: &DIBuilder<'a>, + Lang: c_uint, + File: &'a DIFile, + Producer: *const c_char, + ProducerLen: size_t, + isOptimized: bool, + Flags: *const c_char, + RuntimeVer: c_uint, + SplitName: *const c_char, + SplitNameLen: size_t, + kind: DebugEmissionKind, + DWOId: u64, + SplitDebugInlining: bool, + TableKind: DebugNameTableKind, + ) -> &'a DIDescriptor; + + pub(crate) fn LLVMRustDIBuilderCreateFile<'a>( + Builder: &DIBuilder<'a>, + Filename: *const c_char, + FileNameLen: size_t, + Directory: *const c_char, + DirectoryLen: size_t, + CSKind: ChecksumKind, + Checksum: *const c_char, + ChecksomLen: size_t, + Source: *const c_char, + SourceLen: size_t, + ) -> &'a DIFile; + + pub(crate) fn LLVMRustDIBuilderCreateSubroutineType<'a>( + Builder: &DIBuilder<'a>, + ParameterTypes: &'a DIArray, + ) -> &'a DICompositeType; + + // TODO: updated + pub(crate) fn LLVMRustDIBuilderCreateFunction<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + ScopeLine: c_uint, + Flags: DIFlags, + SPFlags: LLVMRustDISPFlags, + MaybeFn: Option<&'a Value>, + TParam: &'a DIArray, + Decl: Option<&'a DIDescriptor>, + ) -> &'a DISubprogram; + + pub(crate) fn LLVMRustDIBuilderCreateBasicType<'a>( + Builder: &DIBuilder<'a>, + Name: *const c_char, + NameLen: size_t, + SizeInBits: u64, + Encoding: c_uint, + ) -> &'a DIBasicType; + + pub fn LLVMRustDIBuilderCreateTypedef<'a>( + Builder: &DIBuilder<'a>, + Type: &'a DIBasicType, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Scope: Option<&'a DIScope>, + ) -> &'a DIDerivedType; + + pub(crate) fn LLVMRustDIBuilderCreatePointerType<'a>( + Builder: &DIBuilder<'a>, + PointeeTy: &'a DIType, + SizeInBits: u64, + AlignInBits: u32, + AddressSpace: c_uint, + Name: *const c_char, + NameLen: size_t, + ) -> &'a DIDerivedType; + + pub(crate) fn LLVMRustDIBuilderCreateStructType<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIDescriptor>, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + DerivedFrom: Option<&'a DIType>, + Elements: &'a DIArray, + RunTimeLang: c_uint, + VTableHolder: Option<&'a DIType>, + UniqueId: *const c_char, + UniqueIdLen: size_t, + ) -> &'a DICompositeType; + + pub(crate) fn LLVMRustDIBuilderCreateMemberType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + SizeInBits: u64, + AlignInBits: u32, + OffsetInBits: u64, + Flags: DIFlags, + Ty: &'a DIType, + ) -> &'a DIDerivedType; + + pub(crate) fn LLVMRustDIBuilderCreateVariantMemberType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + OffsetInBits: u64, + Discriminant: Option<&'a Value>, + Flags: DIFlags, + Ty: &'a DIType, + ) -> &'a DIType; + + pub(crate) fn LLVMRustDIBuilderCreateLexicalBlock<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + File: &'a DIFile, + Line: c_uint, + Col: c_uint, + ) -> &'a DILexicalBlock; + + pub(crate) fn LLVMRustDIBuilderCreateLexicalBlockFile<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + File: &'a DIFile, + ) -> &'a DILexicalBlock; + + pub(crate) fn LLVMRustDIBuilderCreateStaticVariable<'a>( + Builder: &DIBuilder<'a>, + Context: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + isLocalToUnit: bool, + Val: &'a Value, + Decl: Option<&'a DIDescriptor>, + AlignInBits: u32, + ) -> &'a DIGlobalVariable; + + pub(crate) fn LLVMRustDIBuilderCreateVariable<'a>( + Builder: &DIBuilder<'a>, + Tag: c_uint, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: usize, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + AlwaysPreserve: bool, + Flags: DIFlags, + ArgNo: c_uint, + AlignInBits: u32, + ) -> &'a DIVariable; + + pub(crate) fn LLVMRustDIBuilderCreateArrayType<'a>( + Builder: &DIBuilder<'a>, + Size: u64, + AlignInBits: u32, + Ty: &'a DIType, + Subscripts: &'a DIArray, + ) -> &'a DIType; + + pub(crate) fn LLVMRustDIBuilderCreateVectorType<'a>( + Builder: &DIBuilder<'a>, + Size: u64, + AlignInBits: u32, + Ty: &'a DIType, + Subscripts: &'a DIArray, + ) -> &'a DIType; + + pub(crate) fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( + Builder: &DIBuilder<'a>, + Lo: i64, + Count: i64, + ) -> &'a DISubrange; + + pub(crate) fn LLVMRustDIBuilderGetOrCreateArray<'a>( + Builder: &DIBuilder<'a>, + Ptr: *const Option<&'a DIDescriptor>, + Count: c_uint, + ) -> &'a DIArray; + + pub(crate) fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( + Builder: &DIBuilder<'a>, + Val: &'a Value, + VarInfo: &'a DIVariable, + AddrOps: *const u64, + AddrOpsCount: c_uint, + DL: &'a DILocation, + InsertAtEnd: &'a BasicBlock, + ) -> &'a Value; + + pub(crate) fn LLVMRustDIBuilderCreateEnumerator<'a>( + Builder: &DIBuilder<'a>, + Name: *const c_char, + NameLen: size_t, + Val: i64, + IsUnsigned: bool, + ) -> &'a DIEnumerator; + + pub(crate) fn LLVMRustDIBuilderCreateEnumerationType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Elements: &'a DIArray, + ClassType: &'a DIType, + ) -> &'a DIType; + + pub(crate) fn LLVMRustDIBuilderCreateUnionType<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + Elements: Option<&'a DIArray>, + RunTimeLang: c_uint, + UniqueId: *const c_char, + UniqueIdLen: size_t, + ) -> &'a DIType; + + pub(crate) fn LLVMRustDIBuilderCreateVariantPart<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + Discriminator: Option<&'a DIDerivedType>, + Elements: &'a DIArray, + UniqueId: *const c_char, + UniqueIdLen: size_t, + ) -> &'a DIDerivedType; + + pub(crate) fn LLVMSetUnnamedAddr<'a>(GlobalVar: &'a Value, UnnamedAddr: Bool); + + pub(crate) fn LLVMRustDIBuilderCreateTemplateTypeParameter<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + Ty: &'a DIType, + ) -> &'a DITemplateTypeParameter; + + pub(crate) fn LLVMRustDIBuilderCreateNameSpace<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + ) -> &'a DINameSpace; + + pub(crate) fn LLVMRustDICompositeTypeReplaceArrays<'a>( + Builder: &DIBuilder<'a>, + CompositeType: &'a DIType, + Elements: Option<&'a DIArray>, + Params: Option<&'a DIArray>, + ); + + pub(crate) fn LLVMRustDICompositeTypeSetTypeArray<'a>( + Builder: &DIBuilder<'a>, + CompositeType: &'a DIType, + TypeArray: &'a DIArray, + ); + + pub(crate) fn LLVMRustDIBuilderCreateDebugLocation<'a>( + Line: c_uint, + Column: c_uint, + Scope: &'a DIScope, + InlinedAt: Option<&'a DILocation>, + ) -> &'a DILocation; + + pub(crate) fn LLVMRustDILocationCloneWithBaseDiscriminator<'a>( + Location: &'a DILocation, + BD: c_uint, + ) -> Option<&'a DILocation>; + + pub(crate) fn LLVMRustRunFunctionPassManager(PM: &PassManager, M: &Module); + pub(crate) fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool); + + pub(crate) fn LLVMRustAddBuilderLibraryInfo( + PMB: &PassManagerBuilder, + M: &Module, + DisableSimplifyLibCalls: bool, + ); + + pub(crate) fn LLVMRustConfigurePassManagerBuilder( + PMB: &PassManagerBuilder, + OptLevel: CodeGenOptLevel, + MergeFunctions: bool, + SLPVectorize: bool, + LoopVectorize: bool, + PrepareForThinLTO: bool, + PGOGenPath: *const c_char, + PGOUsePath: *const c_char, + ); + + pub(crate) fn LLVMRustCreateTargetMachine( + TripleStr: *const c_char, + CPU: *const c_char, + Feature: *const c_char, + ABIStr: *const c_char, + RustCM: CodeModel, + RustReloc: RelocMode, + RustOptLevel: CodeGenOptLevel, + RustFloatABIType: FloatABIType, + FunctionSections: bool, + DataSections: bool, + UniqueSectionNames: bool, + TrapUnreachable: bool, + Singlethread: bool, + VerboseAsm: bool, + EmitStackSizeSection: bool, + RelaxELFRelocations: bool, + UseInitArray: bool, + SplitDwarfFile: *const c_char, + OutputObjFile: *const c_char, + DebugInfoCompression: *const c_char, + UseEmulatedTls: bool, + ArgsCstrBuff: *const c_char, + ArgsCstrBuffLen: usize, + ) -> Option<&'static mut TargetMachine>; // TODO: not sure about this + + // TODO: remove me + /*pub(crate) fn LLVMRustAddAnalysisPasses<'a>( + T: &'a TargetMachine, + PM: &'a PassManager, + M: &'a Module, + );*/ + pub(crate) fn LLVMRustPassKind(Pass: &Pass) -> PassKind; + pub(crate) fn LLVMRustFindAndCreatePass( + Pass: *const c_char, + PassNameLen: size_t, + ) -> Option<&'static mut Pass>; + pub(crate) fn LLVMRustAddPass<'a>(PM: &'a PassManager, Pass: &'static mut Pass); + + /// Writes a module to the specified path. Returns 0 on success. + pub(crate) fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int; + + /// Creates a pass manager. + pub(crate) fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; + + /// Creates a function-by-function pass manager + pub(crate) fn LLVMCreateFunctionPassManagerForModule<'a>( + M: &'a Module, + ) -> &'a mut PassManager<'a>; + + /// Disposes a pass manager. + pub(crate) fn LLVMDisposePassManager<'a>(PM: &'a mut PassManager<'a>); + + /// Runs a pass manager on a module. + pub(crate) fn LLVMRunPassManager<'a>(PM: &PassManager<'a>, M: &'a Module) -> Bool; + + pub(crate) fn LLVMTimeTraceProfilerFinish(FileName: *const c_char); + + pub(crate) fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); + + pub(crate) fn LLVMPassManagerBuilderCreate() -> &'static mut PassManagerBuilder; + pub(crate) fn LLVMPassManagerBuilderDispose(PMB: &'static mut PassManagerBuilder); + pub(crate) fn LLVMPassManagerBuilderSetSizeLevel(PMB: &PassManagerBuilder, Value: Bool); + pub(crate) fn LLVMPassManagerBuilderSetDisableUnrollLoops( + PMB: &PassManagerBuilder, + Value: Bool, + ); + pub(crate) fn LLVMPassManagerBuilderUseInlinerWithThreshold( + PMB: &PassManagerBuilder, + threshold: c_uint, + ); + pub(crate) fn LLVMPassManagerBuilderPopulateModulePassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + ); + + pub(crate) fn LLVMPassManagerBuilderPopulateFunctionPassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + ); + pub(crate) fn LLVMPassManagerBuilderPopulateLTOPassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + Internalize: Bool, + RunInliner: Bool, + ); + pub(crate) fn LLVMRustPassManagerBuilderPopulateThinLTOPassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + ); + + // functions that cg_llvm doesnt use but we do. mostly for int_replace. + pub(crate) fn LLVMGetReturnType(FunctionTy: &Type) -> &Type; + pub(crate) fn LLVMGetParams(Fn: &Value, Params: *mut &Value); + pub(crate) fn LLVMGetEntryBasicBlock(Fn: &Value) -> &BasicBlock; + pub(crate) fn LLVMGetNamedFunction(M: &Module, Name: *const c_char) -> &Value; + pub(crate) fn LLVMRustGetFunctionReturnType(V: &Value) -> &Type; + + pub(crate) fn LLVMSetTarget(M: &Module, Triple: *const c_char); + + // Create and destroy contexts. + pub(crate) fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; + pub(crate) fn LLVMContextDispose(C: &'static mut Context); + + // Create modules. + pub(crate) fn LLVMModuleCreateWithNameInContext( + ModuleID: *const c_char, + C: &Context, + ) -> &Module; + + pub(crate) fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); + + pub(crate) fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t); + + /// See llvm::LLVMTypeKind::getTypeID. + pub(crate) fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; + + pub(crate) fn LLVMPrintTypeToString(Val: &Type) -> *mut c_char; + + // Operations on integer types + pub(crate) fn LLVMInt1TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt8TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt16TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt32TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMInt64TypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type; + + pub(crate) fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; + + // Operations on real types + pub(crate) fn LLVMHalfTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type; + pub(crate) fn LLVMFloatTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMDoubleTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMFP128TypeInContext(C: &Context) -> &Type; + + // Operations on function types + pub(crate) fn LLVMFunctionType<'a>( + ReturnType: &'a Type, + ParamTypes: *const &'a Type, + ParamCount: c_uint, + IsVarArg: Bool, + ) -> &'a Type; + pub(crate) fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint; + pub(crate) fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type); + + // Operations on struct types + pub(crate) fn LLVMStructTypeInContext<'a>( + C: &'a Context, + ElementTypes: *const &'a Type, + ElementCount: c_uint, + Packed: Bool, + ) -> &'a Type; + pub(crate) fn LLVMGetStructElementTypes<'a>(StructTy: &'a Type, Dest: *mut &'a Type); + pub(crate) fn LLVMCountStructElementTypes(StructTy: &Type) -> c_uint; + pub(crate) fn LLVMIsPackedStruct(StructTy: &Type) -> Bool; + pub(crate) fn LLVMStructGetTypeAtIndex(StructTy: &Type, i: c_uint) -> &Type; + + // Operations on array, pointer, and vector types (sequence types) + pub(crate) fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type; + //pub(crate) fn LLVMPointerType(ElementType: &Type, AddressSpace: c_uint) -> &Type; + pub(crate) fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; + + pub(crate) fn LLVMGetElementType(Ty: &Type) -> &Type; + pub(crate) fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint; + + // Operations on other types + pub(crate) fn LLVMVoidTypeInContext(C: &Context) -> &Type; + pub(crate) fn LLVMRustMetadataTypeInContext(C: &Context) -> &Type; + + // Operations on all values + pub(crate) fn LLVMIsUndef(Val: &Value) -> Bool; + pub(crate) fn LLVMTypeOf(Val: &Value) -> &Type; + pub(crate) fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char; + pub(crate) fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); + pub(crate) fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); + pub(crate) fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); + pub(crate) fn LLVMPrintValueToString<'a>(Val: &'a Value) -> *mut c_char; + + // Operations on constants of any type + pub(crate) fn LLVMConstNull(Ty: &Type) -> &Value; + pub(crate) fn LLVMGetUndef(Ty: &Type) -> &Value; + + // Operations on metadata + pub(crate) fn LLVMMDStringInContext(C: &Context, Str: *const c_char, SLen: c_uint) -> &Value; + pub(crate) fn LLVMMDNodeInContext<'a>( + C: &'a Context, + Vals: *const &'a Value, + Count: c_uint, + ) -> &'a Value; + pub(crate) fn LLVMAddNamedMetadataOperand<'a>( + M: &'a Module, + Name: *const c_char, + Val: &'a Value, + ); + + // Operations on scalar constants + pub(crate) fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; + pub(crate) fn LLVMConstIntOfArbitraryPrecision( + IntTy: &Type, + Wn: c_uint, + Ws: *const u64, + ) -> &Value; + pub(crate) fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; + pub(crate) fn LLVMConstIntGetZExtValue(ConstantVal: &ConstantInt) -> c_ulonglong; + pub(crate) fn LLVMRustConstInt128Get( + ConstantVal: &ConstantInt, + SExt: bool, + high: &mut u64, + low: &mut u64, + ) -> bool; + + // Operations on composite constants + pub(crate) fn LLVMConstStringInContext( + C: &Context, + Str: *const c_char, + Length: c_uint, + DontNullTerminate: Bool, + ) -> &Value; + pub(crate) fn LLVMConstStructInContext<'a>( + C: &'a Context, + ConstantVals: *const &'a Value, + Count: c_uint, + Packed: Bool, + ) -> &'a Value; + + pub(crate) fn LLVMConstArray<'a>( + ElementTy: &'a Type, + ConstantVals: *const &'a Value, + Length: c_uint, + ) -> &'a Value; + pub(crate) fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; + + // Constant expressions + pub(crate) fn LLVMConstInBoundsGEP2<'a>( + Ty: &'a Type, + ConstantVal: &'a Value, + ConstantIndices: *const &'a Value, + NumIndices: c_uint, + ) -> &'a Value; + // TODO: remove me + //pub(crate) fn LLVMConstZExt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub(crate) fn LLVMConstExtractValue( + AggConstant: &Value, + IdxList: *const c_uint, + NumIdx: c_uint, + ) -> &Value; + + // Operations on global variables, functions, and aliases (globals) + pub(crate) fn LLVMIsDeclaration(Global: &Value) -> Bool; + pub(crate) fn LLVMRustGetLinkage(Global: &Value) -> Linkage; + pub(crate) fn LLVMRustSetLinkage(Global: &Value, RustLinkage: Linkage); + pub(crate) fn LLVMSetSection(Global: &Value, Section: *const c_char); + pub(crate) fn LLVMRustGetVisibility(Global: &Value) -> Visibility; + pub(crate) fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility); + pub(crate) fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); + pub(crate) fn LLVMGetAlignment(Global: &Value) -> c_uint; + pub(crate) fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); + pub(crate) fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); + + // Operations on global variables + pub(crate) fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; + pub(crate) fn LLVMAddGlobal<'a>(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value; + pub(crate) fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>; + pub(crate) fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value; + pub(crate) fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>; + pub(crate) fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>; + pub(crate) fn LLVMDeleteGlobal(GlobalVar: &Value); + pub(crate) fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; + pub(crate) fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); + pub(crate) fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; + pub(crate) fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); + pub(crate) fn LLVMRustGetNamedValue( + M: &Module, + Name: *const c_char, + NameLen: size_t, + ) -> Option<&Value>; + pub(crate) fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); + pub(crate) fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); + + // Operations on functions + pub(crate) fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); + pub(crate) fn LLVMRustAddAlignmentAttr(Fn: &Value, index: c_uint, bytes: u32); + pub(crate) fn LLVMRustAddFunctionAttribute(Fn: &Value, index: c_uint, attr: Attribute); + pub(crate) fn LLVMRustAddFunctionAttributeWithType(Fn: &Value, index: c_uint, attr: Attribute, Ty: &Type); + pub(crate) fn LLVMRustAddFunctionAttrStringValue( + Fn: &Value, + index: c_uint, + Name: *const c_char, + NameLen: size_t, + Value: *const c_char, + ValueLen: size_t, + ); + pub(crate) fn LLVMRustRemoveFunctionAttributes(Fn: &Value, index: c_uint, attr: Attribute); + + // Operations on parameters + pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMCountParams(Fn: &Value) -> c_uint; + pub(crate) fn LLVMGetParam(Fn: &Value, Index: c_uint) -> &Value; + + // Operations on basic blocks + pub(crate) fn LLVMGetBasicBlockParent(BB: &BasicBlock) -> &Value; + pub(crate) fn LLVMAppendBasicBlockInContext<'a>( + C: &'a Context, + Fn: &'a Value, + Name: *const c_char, + ) -> &'a BasicBlock; + + // Operations on instructions + pub(crate) fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; + + // Operations on call sites + pub(crate) fn LLVMRustAddCallSiteAttribute(Instr: &Value, index: c_uint, attr: Attribute); + pub(crate) fn LLVMRustAddCallSiteAttrString(Instr: &Value, index: c_uint, Name: *const c_char); + pub(crate) fn LLVMRustAddAlignmentCallSiteAttr(Instr: &Value, index: c_uint, bytes: u32); + pub(crate) fn LLVMRustAddDereferenceableCallSiteAttr(Instr: &Value, index: c_uint, bytes: u64); + pub(crate) fn LLVMRustAddDereferenceableOrNullCallSiteAttr( + Instr: &Value, + index: c_uint, + bytes: u64, + ); + + // Operations on load/store instructions (only) + pub(crate) fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool); + + // Operations on phi nodes + pub(crate) fn LLVMAddIncoming<'a>( + PhiNode: &'a Value, + IncomingValues: *const &'a Value, + IncomingBlocks: *const &'a BasicBlock, + Count: c_uint, + ); + + // Instruction builders + pub(crate) fn LLVMCreateBuilderInContext<'a>(C: &'a Context) -> &'a mut Builder<'a>; + pub(crate) fn LLVMPositionBuilderAtEnd<'a>(Builder: &Builder<'a>, Block: &'a BasicBlock); + pub(crate) fn LLVMGetInsertBlock<'a>(Builder: &Builder<'a>) -> &'a BasicBlock; + pub(crate) fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); + + pub(crate) fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; + + // Terminators + pub(crate) fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; + pub(crate) fn LLVMBuildRet<'a>(B: &Builder<'a>, V: &'a Value) -> &'a Value; + pub(crate) fn LLVMBuildBr<'a>(B: &Builder<'a>, Dest: &'a BasicBlock) -> &'a Value; + pub(crate) fn LLVMBuildCondBr<'a>( + B: &Builder<'a>, + If: &'a Value, + Then: &'a BasicBlock, + Else: &'a BasicBlock, + ) -> &'a Value; + pub(crate) fn LLVMBuildSwitch<'a>( + B: &Builder<'a>, + V: &'a Value, + Else: &'a BasicBlock, + NumCases: c_uint, + ) -> &'a Value; + + // Add a case to the switch instruction + pub(crate) fn LLVMAddCase<'a>(Switch: &'a Value, OnVal: &'a Value, Dest: &'a BasicBlock); + + // Arithmetic + pub(crate) fn LLVMBuildAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildFAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildFSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildFMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildUDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildExactUDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildSDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildExactSDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildFDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildURem<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildSRem<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildFRem<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildShl<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildLShr<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildAShr<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildNSWAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildNUWAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildNSWSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildNUWSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildNSWMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildNUWMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildAnd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildOr<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildXor<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) + -> &'a Value; + pub(crate) fn LLVMBuildFNeg<'a>( + B: &Builder<'a>, + V: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) + -> &'a Value; + pub(crate) fn LLVMRustSetFastMath(Instr: &Value); + pub(crate) fn LLVMRustSetAlgebraicMath(Instr: &Value); + + // Memory + pub(crate) fn LLVMBuildAlloca<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildArrayAlloca<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Val: &'a Value, + Name: *const c_char, + ) -> &'a Value; + // TODO: remove me + /*pub(crate) fn LLVMBuildLoad<'a>( + B: &Builder<'a>, + PointerVal: &'a Value, + Name: *const c_char, + ) -> &'a Value;*/ + // TODO: do not put llvm v19 code here + pub(crate) fn LLVMBuildLoad2<'a>( + B: &Builder<'a>, + Ty: &'a Type, // New: explicit type parameter + PointerVal: &'a Value, + Name: *const c_char, + ) -> &'a Value; + + pub(crate) fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value; + + pub(crate) fn LLVMBuildGEP2<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Pointer: &'a Value, + Indices: *const &'a Value, + NumIndices: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildInBoundsGEP2<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Pointer: &'a Value, + Indices: *const &'a Value, + NumIndices: c_uint, + Name: *const c_char, + ) -> &'a Value; + + // Casts + pub(crate) fn LLVMBuildTrunc<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildZExt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildSExt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildFPToUI<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildFPToSI<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildUIToFP<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildSIToFP<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildFPTrunc<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildFPExt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildPtrToInt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildIntToPtr<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildBitCast<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildPointerCast<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildIntCast<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + IsSized: bool, + ) -> &'a Value; + + // Comparisons + pub(crate) fn LLVMBuildICmp<'a>( + B: &Builder<'a>, + Op: c_uint, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildFCmp<'a>( + B: &Builder<'a>, + Op: c_uint, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + + // Miscellaneous instructions + pub(crate) fn LLVMBuildPhi<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) + -> &'a Value; + pub(crate) fn LLVMRustGetInstrProfIncrementIntrinsic<'a>(M: &Module) -> &'a Value; + pub(crate) fn LLVMRustBuildMemCpy<'a>( + B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Src: &'a Value, + SrcAlign: c_uint, + Size: &'a Value, + IsVolatile: bool, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildMemMove<'a>( + B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Src: &'a Value, + SrcAlign: c_uint, + Size: &'a Value, + IsVolatile: bool, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildMemSet<'a>( + B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Val: &'a Value, + Size: &'a Value, + IsVolatile: bool, + ) -> &'a Value; + pub(crate) fn LLVMBuildSelect<'a>( + B: &Builder<'a>, + If: &'a Value, + Then: &'a Value, + Else: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildVAArg<'a>( + B: &Builder<'a>, + list: &'a Value, + Ty: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildExtractElement<'a>( + B: &Builder<'a>, + VecVal: &'a Value, + Index: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildInsertElement<'a>( + B: &Builder<'a>, + VecVal: &'a Value, + EltVal: &'a Value, + Index: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildShuffleVector<'a>( + B: &Builder<'a>, + V1: &'a Value, + V2: &'a Value, + Mask: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildExtractValue<'a>( + B: &Builder<'a>, + AggVal: &'a Value, + Index: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub(crate) fn LLVMBuildInsertValue<'a>( + B: &Builder<'a>, + AggVal: &'a Value, + EltVal: &'a Value, + Index: c_uint, + Name: *const c_char, + ) -> &'a Value; + + pub(crate) fn LLVMRustBuildVectorReduceFAdd<'a>( + B: &Builder<'a>, + Acc: &'a Value, + Src: &'a Value, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceFMul<'a>( + B: &Builder<'a>, + Acc: &'a Value, + Src: &'a Value, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceAdd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceMul<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceAnd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceOr<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceXor<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceMin<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsSigned: bool, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceMax<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsSigned: bool, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceFMin<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsNaN: bool, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildVectorReduceFMax<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsNaN: bool, + ) -> &'a Value; + + pub(crate) fn LLVMRustBuildMinNum<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + ) -> &'a Value; + pub(crate) fn LLVMRustBuildMaxNum<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + ) -> &'a Value; + + pub(crate) fn LLVMDisposeMessage(message: *mut c_char); + + /// Returns a string describing the last error caused by an LLVMRust* call. + pub(crate) fn LLVMRustGetLastError() -> *const c_char; + + pub(crate) fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type; + + pub(crate) fn LLVMStructSetBody<'a>( + StructTy: &'a Type, + ElementTypes: *const &'a Type, + ElementCount: c_uint, + Packed: Bool, + ); + + /// Prepares inline assembly. + pub(crate) fn LLVMRustInlineAsm( + Ty: &Type, + AsmString: *const c_char, + AsmStringLen: size_t, + Constraints: *const c_char, + ConstraintsLen: size_t, + SideEffects: Bool, + AlignStack: Bool, + Dialect: AsmDialect, + CanThrow: Bool, + ) -> &Value; + pub(crate) fn LLVMRustInlineAsmVerify( + Ty: &Type, + Constraints: *const c_char, + ConstraintsLen: size_t, + ) -> bool; + + pub(crate) fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; + + pub(crate) fn LLVMRustPrintModule<'a>( + M: &'a Module, + Output: *const c_char, + OutputLen: size_t, + Demangle: extern "C" fn(*const c_char, size_t, *mut c_char, size_t) -> size_t, + ) -> LLVMRustResult; + + pub(crate) fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer; + pub(crate) fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8; + pub(crate) fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; + pub(crate) fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); + pub(crate) fn LLVMRustModuleCost(M: &Module) -> u64; + + pub(crate) fn LLVMRustThinLTOBufferCreate( + M: &Module, + is_thin: Bool, + emit_summary: Bool + ) -> &'static mut ThinLTOBuffer; + pub(crate) fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); + pub(crate) fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; + pub(crate) fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; + pub(crate) fn LLVMRustCreateThinLTOData( + Modules: *const ThinLTOModule, + NumModules: c_uint, + PreservedSymbols: *const *const c_char, + PreservedSymbolsLen: c_uint, + ) -> Option<&'static mut ThinLTOData>; + pub(crate) fn LLVMRustPrepareThinLTOResolveWeak(Data: &ThinLTOData, Module: &Module) -> bool; + pub(crate) fn LLVMRustPrepareThinLTOInternalize(Data: &ThinLTOData, Module: &Module) -> bool; + pub(crate) fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); + pub(crate) fn LLVMRustParseBitcodeForLTO( + Context: &Context, + Data: *const u8, + len: usize, + Identifier: *const c_char, + ) -> Option<&Module>; + pub(crate) fn LLVMRustGetBitcodeSliceFromObjectData( + Data: *const u8, + len: usize, + out_len: &mut usize, + ) -> *const u8; + pub(crate) fn LLVMRustThinLTOGetDICompileUnit( + M: &Module, + CU1: &mut *mut c_void, + CU2: &mut *mut c_void, + ); + pub(crate) fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); + + pub(crate) fn LLVMRustAddDereferenceableAttr(Fn: &Value, index: c_uint, bytes: u64); + pub(crate) fn LLVMRustAddDereferenceableOrNullAttr(Fn: &Value, index: c_uint, bytes: u64); + + pub(crate) fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); + + // TODO: LLVM v19 pass functions shouldn't be here + pub(crate) fn LLVMCreatePassBuilderOptions() -> &'static mut PassBuilderOptions; + pub(crate) fn LLVMDisposePassBuilderOptions(Options: &'static mut PassBuilderOptions); + pub(crate) fn LLVMPassBuilderOptionsSetVerifyEach( + Options: &PassBuilderOptions, + VerifyEach: Bool, + ); + pub(crate) fn LLVMPassBuilderOptionsSetDebugLogging( + Options: &PassBuilderOptions, + DebugLogging: Bool, + ); + pub(crate) fn LLVMRunPasses( + M: &Module, + Passes: *const c_char, + TM: *const TargetMachine, + Options: &PassBuilderOptions, + ) -> Bool; + + pub(crate) fn LLVMVerifyModule( + M: &Module, + Action: LLVMVerifierFailureAction, + OutMessage: *mut *mut c_char, + ) -> Bool; + + pub(crate) fn LLVMDumpModule( + M: &Module, + ); + + pub(crate) fn LLVMRustSetOldDebugFormat(M: &Module); + pub(crate) fn LLVMStripModuleDebugInfo(M: &Module); + + pub(crate) fn LLVMRustVerifyFunction(Fn: &Value, Action: LLVMVerifierFailureAction) -> Bool; +} diff --git a/crates/rustc_codegen_nvvm_v19/src/lto.rs b/crates/rustc_codegen_nvvm_v19/src/lto.rs new file mode 100644 index 00000000..3132d041 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/lto.rs @@ -0,0 +1,196 @@ +use std::ffi::CString; +use std::sync::Arc; + +use rustc_codegen_ssa::{ + ModuleCodegen, + back::{ + lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}, + write::CodegenContext, + }, + traits::{ModuleBufferMethods, ThinBufferMethods}, +}; +use rustc_errors::{DiagCtxtHandle, FatalError}; +use rustc_middle::dep_graph::WorkProduct; +use tracing::{debug, trace}; +use crate::llvm::{self, True}; +use crate::common::AsCCharPtr; +use crate::NvvmCodegenBackend; +use crate::LlvmMod; + +pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); + +unsafe impl Send for ModuleBuffer {} +unsafe impl Sync for ModuleBuffer {} + +impl ModuleBuffer { + pub(crate) fn new(m: &llvm::Module) -> ModuleBuffer { + ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) }) + } +} + +impl ModuleBufferMethods for ModuleBuffer { + fn data(&self) -> &[u8] { + unsafe { + trace!("Retrieving data in module buffer"); + let ptr = llvm::LLVMRustModuleBufferPtr(self.0); + let len = llvm::LLVMRustModuleBufferLen(self.0); + std::slice::from_raw_parts(ptr, len) + } + } +} + +impl Drop for ModuleBuffer { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustModuleBufferFree(&mut *(self.0 as *mut _)); + } + } +} + +pub struct ThinBuffer(&'static mut llvm::ThinLTOBuffer); + +unsafe impl Send for ThinBuffer {} +unsafe impl Sync for ThinBuffer {} + +impl ThinBuffer { + pub(crate) fn new(m: &llvm::Module) -> ThinBuffer { + unsafe { + // TODO: do not hardcode these + let is_thin = True; + let emit_summary = True; + + // TODO: temporary dumping module to try to catch what is invalid + //llvm::LLVMDumpModule(m); + + let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin, emit_summary); + ThinBuffer(buffer) + } + } +} + +impl ThinBufferMethods for ThinBuffer { + fn data(&self) -> &[u8] { + unsafe { + trace!("Retrieving data in thin buffer"); + let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; + + let len = llvm::LLVMRustThinLTOBufferLen(self.0); + + std::slice::from_raw_parts(ptr, len) + } + } + + fn thin_link_data(&self) -> &[u8] { + todo!() + } +} + +impl Drop for ThinBuffer { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _)); + } + } +} + +pub struct ThinData(&'static mut llvm::ThinLTOData); + +unsafe impl Send for ThinData {} +unsafe impl Sync for ThinData {} + +impl Drop for ThinData { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustFreeThinLTOData(&mut *(self.0 as *mut _)); + } + } +} + +// essentially does nothing for now. +pub(crate) fn run_thin( + _cgcx: &CodegenContext, + modules: Vec<(String, ThinBuffer)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, +) -> Result<(Vec>, Vec), FatalError> { + debug!("Running thin LTO"); + let mut thin_buffers = Vec::with_capacity(modules.len()); + let mut module_names = Vec::with_capacity(modules.len() + cached_modules.len()); + // let thin_modules = Vec::with_capacity(modules.len() + cached_modules.len()); + + for (name, buf) in modules { + let cname = CString::new(name.clone()).unwrap(); + // thin_modules.push( + // llvm::ThinLTOModule { + // identifier: cname.as_ptr(), + // data: buf.data().as_ptr(), + // len: buf.data().len() + // } + // ); + thin_buffers.push(buf); + module_names.push(cname); + } + + let mut serialized_modules = Vec::with_capacity(cached_modules.len()); + + for (sm, wp) in cached_modules { + let _slice_u8 = sm.data(); + serialized_modules.push(sm); + module_names.push(CString::new(wp.cgu_name).unwrap()); + } + + let shared = Arc::new(ThinShared { + data: (), + thin_buffers, + serialized_modules, + module_names, + }); + + let mut opt_jobs = vec![]; + for (module_index, _) in shared.module_names.iter().enumerate() { + opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { + shared: shared.clone(), + idx: module_index, + })); + } + + Ok((opt_jobs, vec![])) +} + +pub(crate) unsafe fn optimize_thin( + cgcx: &CodegenContext, + thin_module: ThinModule, +) -> Result, FatalError> { + // essentially does nothing + let dcx = cgcx.create_dcx(); + let dcx = dcx.handle(); + + let module_name = &thin_module.shared.module_names[thin_module.idx]; + + let llcx = unsafe { llvm::LLVMRustContextCreate(cgcx.fewer_names) }; + let llmod = + parse_module(llcx, module_name.to_str().unwrap(), thin_module.data(), dcx)? as *const _; + + let module = + ModuleCodegen::new_regular(thin_module.name().to_string(), LlvmMod { llcx, llmod }); + Ok(module) +} + +pub(crate) fn parse_module<'a>( + cx: &'a llvm::Context, + name: &str, + data: &[u8], + dcx: DiagCtxtHandle<'_>, +) -> Result<&'a llvm::Module, FatalError> { + unsafe { + llvm::LLVMRustParseBitcodeForLTO( + cx, + data.as_ptr(), + data.len(), + name.as_c_char_ptr(), + ) + .ok_or_else(|| { + let msg = "failed to parse bitcode for LTO module"; + crate::back::llvm_err(dcx, msg) + }) + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/mono_item.rs b/crates/rustc_codegen_nvvm_v19/src/mono_item.rs new file mode 100644 index 00000000..d5e2a3dd --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/mono_item.rs @@ -0,0 +1,126 @@ +use crate::abi::FnAbiLlvmExt; +use crate::attributes; +use crate::attributes::NvvmAttributes; +use crate::consts::linkage_to_llvm; +use crate::context::CodegenCx; +use crate::llvm; +use crate::ty::LayoutLlvmExt; +use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::ty::TypeVisitableExt; +use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; +use rustc_middle::ty::{self, Instance}; +use tracing::trace; + +pub(crate) fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { + match linkage { + Visibility::Default => llvm::Visibility::Default, + Visibility::Hidden => llvm::Visibility::Hidden, + Visibility::Protected => llvm::Visibility::Protected, + } +} + +impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { + fn predefine_static( + &self, + def_id: DefId, + linkage: Linkage, + visibility: Visibility, + symbol_name: &str, + ) { + trace!("Predefining static with name `{}`", symbol_name); + let instance = Instance::mono(self.tcx, def_id); + let ty = instance.ty(self.tcx, self.typing_env()); + let llty = self.layout_of(ty).llvm_type(self); + let addrspace = self.static_addrspace(instance); + + let g = self + .define_global(symbol_name, llty, addrspace) + .unwrap_or_else(|| { + self.sess().dcx().span_fatal( + self.tcx.def_span(def_id), + format!("symbol `{}` is already defined", symbol_name), + ) + }); + + unsafe { + llvm::LLVMRustSetLinkage(g, linkage_to_llvm(linkage)); + llvm::LLVMRustSetVisibility(g, visibility_to_llvm(visibility)); + } + + self.instances.borrow_mut().insert(instance, g); + } + + fn predefine_fn( + &self, + instance: Instance<'tcx>, + linkage: Linkage, + visibility: Visibility, + symbol_name: &str, + ) { + trace!( + "Predefining function with name `{}` with linkage `{:?}` and attributes `{:?}`", + symbol_name, + linkage, + self.tcx.codegen_fn_attrs(instance.def_id()) + ); + assert!(!instance.args.has_infer()); + + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + + let lldecl = self.declare_fn(symbol_name, fn_abi.llvm_type(self), Some(fn_abi)); + + unsafe { llvm::LLVMRustSetLinkage(lldecl, linkage_to_llvm(linkage)) }; + + // If we're compiling the compiler-builtins crate, e.g., the equivalent of + // compiler-rt, then we want to implicitly compile everything with hidden + // visibility as we're going to link this object all over the place but + // don't want the symbols to get exported. + if linkage != Linkage::Internal && self.tcx.is_compiler_builtins(LOCAL_CRATE) { + unsafe { + llvm::LLVMRustSetVisibility(lldecl, llvm::Visibility::Hidden); + } + } else { + unsafe { + llvm::LLVMRustSetVisibility(lldecl, visibility_to_llvm(visibility)); + } + } + + attributes::from_fn_attrs(self, lldecl, instance); + + let def_id = instance.def_id(); + let attrs = self.tcx.get_attrs_unchecked(def_id); // TODO: Replace with get_attrs + let nvvm_attrs = NvvmAttributes::parse(self, attrs); + + unsafe { + // if this function is marked as being a kernel, add it + // to nvvm.annotations per the nvvm ir docs. + if nvvm_attrs.kernel { + trace!("Marking function `{:?}` as a kernel", symbol_name); + let kernel = llvm::LLVMMDStringInContext(self.llcx, "kernel".as_ptr().cast(), 6); + let mdvals = &[lldecl, kernel, self.const_i32(1)]; + let node = + llvm::LLVMMDNodeInContext(self.llcx, mdvals.as_ptr(), mdvals.len() as u32); + llvm::LLVMAddNamedMetadataOperand( + self.llmod, + c"nvvm.annotations".as_ptr().cast(), + node, + ); + } + if nvvm_attrs.used { + trace!("Marking function `{:?}` as used", symbol_name); + let mdvals = &[lldecl]; + let node = + llvm::LLVMMDNodeInContext(self.llcx, mdvals.as_ptr(), mdvals.len() as u32); + llvm::LLVMAddNamedMetadataOperand( + self.llmod, + c"cg_nvvm_used".as_ptr().cast(), + node, + ); + } + } + + self.instances.borrow_mut().insert(instance, lldecl); + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/nvvm.rs b/crates/rustc_codegen_nvvm_v19/src/nvvm.rs new file mode 100644 index 00000000..ab0df8a9 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/nvvm.rs @@ -0,0 +1,365 @@ +//! Final steps in codegen, coalescing modules and feeding them to libnvvm. + +use crate::back::demangle_callback; +use crate::builder::unnamed; +use crate::common::AsCCharPtr; +use crate::context::CodegenArgs; +use crate::llvm; +use crate::llvm::{Context, Linkage, Module, Value, Visibility, False, True}; +use crate::lto::ThinBuffer; +use nvvm::{NvvmError, NvvmProgram, LIBDEVICE_BITCODE}; +use rustc_codegen_ssa::traits::ThinBufferMethods; +use rustc_session::{Session, config::DebugInfo}; +use std::fmt::Display; +use std::marker::PhantomData; +use std::ptr; +use tracing::debug; + +// see libintrinsics.ll on what this is. +const LIBINTRINSICS: &[u8] = include_bytes!("../libintrinsics.bc"); + +pub enum CodegenErr { + Nvvm(NvvmError), + Io(std::io::Error), +} + +impl From for CodegenErr { + fn from(v: std::io::Error) -> Self { + Self::Io(v) + } +} + +impl From for CodegenErr { + fn from(v: NvvmError) -> Self { + Self::Nvvm(v) + } +} + +impl Display for CodegenErr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Nvvm(err) => std::fmt::Display::fmt(&err, f), + Self::Io(err) => std::fmt::Display::fmt(&err, f), + } + } +} + +/// Take a list of bitcode module bytes and their names and codegen it +/// into ptx bytes. The final PTX *should* be utf8, but just to be on the safe side +/// it returns a vector of bytes. +/// +/// Note that this will implicitly try to find libdevice and add it, so don't do that +/// step before this. It will fatal error if it cannot find it. +pub fn codegen_bitcode_modules( + args: &CodegenArgs, + sess: &Session, + modules: Vec>, + llcx: &llvm::Context, +) -> Result, CodegenErr> { + debug!("Codegenning bitcode to PTX"); + + // make sure the nvvm ir version is high enough so users don't get confusing compilation errors. + let (ir_major, ir_minor) = nvvm::ir_version(); + if ir_major != 2 || ir_minor != 0 { + sess.dcx() + .fatal(format!("rustc_codegen_nvvm requires nvvm IR version 2.0, got {}.{}", ir_major, ir_minor)); + } + + // TODO: make sure /usr/local/cuda/nvvm/lib64/libnvvm.so is v4 somehow... + + // first, create the nvvm program we will add modules to. + let prog = NvvmProgram::new()?; + + let module = merge_llvm_modules(modules, llcx); + unsafe { + internalize_pass(module, llcx); + dce_pass(module); + + if sess.opts.debuginfo != DebugInfo::None { + cleanup_dicompileunit(module); + } + + let (dbg_major, dbg_minor) = nvvm::dbg_version(); + + // needed for debug info or else nvvm complains about ir version mismatch for some + // reason. It works if you don't use debug info though... + let ty_i32 = llvm::LLVMInt32TypeInContext(llcx); + let ir_major = llvm::LLVMConstInt(ty_i32, ir_major as u64, False); + let ir_minor = llvm::LLVMConstInt(ty_i32, ir_minor as u64, False); + let dbg_major = llvm::LLVMConstInt(ty_i32, dbg_major as u64, False); + let dbg_minor = llvm::LLVMConstInt(ty_i32, dbg_minor as u64, False); + let vals = [ir_major, ir_minor, dbg_major, dbg_minor]; + let node = llvm::LLVMMDNodeInContext(llcx, vals.as_ptr(), vals.len() as u32); + + llvm::LLVMAddNamedMetadataOperand(module, c"nvvmir.version".as_ptr().cast(), node); + + if let Some(path) = &args.final_module_path { + let out = path.to_str().unwrap(); + let result = + llvm::LLVMRustPrintModule(module, out.as_c_char_ptr(), out.len(), demangle_callback); + result + .into_result() + .expect("Failed to write final llvm module output"); + } + } + + let buf = ThinBuffer::new(module); + + prog.add_module(buf.data(), "merged".to_string())?; + prog.add_lazy_module(LIBDEVICE_BITCODE, "libdevice".to_string())?; + prog.add_lazy_module(LIBINTRINSICS, "libintrinsics".to_string())?; + + // for now, while the codegen is young, we always run verification on the program. + // This is to make debugging much easier, libnvvm tends to infinitely loop or segfault on invalid programs + // which makes debugging extremely hard. This way, if a malformed program is created, it is caught before + // giving it to libnvvm. Then to debug codegen failures, we can just ask the user to provide the corresponding llvm ir + // file with --emit=llvm-ir + + let verification_res = prog.verify(&args.nvvm_options); + if verification_res.is_err() { + let log = prog.compiler_log().unwrap().unwrap_or_default(); + let footer = "If you plan to submit a bug report please re-run the codegen with `RUSTFLAGS=\"--emit=llvm-ir\" and include the .ll file corresponding to the .o file mentioned in the log"; + panic!( + "Malformed NVVM IR program rejected by libnvvm, dumping verifier log:\n\n{}\n\n{}", + log, footer + ); + } + + let res = match prog.compile(&args.nvvm_options) { + Ok(b) => b, + Err(error) => { + // this should never happen, if it does, something went really bad or its a bug on libnvvm's end + panic!( + "libnvvm returned an error that was not previously caught by the verifier: {:?}", + error + ); + } + }; + + Ok(res) +} + +unsafe fn cleanup_dicompileunit(module: &llvm::Module) { + unsafe { + let mut cu1 = ptr::null_mut(); + let mut cu2 = ptr::null_mut(); + llvm::LLVMRustThinLTOGetDICompileUnit(module, &mut cu1, &mut cu2); + llvm::LLVMRustThinLTOPatchDICompileUnit(module, cu1); + } +} + +// Merging and DCE (dead code elimination) logic. Inspired a lot by rust-ptx-linker. +// +// This works in a couple of steps starting from the bitcode of every single module (crate), then: +// - Merge all of the modules into a single large module, basically fat LTO. In the future we could probably lazily-load only +// the things we need using dependency graphs, like we used to do for libnvvm. +// - Iterate over every function in the module and: +// - If it is not a kernel and it is not a declaration (i.e. an extern fn) then mark its linkage as internal and its visiblity as default +// - Iterate over every global in the module and: +// - Same as functions, if it is not an external declaration, mark it as internal. +// - run LLVM's global DCE pass, this will remove any functions and globals that are not directly or indirectly used by kernels. + +fn merge_llvm_modules(modules: Vec>, llcx: &Context) -> &Module { + let module = unsafe { crate::create_module(llcx, "merged_modules") }; + for merged_module in modules { + unsafe { + let tmp = llvm::LLVMRustParseBitcodeForLTO( + llcx, + merged_module.as_ptr(), + merged_module.len(), + unnamed(), + ) + .expect("Failed to parse module bitcode"); + llvm::LLVMLinkModules2(module, tmp); + } + } + module +} + +struct FunctionIter<'a, 'll> { + module: PhantomData<&'a &'ll Module>, + next: Option<&'ll Value>, +} + +struct GlobalIter<'a, 'll> { + module: PhantomData<&'a &'ll Module>, + next: Option<&'ll Value>, +} + +impl<'a, 'll> FunctionIter<'a, 'll> { + pub fn new(module: &'a &'ll Module) -> Self { + FunctionIter { + module: PhantomData, + next: unsafe { llvm::LLVMGetFirstFunction(module) }, + } + } +} + +impl<'ll> Iterator for FunctionIter<'_, 'll> { + type Item = &'ll Value; + + fn next(&mut self) -> Option<&'ll Value> { + let next = self.next; + + self.next = match next { + Some(next) => unsafe { llvm::LLVMGetNextFunction(next) }, + None => None, + }; + + next + } +} + +impl<'a, 'll> GlobalIter<'a, 'll> { + pub fn new(module: &'a &'ll Module) -> Self { + GlobalIter { + module: PhantomData, + next: unsafe { llvm::LLVMGetFirstGlobal(module) }, + } + } +} + +impl<'ll> Iterator for GlobalIter<'_, 'll> { + type Item = &'ll Value; + + fn next(&mut self) -> Option<&'ll Value> { + let next = self.next; + + self.next = match next { + Some(next) => unsafe { llvm::LLVMGetNextGlobal(next) }, + None => None, + }; + + next + } +} + +unsafe fn internalize_pass(module: &Module, cx: &Context) { + unsafe { + // collect the values of all the declared kernels + let num_operands = + llvm::LLVMGetNamedMetadataNumOperands(module, c"nvvm.annotations".as_ptr().cast()) as usize; + let mut operands = Vec::with_capacity(num_operands); + llvm::LLVMGetNamedMetadataOperands( + module, + c"nvvm.annotations".as_ptr().cast(), + operands.as_mut_ptr(), + ); + operands.set_len(num_operands); + let mut kernels = Vec::with_capacity(num_operands); + let kernel_str = llvm::LLVMMDStringInContext(cx, "kernel".as_ptr().cast(), 6); + + for mdnode in operands { + let num_operands = llvm::LLVMGetMDNodeNumOperands(mdnode) as usize; + let mut operands = Vec::with_capacity(num_operands); + llvm::LLVMGetMDNodeOperands(mdnode, operands.as_mut_ptr()); + operands.set_len(num_operands); + + if operands.get(1) == Some(&kernel_str) { + kernels.push(operands[0]); + } + } + + // see what functions are marked as externally visible by the user. + let num_operands = + llvm::LLVMGetNamedMetadataNumOperands(module, c"cg_nvvm_used".as_ptr().cast()) as usize; + let mut operands = Vec::with_capacity(num_operands); + llvm::LLVMGetNamedMetadataOperands( + module, + c"cg_nvvm_used".as_ptr().cast(), + operands.as_mut_ptr(), + ); + operands.set_len(num_operands); + let mut used_funcs = Vec::with_capacity(num_operands); + + for mdnode in operands { + let num_operands = llvm::LLVMGetMDNodeNumOperands(mdnode) as usize; + let mut operands = Vec::with_capacity(num_operands); + llvm::LLVMGetMDNodeOperands(mdnode, operands.as_mut_ptr()); + operands.set_len(num_operands); + + used_funcs.push(operands[0]); + } + + let iter = FunctionIter::new(&module); + for func in iter { + let is_kernel = kernels.contains(&func); + let is_decl = llvm::LLVMIsDeclaration(func) == True; + let is_used = used_funcs.contains(&func); + + if !is_decl && !is_kernel { + llvm::LLVMRustSetLinkage(func, Linkage::InternalLinkage); + llvm::LLVMRustSetVisibility(func, Visibility::Default); + } + + // explicitly set it to external just in case the codegen set them to internal for some reason + if is_used { + llvm::LLVMRustSetLinkage(func, Linkage::ExternalLinkage); + llvm::LLVMRustSetVisibility(func, Visibility::Default); + } + } + + let iter = GlobalIter::new(&module); + for func in iter { + let is_decl = llvm::LLVMIsDeclaration(func) == True; + + if !is_decl { + llvm::LLVMRustSetLinkage(func, Linkage::InternalLinkage); + llvm::LLVMRustSetVisibility(func, Visibility::Default); + } + } + } +} + +// TODO: remove llvmv7 code +/*unsafe fn dce_pass(module: &Module) { + unsafe { + let pass_manager = llvm::LLVMCreatePassManager(); + llvm::LLVMAddGlobalDCEPass(pass_manager); + llvm::LLVMRunPassManager(pass_manager, module); + llvm::LLVMDisposePassManager(pass_manager); + } +}*/ + +// TODO: remove llvmv19 code +// TODO: this null_tm transmute thing is really bad +/* +an alternative? + +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Passes/PassPlugin.h" +#include "llvm/Analysis/LoopAnalysisManager.h" +#include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/Transforms/IPO/GlobalDCE.h" +#include "llvm/IR/PassManager.h" + +extern "C" int LLVMRustRunGlobalDCEOnly(LLVMModuleRef M) { + auto *Module = unwrap(M); + + llvm::LoopAnalysisManager LAM; + llvm::FunctionAnalysisManager FAM; + llvm::CGSCCAnalysisManager CGAM; + llvm::ModuleAnalysisManager MAM; + + llvm::PassBuilder PB; + PB.registerModuleAnalyses(MAM); + PB.registerCGSCCAnalyses(CGAM); + PB.registerFunctionAnalyses(FAM); + PB.registerLoopAnalyses(LAM); + PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); + + llvm::ModulePassManager MPM; + MPM.addPass(llvm::GlobalDCEPass()); + + MPM.run(*Module, MAM); + return 0; +} +*/ +unsafe fn dce_pass(module: &Module) { + unsafe { + let options = llvm::LLVMCreatePassBuilderOptions(); + let passes = c"globaldce".as_ptr(); + llvm::LLVMRunPasses(module, passes, std::ptr::null(), options); + llvm::LLVMDisposePassBuilderOptions(options); + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/override_fns.rs b/crates/rustc_codegen_nvvm_v19/src/override_fns.rs new file mode 100644 index 00000000..ed0bcbe6 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/override_fns.rs @@ -0,0 +1,97 @@ +//! Functions for overriding certain functions in certain crates with special +//! codegen-builtin methods. Currently the only use for this is overriding libm functions +//! with libdevice intrinsics (which are much faster and smaller). + +use crate::abi::FnAbiLlvmExt; +use crate::builder::Builder; +use crate::context::CodegenCx; +use crate::llvm; +use rustc_codegen_ssa::mono_item::MonoItemExt; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::mir::mono::MonoItem; +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::{self, Instance}; + +/// Either override or define a function. +pub(crate) fn define_or_override_fn<'tcx>(func: Instance<'tcx>, cx: &CodegenCx<'_, 'tcx>) { + if should_override(func, cx) { + override_libm_function(func, cx); + } else { + MonoItem::define::>(&MonoItem::Fn(func), cx); + } +} + +fn should_override<'tcx>(func: Instance<'tcx>, cx: &CodegenCx<'_, 'tcx>) -> bool { + if !cx.codegen_args.override_libm { + return false; + } + + if cx.tcx.def_kind(func.def_id()) == rustc_hir::def::DefKind::Closure { + // We don't override closures + return false; + } + + // there is no better way to do this without putting some sort of diagnostic/lang item in libm + let func_crate_name = cx.tcx.crate_name(func.def_id().krate); + let func_crate_name = func_crate_name.as_str(); + let local_crate_name = cx.tcx.crate_name(LOCAL_CRATE); + let local_crate_name = local_crate_name.as_str(); + let is_libm = func_crate_name == "libm" || local_crate_name == "libm"; + if !is_libm { + return false; + } + + let sym = cx.tcx.item_name(func.def_id()); + let name = sym.as_str(); + + if is_unsupported_libdevice_fn(name) { + return false; + } + + let libdevice_name = format!("__nv_{}", name); + let ld_fn = if let Some((args, ret)) = cx.intrinsics_map.borrow().get(libdevice_name.as_str()) { + cx.type_func(args, ret) + } else { + return false; + }; + + // Check the function signatures match. + let lm_fn = cx.fn_abi_of_instance(func, ty::List::empty()).llvm_type(cx); + lm_fn == ld_fn +} + +fn is_unsupported_libdevice_fn(name: &str) -> bool { + // libm functions for which libdevice has no intrinsics for. + const UNSUPPORTED: &[&str] = &[ + // doesnt exist + "lgamma_r", + "lgammaf_r", + // different signatures + "sincos", + "sincosf", + ]; + UNSUPPORTED.contains(&name) +} + +fn override_libm_function<'tcx>(func: Instance<'tcx>, cx: &CodegenCx<'_, 'tcx>) { + let name = cx.tcx.item_name(func.def_id()); + let nv_name = format!("__nv_{}", name.as_str()); + let (intrinsic_llfn_ty, intrinsic_llfn) = cx.get_intrinsic(nv_name.as_str()); + + let llfn = cx.get_fn(func); + let start = Builder::append_block(cx, llfn, "start"); + let mut bx = Builder::build(cx, start); + + let params = llvm::get_params(llfn); + let llcall = bx.call( + intrinsic_llfn_ty, + None, + None, + intrinsic_llfn, + ¶ms, + None, + None, + ); + bx.ret(llcall); +} diff --git a/crates/rustc_codegen_nvvm/src/ptxgen.rs b/crates/rustc_codegen_nvvm_v19/src/ptxgen.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/ptxgen.rs rename to crates/rustc_codegen_nvvm_v19/src/ptxgen.rs diff --git a/crates/rustc_codegen_nvvm_v19/src/target.rs b/crates/rustc_codegen_nvvm_v19/src/target.rs new file mode 100644 index 00000000..b8f72c01 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/target.rs @@ -0,0 +1,57 @@ +use crate::llvm; +use crate::llvm::Type; +use rustc_target::spec::{ + LinkerFlavor, MergeFunctions, PanicStrategy, Target, TargetMetadata, TargetOptions, +}; + +//pub const DATA_LAYOUT: &str = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"; +pub const DATA_LAYOUT: &str = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64-a:8:8"; +pub const TARGET_TRIPLE: &str = "nvptx64-nvidia-cuda"; +pub const POINTER_WIDTH: u32 = 64; + +/// The pointer width of the current target +pub(crate) unsafe fn usize_ty(llcx: &'_ llvm::Context) -> &'_ Type { + unsafe { llvm::LLVMInt64TypeInContext(llcx) } +} + +pub fn target() -> Target { + let mut options = TargetOptions::default(); + + options.os = "cuda".into(); + options.vendor = "nvidia".into(); + options.linker_flavor = LinkerFlavor::Ptx; + // nvvm does all the linking for us, but technically its not a linker + options.linker = None; + options.cpu = "sm_120".into(); + options.max_atomic_width = Some(64); + // Unwinding on CUDA is neither feasible nor useful. + options.panic_strategy = PanicStrategy::Abort; + // Needed to use `dylib` and `bin` crate types and the linker. + options.dynamic_linking = true; + options.executables = true; + options.only_cdylib = true; + + // nvvm does all the work of turning the bitcode into ptx + options.obj_is_bitcode = true; + + options.dll_prefix = "".into(); + options.dll_suffix = ".ptx".into(); + options.exe_suffix = ".ptx".into(); + + // Disable MergeFunctions LLVM optimisation pass because it can + // produce kernel functions that call other kernel functions. + // This behavior is not supported by PTX ISA. + options.merge_functions = MergeFunctions::Disabled; + + Target { + arch: "nvptx".into(), + data_layout: DATA_LAYOUT.into(), + llvm_target: "nvptx64-nvidia-cuda".into(), + pointer_width: POINTER_WIDTH, + options, + metadata: TargetMetadata { + description: Some("NVIDIA CUDA".into()), + ..Default::default() + }, + } +} diff --git a/crates/rustc_codegen_nvvm_v19/src/ty.rs b/crates/rustc_codegen_nvvm_v19/src/ty.rs new file mode 100644 index 00000000..5cff38d4 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v19/src/ty.rs @@ -0,0 +1,677 @@ +use crate::abi::{FnAbiLlvmExt, LlvmType}; +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm::{Bool, False, True, Type, Value}; +use libc::c_uint; +use rustc_abi::Primitive::{Float, Int, Pointer}; +use rustc_abi::{ + AddressSpace, Align, BackendRepr, FieldsShape, Integer, PointeeInfo, Reg, Scalar, Size, + TyAbiInterface, Variants, +}; +use rustc_codegen_ssa::common::TypeKind; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_middle::bug; +use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; +use rustc_middle::ty::print::with_no_trimmed_paths; +use rustc_middle::ty::{self, CoroutineArgsExt, Ty, TypeVisitableExt}; +use rustc_target::callconv::{CastTarget, FnAbi}; +use std::ffi::CString; +use std::fmt::{Debug, Write}; +use std::hash::Hash; +use std::ptr; +use tracing::trace; + +impl PartialEq for Type { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self, other) + } +} + +impl Eq for Type {} + +impl Debug for Type { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + unsafe { + let ptr = llvm::LLVMPrintTypeToString(self); + let cstring = CString::from_raw(ptr); + let string = cstring.to_string_lossy(); + f.write_str(&string) + } + } +} + +impl Hash for Type { + fn hash(&self, state: &mut H) { + (self as *const Type).hash(state); + } +} + +impl Type { + // Creates an integer type with the given number of bits, e.g., i24 + pub fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { + unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) } + } +} + +impl<'ll> CodegenCx<'ll, '_> { + pub(crate) fn voidp(&self) -> &'ll Type { + // llvm uses i8* for void ptrs, void* is invalid + let i8_ty = self.type_i8(); + self.type_ptr_to_ext(i8_ty, AddressSpace::DATA) + } + + pub(crate) fn type_named_struct(&self, name: &str) -> &'ll Type { + let name = SmallCStr::new(name); + unsafe { llvm::LLVMStructCreateNamed(self.llcx, name.as_ptr()) } + } + + pub(crate) fn type_struct(&self, els: &[&'ll Type], packed: bool) -> &'ll Type { + unsafe { + llvm::LLVMStructTypeInContext( + self.llcx, + els.as_ptr(), + els.len() as c_uint, + packed as Bool, + ) + } + } + + pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { + unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) } + } + + pub(crate) fn type_void(&self) -> &'ll Type { + unsafe { llvm::LLVMVoidTypeInContext(self.llcx) } + } + + pub(crate) fn type_metadata(&self) -> &'ll Type { + unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } + } + + pub(crate) fn type_i1(&self) -> &'ll Type { + unsafe { llvm::LLVMInt1TypeInContext(self.llcx) } + } + + pub(crate) fn type_i8p(&self) -> &'ll Type { + self.type_i8p_ext(AddressSpace::DATA) + } + + pub(crate) fn type_i8p_ext(&self, address_space: AddressSpace) -> &'ll Type { + self.type_ptr_to_ext(self.type_i8(), address_space) + } + + ///x Creates an integer type with the given number of bits, e.g., i24 + pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type { + unsafe { llvm::LLVMIntTypeInContext(self.llcx, num_bits as c_uint) } + } + + pub(crate) fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { + unsafe { llvm::LLVMVectorType(ty, len as c_uint) } + } + + pub(crate) fn type_ptr_to(&self, _ty: &'ll Type) -> &'ll Type { + /*assert_ne!( + self.type_kind(ty), + TypeKind::Function, + "don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead or explicitly specify an address space if it makes sense" + ); + unsafe { llvm::LLVMPointerType(ty, AddressSpace::DATA.0) }*/ + // LLVMPointerType is deprecated and removed in favor of opaque pointers in modern LLVM. + self.type_ptr() + } + + pub(crate) fn type_ptr_to_ext(&self, _ty: &'ll Type, address_space: AddressSpace) -> &'ll Type { + //unsafe { llvm::LLVMPointerType(ty, address_space.0) } + // LLVMPointerType is deprecated and removed in favor of opaque pointers in modern LLVM. + self.type_ptr_ext(address_space) + } + + pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { + unsafe { + let n_args = llvm::LLVMCountParamTypes(ty) as usize; + let mut args = Vec::with_capacity(n_args); + llvm::LLVMGetParamTypes(ty, args.as_mut_ptr()); + args.set_len(n_args); + args + } + } + + pub(crate) fn type_pointee_for_align(&self, align: Align) -> &'ll Type { + let ity = Integer::approximate_align(self, align); + self.type_from_integer(ity) + } + + /// Return a LLVM type that has at most the required alignment, + /// and exactly the required size, as a best-effort padding array. + pub(crate) fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { + let unit = Integer::approximate_align(self, align); + let size = size.bytes(); + let unit_size = unit.size().bytes(); + assert_eq!(size % unit_size, 0); + self.type_array(self.type_from_integer(unit), size / unit_size) + } + + pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } + } + + pub(crate) fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { + unsafe { llvm::LLVMRustArrayType(ty, len) } + } +} + +impl<'ll, 'tcx> BaseTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn type_i8(&self) -> &'ll Type { + unsafe { llvm::LLVMInt8TypeInContext(self.llcx) } + } + + fn type_i16(&self) -> &'ll Type { + unsafe { llvm::LLVMInt16TypeInContext(self.llcx) } + } + + fn type_i32(&self) -> &'ll Type { + unsafe { llvm::LLVMInt32TypeInContext(self.llcx) } + } + + fn type_i64(&self) -> &'ll Type { + unsafe { llvm::LLVMInt64TypeInContext(self.llcx) } + } + + fn type_i128(&self) -> &'ll Type { + unsafe { llvm::LLVMIntTypeInContext(self.llcx, 128) } + } + + fn type_isize(&self) -> &'ll Type { + self.isize_ty + } + + fn type_f16(&self) -> &'ll Type { + unsafe { llvm::LLVMHalfTypeInContext(self.llcx) } + } + + fn type_f32(&self) -> &'ll Type { + unsafe { llvm::LLVMFloatTypeInContext(self.llcx) } + } + + fn type_f64(&self) -> &'ll Type { + unsafe { llvm::LLVMDoubleTypeInContext(self.llcx) } + } + + fn type_f128(&self) -> &'ll Type { + unsafe { llvm::LLVMFP128TypeInContext(self.llcx) } + } + + fn type_array(&self, ty: Self::Type, len: u64) -> Self::Type { + unsafe { llvm::LLVMRustArrayType(ty, len) } + } + + fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) } + } + + fn type_kind(&self, ty: &'ll Type) -> TypeKind { + unsafe { + let result = llvm::LLVMRustGetTypeKind(ty); + result.to_generic() + } + } + + fn type_ptr(&self) -> Self::Type { + //self.type_ptr_ext(AddressSpace::DATA) + unsafe { llvm::LLVMPointerTypeInContext(self.llcx, AddressSpace::DATA.0) } + } + + fn type_ptr_ext(&self, address_space: AddressSpace) -> Self::Type { + //self.type_ptr_to_ext(self.type_i8(), address_space) + unsafe { llvm::LLVMPointerTypeInContext(self.llcx, address_space.0) } + } + + fn element_type(&self, ty: &'ll Type) -> &'ll Type { + let out = unsafe { llvm::LLVMGetElementType(ty) }; + out + } + + fn vector_length(&self, ty: &'ll Type) -> usize { + unsafe { llvm::LLVMGetVectorSize(ty) as usize } + } + + fn float_width(&self, ty: &'ll Type) -> usize { + match self.type_kind(ty) { + TypeKind::Half | TypeKind::BFloat => 16, + TypeKind::Float => 32, + TypeKind::Double => 64, + TypeKind::X86_FP80 => 80, + TypeKind::FP128 | TypeKind::PPC_FP128 => 128, + ty => bug!("llvm_float_width called on a non-float type: {:?}", ty), + } + } + + fn int_width(&self, ty: &'ll Type) -> u64 { + unsafe { llvm::LLVMGetIntTypeWidth(ty) as u64 } + } + + fn val_ty(&self, v: &'ll Value) -> &'ll Type { + unsafe { llvm::LLVMTypeOf(v) } + } +} + +impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type { + layout.llvm_type(self) + } + fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type { + layout.immediate_llvm_type(self) + } + fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool { + layout.is_llvm_immediate() + } + fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool { + layout.is_llvm_scalar_pair() + } + fn scalar_pair_element_backend_type( + &self, + layout: TyAndLayout<'tcx>, + index: usize, + immediate: bool, + ) -> &'ll Type { + layout.scalar_pair_element_llvm_type(self, index, immediate) + } + fn cast_backend_type(&self, ty: &CastTarget) -> &'ll Type { + ty.llvm_type(self) + } + fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + fn_abi.ptr_to_llvm_type(self) + } + fn reg_backend_type(&self, ty: &Reg) -> &'ll Type { + ty.llvm_type(self) + } + fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + fn_abi.llvm_type(self) + } +} + +// exists in rustc_codegen_llvm but cant use it because its private :'( +pub(crate) trait LayoutLlvmExt<'tcx> { + fn is_llvm_immediate(&self) -> bool; + fn is_llvm_scalar_pair(&self) -> bool; + fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; + fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; + fn scalar_llvm_type_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, scalar: Scalar) -> &'a Type; + fn scalar_pair_element_llvm_type<'a>( + &self, + cx: &CodegenCx<'a, 'tcx>, + index: usize, + immediate: bool, + ) -> &'a Type; + fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option; +} + +impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { + fn is_llvm_immediate(&self) -> bool { + match self.backend_repr { + BackendRepr::Scalar(_) | BackendRepr::Vector { .. } => true, + BackendRepr::ScalarPair(..) => false, + BackendRepr::Memory { .. } => self.is_zst(), + } + } + + fn is_llvm_scalar_pair(&self) -> bool { + match self.backend_repr { + BackendRepr::ScalarPair(..) => true, + BackendRepr::Scalar(_) | BackendRepr::Vector { .. } | BackendRepr::Memory { .. } => { + false + } + } + } + + /// Gets the LLVM type corresponding to a Rust type, i.e., `rustc_middle::ty::Ty`. + /// The pointee type of the pointer in `PlaceRef` is always this type. + /// For sized types, it is also the right LLVM type for an `alloca` + /// containing a value of that type, and most immediates (except `bool`). + /// Unsized types, however, are represented by a "minimal unit", e.g. + /// `[T]` becomes `T`, while `str` and `Trait` turn into `i8` - this + /// is useful for indexing slices, as `&[T]`'s data pointer is `T*`. + /// If the type is an unsized struct, the regular layout is generated, + /// with the inner-most trailing unsized field using the "minimal unit" + /// of that field's type - this is useful for taking the address of + /// that field and ensuring the struct has the right alignment. + fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { + // This must produce the same result for `repr(transparent)` wrappers as for the inner type! + // In other words, this should generally not look at the type at all, but only at the + // layout. + if let BackendRepr::Scalar(scalar) = self.backend_repr { + // Use a different cache for scalars because pointers to DSTs + // can be either fat or thin (data pointers of fat pointers). + if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { + return llty; + } + + let llty = match *self.ty.kind() { + ty::Ref(_, ty, _) | ty::RawPtr(ty, _) => { + cx.type_ptr_to(cx.layout_of(ty).llvm_type(cx)) + } + ty::Adt(def, _) if def.is_box() => { + cx.type_ptr_to(cx.layout_of(self.ty.expect_boxed_ty()).llvm_type(cx)) + } + ty::FnPtr(sig, hdr) => { + cx.fn_ptr_backend_type(cx.fn_abi_of_fn_ptr(sig.with(hdr), ty::List::empty())) + } + _ => self.scalar_llvm_type_at(cx, scalar), + }; + cx.scalar_lltypes.borrow_mut().insert(self.ty, llty); + return llty; + } + + // Check the cache. + let variant_index = match self.variants { + Variants::Single { index } => Some(index), + _ => None, + }; + if let Some(&llty) = cx.lltypes.borrow().get(&(self.ty, variant_index)) { + return llty; + } + + assert!( + !self.ty.has_escaping_bound_vars(), + "{:?} has escaping bound vars", + self.ty + ); + + // Make sure lifetimes are erased, to avoid generating distinct LLVM + // types for Rust types that only differ in the choice of lifetimes. + let normal_ty = cx.tcx.erase_regions(self.ty); + + let mut defer = None; + let llty = if self.ty != normal_ty { + let mut layout = cx.layout_of(normal_ty); + if let Some(v) = variant_index { + layout = layout.for_variant(cx, v); + } + layout.llvm_type(cx) + } else { + uncached_llvm_type(cx, *self, &mut defer) + }; + + cx.lltypes + .borrow_mut() + .insert((self.ty, variant_index), llty); + + if let Some((llty, layout)) = defer { + let (llfields, packed) = struct_llfields(cx, layout); + cx.set_struct_body(llty, &llfields, packed) + } + + llty + } + + fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { + match self.backend_repr { + BackendRepr::Scalar(ref scalar) => { + if scalar.is_bool() { + return cx.type_i1(); + } + } + BackendRepr::ScalarPair(..) => { + // An immediate pair always contains just the two elements, without any padding + // filler, as it should never be stored to memory. + return cx.type_struct( + &[ + self.scalar_pair_element_llvm_type(cx, 0, true), + self.scalar_pair_element_llvm_type(cx, 1, true), + ], + false, + ); + } + _ => {} + }; + self.llvm_type(cx) + } + + fn scalar_llvm_type_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, scalar: Scalar) -> &'a Type { + match scalar.primitive() { + Int(i, _) => cx.type_from_integer(i), + Float(f) => cx.type_from_float(f), + Pointer(address_space) => { + // If we know the alignment, pick something better than i8. + let (pointee, address_space) = if let Some(PointeeInfo { + safe: Some(_), + align, + .. + }) = self.pointee_info_at(cx, Size::ZERO) + { + (cx.type_pointee_for_align(align), address_space) + } else { + (cx.type_i8(), AddressSpace::DATA) + }; + cx.type_ptr_to_ext(pointee, address_space) + } + } + } + + fn scalar_pair_element_llvm_type<'a>( + &self, + cx: &CodegenCx<'a, 'tcx>, + index: usize, + immediate: bool, + ) -> &'a Type { + // This must produce the same result for `repr(transparent)` wrappers as for the inner type! + // In other words, this should generally not look at the type at all, but only at the + // layout. + + // HACK(eddyb) special-case fat pointers until LLVM removes + // pointee types, to avoid bitcasting every `OperandRef::deref`. + match self.ty.kind() { + ty::Ref(..) | ty::RawPtr(..) => { + return self.field(cx, index).llvm_type(cx); + } + ty::Adt(def, _) if def.is_box() => { + let ptr_ty = Ty::new_mut_ptr(cx.tcx, self.ty.boxed_ty().unwrap()); + return cx + .layout_of(ptr_ty) + .scalar_pair_element_llvm_type(cx, index, immediate); + } + // `dyn* Trait` has the same ABI as `*mut dyn Trait` + ty::Dynamic(bounds, region, ty::DynStar) => { + let ptr_ty = + Ty::new_mut_ptr(cx.tcx, Ty::new_dynamic(cx.tcx, bounds, *region, ty::Dyn)); + return cx + .layout_of(ptr_ty) + .scalar_pair_element_llvm_type(cx, index, immediate); + } + _ => {} + } + + let BackendRepr::ScalarPair(a, b) = self.backend_repr else { + bug!( + "TyAndLayout::scalar_pair_element_llty({:?}): not applicable", + self + ); + }; + let scalar = [a, b][index]; + + // Make sure to return the same type `immediate_llvm_type` would when + // dealing with an immediate pair. This means that `(bool, bool)` is + // effectively represented as `{i8, i8}` in memory and two `i1`s as an + // immediate, just like `bool` is typically `i8` in memory and only `i1` + // when immediate. We need to load/store `bool` as `i8` to avoid + // crippling LLVM optimizations or triggering other LLVM bugs with `i1`. + if immediate && scalar.is_bool() { + return cx.type_i1(); + } + + self.scalar_llvm_type_at(cx, scalar) + } + + fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option { + if let Some(&pointee) = cx.pointee_infos.borrow().get(&(self.ty, offset)) { + return pointee; + } + + let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset); + + cx.pointee_infos + .borrow_mut() + .insert((self.ty, offset), result); + result + } +} + +fn uncached_llvm_type<'a, 'tcx>( + cx: &CodegenCx<'a, 'tcx>, + layout: TyAndLayout<'tcx>, + defer: &mut Option<(&'a Type, TyAndLayout<'tcx>)>, +) -> &'a Type { + trace!("Uncached LLVM type of {:?}", layout); + match layout.backend_repr { + BackendRepr::Scalar(_) => bug!("handled elsewhere"), + BackendRepr::Vector { element, count } => { + let element = layout.scalar_llvm_type_at(cx, element); + return cx.type_vector(element, count); + } + BackendRepr::ScalarPair(..) => { + return cx.type_struct( + &[ + layout.scalar_pair_element_llvm_type(cx, 0, false), + layout.scalar_pair_element_llvm_type(cx, 1, false), + ], + false, + ); + } + BackendRepr::Memory { .. } => {} + } + + let name = match layout.ty.kind() { + // FIXME(eddyb) producing readable type names for trait objects can result + // in problematically distinct types due to HRTB and subtyping (see #47638). + // ty::Dynamic(..) | + ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str + // For performance reasons we use names only when emitting LLVM IR. + if !cx.sess().fewer_names() => + { + let mut name = with_no_trimmed_paths!(layout.ty.to_string()); + if let (&ty::Adt(def, _), &Variants::Single { index }) = + (layout.ty.kind(), &layout.variants) + { + if def.is_enum() && !def.variants().is_empty() { + write!(&mut name, "::{}", def.variant(index).name).unwrap(); + } + } + if let (&ty::Coroutine(_, _), &Variants::Single { index }) = + (layout.ty.kind(), &layout.variants) + { + write!(&mut name, "::{}", ty::CoroutineArgs::variant_name(index)).unwrap(); + } + Some(name) + } + _ => None, + }; + + match layout.fields { + FieldsShape::Primitive | FieldsShape::Union(_) => { + let fill = cx.type_padding_filler(layout.size, layout.align.abi); + let packed = false; + match name { + None => cx.type_struct(&[fill], packed), + Some(ref name) => { + let llty = cx.type_named_struct(name); + cx.set_struct_body(llty, &[fill], packed); + llty + } + } + } + FieldsShape::Array { count, .. } => cx.type_array(layout.field(cx, 0).llvm_type(cx), count), + FieldsShape::Arbitrary { .. } => match name { + None => { + let (llfields, packed) = struct_llfields(cx, layout); + cx.type_struct(&llfields, packed) + } + Some(ref name) => { + let llty = cx.type_named_struct(name); + *defer = Some((llty, layout)); + llty + } + }, + } +} + +fn struct_llfields<'a, 'tcx>( + cx: &CodegenCx<'a, 'tcx>, + layout: TyAndLayout<'tcx>, +) -> (Vec<&'a Type>, bool) { + let field_count = layout.fields.count(); + + let mut packed = false; + let mut offset = Size::ZERO; + let mut prev_effective_align = layout.align.abi; + let mut result: Vec<_> = Vec::with_capacity(1 + field_count * 2); + for i in layout.fields.index_by_increasing_offset() { + let target_offset = layout.fields.offset(i); + let field = layout.field(cx, i); + let effective_field_align = layout + .align + .abi + .min(field.align.abi) + .restrict_for_offset(target_offset); + packed |= effective_field_align < field.align.abi; + + assert!(target_offset >= offset); + let padding = target_offset - offset; + if padding != Size::ZERO { + let padding_align = prev_effective_align.min(effective_field_align); + assert_eq!(offset.align_to(padding_align) + padding, target_offset); + result.push(cx.type_padding_filler(padding, padding_align)); + } + + result.push(field.llvm_type(cx)); + offset = target_offset + field.size; + prev_effective_align = effective_field_align; + } + if layout.is_sized() && field_count > 0 { + if offset > layout.size { + bug!( + "layout: {:#?} stride: {:?} offset: {:?}", + layout, + layout.size, + offset + ); + } + let padding = layout.size - offset; + if padding != Size::ZERO { + let padding_align = prev_effective_align; + assert_eq!(offset.align_to(padding_align) + padding, layout.size); + result.push(cx.type_padding_filler(padding, padding_align)); + } + } + + (result, packed) +} + +impl<'tcx> CodegenCx<'_, 'tcx> { + pub fn align_of(&self, ty: Ty<'tcx>) -> Align { + self.layout_of(ty).align.abi + } + + pub fn size_of(&self, ty: Ty<'tcx>) -> Size { + self.layout_of(ty).size + } + + pub fn size_and_align_of(&self, ty: Ty<'tcx>) -> (Size, Align) { + let layout = self.layout_of(ty); + (layout.size, layout.align.abi) + } +} + +impl<'tcx> TypeMembershipCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> { + fn add_type_metadata(&self, _function: Self::Function, _typeid: String) {} + + fn set_type_metadata(&self, _function: Self::Function, _typeid: String) {} + + fn typeid_metadata(&self, _typeid: String) -> Option { + None + } + + fn add_kcfi_type_metadata(&self, _function: Self::Function, _typeid: u32) {} + + fn set_kcfi_type_metadata(&self, _function: Self::Function, _typeid: u32) {} +} diff --git a/crates/rustc_codegen_nvvm_v7/CHANGELOG.md b/crates/rustc_codegen_nvvm_v7/CHANGELOG.md new file mode 100644 index 00000000..327a5668 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v7/CHANGELOG.md @@ -0,0 +1,66 @@ +# Changelog + +Notable changes to this project will be documented in this file. + +## Unreleased + +- Added symbols for cuda_std to link to for warp intrinsics. +- Completely remove support for 32-bit CUDA (it was broken and it is essentially unused nowadays). +- Add a way to export the final llvm ir module fed to libnvvm. + +## 0.2.3 - 1/2/22 + +- Fixed the `raw_eq` intrinsic being undefined. + +## 0.2.2 - 12/5/21 + +- Pass all ADTs directly, fixing certain structs being passed indirectly because they are scalar pairs. + +## 0.2.1 - 12/5/21 + +- Update find_cuda_helper to 0.2 + +## 0.2.0 - 12/5/21 + +### Address Spaces + +CUDA Address Spaces have been mostly implemented. Statics that are not mut statics and do not rely on +interior mutability (are "freeze" types) are placed in constant memory (`__constant__` in CUDA C++), otherwise +they are placed in global memory (`__global__`). Currently this only happens for user-defined statics, not for +codegen-internal globals such as intermediate alloc globals. + +An `#[address_space(...)]` macro has been added to cuda_std to complement this change. However, this macro +is mostly just for advanced users and internal support for things like shared memory. Improper use can +cause undefined behavior, so its use is generally discouraged. + +### Dead Code Elimination + +PTX files no longer include useless functions and globals, we have switched to an alternative +method of codegen for the final steps of the codegen. We no longer lazily-load modules using dependency graphs, +we instead merge all the modules into one then run global DCE on it before giving it to libnvvm. + +This means all of the dead code is gone before it gets to the libnvvm stage, drastically lowering the size of +the built PTX and improving codegen performance. `cuda_std` also has a macro `#[externally_visible]` which can +be used if you want to keep a function around for things like linking multiple PTX files together. + +### Libm override + +The codegen now has the ability to override [`libm`](https://docs.rs/libm/latest/libm/) functions with +[`libdevice`](https://docs.nvidia.com/cuda/libdevice-users-guide/introduction.html#introduction) intrinsics. + +Libdevice is a bitcode library shipped with every CUDA SDK installation which provides float routines that +are optimized for the GPU and for specific GPU architectures. However, these routines are hard to use automatically because +no_std math crates typically use libm for float things. So users often ended up with needlessly slow or large PTX files +because they used "emulated" routines. + +Now, by default (can be disabled in cuda_builder) the codegen will override libm functions with calls to libdevice automatically. +However, if you rely on libm for determinism, you must disable the overriding, since libdevice is not strictly deterministic. +This also makes PTX much smaller generally, in our example path tracer, it slimmed the PTX file from about `3800` LoC to `2300` LoC. + +- Trace-level debug is compiled out for release now, decreasing the size of the codegen dll and improving compile times. +- Updated to nightly 12/4/21 + +## 0.1.1 - 11/26/21 + +- Fix things using the `bswap` intrinsic panicking. +- (internal) Run clippy and clean things up a bit. diff --git a/crates/rustc_codegen_nvvm/Cargo.toml b/crates/rustc_codegen_nvvm_v7/Cargo.toml similarity index 97% rename from crates/rustc_codegen_nvvm/Cargo.toml rename to crates/rustc_codegen_nvvm_v7/Cargo.toml index 825b32c2..f58638fe 100644 --- a/crates/rustc_codegen_nvvm/Cargo.toml +++ b/crates/rustc_codegen_nvvm_v7/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "rustc_codegen_nvvm" +name = "rustc_codegen_nvvm_v7" version = "0.3.0" authors = [ "Riccardo D'Ambrosio ", diff --git a/crates/rustc_codegen_nvvm/build.rs b/crates/rustc_codegen_nvvm_v7/build.rs similarity index 94% rename from crates/rustc_codegen_nvvm/build.rs rename to crates/rustc_codegen_nvvm_v7/build.rs index ba453722..ef441f41 100644 --- a/crates/rustc_codegen_nvvm/build.rs +++ b/crates/rustc_codegen_nvvm_v7/build.rs @@ -138,6 +138,34 @@ pub fn tracked_env_var_os + Display>(key: K) -> Option env::var_os(key) } +fn run_llvm_as() { + // Check if libintrinsics.ll exists + let libintrinsics_path = Path::new("libintrinsics.ll"); + if !libintrinsics_path.exists() { + fail("libintrinsics.ll not found"); + } + + println!("cargo:rerun-if-changed=libintrinsics.ll"); + + let mut cmd = Command::new("llvm-as-7"); + cmd.arg("libintrinsics.ll"); + + let output = match cmd.stderr(Stdio::inherit()).output() { + Ok(status) => status, + Err(e) => fail(&format!( + "failed to execute llvm-as: {:?}\nerror: {}", + cmd, e + )), + }; + + if !output.status.success() { + fail(&format!( + "llvm-as failed: {:?}\nstatus: {}", + cmd, output.status + )); + } +} + fn rustc_llvm_build() { let target = env::var("TARGET").expect("TARGET was not set"); let llvm_config = find_llvm_config(&target); @@ -159,6 +187,9 @@ fn rustc_llvm_build() { println!("cargo:rustc-cfg=llvm_component=\"{component}\""); } + // Run llvm-as on libintrinsics.ll + run_llvm_as(); + // Link in our own LLVM shims, compiled with the same flags as LLVM let mut cmd = Command::new(&llvm_config); cmd.arg("--cxxflags"); diff --git a/crates/rustc_codegen_nvvm/libintrinsics.bc b/crates/rustc_codegen_nvvm_v7/libintrinsics.bc similarity index 98% rename from crates/rustc_codegen_nvvm/libintrinsics.bc rename to crates/rustc_codegen_nvvm_v7/libintrinsics.bc index 1123b2f5..b7078276 100644 Binary files a/crates/rustc_codegen_nvvm/libintrinsics.bc and b/crates/rustc_codegen_nvvm_v7/libintrinsics.bc differ diff --git a/crates/rustc_codegen_nvvm/libintrinsics.ll b/crates/rustc_codegen_nvvm_v7/libintrinsics.ll similarity index 99% rename from crates/rustc_codegen_nvvm/libintrinsics.ll rename to crates/rustc_codegen_nvvm_v7/libintrinsics.ll index 93820294..c3093b7e 100644 --- a/crates/rustc_codegen_nvvm/libintrinsics.ll +++ b/crates/rustc_codegen_nvvm_v7/libintrinsics.ll @@ -329,4 +329,4 @@ start: declare { i32, i1 } @llvm.nvvm.match.all.sync.i64(i32, i64) #1 attributes #0 = { alwaysinline speculatable } -attributes #1 = { alwaysinline } +attributes #1 = { alwaysinline } \ No newline at end of file diff --git a/crates/rustc_codegen_nvvm_v7/rustc_llvm_wrapper/.editorconfig b/crates/rustc_codegen_nvvm_v7/rustc_llvm_wrapper/.editorconfig new file mode 100644 index 00000000..865cd45f --- /dev/null +++ b/crates/rustc_codegen_nvvm_v7/rustc_llvm_wrapper/.editorconfig @@ -0,0 +1,6 @@ +[*.{h,cpp}] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_style = space +indent_size = 2 diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp b/crates/rustc_codegen_nvvm_v7/rustc_llvm_wrapper/PassWrapper.cpp similarity index 100% rename from crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp rename to crates/rustc_codegen_nvvm_v7/rustc_llvm_wrapper/PassWrapper.cpp diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp b/crates/rustc_codegen_nvvm_v7/rustc_llvm_wrapper/RustWrapper.cpp similarity index 100% rename from crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp rename to crates/rustc_codegen_nvvm_v7/rustc_llvm_wrapper/RustWrapper.cpp diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/rustllvm.h b/crates/rustc_codegen_nvvm_v7/rustc_llvm_wrapper/rustllvm.h similarity index 100% rename from crates/rustc_codegen_nvvm/rustc_llvm_wrapper/rustllvm.h rename to crates/rustc_codegen_nvvm_v7/rustc_llvm_wrapper/rustllvm.h diff --git a/crates/rustc_codegen_nvvm/src/abi.rs b/crates/rustc_codegen_nvvm_v7/src/abi.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/abi.rs rename to crates/rustc_codegen_nvvm_v7/src/abi.rs diff --git a/crates/rustc_codegen_nvvm/src/allocator.rs b/crates/rustc_codegen_nvvm_v7/src/allocator.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/allocator.rs rename to crates/rustc_codegen_nvvm_v7/src/allocator.rs diff --git a/crates/rustc_codegen_nvvm/src/asm.rs b/crates/rustc_codegen_nvvm_v7/src/asm.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/asm.rs rename to crates/rustc_codegen_nvvm_v7/src/asm.rs diff --git a/crates/rustc_codegen_nvvm/src/attributes.rs b/crates/rustc_codegen_nvvm_v7/src/attributes.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/attributes.rs rename to crates/rustc_codegen_nvvm_v7/src/attributes.rs diff --git a/crates/rustc_codegen_nvvm/src/back.rs b/crates/rustc_codegen_nvvm_v7/src/back.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/back.rs rename to crates/rustc_codegen_nvvm_v7/src/back.rs diff --git a/crates/rustc_codegen_nvvm/src/builder.rs b/crates/rustc_codegen_nvvm_v7/src/builder.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/builder.rs rename to crates/rustc_codegen_nvvm_v7/src/builder.rs diff --git a/crates/rustc_codegen_nvvm_v7/src/common.rs b/crates/rustc_codegen_nvvm_v7/src/common.rs new file mode 100644 index 00000000..e437aebb --- /dev/null +++ b/crates/rustc_codegen_nvvm_v7/src/common.rs @@ -0,0 +1,19 @@ +use libc::c_char; + +/// Extension trait for explicit casts to `*const c_char`. +pub(crate) trait AsCCharPtr { + /// Equivalent to `self.as_ptr().cast()`, but only casts to `*const c_char`. + fn as_c_char_ptr(&self) -> *const c_char; +} + +impl AsCCharPtr for str { + fn as_c_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } +} + +impl AsCCharPtr for [u8] { + fn as_c_char_ptr(&self) -> *const c_char { + self.as_ptr().cast() + } +} diff --git a/crates/rustc_codegen_nvvm/src/const_ty.rs b/crates/rustc_codegen_nvvm_v7/src/const_ty.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/const_ty.rs rename to crates/rustc_codegen_nvvm_v7/src/const_ty.rs diff --git a/crates/rustc_codegen_nvvm/src/consts.rs b/crates/rustc_codegen_nvvm_v7/src/consts.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/consts.rs rename to crates/rustc_codegen_nvvm_v7/src/consts.rs diff --git a/crates/rustc_codegen_nvvm/src/context.rs b/crates/rustc_codegen_nvvm_v7/src/context.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/context.rs rename to crates/rustc_codegen_nvvm_v7/src/context.rs diff --git a/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs b/crates/rustc_codegen_nvvm_v7/src/ctx_intrinsics.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs rename to crates/rustc_codegen_nvvm_v7/src/ctx_intrinsics.rs diff --git a/crates/rustc_codegen_nvvm_v7/src/debug_info/create_scope_map.rs b/crates/rustc_codegen_nvvm_v7/src/debug_info/create_scope_map.rs new file mode 100644 index 00000000..5b45c693 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v7/src/debug_info/create_scope_map.rs @@ -0,0 +1,182 @@ +use std::collections::hash_map::Entry; + +use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext}; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::fx::FxHashMap; +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::layout::HasTypingEnv; +use rustc_span::BytePos; + +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::{DILocation, DIScope}; +use rustc_middle::mir::{Body, SourceScope}; +use rustc_middle::ty::{self, Instance}; +use rustc_session::config::DebugInfo; + +use rustc_index::Idx; +use rustc_index::bit_set::*; + +use super::metadata::file_metadata; +use super::util::DIB; + +/// Produces DIScope DIEs for each MIR Scope which has variables defined in it. +pub(crate) fn compute_mir_scopes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + mir: &Body<'tcx>, + debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, +) { + // Find all the scopes with variables defined in them. + let variables = if cx.sess().opts.debuginfo == DebugInfo::Full { + // Only consider variables when they're going to be emitted. + let mut vars = DenseBitSet::new_empty(mir.source_scopes.len()); + // FIXME(eddyb) take into account that arguments always have debuginfo, + // irrespective of their name (assuming full debuginfo is enabled). + // NOTE(eddyb) actually, on second thought, those are always in the + // function scope, which always exists. + for var_debug_info in &mir.var_debug_info { + vars.insert(var_debug_info.source_info.scope); + } + Some(vars) + } else { + // Nothing to emit, of course. + None + }; + + let mut instantiated = DenseBitSet::new_empty(mir.source_scopes.len()); + let mut discriminators = FxHashMap::default(); + // Instantiate all scopes. + for idx in 0..mir.source_scopes.len() { + let scope = SourceScope::new(idx); + make_mir_scope( + cx, + instance, + mir, + &variables, + debug_context, + &mut instantiated, + &mut discriminators, + scope, + ); + } +} + +#[allow(clippy::too_many_arguments)] +fn make_mir_scope<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + mir: &Body<'tcx>, + variables: &Option>, + debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, + instantiated: &mut DenseBitSet, + discriminators: &mut FxHashMap, + scope: SourceScope, +) { + if instantiated.contains(scope) { + return; + } + + let scope_data = &mir.source_scopes[scope]; + let parent_scope = if let Some(parent) = scope_data.parent_scope { + make_mir_scope( + cx, + instance, + mir, + variables, + debug_context, + instantiated, + discriminators, + parent, + ); + debug_context.scopes[parent] + } else { + // The root is the function itself. + let loc = cx.lookup_debug_loc(mir.span.lo()); + debug_context.scopes[scope] = DebugScope { + file_start_pos: loc.file.start_pos, + file_end_pos: loc.file.end_position(), + ..debug_context.scopes[scope] + }; + return; + }; + + if let Some(vars) = variables + && !vars.contains(scope) + && scope_data.inlined.is_none() + { + // Do not create a DIScope if there are no variables defined in this + // MIR `SourceScope`, and it's not `inlined`, to avoid debuginfo bloat. + debug_context.scopes[scope] = parent_scope; + instantiated.insert(scope); + return; + } + + let loc = cx.lookup_debug_loc(scope_data.span.lo()); + let file_metadata = file_metadata(cx, &loc.file); + + let dbg_scope = match scope_data.inlined { + Some((callee, _)) => { + // FIXME(eddyb) this would be `self.monomorphize(&callee)` + // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. + let callee = cx.tcx.instantiate_and_normalize_erasing_regions( + instance.args, + cx.typing_env(), + ty::EarlyBinder::bind(callee), + ); + let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); + cx.dbg_scope_fn(callee, callee_fn_abi, None) + } + None => unsafe { + llvm::LLVMRustDIBuilderCreateLexicalBlock( + DIB(cx), + parent_scope.dbg_scope, + file_metadata, + loc.line, + loc.col, + ) + }, + }; + + let inlined_at = scope_data.inlined.map(|(_, callsite_span)| { + let callsite_scope = parent_scope.adjust_dbg_scope_for_span(cx, callsite_span); + let loc = cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span); + + // NB: In order to produce proper debug info for variables (particularly + // arguments) in multiply-inline functions, LLVM expects to see a single + // DILocalVariable with multiple different DILocations in the IR. While + // the source information for each DILocation would be identical, their + // inlinedAt attributes will be unique to the particular callsite. + // + // We generate DILocations here based on the callsite's location in the + // source code. A single location in the source code usually can't + // produce multiple distinct calls so this mostly works, until + // proc-macros get involved. A proc-macro can generate multiple calls + // at the same span, which breaks the assumption that we're going to + // produce a unique DILocation for every scope we process here. We + // have to explicitly add discriminators if we see inlines into the + // same source code location. + // + // Note further that we can't key this hashtable on the span itself, + // because these spans could have distinct SyntaxContexts. We have + // to key on exactly what we're giving to LLVM. + match discriminators.entry(callsite_span.lo()) { + Entry::Occupied(mut o) => { + *o.get_mut() += 1; + unsafe { llvm::LLVMRustDILocationCloneWithBaseDiscriminator(loc, *o.get()) } + .expect("Failed to encode discriminator in DILocation") + } + Entry::Vacant(v) => { + v.insert(0); + loc + } + } + }); + + debug_context.scopes[scope] = DebugScope { + dbg_scope, + inlined_at: inlined_at.or(parent_scope.inlined_at), + file_start_pos: loc.file.start_pos, + file_end_pos: loc.file.end_position(), + }; +} diff --git a/crates/rustc_codegen_nvvm_v7/src/debug_info/dwarf_const.rs b/crates/rustc_codegen_nvvm_v7/src/debug_info/dwarf_const.rs new file mode 100644 index 00000000..f4320950 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v7/src/debug_info/dwarf_const.rs @@ -0,0 +1,35 @@ +//! Definitions of various DWARF-related constants. + +use libc::c_uint; + +/// Helper macro to let us redeclare gimli's constants as our own constants +/// with a different type, with less risk of copy-paste errors. +macro_rules! declare_constant { + ( + $name:ident : $type:ty + ) => { + #[allow(non_upper_case_globals)] + pub(crate) const $name: $type = ::gimli::constants::$name.0 as $type; + + // Assert that as-cast probably hasn't changed the value. + const _: () = assert!($name as i128 == ::gimli::constants::$name.0 as i128); + }; +} + +// DWARF languages. +declare_constant!(DW_LANG_Rust: c_uint); + +// DWARF attribute type encodings. +declare_constant!(DW_ATE_boolean: c_uint); +declare_constant!(DW_ATE_float: c_uint); +declare_constant!(DW_ATE_signed: c_uint); +declare_constant!(DW_ATE_unsigned: c_uint); +declare_constant!(DW_ATE_UTF: c_uint); + +// DWARF expression operators. +declare_constant!(DW_OP_deref: i64); +declare_constant!(DW_OP_plus_uconst: i64); +/// Defined by LLVM in `llvm/include/llvm/BinaryFormat/Dwarf.h`. +/// Double-checked by a static assertion in `RustWrapper.cpp`. +#[allow(non_upper_case_globals)] +pub(crate) const DW_OP_LLVM_fragment: i64 = 0x1000; diff --git a/crates/rustc_codegen_nvvm/src/debug_info/metadata.rs b/crates/rustc_codegen_nvvm_v7/src/debug_info/metadata.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/debug_info/metadata.rs rename to crates/rustc_codegen_nvvm_v7/src/debug_info/metadata.rs diff --git a/crates/rustc_codegen_nvvm/src/debug_info/metadata/enums.rs b/crates/rustc_codegen_nvvm_v7/src/debug_info/metadata/enums.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/debug_info/metadata/enums.rs rename to crates/rustc_codegen_nvvm_v7/src/debug_info/metadata/enums.rs diff --git a/crates/rustc_codegen_nvvm_v7/src/debug_info/metadata/type_map.rs b/crates/rustc_codegen_nvvm_v7/src/debug_info/metadata/type_map.rs new file mode 100644 index 00000000..408145c5 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v7/src/debug_info/metadata/type_map.rs @@ -0,0 +1,311 @@ +use std::cell::RefCell; + +use rustc_abi::{Align, Size, VariantIdx}; +use rustc_data_structures::fingerprint::Fingerprint; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_macros::HashStable; +use rustc_middle::bug; +use rustc_middle::ty::{self, ExistentialTraitRef, Ty, TyCtxt}; + +use super::{DefinitionLocation, SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata}; +use crate::common::AsCCharPtr; +use crate::context::CodegenCx; +use crate::debug_info::util::{DIB, create_DIArray, debug_context}; +use crate::llvm; +use crate::llvm::debuginfo::{DIFlags, DIScope, DIType}; + +mod private { + use rustc_macros::HashStable; + + // This type cannot be constructed outside of this module because + // it has a private field. We make use of this in order to prevent + // `UniqueTypeId` from being constructed directly, without asserting + // the preconditions. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] + pub struct HiddenZst; +} + +/// A unique identifier for anything that we create a debuginfo node for. +/// The types it contains are expected to already be normalized (which +/// is debug_asserted in the constructors). +/// +/// Note that there are some things that only show up in debuginfo, like +/// the separate type descriptions for each enum variant. These get an ID +/// too because they have their own debuginfo node in LLVM IR. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] +pub(super) enum UniqueTypeId<'tcx> { + /// The ID of a regular type as it shows up at the language level. + Ty(Ty<'tcx>, private::HiddenZst), + /// The ID for the single DW_TAG_variant_part nested inside the top-level + /// DW_TAG_structure_type that describes enums and generators. + VariantPart(Ty<'tcx>, private::HiddenZst), + /// The ID for the artificial struct type describing a single enum variant. + VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst), + /// The ID of the artificial type we create for VTables. + VTableTy( + Ty<'tcx>, + Option>, + private::HiddenZst, + ), +} + +impl<'tcx> UniqueTypeId<'tcx> { + pub fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self { + assert_eq!( + t, + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), t) + ); + UniqueTypeId::Ty(t, private::HiddenZst) + } + + pub fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self { + assert_eq!( + enum_ty, + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), enum_ty) + ); + UniqueTypeId::VariantPart(enum_ty, private::HiddenZst) + } + + pub fn for_enum_variant_struct_type( + tcx: TyCtxt<'tcx>, + enum_ty: Ty<'tcx>, + variant_idx: VariantIdx, + ) -> Self { + assert_eq!( + enum_ty, + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), enum_ty) + ); + UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst) + } + + pub fn for_vtable_ty( + tcx: TyCtxt<'tcx>, + self_type: Ty<'tcx>, + implemented_trait: Option>, + ) -> Self { + assert_eq!( + self_type, + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), self_type) + ); + assert_eq!( + implemented_trait, + tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), implemented_trait) + ); + UniqueTypeId::VTableTy(self_type, implemented_trait, private::HiddenZst) + } + + /// Generates a string version of this [UniqueTypeId], which can be used as the `UniqueId` + /// argument of the various `LLVMRustDIBuilderCreate*Type()` methods. + /// + /// Right now this takes the form of a hex-encoded opaque hash value. + pub fn generate_unique_id_string(self, tcx: TyCtxt<'tcx>) -> String { + let mut hasher = StableHasher::new(); + tcx.with_stable_hashing_context(|mut hcx| { + hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher)) + }); + hasher.finish::().to_hex() + } + + pub fn expect_ty(self) -> Ty<'tcx> { + match self { + UniqueTypeId::Ty(ty, _) => ty, + _ => bug!("Expected `UniqueTypeId::Ty` but found `{:?}`", self), + } + } +} + +/// The `TypeMap` is where the debug context holds the type metadata nodes +/// created so far. The debuginfo nodes are identified by `UniqueTypeId`. +#[derive(Default)] +pub(crate) struct TypeMap<'ll, 'tcx> { + pub(super) unique_id_to_di_node: RefCell, &'ll DIType>>, +} + +impl<'ll, 'tcx> TypeMap<'ll, 'tcx> { + /// Adds a `UniqueTypeId` to metadata mapping to the `TypeMap`. The method will + /// fail if the mapping already exists. + pub(super) fn insert(&self, unique_type_id: UniqueTypeId<'tcx>, metadata: &'ll DIType) { + if self + .unique_id_to_di_node + .borrow_mut() + .insert(unique_type_id, metadata) + .is_some() + { + bug!( + "type metadata for unique ID '{:?}' is already in the `TypeMap`!", + unique_type_id + ); + } + } + + pub(super) fn di_node_for_unique_id( + &self, + unique_type_id: UniqueTypeId<'tcx>, + ) -> Option<&'ll DIType> { + self.unique_id_to_di_node + .borrow() + .get(&unique_type_id) + .cloned() + } +} + +pub(crate) struct DINodeCreationResult<'ll> { + pub di_node: &'ll DIType, + pub already_stored_in_typemap: bool, +} + +impl<'ll> DINodeCreationResult<'ll> { + pub(crate) fn new(di_node: &'ll DIType, already_stored_in_typemap: bool) -> Self { + DINodeCreationResult { + di_node, + already_stored_in_typemap, + } + } +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub(crate) enum Stub<'ll> { + Struct, + Union, + VTableTy { vtable_holder: &'ll DIType }, +} + +pub(crate) struct StubInfo<'ll, 'tcx> { + metadata: &'ll DIType, + unique_type_id: UniqueTypeId<'tcx>, +} + +impl<'ll, 'tcx> StubInfo<'ll, 'tcx> { + pub(super) fn new( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, + build: impl FnOnce(&CodegenCx<'ll, 'tcx>, /* unique_type_id_str: */ &str) -> &'ll DIType, + ) -> StubInfo<'ll, 'tcx> { + let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); + let di_node = build(cx, &unique_type_id_str); + StubInfo { + metadata: di_node, + unique_type_id, + } + } +} + +/// Create a stub debuginfo node onto which fields and nested types can be attached. +#[allow(clippy::too_many_arguments)] +pub(super) fn stub<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + kind: Stub<'ll>, + unique_type_id: UniqueTypeId<'tcx>, + name: &str, + def_location: Option>, + (size, align): (Size, Align), + containing_scope: Option<&'ll DIScope>, + flags: DIFlags, +) -> StubInfo<'ll, 'tcx> { + let empty_array = create_DIArray(DIB(cx), &[]); + let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); + + let (file_metadata, line_number) = if let Some(def_location) = def_location { + (def_location.0, def_location.1) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + + let metadata = match kind { + Stub::Struct | Stub::VTableTy { .. } => { + let vtable_holder = match kind { + Stub::VTableTy { vtable_holder } => Some(vtable_holder), + _ => None, + }; + unsafe { + llvm::LLVMRustDIBuilderCreateStructType( + DIB(cx), + containing_scope, + name.as_c_char_ptr(), + name.len(), + file_metadata, + line_number, + size.bits(), + align.bits() as u32, + flags, + None, + empty_array, + 0, + vtable_holder, + unique_type_id_str.as_c_char_ptr(), + unique_type_id_str.len(), + ) + } + } + Stub::Union => unsafe { + llvm::LLVMRustDIBuilderCreateUnionType( + DIB(cx), + containing_scope, + name.as_c_char_ptr(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + flags, + Some(empty_array), + 0, + unique_type_id_str.as_c_char_ptr(), + unique_type_id_str.len(), + ) + }, + }; + StubInfo { + metadata, + unique_type_id, + } +} + +/// This function enables creating debuginfo nodes that can recursively refer to themselves. +/// It will first insert the given stub into the type map and only then execute the `members` +/// and `generics` closures passed in. These closures have access to the stub so they can +/// directly attach fields to them. If the type of a field transitively refers back +/// to the type currently being built, the stub will already be found in the type map, +/// which effectively breaks the recursion cycle. +pub(super) fn build_type_with_children<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + stub_info: StubInfo<'ll, 'tcx>, + members: impl FnOnce(&CodegenCx<'ll, 'tcx>, &'ll DIType) -> SmallVec<&'ll DIType>, + generics: impl FnOnce(&CodegenCx<'ll, 'tcx>) -> SmallVec<&'ll DIType>, +) -> DINodeCreationResult<'ll> { + assert_eq!( + debug_context(cx) + .type_map + .di_node_for_unique_id(stub_info.unique_type_id), + None + ); + + debug_context(cx) + .type_map + .insert(stub_info.unique_type_id, stub_info.metadata); + + let members: SmallVec<_> = members(cx, stub_info.metadata) + .into_iter() + .map(Some) + .collect(); + let generics: SmallVec> = generics(cx).into_iter().map(Some).collect(); + + if !(members.is_empty() && generics.is_empty()) { + unsafe { + let members_array = create_DIArray(DIB(cx), &members[..]); + let generics_array = create_DIArray(DIB(cx), &generics[..]); + llvm::LLVMRustDICompositeTypeReplaceArrays( + DIB(cx), + stub_info.metadata, + Some(members_array), + Some(generics_array), + ); + } + } + + DINodeCreationResult { + di_node: stub_info.metadata, + already_stored_in_typemap: true, + } +} diff --git a/crates/rustc_codegen_nvvm/src/debug_info/mod.rs b/crates/rustc_codegen_nvvm_v7/src/debug_info/mod.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/debug_info/mod.rs rename to crates/rustc_codegen_nvvm_v7/src/debug_info/mod.rs diff --git a/crates/rustc_codegen_nvvm_v7/src/debug_info/namespace.rs b/crates/rustc_codegen_nvvm_v7/src/debug_info/namespace.rs new file mode 100644 index 00000000..9e85b6b7 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v7/src/debug_info/namespace.rs @@ -0,0 +1,58 @@ +// Namespace Handling. + +use rustc_codegen_ssa::debuginfo::type_names; +use rustc_middle::ty::{self, Instance}; + +use crate::common::AsCCharPtr; +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::DIScope; +use rustc_hir::def_id::DefId; + +use super::util::{DIB, debug_context}; + +pub(crate) fn mangled_name_of_instance<'tcx>( + cx: &CodegenCx<'_, 'tcx>, + instance: Instance<'tcx>, +) -> ty::SymbolName<'tcx> { + let tcx = cx.tcx; + tcx.symbol_name(instance) +} + +pub(crate) fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { + if let Some(&scope) = debug_context(cx).namespace_map.borrow().get(&def_id) { + return scope; + } + + let def_key = cx.tcx.def_key(def_id); + let parent_scope = def_key.parent.map(|parent| { + item_namespace( + cx, + DefId { + krate: def_id.krate, + index: parent, + }, + ) + }); + + let namespace_name_string = { + let mut output = String::new(); + type_names::push_item_name(cx.tcx, def_id, false, &mut output); + output + }; + + let scope = unsafe { + llvm::LLVMRustDIBuilderCreateNameSpace( + DIB(cx), + parent_scope, + namespace_name_string.as_c_char_ptr(), + namespace_name_string.len(), + ) + }; + + debug_context(cx) + .namespace_map + .borrow_mut() + .insert(def_id, scope); + scope +} diff --git a/crates/rustc_codegen_nvvm_v7/src/debug_info/util.rs b/crates/rustc_codegen_nvvm_v7/src/debug_info/util.rs new file mode 100644 index 00000000..d8ab24e9 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v7/src/debug_info/util.rs @@ -0,0 +1,96 @@ +use rustc_hir::def_id::DefId; +use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; +use rustc_middle::ty::{self, Ty}; +use tracing::trace; + +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::{DIArray, DIBuilder, DIDescriptor, DIScope}; + +use super::CodegenUnitDebugContext; +use super::namespace::item_namespace; + +pub(crate) fn is_node_local_to_unit(cx: &CodegenCx<'_, '_>, def_id: DefId) -> bool { + // The is_local_to_unit flag indicates whether a function is local to the + // current compilation unit (i.e., if it is *static* in the C-sense). The + // *reachable* set should provide a good approximation of this, as it + // contains everything that might leak out of the current crate (by being + // externally visible or by being inlined into something externally + // visible). It might better to use the `exported_items` set from + // `driver::CrateAnalysis` in the future, but (atm) this set is not + // available in the codegen pass. + !cx.tcx.is_reachable_non_generic(def_id) +} + +#[allow(non_snake_case)] +pub(crate) fn create_DIArray<'ll>( + builder: &DIBuilder<'ll>, + arr: &[Option<&'ll DIDescriptor>], +) -> &'ll DIArray { + unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) } +} + +#[inline] +pub(crate) fn debug_context<'a, 'll, 'tcx>( + cx: &'a CodegenCx<'ll, 'tcx>, +) -> &'a CodegenUnitDebugContext<'ll, 'tcx> { + cx.dbg_cx.as_ref().unwrap() +} + +#[inline] +#[allow(non_snake_case)] +pub(crate) fn DIB<'a, 'll>(cx: &'a CodegenCx<'ll, '_>) -> &'a DIBuilder<'ll> { + cx.dbg_cx.as_ref().unwrap().builder +} + +pub(crate) fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { + item_namespace(cx, cx.tcx.parent(def_id)) +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum WidePtrKind { + Slice, + Dyn, +} + +/// Determines if `pointee_ty` is slice-like or trait-object-like, i.e. +/// if the second field of the wide pointer is a length or a vtable-pointer. +/// If `pointee_ty` does not require a wide pointer (because it is Sized) then +/// the function returns `None`. +pub(crate) fn wide_pointer_kind<'tcx>( + cx: &CodegenCx<'_, 'tcx>, + pointee_ty: Ty<'tcx>, +) -> Option { + let pointee_tail_ty = cx.tcx.struct_tail_for_codegen(pointee_ty, cx.typing_env()); + let layout = cx.layout_of(pointee_tail_ty); + trace!( + "wide_pointer_kind: {:?} has layout {:?} (is_unsized? {})", + pointee_tail_ty, + layout, + layout.is_unsized() + ); + + if layout.is_sized() { + return None; + } + + match *pointee_tail_ty.kind() { + ty::Str | ty::Slice(_) => Some(WidePtrKind::Slice), + ty::Dynamic(..) => Some(WidePtrKind::Dyn), + ty::Foreign(_) => { + // Assert that pointers to foreign types really are thin: + assert_eq!( + cx.size_of(Ty::new_imm_ptr(cx.tcx, pointee_tail_ty)), + cx.size_of(Ty::new_imm_ptr(cx.tcx, cx.tcx.types.u8)) + ); + None + } + _ => { + // For all other pointee types we should already have returned None + // at the beginning of the function. + panic!( + "wide_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {pointee_tail_ty:?}" + ) + } + } +} diff --git a/crates/rustc_codegen_nvvm/src/init.rs b/crates/rustc_codegen_nvvm_v7/src/init.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/init.rs rename to crates/rustc_codegen_nvvm_v7/src/init.rs diff --git a/crates/rustc_codegen_nvvm/src/int_replace.rs b/crates/rustc_codegen_nvvm_v7/src/int_replace.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/int_replace.rs rename to crates/rustc_codegen_nvvm_v7/src/int_replace.rs diff --git a/crates/rustc_codegen_nvvm/src/intrinsic.rs b/crates/rustc_codegen_nvvm_v7/src/intrinsic.rs similarity index 99% rename from crates/rustc_codegen_nvvm/src/intrinsic.rs rename to crates/rustc_codegen_nvvm_v7/src/intrinsic.rs index b6a7c28c..fb903529 100644 --- a/crates/rustc_codegen_nvvm/src/intrinsic.rs +++ b/crates/rustc_codegen_nvvm_v7/src/intrinsic.rs @@ -296,16 +296,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } } sym::volatile_load | sym::unaligned_volatile_load => { + let tp_ty = fn_args.type_at(0); let mut ptr = args[0].immediate(); - // Handle cast if the ABI requires it if let PassMode::Cast { cast: ty, .. } = &fn_abi.ret.mode { ptr = self.pointercast(ptr, self.type_ptr_to(ty.llvm_type(self))); } - let load = self.volatile_load(result.layout.llvm_type(self), ptr); + let load = self.volatile_load(self.type_i1(), ptr); let align = if name == sym::unaligned_volatile_load { 1 } else { - result.layout.align.abi.bytes() as u32 + self.align_of(tp_ty).bytes() as u32 }; unsafe { llvm::LLVMSetAlignment(load, align); diff --git a/crates/rustc_codegen_nvvm/src/lib.rs b/crates/rustc_codegen_nvvm_v7/src/lib.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/lib.rs rename to crates/rustc_codegen_nvvm_v7/src/lib.rs diff --git a/crates/rustc_codegen_nvvm/src/link.rs b/crates/rustc_codegen_nvvm_v7/src/link.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/link.rs rename to crates/rustc_codegen_nvvm_v7/src/link.rs diff --git a/crates/rustc_codegen_nvvm/src/llvm.rs b/crates/rustc_codegen_nvvm_v7/src/llvm.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/llvm.rs rename to crates/rustc_codegen_nvvm_v7/src/llvm.rs diff --git a/crates/rustc_codegen_nvvm/src/lto.rs b/crates/rustc_codegen_nvvm_v7/src/lto.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/lto.rs rename to crates/rustc_codegen_nvvm_v7/src/lto.rs diff --git a/crates/rustc_codegen_nvvm/src/mono_item.rs b/crates/rustc_codegen_nvvm_v7/src/mono_item.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/mono_item.rs rename to crates/rustc_codegen_nvvm_v7/src/mono_item.rs diff --git a/crates/rustc_codegen_nvvm/src/nvvm.rs b/crates/rustc_codegen_nvvm_v7/src/nvvm.rs similarity index 99% rename from crates/rustc_codegen_nvvm/src/nvvm.rs rename to crates/rustc_codegen_nvvm_v7/src/nvvm.rs index 2bee80e3..1169a7db 100644 --- a/crates/rustc_codegen_nvvm/src/nvvm.rs +++ b/crates/rustc_codegen_nvvm_v7/src/nvvm.rs @@ -113,7 +113,7 @@ pub fn codegen_bitcode_modules( // giving it to libnvvm. Then to debug codegen failures, we can just ask the user to provide the corresponding llvm ir // file with --emit=llvm-ir - let verification_res = prog.verify(); + let verification_res = prog.verify(&args.nvvm_options); if verification_res.is_err() { let log = prog.compiler_log().unwrap().unwrap_or_default(); let footer = "If you plan to submit a bug report please re-run the codegen with `RUSTFLAGS=\"--emit=llvm-ir\" and include the .ll file corresponding to the .o file mentioned in the log"; diff --git a/crates/rustc_codegen_nvvm/src/override_fns.rs b/crates/rustc_codegen_nvvm_v7/src/override_fns.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/override_fns.rs rename to crates/rustc_codegen_nvvm_v7/src/override_fns.rs diff --git a/crates/rustc_codegen_nvvm/src/ptx_filter.rs b/crates/rustc_codegen_nvvm_v7/src/ptx_filter.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/ptx_filter.rs rename to crates/rustc_codegen_nvvm_v7/src/ptx_filter.rs diff --git a/crates/rustc_codegen_nvvm_v7/src/ptxgen.rs b/crates/rustc_codegen_nvvm_v7/src/ptxgen.rs new file mode 100644 index 00000000..b7ddda78 --- /dev/null +++ b/crates/rustc_codegen_nvvm_v7/src/ptxgen.rs @@ -0,0 +1,3 @@ +//! The final stages of compilation, gathering the llvm bitcode from rlibs and generating a ptx file. + +// diff --git a/crates/rustc_codegen_nvvm/src/target.rs b/crates/rustc_codegen_nvvm_v7/src/target.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/target.rs rename to crates/rustc_codegen_nvvm_v7/src/target.rs diff --git a/crates/rustc_codegen_nvvm/src/ty.rs b/crates/rustc_codegen_nvvm_v7/src/ty.rs similarity index 100% rename from crates/rustc_codegen_nvvm/src/ty.rs rename to crates/rustc_codegen_nvvm_v7/src/ty.rs diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index b64ae395..1dcaac05 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -4,8 +4,14 @@ version = "0.0.0" edition = "2021" license = "MIT" +[features] +default = ["nvvm-v7"] +nvvm-v7 = ["dep:rustc_codegen_nvvm_v7"] +nvvm-v19 = ["dep:rustc_codegen_nvvm_v19"] + [dependencies] pico-args = "0.4.2" rayon = "1.10" regex = "1.11.1" -rustc_codegen_nvvm = { path = "../crates/rustc_codegen_nvvm" } +rustc_codegen_nvvm_v7 = { path = "../crates/rustc_codegen_nvvm_v7", optional = true } +rustc_codegen_nvvm_v19 = { path = "../crates/rustc_codegen_nvvm_v19", optional = true } \ No newline at end of file