Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ members = [
"crates/symbol-check",
"crates/util",
"libm",
"libm-test",
"libm-test"
]

default-members = [
Expand All @@ -26,6 +26,7 @@ exclude = [
# and `mangled-names` disabled, which is the opposite of what is needed for
# other tests, so it makes sense to keep it out of the workspace.
"builtins-test-intrinsics",
"rustc-std-workspace-compiler-builtins",
]

[profile.release]
Expand Down
7 changes: 1 addition & 6 deletions compiler-builtins/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,6 @@ bench = false
doctest = false
test = false

[dependencies]
# For more information on this dependency see
# https://github.com/rust-lang/rust/tree/master/library/rustc-std-workspace-core
core = { version = "1.0.1", optional = true, package = "rustc-std-workspace-core" }

[build-dependencies]
cc = { optional = true, version = "1.2" }

Expand Down Expand Up @@ -50,7 +45,7 @@ mem = []
mangled-names = []

# Only used in the compiler's build system
rustc-dep-of-std = ["compiler-builtins", "dep:core"]
rustc-dep-of-std = ["compiler-builtins"]

# This makes certain traits and function specializations public that
# are not normally public but are required by the `builtins-test`
Expand Down
5 changes: 3 additions & 2 deletions compiler-builtins/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ fn main() {

println!("cargo:compiler-rt={}", cwd.join("compiler-rt").display());

println!("cargo::rustc-check-cfg=cfg(kernel_user_helpers)");
println!("cargo::rustc-check-cfg=cfg(feature, values(\"mem-unaligned\"))");

// Emscripten's runtime includes all the builtins
if target.os == "emscripten" {
return;
Expand All @@ -47,7 +50,6 @@ fn main() {
}

// These targets have hardware unaligned access support.
println!("cargo::rustc-check-cfg=cfg(feature, values(\"mem-unaligned\"))");
if target.arch.contains("x86_64")
|| target.arch.contains("x86")
|| target.arch.contains("aarch64")
Expand Down Expand Up @@ -78,7 +80,6 @@ fn main() {
// Only emit the ARM Linux atomic emulation on pre-ARMv6 architectures. This
// includes the old androideabi. It is deprecated but it is available as a
// rustc target (arm-linux-androideabi).
println!("cargo::rustc-check-cfg=cfg(kernel_user_helpers)");
if llvm_target[0] == "armv4t"
|| llvm_target[0] == "armv5te"
|| target.triple == "arm-linux-androideabi"
Expand Down
2 changes: 1 addition & 1 deletion compiler-builtins/src/aarch64_linux.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
//! To avoid breaking backwards compat, C toolchains introduced a concept of "outlined atomics",
//! where atomic operations call into the compiler runtime to dispatch between two depending on
//! which is supported on the current CPU.
//! See https://community.arm.com/arm-community-blogs/b/tools-software-ides-blog/posts/making-the-most-of-the-arm-architecture-in-gcc-10#:~:text=out%20of%20line%20atomics for more discussion.
//! See <https://community.arm.com/arm-community-blogs/b/tools-software-ides-blog/posts/making-the-most-of-the-arm-architecture-in-gcc-10#:~:text=out%20of%20line%20atomics> for more discussion.
//!
//! Currently we only support LL/SC, because LSE requires `getauxval` from libc in order to do runtime detection.
//! Use the `compiler-rt` intrinsics if you want LSE support.
Expand Down
40 changes: 30 additions & 10 deletions compiler-builtins/src/arm_linux.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,17 @@ use core::{arch, mem};
// Kernel-provided user-mode helper functions:
// https://www.kernel.org/doc/Documentation/arm/kernel_user_helpers.txt
unsafe fn __kuser_cmpxchg(oldval: u32, newval: u32, ptr: *mut u32) -> bool {
let f: extern "C" fn(u32, u32, *mut u32) -> u32 = mem::transmute(0xffff0fc0usize as *const ());
// FIXME(volatile): the third parameter is a volatile pointer
// SAFETY: kernel docs specify a known address with the given signature
let f = unsafe {
mem::transmute::<_, extern "C" fn(u32, u32, *mut u32) -> u32>(0xffff0fc0usize as *const ())
};
f(oldval, newval, ptr) == 0
}

unsafe fn __kuser_memory_barrier() {
let f: extern "C" fn() = mem::transmute(0xffff0fa0usize as *const ());
// SAFETY: kernel docs specify a known address with the given signature
let f = unsafe { mem::transmute::<_, extern "C" fn()>(0xffff0fa0usize as *const ()) };
f();
}

Expand Down Expand Up @@ -67,8 +72,10 @@ fn insert_aligned(aligned: u32, val: u32, shift: u32, mask: u32) -> u32 {
/// - if `size_of::<T>() == 2`, `ptr` or `ptr` offset by 2 bytes must be valid for a relaxed atomic
/// read of 2 bytes.
/// - if `size_of::<T>() == 4`, `ptr` must be valid for a relaxed atomic read of 4 bytes.
// FIXME: assert some of the preconditions in debug mode
unsafe fn atomic_load_aligned<T>(ptr: *mut u32) -> u32 {
if mem::size_of::<T>() == 4 {
const { assert!(size_of::<T>() <= 4) };
if size_of::<T>() == 4 {
// SAFETY: As `T` has a size of 4, the caller garantees this is sound.
unsafe { AtomicU32::from_ptr(ptr).load(Ordering::Relaxed) }
} else {
Expand Down Expand Up @@ -100,11 +107,13 @@ unsafe fn atomic_rmw<T, F: Fn(u32) -> u32, G: Fn(u32, u32) -> u32>(ptr: *mut T,
let (shift, mask) = get_shift_mask(ptr);

loop {
let curval_aligned = atomic_load_aligned::<T>(aligned_ptr);
// FIXME(safety): preconditions review needed
let curval_aligned = unsafe { atomic_load_aligned::<T>(aligned_ptr) };
let curval = extract_aligned(curval_aligned, shift, mask);
let newval = f(curval);
let newval_aligned = insert_aligned(curval_aligned, newval, shift, mask);
if __kuser_cmpxchg(curval_aligned, newval_aligned, aligned_ptr) {
// FIXME(safety): preconditions review needed
if unsafe { __kuser_cmpxchg(curval_aligned, newval_aligned, aligned_ptr) } {
return g(curval, newval);
}
}
Expand All @@ -116,13 +125,15 @@ unsafe fn atomic_cmpxchg<T>(ptr: *mut T, oldval: u32, newval: u32) -> u32 {
let (shift, mask) = get_shift_mask(ptr);

loop {
let curval_aligned = atomic_load_aligned::<T>(aligned_ptr);
// FIXME(safety): preconditions review needed
let curval_aligned = unsafe { atomic_load_aligned::<T>(aligned_ptr) };
let curval = extract_aligned(curval_aligned, shift, mask);
if curval != oldval {
return curval;
}
let newval_aligned = insert_aligned(curval_aligned, newval, shift, mask);
if __kuser_cmpxchg(curval_aligned, newval_aligned, aligned_ptr) {
// FIXME(safety): preconditions review needed
if unsafe { __kuser_cmpxchg(curval_aligned, newval_aligned, aligned_ptr) } {
return oldval;
}
}
Expand All @@ -132,7 +143,14 @@ macro_rules! atomic_rmw {
($name:ident, $ty:ty, $op:expr, $fetch:expr) => {
intrinsics! {
pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty {
atomic_rmw(ptr, |x| $op(x as $ty, val) as u32, |old, new| $fetch(old, new)) as $ty
// FIXME(safety): preconditions review needed
unsafe {
atomic_rmw(
ptr,
|x| $op(x as $ty, val) as u32,
|old, new| $fetch(old, new)
) as $ty
}
}
}
};
Expand All @@ -149,7 +167,8 @@ macro_rules! atomic_cmpxchg {
($name:ident, $ty:ty) => {
intrinsics! {
pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty {
atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty
// FIXME(safety): preconditions review needed
unsafe { atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty }
}
}
};
Expand Down Expand Up @@ -285,6 +304,7 @@ atomic_cmpxchg!(__sync_val_compare_and_swap_4, u32);

intrinsics! {
pub unsafe extern "C" fn __sync_synchronize() {
__kuser_memory_barrier();
// SAFETY: preconditions are the same as the calling function.
unsafe { __kuser_memory_barrier() };
}
}
6 changes: 3 additions & 3 deletions compiler-builtins/src/mem/x86_64.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, count: usize) {
"rep movsb",
"sub $7, %rsi",
"sub $7, %rdi",
"mov {qword_count}, %rcx",
"mov {qword_count:r}, %rcx",
"rep movsq",
"test {pre_byte_count:e}, {pre_byte_count:e}",
"add $7, %rsi",
Expand Down Expand Up @@ -212,7 +212,7 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize {
let x = {
let r;
asm!(
"movdqa ({addr}), {dest}",
"movdqa ({addr:r}), {dest}",
addr = in(reg) s,
dest = out(xmm_reg) r,
options(att_syntax, nostack),
Expand All @@ -232,7 +232,7 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize {
let x = {
let r;
asm!(
"movdqa ({addr}), {dest}",
"movdqa ({addr:r}), {dest}",
addr = in(reg) s,
dest = out(xmm_reg) r,
options(att_syntax, nostack),
Expand Down
2 changes: 1 addition & 1 deletion rust-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
df8102fe5f24f28a918660b0cd918d7331c3896e
d13a431a6cc69cd65efe7c3eb7808251d6fd7a46
56 changes: 56 additions & 0 deletions rustc-std-workspace-compiler-builtins/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
[package]
name = "compiler_builtins"
version = "0.1.160"
license = "MIT AND Apache-2.0 WITH LLVM-exception AND (MIT OR Apache-2.0)"
edition = "2024"
publish = false
links = "compiler-rt"
include = ["../compiler-builtins/src"]
build = "../compiler-builtins/build.rs"

[lib]
bench = false
doctest = false
test = false

[dependencies]
core = { path = "../../core", optional = true }

[build-dependencies]
cc = { optional = true, version = "1.2" }

[features]
default = ["compiler-builtins"]

# Enable compilation of C code in compiler-rt, filling in some more optimized
# implementations and also filling in unimplemented intrinsics
c = ["dep:cc"]

# Workaround for the Cranelift codegen backend. Disables any implementations
# which use inline assembly and fall back to pure Rust versions (if available).
no-asm = []

# Workaround for codegen backends which haven't yet implemented `f16` and
# `f128` support. Disabled any intrinsics which use those types.
no-f16-f128 = []

# Flag this library as the unstable compiler-builtins lib
compiler-builtins = []

# Generate memory-related intrinsics like memcpy
mem = []

# Mangle all names so this can be linked in with other versions or other
# compiler-rt implementations. Also used for testing
mangled-names = []

# Only used in the compiler's build system
rustc-dep-of-std = ["compiler-builtins"]

# This makes certain traits and function specializations public that
# are not normally public but are required by the `builtins-test`
unstable-public-internals = []

[lints.rust]
# The cygwin config can be dropped after our benchmark toolchain is bumped
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bootstrap)', 'cfg(target_os, values("cygwin"))'] }
14 changes: 14 additions & 0 deletions rustc-std-workspace-compiler-builtins/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
pub fn add(left: u64, right: u64) -> u64 {
left + right
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn it_works() {
let result = add(2, 2);
assert_eq!(result, 4);
}
}
Loading