diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index e320ba75f611..cbed2c3217bf 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -2,7 +2,7 @@ use crate::{ compiled_blob::CompiledBlob, - memory::{BranchProtection, JITMemoryProvider, SystemMemoryProvider}, + memory::{BranchProtection, JITMemoryKind, JITMemoryProvider, SystemMemoryProvider}, }; use cranelift_codegen::binemit::Reloc; use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa}; @@ -19,7 +19,6 @@ use std::cell::RefCell; use std::collections::HashMap; use std::ffi::CString; use std::io::Write; -use std::ptr; use target_lexicon::PointerWidth; const WRITABLE_DATA_ALIGNMENT: u64 = 0x8; @@ -217,7 +216,7 @@ impl JITModule { let (name, linkage) = if ModuleDeclarations::is_function(name) { let func_id = FuncId::from_name(name); match &self.compiled_functions[func_id] { - Some(compiled) => return compiled.ptr, + Some(compiled) => return compiled.ptr(), None => { let decl = self.declarations.get_function_decl(func_id); (&decl.name, decl.linkage) @@ -226,7 +225,7 @@ impl JITModule { } else { let data_id = DataId::from_name(name); match &self.compiled_data_objects[data_id] { - Some(compiled) => return compiled.ptr, + Some(compiled) => return compiled.ptr(), None => { let decl = self.declarations.get_data_decl(data_id); (&decl.name, decl.linkage) @@ -251,7 +250,7 @@ impl JITModule { } ModuleRelocTarget::FunctionOffset(func_id, offset) => { match &self.compiled_functions[*func_id] { - Some(compiled) => return compiled.ptr.wrapping_add(*offset as usize), + Some(compiled) => return compiled.ptr().wrapping_add(*offset as usize), None => todo!(), } } @@ -271,7 +270,7 @@ impl JITModule { ); info.as_ref() .expect("function must be compiled before it can be finalized") - .ptr + .ptr() } /// Returns the address and size of a finalized data object. @@ -288,10 +287,10 @@ impl JITModule { .as_ref() .expect("data object must be compiled before it can be finalized"); - (compiled.ptr, compiled.size) + (compiled.ptr(), compiled.size()) } - fn record_function_for_perf(&self, ptr: *mut u8, size: usize, name: &str) { + fn record_function_for_perf(&self, ptr: *const u8, size: usize, name: &str) { // The Linux perf tool supports JIT code via a /tmp/perf-$PID.map file, // which contains memory regions and their associated names. If we // are profiling with perf and saving binaries to PERF_BUILDID_DIR @@ -407,8 +406,7 @@ impl JITModule { let data = self.compiled_functions[func] .as_ref() .unwrap() - .exception_data - .as_ref()?; + .wasmtime_exception_data()?; let exception_table = wasmtime_unwinder::ExceptionTable::parse(data).ok()?; Some((start, exception_table)) } @@ -485,22 +483,9 @@ impl Module for JITModule { let alignment = res.buffer.alignment as u64; let compiled_code = ctx.compiled_code().unwrap(); - let size = compiled_code.code_info().total_size as usize; let align = alignment .max(self.isa.function_alignment().minimum as u64) .max(self.isa.symbol_alignment()); - let ptr = - self.memory - .allocate_readexec(size, align) - .map_err(|e| ModuleError::Allocation { - message: "unable to alloc function", - err: e, - })?; - - { - let mem = unsafe { std::slice::from_raw_parts_mut(ptr, size) }; - mem.copy_from_slice(compiled_code.code_buffer()); - } let relocs = compiled_code .buffer @@ -509,22 +494,8 @@ impl Module for JITModule { .map(|reloc| ModuleReloc::from_mach_reloc(reloc, &ctx.func, id)) .collect(); - self.record_function_for_perf(ptr, size, &decl.linkage_name(id)); - self.compiled_functions[id] = Some(CompiledBlob { - ptr, - size, - relocs, - #[cfg(feature = "wasmtime-unwinder")] - exception_data: None, - }); - - let range_start = ptr as usize; - let range_end = range_start + size; - // These will be sorted when we finalize. - self.code_ranges.push((range_start, range_end, id)); - #[cfg(feature = "wasmtime-unwinder")] - { + let wasmtime_exception_data = { let mut exception_builder = wasmtime_unwinder::ExceptionTableBuilder::default(); exception_builder .add_func(0, compiled_code.buffer.call_sites()) @@ -533,9 +504,25 @@ impl Module for JITModule { "Invalid exception data".into(), )) })?; - self.compiled_functions[id].as_mut().unwrap().exception_data = - Some(exception_builder.to_vec()); - } + Some(exception_builder.to_vec()) + }; + + let blob = self.compiled_functions[id].insert(CompiledBlob::new( + &mut *self.memory, + compiled_code.code_buffer(), + align, + relocs, + #[cfg(feature = "wasmtime-unwinder")] + wasmtime_exception_data, + JITMemoryKind::Executable, + )?); + let (ptr, size) = (blob.ptr(), blob.size()); + self.record_function_for_perf(ptr, size, &decl.linkage_name(id)); + + let range_start = ptr.addr(); + let range_end = range_start + size; + // These will be sorted when we finalize. + self.code_ranges.push((range_start, range_end, id)); self.functions_to_finalize.push(id); @@ -563,30 +550,21 @@ impl Module for JITModule { )); } - let size = bytes.len(); let align = alignment .max(self.isa.function_alignment().minimum as u64) .max(self.isa.symbol_alignment()); - let ptr = - self.memory - .allocate_readexec(size, align) - .map_err(|e| ModuleError::Allocation { - message: "unable to alloc function bytes", - err: e, - })?; - - unsafe { - ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, size); - } - self.record_function_for_perf(ptr, size, &decl.linkage_name(id)); - self.compiled_functions[id] = Some(CompiledBlob { - ptr, - size, - relocs: relocs.to_owned(), + let blob = self.compiled_functions[id].insert(CompiledBlob::new( + &mut *self.memory, + bytes, + align, + relocs.to_owned(), #[cfg(feature = "wasmtime-unwinder")] - exception_data: None, - }); + None, + JITMemoryKind::Executable, + )?); + let (ptr, size) = (blob.ptr(), blob.size()); + self.record_function_for_perf(ptr, size, &decl.linkage_name(id)); self.functions_to_finalize.push(id); @@ -620,52 +598,18 @@ impl Module for JITModule { used: _, } = data; - // Make sure to allocate at least 1 byte. Allocating 0 bytes is UB. Previously a dummy - // value was used, however as it turns out this will cause pc-relative relocations to - // fail on architectures where pc-relative offsets are range restricted as the dummy - // value is not close enough to the code that has the pc-relative relocation. - let alloc_size = std::cmp::max(init.size(), 1); - - let ptr = if decl.writable { - self.memory - .allocate_readwrite(alloc_size, align.unwrap_or(WRITABLE_DATA_ALIGNMENT)) - .map_err(|e| ModuleError::Allocation { - message: "unable to alloc writable data", - err: e, - })? + let (align, kind) = if decl.writable { + ( + align.unwrap_or(WRITABLE_DATA_ALIGNMENT), + JITMemoryKind::Writable, + ) } else { - self.memory - .allocate_readonly(alloc_size, align.unwrap_or(READONLY_DATA_ALIGNMENT)) - .map_err(|e| ModuleError::Allocation { - message: "unable to alloc readonly data", - err: e, - })? + ( + align.unwrap_or(READONLY_DATA_ALIGNMENT), + JITMemoryKind::ReadOnly, + ) }; - if ptr.is_null() { - // FIXME pass a Layout to allocate and only compute the layout once. - std::alloc::handle_alloc_error( - std::alloc::Layout::from_size_align( - alloc_size, - align.unwrap_or(READONLY_DATA_ALIGNMENT).try_into().unwrap(), - ) - .unwrap(), - ); - } - - match *init { - Init::Uninitialized => { - panic!("data is not initialized yet"); - } - Init::Zeros { size } => { - unsafe { ptr::write_bytes(ptr, 0, size) }; - } - Init::Bytes { ref contents } => { - let src = contents.as_ptr(); - unsafe { ptr::copy_nonoverlapping(src, ptr, contents.len()) }; - } - } - let pointer_reloc = match self.isa.triple().pointer_width().unwrap() { PointerWidth::U16 => panic!(), PointerWidth::U32 => Reloc::Abs4, @@ -673,44 +617,43 @@ impl Module for JITModule { }; let relocs = data.all_relocs(pointer_reloc).collect::>(); - self.compiled_data_objects[id] = Some(CompiledBlob { - ptr, - size: init.size(), - relocs, - #[cfg(feature = "wasmtime-unwinder")] - exception_data: None, + self.compiled_data_objects[id] = Some(match *init { + Init::Uninitialized => { + panic!("data is not initialized yet"); + } + Init::Zeros { size } => CompiledBlob::new_zeroed( + &mut *self.memory, + size.max(1), + align, + relocs, + #[cfg(feature = "wasmtime-unwinder")] + None, + kind, + )?, + Init::Bytes { ref contents } => CompiledBlob::new( + &mut *self.memory, + if contents.is_empty() { + // Make sure to allocate at least 1 byte. Allocating 0 bytes is UB. Previously + // a dummy value was used, however as it turns out this will cause pc-relative + // relocations to fail on architectures where pc-relative offsets are range + // restricted as the dummy value is not close enough to the code that has the + // pc-relative relocation. + &[0] + } else { + &contents[..] + }, + align, + relocs, + #[cfg(feature = "wasmtime-unwinder")] + None, + kind, + )?, }); + self.data_objects_to_finalize.push(id); Ok(()) } - - fn get_name(&self, name: &str) -> Option { - self.declarations().get_name(name) - } - - fn target_config(&self) -> cranelift_codegen::isa::TargetFrontendConfig { - self.isa().frontend_config() - } - - fn make_context(&self) -> cranelift_codegen::Context { - let mut ctx = cranelift_codegen::Context::new(); - ctx.func.signature.call_conv = self.isa().default_call_conv(); - ctx - } - - fn clear_context(&self, ctx: &mut cranelift_codegen::Context) { - ctx.clear(); - ctx.func.signature.call_conv = self.isa().default_call_conv(); - } - - fn make_signature(&self) -> ir::Signature { - ir::Signature::new(self.isa().default_call_conv()) - } - - fn clear_signature(&self, sig: &mut ir::Signature) { - sig.clear(self.isa().default_call_conv()); - } } #[cfg(not(windows))] @@ -728,6 +671,7 @@ fn lookup_with_dlsym(name: &str) -> Option<*const u8> { #[cfg(windows)] fn lookup_with_dlsym(name: &str) -> Option<*const u8> { use std::os::windows::io::RawHandle; + use std::ptr; use windows_sys::Win32::Foundation::HMODULE; use windows_sys::Win32::System::LibraryLoader; @@ -763,13 +707,7 @@ fn use_bti(isa_flags: &Vec) -> bool { .map_or(false, |f| f.as_bool().unwrap_or(false)) } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_jit_module_is_send() { - fn assert_is_send() {} - assert_is_send::(); - } -} +const _ASSERT_JIT_MODULE_IS_SEND: () = { + const fn assert_is_send() {} + assert_is_send::(); +}; diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs index 003d6f00da43..81e1bc707d84 100644 --- a/cranelift/jit/src/compiled_blob.rs +++ b/cranelift/jit/src/compiled_blob.rs @@ -1,6 +1,12 @@ +use std::ptr; + use cranelift_codegen::binemit::Reloc; -use cranelift_module::ModuleReloc; -use cranelift_module::ModuleRelocTarget; +use cranelift_module::{ModuleError, ModuleReloc, ModuleRelocTarget, ModuleResult}; + +use crate::JITMemoryProvider; +use crate::memory::JITMemoryKind; + +const VENEER_SIZE: usize = 24; // ldr + br + pointer /// Reads a 32bit instruction at `iptr`, and writes it again after /// being altered by `modifier` @@ -12,22 +18,97 @@ unsafe fn modify_inst32(iptr: *mut u32, modifier: impl FnOnce(u32) -> u32) { #[derive(Clone)] pub(crate) struct CompiledBlob { - pub(crate) ptr: *mut u8, - pub(crate) size: usize, - pub(crate) relocs: Vec, + ptr: *mut u8, + size: usize, + relocs: Vec, + veneer_count: usize, #[cfg(feature = "wasmtime-unwinder")] - pub(crate) exception_data: Option>, + wasmtime_exception_data: Option>, } unsafe impl Send for CompiledBlob {} impl CompiledBlob { + pub(crate) fn new( + memory: &mut dyn JITMemoryProvider, + data: &[u8], + align: u64, + relocs: Vec, + #[cfg(feature = "wasmtime-unwinder")] wasmtime_exception_data: Option>, + kind: JITMemoryKind, + ) -> ModuleResult { + // Reserve veneers for all function calls just in case + let mut veneer_count = 0; + for reloc in &relocs { + match reloc.kind { + Reloc::Arm64Call => veneer_count += 1, + _ => {} + } + } + + let ptr = memory + .allocate(data.len() + veneer_count * VENEER_SIZE, align, kind) + .map_err(|e| ModuleError::Allocation { err: e })?; + + unsafe { + ptr::copy_nonoverlapping(data.as_ptr(), ptr, data.len()); + } + + Ok(CompiledBlob { + ptr, + size: data.len(), + relocs, + veneer_count, + #[cfg(feature = "wasmtime-unwinder")] + wasmtime_exception_data, + }) + } + + pub(crate) fn new_zeroed( + memory: &mut dyn JITMemoryProvider, + size: usize, + align: u64, + relocs: Vec, + #[cfg(feature = "wasmtime-unwinder")] wasmtime_exception_data: Option>, + kind: JITMemoryKind, + ) -> ModuleResult { + let ptr = memory + .allocate(size, align, kind) + .map_err(|e| ModuleError::Allocation { err: e })?; + + unsafe { ptr::write_bytes(ptr, 0, size) }; + + Ok(CompiledBlob { + ptr, + size, + relocs, + veneer_count: 0, + #[cfg(feature = "wasmtime-unwinder")] + wasmtime_exception_data, + }) + } + + pub(crate) fn ptr(&self) -> *const u8 { + self.ptr + } + + pub(crate) fn size(&self) -> usize { + self.size + } + + #[cfg(feature = "wasmtime-unwinder")] + pub(crate) fn wasmtime_exception_data(&self) -> Option<&[u8]> { + self.wasmtime_exception_data.as_deref() + } + pub(crate) fn perform_relocations( &self, get_address: impl Fn(&ModuleRelocTarget) -> *const u8, ) { use std::ptr::write_unaligned; + let mut next_veneer_idx = 0; + for ( i, &ModuleReloc { @@ -75,21 +156,53 @@ impl CompiledBlob { } Reloc::Arm64Call => { let base = get_address(name); + let what = unsafe { base.offset(isize::try_from(addend).unwrap()) }; // The instruction is 32 bits long. let iptr = at as *mut u32; + // The offset encoded in the `bl` instruction is the // number of bytes divided by 4. - let diff = ((base as isize) - (at as isize)) >> 2; + let diff = ((what as isize) - (at as isize)) >> 2; // Sign propagating right shift disposes of the // included bits, so the result is expected to be - // either all sign bits or 0, depending on if the original - // value was negative or positive. - assert!((diff >> 26 == -1) || (diff >> 26 == 0)); - // The lower 26 bits of the `bl` instruction form the - // immediate offset argument. - let chop = 32 - 26; - let imm26 = (diff as u32) << chop >> chop; - unsafe { modify_inst32(iptr, |inst| inst | imm26) }; + // either all sign bits or 0 when in-range, depending + // on if the original value was negative or positive. + if (diff >> 25 == -1) || (diff >> 25 == 0) { + // The lower 26 bits of the `bl` instruction form the + // immediate offset argument. + let chop = 32 - 26; + let imm26 = (diff as u32) << chop >> chop; + unsafe { modify_inst32(iptr, |inst| inst | imm26) }; + } else { + // If the target is out of range for a direct call, insert a veneer at the + // end of the function. + let veneer_idx = next_veneer_idx; + next_veneer_idx += 1; + assert!(veneer_idx <= self.veneer_count); + let veneer = + unsafe { self.ptr.byte_add(self.size + veneer_idx * VENEER_SIZE) }; + + // Write the veneer + // x16 is reserved as scratch register to be used by veneers and PLT entries + unsafe { + write_unaligned( + veneer.cast::(), + 0x58000050, // ldr x16, 0x8 + ); + write_unaligned( + veneer.byte_add(4).cast::(), + 0xd61f0200, // br x16 + ); + write_unaligned(veneer.byte_add(8).cast::(), what.addr() as u64); + }; + + // Set the veneer as target of the call + let diff = ((veneer as isize) - (at as isize)) >> 2; + assert!((diff >> 25 == -1) || (diff >> 25 == 0)); + let chop = 32 - 26; + let imm26 = (diff as u32) << chop >> chop; + unsafe { modify_inst32(iptr, |inst| inst | imm26) }; + } } Reloc::Aarch64AdrGotPage21 => { panic!("GOT relocation shouldn't be generated when !is_pic"); diff --git a/cranelift/jit/src/memory/arena.rs b/cranelift/jit/src/memory/arena.rs index 1afb93f7cf16..00fdbbc88c68 100644 --- a/cranelift/jit/src/memory/arena.rs +++ b/cranelift/jit/src/memory/arena.rs @@ -4,7 +4,7 @@ use std::ptr; use cranelift_module::ModuleResult; -use super::{BranchProtection, JITMemoryProvider}; +use super::{BranchProtection, JITMemoryKind, JITMemoryProvider}; fn align_up(addr: usize, align: usize) -> usize { debug_assert!(align.is_power_of_two()); @@ -123,7 +123,7 @@ impl ArenaMemoryProvider { }) } - fn allocate( + fn allocate_inner( &mut self, size: usize, align: u64, @@ -221,16 +221,16 @@ impl Drop for ArenaMemoryProvider { } impl JITMemoryProvider for ArenaMemoryProvider { - fn allocate_readexec(&mut self, size: usize, align: u64) -> io::Result<*mut u8> { - self.allocate(size, align, region::Protection::READ_EXECUTE) - } - - fn allocate_readwrite(&mut self, size: usize, align: u64) -> io::Result<*mut u8> { - self.allocate(size, align, region::Protection::READ_WRITE) - } - - fn allocate_readonly(&mut self, size: usize, align: u64) -> io::Result<*mut u8> { - self.allocate(size, align, region::Protection::READ) + fn allocate(&mut self, size: usize, align: u64, kind: JITMemoryKind) -> io::Result<*mut u8> { + self.allocate_inner( + size, + align, + match kind { + JITMemoryKind::Executable => region::Protection::READ_EXECUTE, + JITMemoryKind::Writable => region::Protection::READ_WRITE, + JITMemoryKind::ReadOnly => region::Protection::READ, + }, + ) } unsafe fn free_memory(&mut self) { @@ -254,7 +254,9 @@ mod tests { for align_log2 in 0..8 { let align = 1usize << align_log2; for size in 1..128 { - let ptr = arena.allocate_readwrite(size, align as u64).unwrap(); + let ptr = arena + .allocate(size, align as u64, JITMemoryKind::Writable) + .unwrap(); // assert!(ptr.is_aligned_to(align)); assert_eq!(ptr.addr() % align, 0); } @@ -269,7 +271,7 @@ mod tests { // platforms. Physical memory should be committed as we go. let reserve_size = 1 << 40; let mut arena = ArenaMemoryProvider::new_with_size(reserve_size).unwrap(); - let ptr = arena.allocate_readwrite(1, 1).unwrap(); + let ptr = arena.allocate(1, 1, JITMemoryKind::Writable).unwrap(); assert_eq!(ptr.addr(), arena.ptr.addr()); arena.finalize(BranchProtection::None); unsafe { ptr.write_volatile(42) }; @@ -280,8 +282,10 @@ mod tests { fn over_capacity() { let mut arena = ArenaMemoryProvider::new_with_size(1 << 20).unwrap(); // 1 MB - let _ = arena.allocate_readwrite(900_000, 1).unwrap(); - let _ = arena.allocate_readwrite(200_000, 1).unwrap_err(); + let _ = arena.allocate(900_000, 1, JITMemoryKind::Writable).unwrap(); + let _ = arena + .allocate(200_000, 1, JITMemoryKind::Writable) + .unwrap_err(); } #[test] diff --git a/cranelift/jit/src/memory/mod.rs b/cranelift/jit/src/memory/mod.rs index 9052d2da56e6..7a9e8b77eb47 100644 --- a/cranelift/jit/src/memory/mod.rs +++ b/cranelift/jit/src/memory/mod.rs @@ -16,14 +16,19 @@ pub enum BranchProtection { BTI, } -/// A provider of memory for the JIT. -pub trait JITMemoryProvider { +pub enum JITMemoryKind { /// Allocate memory that will be executable once finalized. - fn allocate_readexec(&mut self, size: usize, align: u64) -> io::Result<*mut u8>; + Executable, /// Allocate writable memory. - fn allocate_readwrite(&mut self, size: usize, align: u64) -> io::Result<*mut u8>; + Writable, /// Allocate memory that will be read-only once finalized. - fn allocate_readonly(&mut self, size: usize, align: u64) -> io::Result<*mut u8>; + ReadOnly, +} + +/// A provider of memory for the JIT. +pub trait JITMemoryProvider { + /// Allocate memory + fn allocate(&mut self, size: usize, align: u64, kind: JITMemoryKind) -> io::Result<*mut u8>; /// Free the memory region. unsafe fn free_memory(&mut self); diff --git a/cranelift/jit/src/memory/system.rs b/cranelift/jit/src/memory/system.rs index 08f8f0fd0a4a..bd845aef4a17 100644 --- a/cranelift/jit/src/memory/system.rs +++ b/cranelift/jit/src/memory/system.rs @@ -9,8 +9,7 @@ use std::io; use std::mem; use std::ptr; -use super::BranchProtection; -use super::JITMemoryProvider; +use super::{BranchProtection, JITMemoryKind, JITMemoryProvider}; /// A simple struct consisting of a pointer and length. struct PtrLen { @@ -263,15 +262,11 @@ impl JITMemoryProvider for SystemMemoryProvider { self.code.set_readable_and_executable(branch_protection) } - fn allocate_readexec(&mut self, size: usize, align: u64) -> io::Result<*mut u8> { - self.code.allocate(size, align) - } - - fn allocate_readwrite(&mut self, size: usize, align: u64) -> io::Result<*mut u8> { - self.writable.allocate(size, align) - } - - fn allocate_readonly(&mut self, size: usize, align: u64) -> io::Result<*mut u8> { - self.readonly.allocate(size, align) + fn allocate(&mut self, size: usize, align: u64, kind: JITMemoryKind) -> io::Result<*mut u8> { + match kind { + JITMemoryKind::Executable => self.code.allocate(size, align), + JITMemoryKind::Writable => self.writable.allocate(size, align), + JITMemoryKind::ReadOnly => self.readonly.allocate(size, align), + } } } diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rs index 79dcc1efec6a..526f998561a8 100644 --- a/cranelift/module/src/module.rs +++ b/cranelift/module/src/module.rs @@ -291,8 +291,6 @@ pub enum ModuleError { /// Memory allocation failure from a backend Allocation { - /// Tell where the allocation came from - message: &'static str, /// Io error the allocation failed with err: std::io::Error, }, @@ -321,7 +319,7 @@ impl std::error::Error for ModuleError { | Self::DuplicateDefinition { .. } | Self::InvalidImportDefinition { .. } => None, Self::Compilation(source) => Some(source), - Self::Allocation { err: source, .. } => Some(source), + Self::Allocation { err: source } => Some(source), Self::Backend(source) => Some(&**source), Self::Flag(source) => Some(source), } @@ -355,8 +353,8 @@ impl std::fmt::Display for ModuleError { Self::Compilation(err) => { write!(f, "Compilation error: {err}") } - Self::Allocation { message, err } => { - write!(f, "Allocation error: {message}: {err}") + Self::Allocation { err } => { + write!(f, "Allocation error: {err}") } Self::Backend(err) => write!(f, "Backend error: {err}"), Self::Flag(err) => write!(f, "Flag error: {err}"),