From d58809f0bbe8d19675e3c877e6e04e2ca5149c20 Mon Sep 17 00:00:00 2001 From: Leonard Chan Date: Thu, 31 Jul 2025 11:52:39 -0700 Subject: [PATCH] [WIP] Relative VTables for Rust This is a WIP patch for implementing https://github.com/rust-lang/compiler-team/issues/903. It adds a new unstable flag `-Zexperimental-relative-rust-abi-vtables` that makes vtables PIC-friendly. This is only supported for LLVM codegen and not supported for other backends. Early feedback on this is welcome. I'm not sure if how I implemented it is the best way of doing so since much of the actual vtable emission is heavily done during LLVM codegen. That is, the vtable to MIR looks like a normal table of pointers and byte arrays and I really only make the vtables relative on the codegen level. Locally, I can build the stage 1 compiler and runtimes with relative vtables, but I couldn't figure out how to tell the build system to only build stage 1 binaries with this flag, so I work around this by unconditionally enabling relative vtables in rustc. The end goal I think we'd like is either something akin to multilibs in clang where the compiler chooses which runtimes to use based off compilation flags, or binding this ABI to the target and have it be part of the default ABI for that target (just like how relative vtables are the default for Fuchsia in C++ with Clang). I think the later is what target modifiers do (https://github.com/rust-lang/rust/issues/136966). Action Items: - I'm still experimenting with building Fuchsia with this to assert it works e2e and I still need to do some measurements to see if this is still worth pursuing. - More work will still be needed to ensure the correct relative intrinsics are emitted with CFI and LTO. Rn I'm experimenting on a normal build. --- compiler/rustc_abi/src/lib.rs | 29 +++- compiler/rustc_codegen_llvm/src/builder.rs | 4 + compiler/rustc_codegen_llvm/src/common.rs | 47 ++++++- compiler/rustc_codegen_llvm/src/consts.rs | 128 +++++++++++++++--- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 9 ++ compiler/rustc_codegen_llvm/src/type_.rs | 7 + compiler/rustc_codegen_ssa/src/base.rs | 23 +++- compiler/rustc_codegen_ssa/src/meth.rs | 87 ++++++++++-- compiler/rustc_codegen_ssa/src/mir/block.rs | 18 ++- compiler/rustc_codegen_ssa/src/size_of_val.rs | 14 +- .../rustc_codegen_ssa/src/traits/builder.rs | 1 + .../rustc_codegen_ssa/src/traits/consts.rs | 1 + .../rustc_codegen_ssa/src/traits/type_.rs | 1 + .../rustc_llvm/llvm-wrapper/RustWrapper.cpp | 12 ++ compiler/rustc_session/src/config.rs | 4 + compiler/rustc_session/src/options.rs | 3 + compiler/rustc_ty_utils/src/abi.rs | 29 +++- src/doc/rustc/src/codegen-options/index.md | 4 + .../relative-vtables/simple-vtable.rs | 79 +++++++++++ 19 files changed, 452 insertions(+), 48 deletions(-) create mode 100644 tests/codegen-llvm/relative-vtables/simple-vtable.rs diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 14e256b8045df..dcb7b44393c5d 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -43,7 +43,7 @@ use std::fmt; #[cfg(feature = "nightly")] use std::iter::Step; use std::num::{NonZeroUsize, ParseIntError}; -use std::ops::{Add, AddAssign, Deref, Mul, RangeFull, RangeInclusive, Sub}; +use std::ops::{Add, AddAssign, Deref, Div, Mul, RangeFull, RangeInclusive, Sub}; use std::str::FromStr; use bitflags::bitflags; @@ -819,6 +819,14 @@ impl Size { if bytes < dl.obj_size_bound() { Some(Size::from_bytes(bytes)) } else { None } } + #[inline] + pub fn checked_div(self, count: u64, cx: &C) -> Option { + let dl = cx.data_layout(); + + let bytes = self.bytes().checked_div(count)?; + if bytes < dl.obj_size_bound() { Some(Size::from_bytes(bytes)) } else { None } + } + /// Truncates `value` to `self` bits and then sign-extends it to 128 bits /// (i.e., if it is negative, fill with 1's on the left). #[inline] @@ -906,6 +914,25 @@ impl Mul for Size { } } +impl Div for u64 { + type Output = Size; + #[inline] + fn div(self, size: Size) -> Size { + size / self + } +} + +impl Div for Size { + type Output = Size; + #[inline] + fn div(self, count: u64) -> Size { + match self.bytes().checked_div(count) { + Some(bytes) => Size::from_bytes(bytes), + None => panic!("Size::div: {} / {} doesn't fit in u64", self.bytes(), count), + } + } +} + impl AddAssign for Size { #[inline] fn add_assign(&mut self, other: Size) { diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index da2a153d819f9..df26501da31c7 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -614,6 +614,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn load_relative(&mut self, ptr: &'ll Value, byte_offset: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMBuildLoadRelative(self.llbuilder, ptr, byte_offset) } + } + fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { unsafe { let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index f29fefb66f0fe..13d2cf3cec269 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -287,8 +287,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { self.const_bitcast(llval, llty) }; } else { - let init = - const_alloc_to_llvm(self, alloc.inner(), /*static*/ false); + let init = const_alloc_to_llvm( + self, + alloc.inner(), + /*static*/ false, + /*vtable_base*/ None, + ); let alloc = alloc.inner(); let value = match alloc.mutability { Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), @@ -320,7 +324,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { }), ))) .unwrap_memory(); - let init = const_alloc_to_llvm(self, alloc.inner(), /*static*/ false); + let init = const_alloc_to_llvm( + self, + alloc.inner(), + /*static*/ false, + /*vtable_base*/ None, + ); self.static_addr_of_impl(init, alloc.inner().align, None) } GlobalAlloc::Static(def_id) => { @@ -354,7 +363,37 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { } fn const_data_from_alloc(&self, alloc: ConstAllocation<'_>) -> Self::Value { - const_alloc_to_llvm(self, alloc.inner(), /*static*/ false) + const_alloc_to_llvm(self, alloc.inner(), /*static*/ false, /*vtable_base*/ None) + } + + fn construct_vtable( + &self, + vtable_allocation: ConstAllocation<'_>, + num_entries: u64, + ) -> Self::Value { + // When constructing relative vtables, we need to create the global first before creating + // the initializer so the initializer has references to the global we will bind it to. + // Regular vtables aren't self-referential so we can just create the initializer on its + // own. + if self.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let llty = self.type_array(self.type_i32(), num_entries); + let vtable = self.static_addr_of_mut_from_type( + llty, + self.data_layout().i32_align.abi, + Some("vtable"), + ); + let init = const_alloc_to_llvm( + self, + vtable_allocation.inner(), + /*static*/ false, + Some(vtable), + ); + self.static_addr_of_impl_for_gv(init, vtable) + } else { + let vtable_const = self.const_data_from_alloc(vtable_allocation); + let align = self.data_layout().pointer_align().abi; + self.static_addr_of(vtable_const, align, Some("vtable")) + } } fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value { diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 0b96b63bc8573..79f92139baf22 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,6 +1,6 @@ use std::ops::Range; -use rustc_abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; +use rustc_abi::{Align, Endian, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; use rustc_codegen_ssa::common; use rustc_codegen_ssa::traits::*; use rustc_hir::LangItem; @@ -28,6 +28,7 @@ pub(crate) fn const_alloc_to_llvm<'ll>( cx: &CodegenCx<'ll, '_>, alloc: &Allocation, is_static: bool, + vtable_base: Option<&'ll Value>, ) -> &'ll Value { // We expect that callers of const_alloc_to_llvm will instead directly codegen a pointer or // integer for any &ZST where the ZST is a constant (i.e. not a static). We should never be @@ -43,6 +44,8 @@ pub(crate) fn const_alloc_to_llvm<'ll>( let dl = cx.data_layout(); let pointer_size = dl.pointer_size(); let pointer_size_bytes = pointer_size.bytes() as usize; + let use_relative_layout = cx.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables + && vtable_base.is_some(); // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range` // must be within the bounds of `alloc` and not contain or overlap a pointer provenance. @@ -51,7 +54,11 @@ pub(crate) fn const_alloc_to_llvm<'ll>( cx: &'a CodegenCx<'ll, 'b>, alloc: &'a Allocation, range: Range, + use_relative_layout: bool, ) { + let dl = cx.data_layout(); + let pointer_size = dl.pointer_size(); + let pointer_size_bytes = pointer_size.bytes() as usize; let chunks = alloc.init_mask().range_as_init_chunks(range.clone().into()); let chunk_to_llval = move |chunk| match chunk { @@ -74,7 +81,43 @@ pub(crate) fn const_alloc_to_llvm<'ll>( let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; if allow_uninit_chunks { - llvals.extend(chunks.map(chunk_to_llval)); + if use_relative_layout { + // Rather than being stored as a struct of pointers or byte-arrays, a relative + // vtable is a pure i32 array, so its components must be chunks of i32s. Here we + // explicitly group any sequence of bytes into i32s. + // + // Normally we can only do this if an 8-byte constant can fit into 4 bytes. + for chunk in chunks { + match chunk { + InitChunk::Init(range) => { + let range = + (range.start.bytes() as usize)..(range.end.bytes() as usize); + let bytes = + alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); + for bytes in bytes.chunks_exact(pointer_size_bytes) { + assert!( + bytes[4..pointer_size_bytes].iter().all(|&x| x == 0), + "Cannot fit constant into 4-bytes: {:?}", + bytes + ); + let bytes: [u8; 4] = bytes[0..4].try_into().unwrap(); + let val: u32 = match dl.endian { + Endian::Big => u32::from_be_bytes(bytes), + Endian::Little => u32::from_le_bytes(bytes), + }; + llvals.push(cx.const_u32(val)); + } + } + InitChunk::Uninit(range) => { + let len = range.end.bytes() - range.start.bytes(); + let val = cx.const_undef(cx.type_array(cx.type_i8(), len / 2)); + llvals.push(val); + } + }; + } + } else { + llvals.extend(chunks.map(chunk_to_llval)); + } } else { // If this allocation contains any uninit bytes, codegen as if it was initialized // (using some arbitrary value for uninit bytes). @@ -92,7 +135,13 @@ pub(crate) fn const_alloc_to_llvm<'ll>( // This `inspect` is okay since we have checked that there is no provenance, it // is within the bounds of the allocation, and it doesn't affect interpreter execution // (we inspect the result after interpreter execution). - append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); + append_chunks_of_init_and_uninit_bytes( + &mut llvals, + cx, + alloc, + next_offset..offset, + use_relative_layout, + ); } let ptr_offset = read_target_uint( dl.endian, @@ -108,14 +157,34 @@ pub(crate) fn const_alloc_to_llvm<'ll>( let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); - llvals.push(cx.scalar_to_backend( - InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx), - Scalar::Initialized { - value: Primitive::Pointer(address_space), - valid_range: WrappingRange::full(pointer_size), - }, - cx.type_ptr_ext(address_space), - )); + let s = { + let scalar = cx.scalar_to_backend( + InterpScalar::from_pointer( + Pointer::new(prov, Size::from_bytes(ptr_offset)), + &cx.tcx, + ), + Scalar::Initialized { + value: Primitive::Pointer(address_space), + valid_range: WrappingRange::full(pointer_size), + }, + cx.type_ptr_ext(address_space), + ); + + if use_relative_layout { + unsafe { + let fptr = llvm::LLVMDSOLocalEquivalent(scalar); + let sub = llvm::LLVMConstSub( + llvm::LLVMConstPtrToInt(fptr, cx.type_i64()), + llvm::LLVMConstPtrToInt(vtable_base.unwrap(), cx.type_i64()), + ); + llvm::LLVMConstTrunc(sub, cx.type_i32()) + } + } else { + scalar + } + }; + + llvals.push(s); next_offset = offset + pointer_size_bytes; } if alloc.len() >= next_offset { @@ -123,7 +192,7 @@ pub(crate) fn const_alloc_to_llvm<'ll>( // This `inspect` is okay since we have check that it is after all provenance, it is // within the bounds of the allocation, and it doesn't affect interpreter execution (we // inspect the result after interpreter execution). - append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); + append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range, use_relative_layout); } // Avoid wrapping in a struct if there is only a single value. This ensures @@ -131,7 +200,13 @@ pub(crate) fn const_alloc_to_llvm<'ll>( // is a valid C string. LLVM only considers bare arrays for this optimization, // not arrays wrapped in a struct. LLVM handles this at: // https://github.com/rust-lang/llvm-project/blob/acaea3d2bb8f351b740db7ebce7d7a40b9e21488/llvm/lib/Target/TargetLoweringObjectFile.cpp#L249-L280 - if let &[data] = &*llvals { data } else { cx.const_struct(&llvals, true) } + if let &[data] = &*llvals { + data + } else if use_relative_layout { + cx.const_array(cx.type_i32(), &llvals) + } else { + cx.const_struct(&llvals, true) + } } fn codegen_static_initializer<'ll, 'tcx>( @@ -139,7 +214,7 @@ fn codegen_static_initializer<'ll, 'tcx>( def_id: DefId, ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { let alloc = cx.tcx.eval_static_initializer(def_id)?; - Ok((const_alloc_to_llvm(cx, alloc.inner(), /*static*/ true), alloc)) + Ok((const_alloc_to_llvm(cx, alloc.inner(), /*static*/ true, /*vtable_base*/ None), alloc)) } fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) { @@ -232,19 +307,29 @@ impl<'ll> CodegenCx<'ll, '_> { cv: &'ll Value, align: Align, kind: Option<&str>, + ) -> &'ll Value { + let gv = self.static_addr_of_mut_from_type(self.val_ty(cv), align, kind); + llvm::set_initializer(gv, cv); + gv + } + + pub(crate) fn static_addr_of_mut_from_type( + &self, + ty: &'ll Type, + align: Align, + kind: Option<&str>, ) -> &'ll Value { let gv = match kind { Some(kind) if !self.tcx.sess.fewer_names() => { let name = self.generate_local_symbol_name(kind); - let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { + let gv = self.define_global(&name, ty).unwrap_or_else(|| { bug!("symbol `{}` is already defined", name); }); llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage); gv } - _ => self.define_private_global(self.val_ty(cv)), + _ => self.define_private_global(ty), }; - llvm::set_initializer(gv, cv); set_global_alignment(self, gv, align); llvm::set_unnamed_address(gv, llvm::UnnamedAddr::Global); gv @@ -277,6 +362,15 @@ impl<'ll> CodegenCx<'ll, '_> { gv } + pub(crate) fn static_addr_of_impl_for_gv(&self, cv: &'ll Value, gv: &'ll Value) -> &'ll Value { + assert!(!self.const_globals.borrow().contains_key(&cv)); + let mut binding = self.const_globals.borrow_mut(); + binding.insert(cv, gv); + llvm::set_initializer(gv, cv); + llvm::set_global_constant(gv, true); + gv + } + #[instrument(level = "debug", skip(self))] pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { let instance = Instance::mono(self.tcx, def_id); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 2443194ff4832..a08f423732f52 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1169,6 +1169,8 @@ unsafe extern "C" { pub(crate) fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; pub(crate) fn LLVMGetConstOpcode(ConstantVal: &Value) -> Opcode; pub(crate) fn LLVMIsAConstantExpr(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMConstSub<'a>(LHS: &'a Value, RHS: &'a Value) -> &'a Value; + pub(crate) fn LLVMConstTrunc<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; // Operations on global variables, functions, and aliases (globals) pub(crate) fn LLVMIsDeclaration(Global: &Value) -> Bool; @@ -1198,6 +1200,13 @@ unsafe extern "C" { pub(crate) safe fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); pub(crate) safe fn LLVMRustSetTailCallKind(CallInst: &Value, Kind: TailCallKind); + pub(crate) fn LLVMDSOLocalEquivalent(GlobalVar: &Value) -> &Value; + pub(crate) fn LLVMBuildLoadRelative<'a>( + Builder: &Builder<'a>, + Ptr: &'a Value, + ByteOffset: &'a Value, + ) -> &'a Value; + // Operations on attributes pub(crate) fn LLVMCreateStringAttribute( C: &Context, diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 893655031388c..af011b18dd8e1 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -292,6 +292,13 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { fn_abi.ptr_to_llvm_type(self) } + fn vtable_component_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + if self.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + self.type_i32() + } else { + fn_abi.ptr_to_llvm_type(self) + } + } fn reg_backend_type(&self, ty: &Reg) -> &'ll Type { ty.llvm_type(self) } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index b4556ced0b3fb..4aa44472c2a9d 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -202,7 +202,28 @@ fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if let Some(entry_idx) = vptr_entry_idx { let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = u64::try_from(entry_idx).unwrap() * ptr_size.bytes(); - load_vtable(bx, old_info, bx.type_ptr(), vtable_byte_offset, source, true) + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let val = load_vtable( + bx, + old_info, + bx.type_ptr(), + vtable_byte_offset / 2, + source, + true, + /*load_relative*/ true, + ); + bx.zext(val, bx.type_isize()) + } else { + load_vtable( + bx, + old_info, + bx.type_ptr(), + vtable_byte_offset, + source, + true, + /*load_relative*/ false, + ) + } } else { old_info } diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 34ad35a729b98..255490ffc48ce 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -25,12 +25,32 @@ impl<'a, 'tcx> VirtualIndex { ) -> Bx::Value { // Load the function pointer from the object. debug!("get_fn({llvtable:?}, {ty:?}, {self:?})"); - - let llty = bx.fn_ptr_backend_type(fn_abi); let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = self.0 * ptr_size.bytes(); - load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, nonnull) + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let llty = bx.vtable_component_type(fn_abi); + load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset / 2, + ty, + nonnull, + /*load_relative*/ true, + ) + } else { + let llty = bx.fn_ptr_backend_type(fn_abi); + load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset, + ty, + nonnull, + /*load_relative*/ false, + ) + } } pub(crate) fn get_optional_fn>( @@ -61,12 +81,33 @@ impl<'a, 'tcx> VirtualIndex { ) -> Bx::Value { // Load the data pointer from the object. debug!("get_int({:?}, {:?})", llvtable, self); - - let llty = bx.type_isize(); let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = self.0 * ptr_size.bytes(); - load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, false) + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let llty = bx.type_i32(); + let val = load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset / 2, + ty, + false, + /*load_relative*/ false, + ); + bx.zext(val, bx.type_isize()) + } else { + let llty = bx.type_isize(); + load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset, + ty, + false, + /*load_relative*/ false, + ) + } } } @@ -114,9 +155,17 @@ pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( let vtable_alloc_id = tcx.vtable_allocation((ty, trait_ref)); let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory(); - let vtable_const = cx.const_data_from_alloc(vtable_allocation); - let align = cx.data_layout().pointer_align().abi; - let vtable = cx.static_addr_of(vtable_const, align, Some("vtable")); + let num_entries = { + if let Some(trait_ref) = trait_ref { + let trait_ref = trait_ref.with_self_ty(tcx, ty); + let trait_ref = tcx.erase_regions(trait_ref); + tcx.vtable_entries(trait_ref) + } else { + TyCtxt::COMMON_VTABLE_ENTRIES + } + } + .len(); + let vtable = cx.construct_vtable(vtable_allocation, num_entries as u64); cx.apply_vcall_visibility_metadata(ty, trait_ref, vtable); cx.create_vtable_debuginfo(ty, trait_ref, vtable); @@ -132,6 +181,7 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( vtable_byte_offset: u64, ty: Ty<'tcx>, nonnull: bool, + load_relative: bool, ) -> Bx::Value { let ptr_align = bx.data_layout().pointer_align().abi; @@ -141,6 +191,7 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if let Some(trait_ref) = dyn_trait_in_self(bx.tcx(), ty) { let typeid = bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), trait_ref).as_bytes()).unwrap(); + // FIXME: Add correct intrinsic for RV here. let func = bx.type_checked_load(llvtable, vtable_byte_offset, typeid); return func; } else if nonnull { @@ -148,11 +199,23 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } - let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); - let ptr = bx.load(llty, gep, ptr_align); + let ptr = if load_relative { + bx.load_relative(llvtable, bx.const_i32(vtable_byte_offset.try_into().unwrap())) + } else { + let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); + bx.load(llty, gep, ptr_align) + }; + // VTable loads are invariant. bx.set_invariant_load(ptr); - if nonnull { + // FIXME: The verifier complains with + // + // nonnull applies only to load instructions, use attributes for calls or invokes + // @llvm.load.relative.i32 (ptr nonnull %13, i32 12), !dbg !4323, !invariant.load !27, !noalias !27, !nonnull !27 + // + // For now, do not mark the load relative intrinsic with nonnull, but I think it should be fine + // to do so since it's effectively a load. + if nonnull && !load_relative { bx.nonnull_metadata(ptr); } ptr diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index e96590441fa42..31eb5807ce647 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -607,7 +607,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { args1 = [place.val.llval]; &args1[..] }; - let (maybe_null, drop_fn, fn_abi, drop_instance) = match ty.kind() { + let (maybe_null, drop_fn, fn_abi, drop_instance, vtable) = match ty.kind() { // FIXME(eddyb) perhaps move some of this logic into // `Instance::resolve_drop_in_place`? ty::Dynamic(_, _, ty::Dyn) => { @@ -640,6 +640,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { .get_optional_fn(bx, vtable, ty, fn_abi), fn_abi, virtual_drop, + Some(vtable), ) } _ => ( @@ -647,6 +648,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty()), drop_fn, + None, ), }; @@ -654,10 +656,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // generated for no-op drops. if maybe_null { let is_not_null = bx.append_sibling_block("is_not_null"); - let llty = bx.fn_ptr_backend_type(fn_abi); - let null = bx.const_null(llty); let non_null = - bx.icmp(base::bin_op_to_icmp_predicate(mir::BinOp::Ne, false), drop_fn, null); + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + bx.icmp( + base::bin_op_to_icmp_predicate(mir::BinOp::Ne, /*signed*/ false), + drop_fn, + vtable.unwrap(), + ) + } else { + let llty = bx.fn_ptr_backend_type(fn_abi); + let null = bx.const_null(llty); + bx.icmp(base::bin_op_to_icmp_predicate(mir::BinOp::Ne, false), drop_fn, null) + }; bx.cond_br(non_null, is_not_null, helper.llbb_with_cleanup(self, target)); bx.switch_to_block(is_not_null); self.set_debug_loc(bx, *source_info); diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 577012151e49f..7470f98253bcd 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -33,11 +33,15 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let align = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_ALIGN) .get_usize(bx, vtable, t); - // Size is always <= isize::MAX. - let size_bound = bx.data_layout().ptr_sized_integer().signed_max() as u128; - bx.range_metadata(size, WrappingRange { start: 0, end: size_bound }); - // Alignment is always nonzero. - bx.range_metadata(align, WrappingRange { start: 1, end: !0 }); + // FIXME: The range metadatat can only be applied to load instructions, but it probably + // can also be changed to apply to the load_relative intrinsic. + if !bx.tcx().sess.opts.unstable_opts.experimental_relative_rust_abi_vtables { + // Size is always <= isize::MAX. + let size_bound = bx.data_layout().ptr_sized_integer().signed_max() as u128; + bx.range_metadata(size, WrappingRange { start: 0, end: size_bound }); + // Alignment is always nonzero. + bx.range_metadata(align, WrappingRange { start: 1, end: !0 }); + } (size, align) } diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 4b18146863bf4..b217f8bc3a4e8 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -236,6 +236,7 @@ pub trait BuilderMethods<'a, 'tcx>: fn alloca(&mut self, size: Size, align: Align) -> Self::Value; fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value; + fn load_relative(&mut self, ptr: Self::Value, byte_offset: Self::Value) -> Self::Value; fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value; fn atomic_load( &mut self, diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs index d83a04d814be3..1e249ffac1141 100644 --- a/compiler/rustc_codegen_ssa/src/traits/consts.rs +++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs @@ -38,6 +38,7 @@ pub trait ConstCodegenMethods: BackendTypes { fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option; fn const_data_from_alloc(&self, alloc: ConstAllocation<'_>) -> Self::Value; + fn construct_vtable(&self, alloc: ConstAllocation<'_>, num_entries: u64) -> Self::Value; fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value; diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 32c24965e1bf6..bb1f814338e23 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -116,6 +116,7 @@ pub trait LayoutTypeCodegenMethods<'tcx>: BackendTypes { fn cast_backend_type(&self, ty: &CastTarget) -> Self::Type; fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; + fn vtable_component_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; fn reg_backend_type(&self, ty: &Reg) -> Self::Type; /// The backend type used for a rust type when it's in an SSA register. /// diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 588d867bbbf4e..fda6ecfe2deef 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -151,6 +151,10 @@ extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames) { return wrap(ctx); } +extern "C" LLVMValueRef LLVMDSOLocalEquivalent(LLVMValueRef GlobalVal) { + return wrap(DSOLocalEquivalent::get(unwrap(GlobalVal))); +} + extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, const char *Target) { #if LLVM_VERSION_GE(21, 0) @@ -613,6 +617,14 @@ LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source, return wrap(LI); } +extern "C" LLVMValueRef LLVMBuildLoadRelative(LLVMBuilderRef B, LLVMValueRef Ptr, + LLVMValueRef ByteOffset) { + Type *Int32Ty = Type::getInt32Ty(unwrap(B)->getContext()); + Value *call = unwrap(B)->CreateIntrinsic( + Intrinsic::load_relative, {Int32Ty}, {unwrap(Ptr), unwrap(ByteOffset)}); + return wrap(call); +} + extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B, LLVMValueRef V, LLVMValueRef Target, diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index cfeadf3c7595a..65ed4a3020f10 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -2802,6 +2802,10 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M early_dcx.early_fatal("can't dump dependency graph without `-Z query-dep-graph`"); } + if target_triple.tuple().ends_with("fuchsia") { + unstable_opts.experimental_relative_rust_abi_vtables = true; + } + let logical_env = parse_logical_env(early_dcx, matches); let sysroot = Sysroot::new(matches.opt_str("sysroot").map(PathBuf::from)); diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 880b08d444414..0bda97ee132c9 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2241,6 +2241,9 @@ options! { "enforce the type length limit when monomorphizing instances in codegen"), experimental_default_bounds: bool = (false, parse_bool, [TRACKED], "enable default bounds for experimental group of auto traits"), + // Change this to true to unconditionally build all runtimes with relative vtables. + experimental_relative_rust_abi_vtables: bool = (false, parse_bool, [TRACKED], + "use the relative layout for vtables"), export_executable_symbols: bool = (false, parse_bool, [TRACKED], "export symbols from executables, as if they were dynamic libraries"), external_clangrt: bool = (false, parse_bool, [UNTRACKED], diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index af2e000e34028..7d5c5ad792100 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -309,8 +309,21 @@ fn adjust_for_rust_scalar<'tcx>( None }; if let Some(kind) = kind { - attrs.pointee_align = - Some(pointee.align.min(cx.tcx().sess.target.max_reliable_alignment())); + let is_vtable = if let ty::Ref(_, ty, _) = *layout.ty.kind() { + ty.is_trait() + && matches!(kind, PointerKind::SharedRef { frozen: true }) + && !is_return + } else { + false + }; + + if cx.tcx().sess.opts.unstable_opts.experimental_relative_rust_abi_vtables && is_vtable + { + attrs.pointee_align = Some(cx.tcx().data_layout.i32_align.abi); + } else { + attrs.pointee_align = + Some(pointee.align.min(cx.tcx().sess.target.max_reliable_alignment())); + } // `Box` are not necessarily dereferenceable for the entire duration of the function as // they can be deallocated at any time. Same for non-frozen shared references (see @@ -322,8 +335,16 @@ fn adjust_for_rust_scalar<'tcx>( PointerKind::Box { .. } | PointerKind::SharedRef { frozen: false } | PointerKind::MutableRef { unpin: false } => Size::ZERO, - PointerKind::SharedRef { frozen: true } - | PointerKind::MutableRef { unpin: true } => pointee.size, + PointerKind::SharedRef { frozen: true } => { + if cx.tcx().sess.opts.unstable_opts.experimental_relative_rust_abi_vtables + && is_vtable + { + pointee.size / 2 + } else { + pointee.size + } + } + PointerKind::MutableRef { unpin: true } => pointee.size, }; // The aliasing rules for `Box` are still not decided, but currently we emit diff --git a/src/doc/rustc/src/codegen-options/index.md b/src/doc/rustc/src/codegen-options/index.md index 07eafdf4c4c62..26b220a67c369 100644 --- a/src/doc/rustc/src/codegen-options/index.md +++ b/src/doc/rustc/src/codegen-options/index.md @@ -162,6 +162,10 @@ This option allows you to put extra data in each output filename. It takes a string to add as a suffix to the filename. See the [`--emit` flag][option-emit] for more information. +## experimental-relative-rust-abi-vtables + +This option uses the relative layout for vtables. + ## force-frame-pointers This flag forces the use of frame pointers. It takes one of the following diff --git a/tests/codegen-llvm/relative-vtables/simple-vtable.rs b/tests/codegen-llvm/relative-vtables/simple-vtable.rs new file mode 100644 index 0000000000000..8cd5329051384 --- /dev/null +++ b/tests/codegen-llvm/relative-vtables/simple-vtable.rs @@ -0,0 +1,79 @@ +//@ compile-flags: -Zexperimental-relative-rust-abi-vtables=y + +#![crate_type = "lib"] + +// CHECK: @vtable.0 = private {{.*}}constant [5 x i32] [ +// CHECK-SAME: i32 0, +// CHECK-SAME: i32 4, +// CHECK-SAME: i32 4, +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT2_FOO:@".*Struct2\$.*foo.*"]] to i64), i64 ptrtoint (ptr @vtable.0 to i64)) to i32), +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT2_BAR:@".*Struct2\$.*bar.*"]] to i64), i64 ptrtoint (ptr @vtable.0 to i64)) to i32) +// CHECK-SAME: ], align 4 + +// CHECK: @vtable.1 = private {{.*}}constant [5 x i32] [ +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[VT1_DROP_IN_PLACE:\@".*drop_in_place[^\"]*"]] to i64), i64 ptrtoint (ptr @vtable.1 to i64)) to i32), +// CHECK-SAME: i32 24, +// CHECK-SAME: i32 8, +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT_FOO:@".*Struct\$.*foo.*"]] to i64), i64 ptrtoint (ptr @vtable.1 to i64)) to i32), +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT_BAR:@".*Struct\$.*bar.*"]] to i64), i64 ptrtoint (ptr @vtable.1 to i64)) to i32) +// CHECK-SAME: ], align 4 + +// CHECK-DAG: define {{.*}}void [[STRUCT2_FOO]](ptr +// CHECK-DAG: define {{.*}}i32 [[STRUCT2_BAR]](ptr +// CHECK-DAG: define {{.*}}void [[STRUCT_FOO]](ptr +// CHECK-DAG: define {{.*}}i32 [[STRUCT_BAR]](ptr + +trait MyTrait { + fn foo(&self); + fn bar(&self) -> u32; +} + +struct Struct { + s: String, +} + +struct Struct2 { + u: u32, +} + +impl MyTrait for Struct { + fn foo(&self) { + println!("Struct foo {}", self.s); + } + fn bar(&self) -> u32 { + 42 + } +} + +impl MyTrait for Struct2 { + fn foo(&self) { + println!("Struct2 foo {}", self.bar()); + } + fn bar(&self) -> u32 { + self.u + } +} + +/// This is only here to manifest the vtables. +pub fn create_struct(b: bool) -> Box { + if b { Box::new(Struct { s: "abc".to_string() }) } else { Box::new(Struct2 { u: 1 }) } +} + +// CHECK-LABEL: define void @_ZN13simple_vtable10invoke_foo17hebe38af67b23d609E( +// CHECK-SAME: ptr noundef nonnull align 1 [[DATA:%.*]], +// CHECK-SAME: ptr noalias noundef readonly align 4 dereferenceable(20) [[VTABLE:%.*]]) {{.*}}{ +// CHECK-NEXT: start: +// CHECK-NEXT: [[FUNC:%.*]] = tail call ptr @llvm.load.relative.i32(ptr nonnull [[VTABLE]], i32 12), !invariant.load +// CHECK-NEXT: tail call void [[FUNC]](ptr noundef nonnull align 1 [[DATA]]) +// CHECK-NEXT: ret void +// CHECK-NEXT: } +pub fn invoke_foo(x: &dyn MyTrait) { + x.foo(); +} + +// CHECK-LABEL: define void @_ZN13simple_vtable11invoke_drop17ha1ff356bc5a4b987E(i1 noundef zeroext %b) {{.*}}{ +// CHECK: [[BOX:%.*]] = tail call { ptr, ptr } @_ZN13simple_vtable13create_struct{{.*}}(i1 noundef zeroext %b) +pub fn invoke_drop(b: bool) { + let bx = create_struct(b); + invoke_foo(&*bx); +}