diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 14e256b8045df..dcb7b44393c5d 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -43,7 +43,7 @@ use std::fmt; #[cfg(feature = "nightly")] use std::iter::Step; use std::num::{NonZeroUsize, ParseIntError}; -use std::ops::{Add, AddAssign, Deref, Mul, RangeFull, RangeInclusive, Sub}; +use std::ops::{Add, AddAssign, Deref, Div, Mul, RangeFull, RangeInclusive, Sub}; use std::str::FromStr; use bitflags::bitflags; @@ -819,6 +819,14 @@ impl Size { if bytes < dl.obj_size_bound() { Some(Size::from_bytes(bytes)) } else { None } } + #[inline] + pub fn checked_div(self, count: u64, cx: &C) -> Option { + let dl = cx.data_layout(); + + let bytes = self.bytes().checked_div(count)?; + if bytes < dl.obj_size_bound() { Some(Size::from_bytes(bytes)) } else { None } + } + /// Truncates `value` to `self` bits and then sign-extends it to 128 bits /// (i.e., if it is negative, fill with 1's on the left). #[inline] @@ -906,6 +914,25 @@ impl Mul for Size { } } +impl Div for u64 { + type Output = Size; + #[inline] + fn div(self, size: Size) -> Size { + size / self + } +} + +impl Div for Size { + type Output = Size; + #[inline] + fn div(self, count: u64) -> Size { + match self.bytes().checked_div(count) { + Some(bytes) => Size::from_bytes(bytes), + None => panic!("Size::div: {} / {} doesn't fit in u64", self.bytes(), count), + } + } +} + impl AddAssign for Size { #[inline] fn add_assign(&mut self, other: Size) { diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index da2a153d819f9..df26501da31c7 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -614,6 +614,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { } } + fn load_relative(&mut self, ptr: &'ll Value, byte_offset: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMBuildLoadRelative(self.llbuilder, ptr, byte_offset) } + } + fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { unsafe { let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs index f29fefb66f0fe..13d2cf3cec269 100644 --- a/compiler/rustc_codegen_llvm/src/common.rs +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -287,8 +287,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { self.const_bitcast(llval, llty) }; } else { - let init = - const_alloc_to_llvm(self, alloc.inner(), /*static*/ false); + let init = const_alloc_to_llvm( + self, + alloc.inner(), + /*static*/ false, + /*vtable_base*/ None, + ); let alloc = alloc.inner(); let value = match alloc.mutability { Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), @@ -320,7 +324,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { }), ))) .unwrap_memory(); - let init = const_alloc_to_llvm(self, alloc.inner(), /*static*/ false); + let init = const_alloc_to_llvm( + self, + alloc.inner(), + /*static*/ false, + /*vtable_base*/ None, + ); self.static_addr_of_impl(init, alloc.inner().align, None) } GlobalAlloc::Static(def_id) => { @@ -354,7 +363,37 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> { } fn const_data_from_alloc(&self, alloc: ConstAllocation<'_>) -> Self::Value { - const_alloc_to_llvm(self, alloc.inner(), /*static*/ false) + const_alloc_to_llvm(self, alloc.inner(), /*static*/ false, /*vtable_base*/ None) + } + + fn construct_vtable( + &self, + vtable_allocation: ConstAllocation<'_>, + num_entries: u64, + ) -> Self::Value { + // When constructing relative vtables, we need to create the global first before creating + // the initializer so the initializer has references to the global we will bind it to. + // Regular vtables aren't self-referential so we can just create the initializer on its + // own. + if self.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let llty = self.type_array(self.type_i32(), num_entries); + let vtable = self.static_addr_of_mut_from_type( + llty, + self.data_layout().i32_align.abi, + Some("vtable"), + ); + let init = const_alloc_to_llvm( + self, + vtable_allocation.inner(), + /*static*/ false, + Some(vtable), + ); + self.static_addr_of_impl_for_gv(init, vtable) + } else { + let vtable_const = self.const_data_from_alloc(vtable_allocation); + let align = self.data_layout().pointer_align().abi; + self.static_addr_of(vtable_const, align, Some("vtable")) + } } fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value { diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs index 0b96b63bc8573..79f92139baf22 100644 --- a/compiler/rustc_codegen_llvm/src/consts.rs +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -1,6 +1,6 @@ use std::ops::Range; -use rustc_abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; +use rustc_abi::{Align, Endian, HasDataLayout, Primitive, Scalar, Size, WrappingRange}; use rustc_codegen_ssa::common; use rustc_codegen_ssa::traits::*; use rustc_hir::LangItem; @@ -28,6 +28,7 @@ pub(crate) fn const_alloc_to_llvm<'ll>( cx: &CodegenCx<'ll, '_>, alloc: &Allocation, is_static: bool, + vtable_base: Option<&'ll Value>, ) -> &'ll Value { // We expect that callers of const_alloc_to_llvm will instead directly codegen a pointer or // integer for any &ZST where the ZST is a constant (i.e. not a static). We should never be @@ -43,6 +44,8 @@ pub(crate) fn const_alloc_to_llvm<'ll>( let dl = cx.data_layout(); let pointer_size = dl.pointer_size(); let pointer_size_bytes = pointer_size.bytes() as usize; + let use_relative_layout = cx.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables + && vtable_base.is_some(); // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range` // must be within the bounds of `alloc` and not contain or overlap a pointer provenance. @@ -51,7 +54,11 @@ pub(crate) fn const_alloc_to_llvm<'ll>( cx: &'a CodegenCx<'ll, 'b>, alloc: &'a Allocation, range: Range, + use_relative_layout: bool, ) { + let dl = cx.data_layout(); + let pointer_size = dl.pointer_size(); + let pointer_size_bytes = pointer_size.bytes() as usize; let chunks = alloc.init_mask().range_as_init_chunks(range.clone().into()); let chunk_to_llval = move |chunk| match chunk { @@ -74,7 +81,43 @@ pub(crate) fn const_alloc_to_llvm<'ll>( let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; if allow_uninit_chunks { - llvals.extend(chunks.map(chunk_to_llval)); + if use_relative_layout { + // Rather than being stored as a struct of pointers or byte-arrays, a relative + // vtable is a pure i32 array, so its components must be chunks of i32s. Here we + // explicitly group any sequence of bytes into i32s. + // + // Normally we can only do this if an 8-byte constant can fit into 4 bytes. + for chunk in chunks { + match chunk { + InitChunk::Init(range) => { + let range = + (range.start.bytes() as usize)..(range.end.bytes() as usize); + let bytes = + alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); + for bytes in bytes.chunks_exact(pointer_size_bytes) { + assert!( + bytes[4..pointer_size_bytes].iter().all(|&x| x == 0), + "Cannot fit constant into 4-bytes: {:?}", + bytes + ); + let bytes: [u8; 4] = bytes[0..4].try_into().unwrap(); + let val: u32 = match dl.endian { + Endian::Big => u32::from_be_bytes(bytes), + Endian::Little => u32::from_le_bytes(bytes), + }; + llvals.push(cx.const_u32(val)); + } + } + InitChunk::Uninit(range) => { + let len = range.end.bytes() - range.start.bytes(); + let val = cx.const_undef(cx.type_array(cx.type_i8(), len / 2)); + llvals.push(val); + } + }; + } + } else { + llvals.extend(chunks.map(chunk_to_llval)); + } } else { // If this allocation contains any uninit bytes, codegen as if it was initialized // (using some arbitrary value for uninit bytes). @@ -92,7 +135,13 @@ pub(crate) fn const_alloc_to_llvm<'ll>( // This `inspect` is okay since we have checked that there is no provenance, it // is within the bounds of the allocation, and it doesn't affect interpreter execution // (we inspect the result after interpreter execution). - append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); + append_chunks_of_init_and_uninit_bytes( + &mut llvals, + cx, + alloc, + next_offset..offset, + use_relative_layout, + ); } let ptr_offset = read_target_uint( dl.endian, @@ -108,14 +157,34 @@ pub(crate) fn const_alloc_to_llvm<'ll>( let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); - llvals.push(cx.scalar_to_backend( - InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx), - Scalar::Initialized { - value: Primitive::Pointer(address_space), - valid_range: WrappingRange::full(pointer_size), - }, - cx.type_ptr_ext(address_space), - )); + let s = { + let scalar = cx.scalar_to_backend( + InterpScalar::from_pointer( + Pointer::new(prov, Size::from_bytes(ptr_offset)), + &cx.tcx, + ), + Scalar::Initialized { + value: Primitive::Pointer(address_space), + valid_range: WrappingRange::full(pointer_size), + }, + cx.type_ptr_ext(address_space), + ); + + if use_relative_layout { + unsafe { + let fptr = llvm::LLVMDSOLocalEquivalent(scalar); + let sub = llvm::LLVMConstSub( + llvm::LLVMConstPtrToInt(fptr, cx.type_i64()), + llvm::LLVMConstPtrToInt(vtable_base.unwrap(), cx.type_i64()), + ); + llvm::LLVMConstTrunc(sub, cx.type_i32()) + } + } else { + scalar + } + }; + + llvals.push(s); next_offset = offset + pointer_size_bytes; } if alloc.len() >= next_offset { @@ -123,7 +192,7 @@ pub(crate) fn const_alloc_to_llvm<'ll>( // This `inspect` is okay since we have check that it is after all provenance, it is // within the bounds of the allocation, and it doesn't affect interpreter execution (we // inspect the result after interpreter execution). - append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); + append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range, use_relative_layout); } // Avoid wrapping in a struct if there is only a single value. This ensures @@ -131,7 +200,13 @@ pub(crate) fn const_alloc_to_llvm<'ll>( // is a valid C string. LLVM only considers bare arrays for this optimization, // not arrays wrapped in a struct. LLVM handles this at: // https://github.com/rust-lang/llvm-project/blob/acaea3d2bb8f351b740db7ebce7d7a40b9e21488/llvm/lib/Target/TargetLoweringObjectFile.cpp#L249-L280 - if let &[data] = &*llvals { data } else { cx.const_struct(&llvals, true) } + if let &[data] = &*llvals { + data + } else if use_relative_layout { + cx.const_array(cx.type_i32(), &llvals) + } else { + cx.const_struct(&llvals, true) + } } fn codegen_static_initializer<'ll, 'tcx>( @@ -139,7 +214,7 @@ fn codegen_static_initializer<'ll, 'tcx>( def_id: DefId, ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { let alloc = cx.tcx.eval_static_initializer(def_id)?; - Ok((const_alloc_to_llvm(cx, alloc.inner(), /*static*/ true), alloc)) + Ok((const_alloc_to_llvm(cx, alloc.inner(), /*static*/ true, /*vtable_base*/ None), alloc)) } fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) { @@ -232,19 +307,29 @@ impl<'ll> CodegenCx<'ll, '_> { cv: &'ll Value, align: Align, kind: Option<&str>, + ) -> &'ll Value { + let gv = self.static_addr_of_mut_from_type(self.val_ty(cv), align, kind); + llvm::set_initializer(gv, cv); + gv + } + + pub(crate) fn static_addr_of_mut_from_type( + &self, + ty: &'ll Type, + align: Align, + kind: Option<&str>, ) -> &'ll Value { let gv = match kind { Some(kind) if !self.tcx.sess.fewer_names() => { let name = self.generate_local_symbol_name(kind); - let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { + let gv = self.define_global(&name, ty).unwrap_or_else(|| { bug!("symbol `{}` is already defined", name); }); llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage); gv } - _ => self.define_private_global(self.val_ty(cv)), + _ => self.define_private_global(ty), }; - llvm::set_initializer(gv, cv); set_global_alignment(self, gv, align); llvm::set_unnamed_address(gv, llvm::UnnamedAddr::Global); gv @@ -277,6 +362,15 @@ impl<'ll> CodegenCx<'ll, '_> { gv } + pub(crate) fn static_addr_of_impl_for_gv(&self, cv: &'ll Value, gv: &'ll Value) -> &'ll Value { + assert!(!self.const_globals.borrow().contains_key(&cv)); + let mut binding = self.const_globals.borrow_mut(); + binding.insert(cv, gv); + llvm::set_initializer(gv, cv); + llvm::set_global_constant(gv, true); + gv + } + #[instrument(level = "debug", skip(self))] pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { let instance = Instance::mono(self.tcx, def_id); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 2443194ff4832..a08f423732f52 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1169,6 +1169,8 @@ unsafe extern "C" { pub(crate) fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; pub(crate) fn LLVMGetConstOpcode(ConstantVal: &Value) -> Opcode; pub(crate) fn LLVMIsAConstantExpr(Val: &Value) -> Option<&Value>; + pub(crate) fn LLVMConstSub<'a>(LHS: &'a Value, RHS: &'a Value) -> &'a Value; + pub(crate) fn LLVMConstTrunc<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; // Operations on global variables, functions, and aliases (globals) pub(crate) fn LLVMIsDeclaration(Global: &Value) -> Bool; @@ -1198,6 +1200,13 @@ unsafe extern "C" { pub(crate) safe fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); pub(crate) safe fn LLVMRustSetTailCallKind(CallInst: &Value, Kind: TailCallKind); + pub(crate) fn LLVMDSOLocalEquivalent(GlobalVar: &Value) -> &Value; + pub(crate) fn LLVMBuildLoadRelative<'a>( + Builder: &Builder<'a>, + Ptr: &'a Value, + ByteOffset: &'a Value, + ) -> &'a Value; + // Operations on attributes pub(crate) fn LLVMCreateStringAttribute( C: &Context, diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 893655031388c..af011b18dd8e1 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -292,6 +292,13 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { fn_abi.ptr_to_llvm_type(self) } + fn vtable_component_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + if self.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + self.type_i32() + } else { + fn_abi.ptr_to_llvm_type(self) + } + } fn reg_backend_type(&self, ty: &Reg) -> &'ll Type { ty.llvm_type(self) } diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index b4556ced0b3fb..4aa44472c2a9d 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -202,7 +202,28 @@ fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if let Some(entry_idx) = vptr_entry_idx { let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = u64::try_from(entry_idx).unwrap() * ptr_size.bytes(); - load_vtable(bx, old_info, bx.type_ptr(), vtable_byte_offset, source, true) + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let val = load_vtable( + bx, + old_info, + bx.type_ptr(), + vtable_byte_offset / 2, + source, + true, + /*load_relative*/ true, + ); + bx.zext(val, bx.type_isize()) + } else { + load_vtable( + bx, + old_info, + bx.type_ptr(), + vtable_byte_offset, + source, + true, + /*load_relative*/ false, + ) + } } else { old_info } diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs index 34ad35a729b98..255490ffc48ce 100644 --- a/compiler/rustc_codegen_ssa/src/meth.rs +++ b/compiler/rustc_codegen_ssa/src/meth.rs @@ -25,12 +25,32 @@ impl<'a, 'tcx> VirtualIndex { ) -> Bx::Value { // Load the function pointer from the object. debug!("get_fn({llvtable:?}, {ty:?}, {self:?})"); - - let llty = bx.fn_ptr_backend_type(fn_abi); let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = self.0 * ptr_size.bytes(); - load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, nonnull) + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let llty = bx.vtable_component_type(fn_abi); + load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset / 2, + ty, + nonnull, + /*load_relative*/ true, + ) + } else { + let llty = bx.fn_ptr_backend_type(fn_abi); + load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset, + ty, + nonnull, + /*load_relative*/ false, + ) + } } pub(crate) fn get_optional_fn>( @@ -61,12 +81,33 @@ impl<'a, 'tcx> VirtualIndex { ) -> Bx::Value { // Load the data pointer from the object. debug!("get_int({:?}, {:?})", llvtable, self); - - let llty = bx.type_isize(); let ptr_size = bx.data_layout().pointer_size(); let vtable_byte_offset = self.0 * ptr_size.bytes(); - load_vtable(bx, llvtable, llty, vtable_byte_offset, ty, false) + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + let llty = bx.type_i32(); + let val = load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset / 2, + ty, + false, + /*load_relative*/ false, + ); + bx.zext(val, bx.type_isize()) + } else { + let llty = bx.type_isize(); + load_vtable( + bx, + llvtable, + llty, + vtable_byte_offset, + ty, + false, + /*load_relative*/ false, + ) + } } } @@ -114,9 +155,17 @@ pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>( let vtable_alloc_id = tcx.vtable_allocation((ty, trait_ref)); let vtable_allocation = tcx.global_alloc(vtable_alloc_id).unwrap_memory(); - let vtable_const = cx.const_data_from_alloc(vtable_allocation); - let align = cx.data_layout().pointer_align().abi; - let vtable = cx.static_addr_of(vtable_const, align, Some("vtable")); + let num_entries = { + if let Some(trait_ref) = trait_ref { + let trait_ref = trait_ref.with_self_ty(tcx, ty); + let trait_ref = tcx.erase_regions(trait_ref); + tcx.vtable_entries(trait_ref) + } else { + TyCtxt::COMMON_VTABLE_ENTRIES + } + } + .len(); + let vtable = cx.construct_vtable(vtable_allocation, num_entries as u64); cx.apply_vcall_visibility_metadata(ty, trait_ref, vtable); cx.create_vtable_debuginfo(ty, trait_ref, vtable); @@ -132,6 +181,7 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( vtable_byte_offset: u64, ty: Ty<'tcx>, nonnull: bool, + load_relative: bool, ) -> Bx::Value { let ptr_align = bx.data_layout().pointer_align().abi; @@ -141,6 +191,7 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( if let Some(trait_ref) = dyn_trait_in_self(bx.tcx(), ty) { let typeid = bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), trait_ref).as_bytes()).unwrap(); + // FIXME: Add correct intrinsic for RV here. let func = bx.type_checked_load(llvtable, vtable_byte_offset, typeid); return func; } else if nonnull { @@ -148,11 +199,23 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( } } - let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); - let ptr = bx.load(llty, gep, ptr_align); + let ptr = if load_relative { + bx.load_relative(llvtable, bx.const_i32(vtable_byte_offset.try_into().unwrap())) + } else { + let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset)); + bx.load(llty, gep, ptr_align) + }; + // VTable loads are invariant. bx.set_invariant_load(ptr); - if nonnull { + // FIXME: The verifier complains with + // + // nonnull applies only to load instructions, use attributes for calls or invokes + // @llvm.load.relative.i32 (ptr nonnull %13, i32 12), !dbg !4323, !invariant.load !27, !noalias !27, !nonnull !27 + // + // For now, do not mark the load relative intrinsic with nonnull, but I think it should be fine + // to do so since it's effectively a load. + if nonnull && !load_relative { bx.nonnull_metadata(ptr); } ptr diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index e96590441fa42..31eb5807ce647 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -607,7 +607,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { args1 = [place.val.llval]; &args1[..] }; - let (maybe_null, drop_fn, fn_abi, drop_instance) = match ty.kind() { + let (maybe_null, drop_fn, fn_abi, drop_instance, vtable) = match ty.kind() { // FIXME(eddyb) perhaps move some of this logic into // `Instance::resolve_drop_in_place`? ty::Dynamic(_, _, ty::Dyn) => { @@ -640,6 +640,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { .get_optional_fn(bx, vtable, ty, fn_abi), fn_abi, virtual_drop, + Some(vtable), ) } _ => ( @@ -647,6 +648,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty()), drop_fn, + None, ), }; @@ -654,10 +656,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // generated for no-op drops. if maybe_null { let is_not_null = bx.append_sibling_block("is_not_null"); - let llty = bx.fn_ptr_backend_type(fn_abi); - let null = bx.const_null(llty); let non_null = - bx.icmp(base::bin_op_to_icmp_predicate(mir::BinOp::Ne, false), drop_fn, null); + if bx.cx().sess().opts.unstable_opts.experimental_relative_rust_abi_vtables { + bx.icmp( + base::bin_op_to_icmp_predicate(mir::BinOp::Ne, /*signed*/ false), + drop_fn, + vtable.unwrap(), + ) + } else { + let llty = bx.fn_ptr_backend_type(fn_abi); + let null = bx.const_null(llty); + bx.icmp(base::bin_op_to_icmp_predicate(mir::BinOp::Ne, false), drop_fn, null) + }; bx.cond_br(non_null, is_not_null, helper.llbb_with_cleanup(self, target)); bx.switch_to_block(is_not_null); self.set_debug_loc(bx, *source_info); diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 577012151e49f..7470f98253bcd 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -33,11 +33,15 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( let align = meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_ALIGN) .get_usize(bx, vtable, t); - // Size is always <= isize::MAX. - let size_bound = bx.data_layout().ptr_sized_integer().signed_max() as u128; - bx.range_metadata(size, WrappingRange { start: 0, end: size_bound }); - // Alignment is always nonzero. - bx.range_metadata(align, WrappingRange { start: 1, end: !0 }); + // FIXME: The range metadatat can only be applied to load instructions, but it probably + // can also be changed to apply to the load_relative intrinsic. + if !bx.tcx().sess.opts.unstable_opts.experimental_relative_rust_abi_vtables { + // Size is always <= isize::MAX. + let size_bound = bx.data_layout().ptr_sized_integer().signed_max() as u128; + bx.range_metadata(size, WrappingRange { start: 0, end: size_bound }); + // Alignment is always nonzero. + bx.range_metadata(align, WrappingRange { start: 1, end: !0 }); + } (size, align) } diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index 4b18146863bf4..b217f8bc3a4e8 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -236,6 +236,7 @@ pub trait BuilderMethods<'a, 'tcx>: fn alloca(&mut self, size: Size, align: Align) -> Self::Value; fn load(&mut self, ty: Self::Type, ptr: Self::Value, align: Align) -> Self::Value; + fn load_relative(&mut self, ptr: Self::Value, byte_offset: Self::Value) -> Self::Value; fn volatile_load(&mut self, ty: Self::Type, ptr: Self::Value) -> Self::Value; fn atomic_load( &mut self, diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs index d83a04d814be3..1e249ffac1141 100644 --- a/compiler/rustc_codegen_ssa/src/traits/consts.rs +++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs @@ -38,6 +38,7 @@ pub trait ConstCodegenMethods: BackendTypes { fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option; fn const_data_from_alloc(&self, alloc: ConstAllocation<'_>) -> Self::Value; + fn construct_vtable(&self, alloc: ConstAllocation<'_>, num_entries: u64) -> Self::Value; fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value; diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs index 32c24965e1bf6..bb1f814338e23 100644 --- a/compiler/rustc_codegen_ssa/src/traits/type_.rs +++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs @@ -116,6 +116,7 @@ pub trait LayoutTypeCodegenMethods<'tcx>: BackendTypes { fn cast_backend_type(&self, ty: &CastTarget) -> Self::Type; fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; + fn vtable_component_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Self::Type; fn reg_backend_type(&self, ty: &Reg) -> Self::Type; /// The backend type used for a rust type when it's in an SSA register. /// diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 588d867bbbf4e..fda6ecfe2deef 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -151,6 +151,10 @@ extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames) { return wrap(ctx); } +extern "C" LLVMValueRef LLVMDSOLocalEquivalent(LLVMValueRef GlobalVal) { + return wrap(DSOLocalEquivalent::get(unwrap(GlobalVal))); +} + extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M, const char *Target) { #if LLVM_VERSION_GE(21, 0) @@ -613,6 +617,14 @@ LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Source, return wrap(LI); } +extern "C" LLVMValueRef LLVMBuildLoadRelative(LLVMBuilderRef B, LLVMValueRef Ptr, + LLVMValueRef ByteOffset) { + Type *Int32Ty = Type::getInt32Ty(unwrap(B)->getContext()); + Value *call = unwrap(B)->CreateIntrinsic( + Intrinsic::load_relative, {Int32Ty}, {unwrap(Ptr), unwrap(ByteOffset)}); + return wrap(call); +} + extern "C" LLVMValueRef LLVMRustBuildAtomicStore(LLVMBuilderRef B, LLVMValueRef V, LLVMValueRef Target, diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index cfeadf3c7595a..65ed4a3020f10 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -2802,6 +2802,10 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M early_dcx.early_fatal("can't dump dependency graph without `-Z query-dep-graph`"); } + if target_triple.tuple().ends_with("fuchsia") { + unstable_opts.experimental_relative_rust_abi_vtables = true; + } + let logical_env = parse_logical_env(early_dcx, matches); let sysroot = Sysroot::new(matches.opt_str("sysroot").map(PathBuf::from)); diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 880b08d444414..0bda97ee132c9 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -2241,6 +2241,9 @@ options! { "enforce the type length limit when monomorphizing instances in codegen"), experimental_default_bounds: bool = (false, parse_bool, [TRACKED], "enable default bounds for experimental group of auto traits"), + // Change this to true to unconditionally build all runtimes with relative vtables. + experimental_relative_rust_abi_vtables: bool = (false, parse_bool, [TRACKED], + "use the relative layout for vtables"), export_executable_symbols: bool = (false, parse_bool, [TRACKED], "export symbols from executables, as if they were dynamic libraries"), external_clangrt: bool = (false, parse_bool, [UNTRACKED], diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index af2e000e34028..7d5c5ad792100 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -309,8 +309,21 @@ fn adjust_for_rust_scalar<'tcx>( None }; if let Some(kind) = kind { - attrs.pointee_align = - Some(pointee.align.min(cx.tcx().sess.target.max_reliable_alignment())); + let is_vtable = if let ty::Ref(_, ty, _) = *layout.ty.kind() { + ty.is_trait() + && matches!(kind, PointerKind::SharedRef { frozen: true }) + && !is_return + } else { + false + }; + + if cx.tcx().sess.opts.unstable_opts.experimental_relative_rust_abi_vtables && is_vtable + { + attrs.pointee_align = Some(cx.tcx().data_layout.i32_align.abi); + } else { + attrs.pointee_align = + Some(pointee.align.min(cx.tcx().sess.target.max_reliable_alignment())); + } // `Box` are not necessarily dereferenceable for the entire duration of the function as // they can be deallocated at any time. Same for non-frozen shared references (see @@ -322,8 +335,16 @@ fn adjust_for_rust_scalar<'tcx>( PointerKind::Box { .. } | PointerKind::SharedRef { frozen: false } | PointerKind::MutableRef { unpin: false } => Size::ZERO, - PointerKind::SharedRef { frozen: true } - | PointerKind::MutableRef { unpin: true } => pointee.size, + PointerKind::SharedRef { frozen: true } => { + if cx.tcx().sess.opts.unstable_opts.experimental_relative_rust_abi_vtables + && is_vtable + { + pointee.size / 2 + } else { + pointee.size + } + } + PointerKind::MutableRef { unpin: true } => pointee.size, }; // The aliasing rules for `Box` are still not decided, but currently we emit diff --git a/src/doc/rustc/src/codegen-options/index.md b/src/doc/rustc/src/codegen-options/index.md index 07eafdf4c4c62..26b220a67c369 100644 --- a/src/doc/rustc/src/codegen-options/index.md +++ b/src/doc/rustc/src/codegen-options/index.md @@ -162,6 +162,10 @@ This option allows you to put extra data in each output filename. It takes a string to add as a suffix to the filename. See the [`--emit` flag][option-emit] for more information. +## experimental-relative-rust-abi-vtables + +This option uses the relative layout for vtables. + ## force-frame-pointers This flag forces the use of frame pointers. It takes one of the following diff --git a/tests/codegen-llvm/relative-vtables/simple-vtable.rs b/tests/codegen-llvm/relative-vtables/simple-vtable.rs new file mode 100644 index 0000000000000..8cd5329051384 --- /dev/null +++ b/tests/codegen-llvm/relative-vtables/simple-vtable.rs @@ -0,0 +1,79 @@ +//@ compile-flags: -Zexperimental-relative-rust-abi-vtables=y + +#![crate_type = "lib"] + +// CHECK: @vtable.0 = private {{.*}}constant [5 x i32] [ +// CHECK-SAME: i32 0, +// CHECK-SAME: i32 4, +// CHECK-SAME: i32 4, +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT2_FOO:@".*Struct2\$.*foo.*"]] to i64), i64 ptrtoint (ptr @vtable.0 to i64)) to i32), +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT2_BAR:@".*Struct2\$.*bar.*"]] to i64), i64 ptrtoint (ptr @vtable.0 to i64)) to i32) +// CHECK-SAME: ], align 4 + +// CHECK: @vtable.1 = private {{.*}}constant [5 x i32] [ +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[VT1_DROP_IN_PLACE:\@".*drop_in_place[^\"]*"]] to i64), i64 ptrtoint (ptr @vtable.1 to i64)) to i32), +// CHECK-SAME: i32 24, +// CHECK-SAME: i32 8, +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT_FOO:@".*Struct\$.*foo.*"]] to i64), i64 ptrtoint (ptr @vtable.1 to i64)) to i32), +// CHECK-SAME: i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent [[STRUCT_BAR:@".*Struct\$.*bar.*"]] to i64), i64 ptrtoint (ptr @vtable.1 to i64)) to i32) +// CHECK-SAME: ], align 4 + +// CHECK-DAG: define {{.*}}void [[STRUCT2_FOO]](ptr +// CHECK-DAG: define {{.*}}i32 [[STRUCT2_BAR]](ptr +// CHECK-DAG: define {{.*}}void [[STRUCT_FOO]](ptr +// CHECK-DAG: define {{.*}}i32 [[STRUCT_BAR]](ptr + +trait MyTrait { + fn foo(&self); + fn bar(&self) -> u32; +} + +struct Struct { + s: String, +} + +struct Struct2 { + u: u32, +} + +impl MyTrait for Struct { + fn foo(&self) { + println!("Struct foo {}", self.s); + } + fn bar(&self) -> u32 { + 42 + } +} + +impl MyTrait for Struct2 { + fn foo(&self) { + println!("Struct2 foo {}", self.bar()); + } + fn bar(&self) -> u32 { + self.u + } +} + +/// This is only here to manifest the vtables. +pub fn create_struct(b: bool) -> Box { + if b { Box::new(Struct { s: "abc".to_string() }) } else { Box::new(Struct2 { u: 1 }) } +} + +// CHECK-LABEL: define void @_ZN13simple_vtable10invoke_foo17hebe38af67b23d609E( +// CHECK-SAME: ptr noundef nonnull align 1 [[DATA:%.*]], +// CHECK-SAME: ptr noalias noundef readonly align 4 dereferenceable(20) [[VTABLE:%.*]]) {{.*}}{ +// CHECK-NEXT: start: +// CHECK-NEXT: [[FUNC:%.*]] = tail call ptr @llvm.load.relative.i32(ptr nonnull [[VTABLE]], i32 12), !invariant.load +// CHECK-NEXT: tail call void [[FUNC]](ptr noundef nonnull align 1 [[DATA]]) +// CHECK-NEXT: ret void +// CHECK-NEXT: } +pub fn invoke_foo(x: &dyn MyTrait) { + x.foo(); +} + +// CHECK-LABEL: define void @_ZN13simple_vtable11invoke_drop17ha1ff356bc5a4b987E(i1 noundef zeroext %b) {{.*}}{ +// CHECK: [[BOX:%.*]] = tail call { ptr, ptr } @_ZN13simple_vtable13create_struct{{.*}}(i1 noundef zeroext %b) +pub fn invoke_drop(b: bool) { + let bx = create_struct(b); + invoke_foo(&*bx); +}