[WIP] Relative VTables for Rust

This is a WIP patch for implementing
rust-lang/compiler-team#903. It adds a new
unstable flag `-Zexperimental-relative-rust-abi-vtables` that makes
vtables PIC-friendly. This is only supported for LLVM codegen and
not supported for other backends.

Early feedback on this is welcome. I'm not sure if how I implemented it
is the best way of doing so since much of the actual vtable emission is
heavily done during LLVM codegen. That is, the vtable to MIR looks like
a normal table of pointers and byte arrays and I really only make the
vtables relative on the codegen level.

Locally, I can build the stage 1 compiler and runtimes with relative
vtables, but I couldn't figure out how to tell the build system to only
build stage 1 binaries with this flag, so I work around this by
unconditionally enabling relative vtables in rustc. The end goal I think
we'd like is either something akin to multilibs in clang where the
compiler chooses which runtimes to use based off compilation flags, or
binding this ABI to the target and have it be part of the default ABI
for that target (just like how relative vtables are the default for
Fuchsia in C++ with Clang). I think the later is what target modifiers
do (#136966).

Action Items:

- I'm still experimenting with building Fuchsia with this to assert it
  works e2e and I still need to do some measurements to see if this is
  still worth pursuing.
- More work will still be needed to ensure the correct relative
  intrinsics are emitted with CFI and LTO. Rn I'm experimenting on a normal
  build.

Author: PiJoules

compiler/rustc_abi/src/lib.rs

@@ -43,7 +43,7 @@ use std::fmt;
 #[cfg(feature = "nightly")]
 use std::iter::Step;
 use std::num::{NonZeroUsize, ParseIntError};
-use std::ops::{Add, AddAssign, Deref, Mul, RangeFull, RangeInclusive, Sub};
+use std::ops::{Add, AddAssign, Deref, Div, Mul, RangeFull, RangeInclusive, Sub};
 use std::str::FromStr;
 
 use bitflags::bitflags;
@@ -819,6 +819,14 @@ impl Size {
         if bytes < dl.obj_size_bound() { Some(Size::from_bytes(bytes)) } else { None }
     }
 
+    #[inline]
+    pub fn checked_div<C: HasDataLayout>(self, count: u64, cx: &C) -> Option<Size> {
+        let dl = cx.data_layout();
+
+        let bytes = self.bytes().checked_div(count)?;
+        if bytes < dl.obj_size_bound() { Some(Size::from_bytes(bytes)) } else { None }
+    }
+
     /// Truncates `value` to `self` bits and then sign-extends it to 128 bits
     /// (i.e., if it is negative, fill with 1's on the left).
     #[inline]
@@ -906,6 +914,25 @@ impl Mul<u64> for Size {
     }
 }
 
+impl Div<Size> for u64 {
+    type Output = Size;
+    #[inline]
+    fn div(self, size: Size) -> Size {
+        size / self
+    }
+}
+
+impl Div<u64> for Size {
+    type Output = Size;
+    #[inline]
+    fn div(self, count: u64) -> Size {
+        match self.bytes().checked_div(count) {
+            Some(bytes) => Size::from_bytes(bytes),
+            None => panic!("Size::div: {} / {} doesn't fit in u64", self.bytes(), count),
+        }
+    }
+}
+
 impl AddAssign for Size {
     #[inline]
     fn add_assign(&mut self, other: Size) {

compiler/rustc_codegen_llvm/src/builder.rs

@@ -614,6 +614,10 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         }
     }
 
+    fn load_relative(&mut self, ptr: &'ll Value, byte_offset: &'ll Value) -> &'ll Value {
+        unsafe { llvm::LLVMBuildLoadRelative(self.llbuilder, ptr, byte_offset) }
+    }
+
     fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
         unsafe {
             let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);

compiler/rustc_codegen_llvm/src/common.rs

@@ -287,8 +287,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
                                 self.const_bitcast(llval, llty)
                             };
                         } else {
-                            let init =
-                                const_alloc_to_llvm(self, alloc.inner(), /*static*/ false);
+                            let init = const_alloc_to_llvm(
+                                self,
+                                alloc.inner(),
+                                /*static*/ false,
+                                /*vtable_base*/ None,
+                            );
                             let alloc = alloc.inner();
                             let value = match alloc.mutability {
                                 Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
@@ -320,7 +324,12 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
                                 }),
                             )))
                             .unwrap_memory();
-                        let init = const_alloc_to_llvm(self, alloc.inner(), /*static*/ false);
+                        let init = const_alloc_to_llvm(
+                            self,
+                            alloc.inner(),
+                            /*static*/ false,
+                            /*vtable_base*/ None,
+                        );
                         self.static_addr_of_impl(init, alloc.inner().align, None)
                     }
                     GlobalAlloc::Static(def_id) => {
@@ -354,7 +363,37 @@ impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
     }
 
     fn const_data_from_alloc(&self, alloc: ConstAllocation<'_>) -> Self::Value {
-        const_alloc_to_llvm(self, alloc.inner(), /*static*/ false)
+        const_alloc_to_llvm(self, alloc.inner(), /*static*/ false, /*vtable_base*/ None)
+    }
+
+    fn construct_vtable(
+        &self,
+        vtable_allocation: ConstAllocation<'_>,
+        num_entries: u64,
+    ) -> Self::Value {
+        // When constructing relative vtables, we need to create the global first before creating
+        // the initializer so the initializer has references to the global we will bind it to.
+        // Regular vtables aren't self-referential so we can just create the initializer on its
+        // own.
+        if self.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables {
+            let llty = self.type_array(self.type_i32(), num_entries);
+            let vtable = self.static_addr_of_mut_from_type(
+                llty,
+                self.data_layout().i32_align.abi,
+                Some("vtable"),
+            );
+            let init = const_alloc_to_llvm(
+                self,
+                vtable_allocation.inner(),
+                /*static*/ false,
+                Some(vtable),
+            );
+            self.static_addr_of_impl_for_gv(init, vtable)
+        } else {
+            let vtable_const = self.const_data_from_alloc(vtable_allocation);
+            let align = self.data_layout().pointer_align().abi;
+            self.static_addr_of(vtable_const, align, Some("vtable"))
+        }
     }
 
     fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value {

compiler/rustc_codegen_llvm/src/consts.rs

@@ -1,6 +1,6 @@
 use std::ops::Range;
 
-use rustc_abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange};
+use rustc_abi::{Align, Endian, HasDataLayout, Primitive, Scalar, Size, WrappingRange};
 use rustc_codegen_ssa::common;
 use rustc_codegen_ssa::traits::*;
 use rustc_hir::LangItem;
@@ -28,6 +28,7 @@ pub(crate) fn const_alloc_to_llvm<'ll>(
     cx: &CodegenCx<'ll, '_>,
     alloc: &Allocation,
     is_static: bool,
+    vtable_base: Option<&'ll Value>,
 ) -> &'ll Value {
     // We expect that callers of const_alloc_to_llvm will instead directly codegen a pointer or
     // integer for any &ZST where the ZST is a constant (i.e. not a static). We should never be
@@ -43,6 +44,8 @@ pub(crate) fn const_alloc_to_llvm<'ll>(
     let dl = cx.data_layout();
     let pointer_size = dl.pointer_size();
     let pointer_size_bytes = pointer_size.bytes() as usize;
+    let use_relative_layout = cx.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables
+        && vtable_base.is_some();
 
     // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range`
     // must be within the bounds of `alloc` and not contain or overlap a pointer provenance.
@@ -51,7 +54,11 @@ pub(crate) fn const_alloc_to_llvm<'ll>(
         cx: &'a CodegenCx<'ll, 'b>,
         alloc: &'a Allocation,
         range: Range<usize>,
+        use_relative_layout: bool,
     ) {
+        let dl = cx.data_layout();
+        let pointer_size = dl.pointer_size();
+        let pointer_size_bytes = pointer_size.bytes() as usize;
         let chunks = alloc.init_mask().range_as_init_chunks(range.clone().into());
 
         let chunk_to_llval = move |chunk| match chunk {
@@ -74,7 +81,43 @@ pub(crate) fn const_alloc_to_llvm<'ll>(
         let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max;
 
         if allow_uninit_chunks {
-            llvals.extend(chunks.map(chunk_to_llval));
+            if use_relative_layout {
+                // Rather than being stored as a struct of pointers or byte-arrays, a relative
+                // vtable is a pure i32 array, so its components must be chunks of i32s. Here we
+                // explicitly group any sequence of bytes into i32s.
+                //
+                // Normally we can only do this if an 8-byte constant can fit into 4 bytes.
+                for chunk in chunks {
+                    match chunk {
+                        InitChunk::Init(range) => {
+                            let range =
+                                (range.start.bytes() as usize)..(range.end.bytes() as usize);
+                            let bytes =
+                                alloc.inspect_with_uninit_and_ptr_outside_interpreter(range);
+                            for bytes in bytes.chunks_exact(pointer_size_bytes) {
+                                assert!(
+                                    bytes[4..pointer_size_bytes].iter().all(|&x| x == 0),
+                                    "Cannot fit constant into 4-bytes: {:?}",
+                                    bytes
+                                );
+                                let bytes: [u8; 4] = bytes[0..4].try_into().unwrap();
+                                let val: u32 = match dl.endian {
+                                    Endian::Big => u32::from_be_bytes(bytes),
+                                    Endian::Little => u32::from_le_bytes(bytes),
+                                };
+                                llvals.push(cx.const_u32(val));
+                            }
+                        }
+                        InitChunk::Uninit(range) => {
+                            let len = range.end.bytes() - range.start.bytes();
+                            let val = cx.const_undef(cx.type_array(cx.type_i8(), len / 2));
+                            llvals.push(val);
+                        }
+                    };
+                }
+            } else {
+                llvals.extend(chunks.map(chunk_to_llval));
+            }
         } else {
             // If this allocation contains any uninit bytes, codegen as if it was initialized
             // (using some arbitrary value for uninit bytes).
@@ -92,7 +135,13 @@ pub(crate) fn const_alloc_to_llvm<'ll>(
             // This `inspect` is okay since we have checked that there is no provenance, it
             // is within the bounds of the allocation, and it doesn't affect interpreter execution
             // (we inspect the result after interpreter execution).
-            append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset);
+            append_chunks_of_init_and_uninit_bytes(
+                &mut llvals,
+                cx,
+                alloc,
+                next_offset..offset,
+                use_relative_layout,
+            );
         }
         let ptr_offset = read_target_uint(
             dl.endian,
@@ -108,38 +157,64 @@ pub(crate) fn const_alloc_to_llvm<'ll>(
 
         let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx);
 
-        llvals.push(cx.scalar_to_backend(
-            InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx),
-            Scalar::Initialized {
-                value: Primitive::Pointer(address_space),
-                valid_range: WrappingRange::full(pointer_size),
-            },
-            cx.type_ptr_ext(address_space),
-        ));
+        let s = {
+            let scalar = cx.scalar_to_backend(
+                InterpScalar::from_pointer(
+                    Pointer::new(prov, Size::from_bytes(ptr_offset)),
+                    &cx.tcx,
+                ),
+                Scalar::Initialized {
+                    value: Primitive::Pointer(address_space),
+                    valid_range: WrappingRange::full(pointer_size),
+                },
+                cx.type_ptr_ext(address_space),
+            );
+
+            if use_relative_layout {
+                unsafe {
+                    let fptr = llvm::LLVMDSOLocalEquivalent(scalar);
+                    let sub = llvm::LLVMConstSub(
+                        llvm::LLVMConstPtrToInt(fptr, cx.type_i64()),
+                        llvm::LLVMConstPtrToInt(vtable_base.unwrap(), cx.type_i64()),
+                    );
+                    llvm::LLVMConstTrunc(sub, cx.type_i32())
+                }
+            } else {
+                scalar
+            }
+        };
+
+        llvals.push(s);
         next_offset = offset + pointer_size_bytes;
     }
     if alloc.len() >= next_offset {
         let range = next_offset..alloc.len();
         // This `inspect` is okay since we have check that it is after all provenance, it is
         // within the bounds of the allocation, and it doesn't affect interpreter execution (we
         // inspect the result after interpreter execution).
-        append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range);
+        append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range, use_relative_layout);
     }
 
     // Avoid wrapping in a struct if there is only a single value. This ensures
     // that LLVM is able to perform the string merging optimization if the constant
     // is a valid C string. LLVM only considers bare arrays for this optimization,
     // not arrays wrapped in a struct. LLVM handles this at:
     // https://github.com/rust-lang/llvm-project/blob/acaea3d2bb8f351b740db7ebce7d7a40b9e21488/llvm/lib/Target/TargetLoweringObjectFile.cpp#L249-L280
-    if let &[data] = &*llvals { data } else { cx.const_struct(&llvals, true) }
+    if let &[data] = &*llvals {
+        data
+    } else if use_relative_layout {
+        cx.const_array(cx.type_i32(), &llvals)
+    } else {
+        cx.const_struct(&llvals, true)
+    }
 }
 
 fn codegen_static_initializer<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     def_id: DefId,
 ) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> {
     let alloc = cx.tcx.eval_static_initializer(def_id)?;
-    Ok((const_alloc_to_llvm(cx, alloc.inner(), /*static*/ true), alloc))
+    Ok((const_alloc_to_llvm(cx, alloc.inner(), /*static*/ true, /*vtable_base*/ None), alloc))
 }
 
 fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) {
@@ -232,19 +307,29 @@ impl<'ll> CodegenCx<'ll, '_> {
         cv: &'ll Value,
         align: Align,
         kind: Option<&str>,
+    ) -> &'ll Value {
+        let gv = self.static_addr_of_mut_from_type(self.val_ty(cv), align, kind);
+        llvm::set_initializer(gv, cv);
+        gv
+    }
+
+    pub(crate) fn static_addr_of_mut_from_type(
+        &self,
+        ty: &'ll Type,
+        align: Align,
+        kind: Option<&str>,
     ) -> &'ll Value {
         let gv = match kind {
             Some(kind) if !self.tcx.sess.fewer_names() => {
                 let name = self.generate_local_symbol_name(kind);
-                let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| {
+                let gv = self.define_global(&name, ty).unwrap_or_else(|| {
                     bug!("symbol `{}` is already defined", name);
                 });
                 llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage);
                 gv
             }
-            _ => self.define_private_global(self.val_ty(cv)),
+            _ => self.define_private_global(ty),
         };
-        llvm::set_initializer(gv, cv);
         set_global_alignment(self, gv, align);
         llvm::set_unnamed_address(gv, llvm::UnnamedAddr::Global);
         gv
@@ -277,6 +362,15 @@ impl<'ll> CodegenCx<'ll, '_> {
         gv
     }
 
+    pub(crate) fn static_addr_of_impl_for_gv(&self, cv: &'ll Value, gv: &'ll Value) -> &'ll Value {
+        assert!(!self.const_globals.borrow().contains_key(&cv));
+        let mut binding = self.const_globals.borrow_mut();
+        binding.insert(cv, gv);
+        llvm::set_initializer(gv, cv);
+        llvm::set_global_constant(gv, true);
+        gv
+    }
+
     #[instrument(level = "debug", skip(self))]
     pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value {
         let instance = Instance::mono(self.tcx, def_id);

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1169,6 +1169,8 @@ unsafe extern "C" {
     pub(crate) fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>;
     pub(crate) fn LLVMGetConstOpcode(ConstantVal: &Value) -> Opcode;
     pub(crate) fn LLVMIsAConstantExpr(Val: &Value) -> Option<&Value>;
+    pub(crate) fn LLVMConstSub<'a>(LHS: &'a Value, RHS: &'a Value) -> &'a Value;
+    pub(crate) fn LLVMConstTrunc<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value;
 
     // Operations on global variables, functions, and aliases (globals)
     pub(crate) fn LLVMIsDeclaration(Global: &Value) -> Bool;
@@ -1198,6 +1200,13 @@ unsafe extern "C" {
     pub(crate) safe fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool);
     pub(crate) safe fn LLVMRustSetTailCallKind(CallInst: &Value, Kind: TailCallKind);
 
+    pub(crate) fn LLVMDSOLocalEquivalent(GlobalVar: &Value) -> &Value;
+    pub(crate) fn LLVMBuildLoadRelative<'a>(
+        Builder: &Builder<'a>,
+        Ptr: &'a Value,
+        ByteOffset: &'a Value,
+    ) -> &'a Value;
+
     // Operations on attributes
     pub(crate) fn LLVMCreateStringAttribute(
         C: &Context,

compiler/rustc_codegen_llvm/src/type_.rs

@@ -292,6 +292,13 @@ impl<'ll, 'tcx> LayoutTypeCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
         fn_abi.ptr_to_llvm_type(self)
     }
+    fn vtable_component_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
+        if self.sess().opts.unstable_opts.experimental_relative_rust_abi_vtables {
+            self.type_i32()
+        } else {
+            fn_abi.ptr_to_llvm_type(self)
+        }
+    }
     fn reg_backend_type(&self, ty: &Reg) -> &'ll Type {
         ty.llvm_type(self)
     }