Skip to content

Commit e1c1d0f

Browse files
flip1995flip1995
committed
Add llvm.type.checked.load intrinsic
Add the intrinsic declare {i8*, i1} @llvm.type.checked.load(i8* %ptr, i32 %offset, metadata %type) This is used in the VFE optimization when lowering loading functions from vtables to LLVM IR. The `metadata` is used to map the function to all vtables this function could belong to. This ensures that functions from vtables that might be used somewhere won't get removed.
1 parent d55787a commit e1c1d0f

File tree

6 files changed

+88
-18
lines changed

6 files changed

+88
-18
lines changed

‎compiler/rustc_codegen_gcc/src/intrinsic/mod.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,16 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
356356
self.context.new_rvalue_from_int(self.int_type, 0)
357357
}
358358

359+
fn type_checked_load(
360+
&mut self,
361+
_llvtable: Self::Value,
362+
_vtable_byte_offset: u64,
363+
_typeid: Self::Value,
364+
) -> Self::Value {
365+
// Unsupported.
366+
self.context.new_rvalue_from_int(self.int_type, 0)
367+
}
368+
359369
fn va_start(&mut self, _va_list: RValue<'gcc>) -> RValue<'gcc> {
360370
unimplemented!();
361371
}

‎compiler/rustc_codegen_llvm/src/context.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -665,6 +665,7 @@ impl<'ll> CodegenCx<'ll, '_> {
665665
let t_isize = self.type_isize();
666666
let t_f32 = self.type_f32();
667667
let t_f64 = self.type_f64();
668+
let t_metadata = self.type_metadata();
668669

669670
ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32);
670671
ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32);
@@ -890,11 +891,12 @@ impl<'ll> CodegenCx<'ll, '_> {
890891
ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void);
891892
}
892893

893-
ifn!("llvm.type.test", fn(i8p, self.type_metadata()) -> i1);
894+
ifn!("llvm.type.test", fn(i8p, t_metadata) -> i1);
895+
ifn!("llvm.type.checked.load", fn(i8p, t_i32, t_metadata) -> mk_struct! {i8p, i1});
894896

895897
if self.sess().opts.debuginfo != DebugInfo::None {
896-
ifn!("llvm.dbg.declare", fn(self.type_metadata(), self.type_metadata()) -> void);
897-
ifn!("llvm.dbg.value", fn(self.type_metadata(), t_i64, self.type_metadata()) -> void);
898+
ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void);
899+
ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void);
898900
}
899901
None
900902
}

‎compiler/rustc_codegen_llvm/src/intrinsic.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,6 +406,16 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
406406
self.call_intrinsic("llvm.type.test", &[bitcast, typeid])
407407
}
408408

409+
fn type_checked_load(
410+
&mut self,
411+
llvtable: &'ll Value,
412+
vtable_byte_offset: u64,
413+
typeid: &'ll Value,
414+
) -> Self::Value {
415+
let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32);
416+
self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid])
417+
}
418+
409419
fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value {
410420
self.call_intrinsic("llvm.va_start", &[va_list])
411421
}

‎compiler/rustc_codegen_ssa/src/meth.rs‎

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
use crate::traits::*;
22

3-
use rustc_middle::ty::{self, Ty};
3+
use rustc_middle::ty::{self, subst::GenericArgKind, ExistentialPredicate, Ty, TyCtxt};
4+
use rustc_session::config::Lto;
5+
use rustc_symbol_mangling::typeid_for_trait_ref;
46
use rustc_target::abi::call::FnAbi;
57

68
#[derive(Copy, Clone, Debug)]
@@ -15,20 +17,32 @@ impl<'a, 'tcx> VirtualIndex {
1517
self,
1618
bx: &mut Bx,
1719
llvtable: Bx::Value,
20+
ty: Ty<'tcx>,
1821
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
1922
) -> Bx::Value {
2023
// Load the data pointer from the object.
21-
debug!("get_fn({:?}, {:?})", llvtable, self);
22-
24+
debug!("get_fn({llvtable:?}, {ty:?}, {self:?})");
2325
let llty = bx.fn_ptr_backend_type(fn_abi);
2426
let llvtable = bx.pointercast(llvtable, bx.type_ptr_to(llty));
25-
let ptr_align = bx.tcx().data_layout.pointer_align.abi;
26-
let gep = bx.inbounds_gep(llty, llvtable, &[bx.const_usize(self.0)]);
27-
let ptr = bx.load(llty, gep, ptr_align);
28-
bx.nonnull_metadata(ptr);
29-
// Vtable loads are invariant.
30-
bx.set_invariant_load(ptr);
31-
ptr
27+
28+
if bx.cx().sess().opts.debugging_opts.virtual_function_elimination
29+
&& bx.cx().sess().lto() == Lto::Fat
30+
{
31+
let typeid =
32+
bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), get_trait_ref(bx.tcx(), ty)));
33+
let vtable_byte_offset = self.0 * bx.data_layout().pointer_size.bytes();
34+
let type_checked_load = bx.type_checked_load(llvtable, vtable_byte_offset, typeid);
35+
let func = bx.extract_value(type_checked_load, 0);
36+
bx.pointercast(func, llty)
37+
} else {
38+
let ptr_align = bx.tcx().data_layout.pointer_align.abi;
39+
let gep = bx.inbounds_gep(llty, llvtable, &[bx.const_usize(self.0)]);
40+
let ptr = bx.load(llty, gep, ptr_align);
41+
bx.nonnull_metadata(ptr);
42+
// Vtable loads are invariant.
43+
bx.set_invariant_load(ptr);
44+
ptr
45+
}
3246
}
3347

3448
pub fn get_usize<Bx: BuilderMethods<'a, 'tcx>>(
@@ -50,6 +64,24 @@ impl<'a, 'tcx> VirtualIndex {
5064
}
5165
}
5266

67+
fn get_trait_ref<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> {
68+
for arg in ty.peel_refs().walk() {
69+
if let GenericArgKind::Type(ty) = arg.unpack() {
70+
if let ty::Dynamic(trait_refs, _) = ty.kind() {
71+
return trait_refs[0].map_bound(|trait_ref| match trait_ref {
72+
ExistentialPredicate::Trait(tr) => tr,
73+
ExistentialPredicate::Projection(proj) => proj.trait_ref(tcx),
74+
ExistentialPredicate::AutoTrait(_) => {
75+
bug!("auto traits don't have functions")
76+
}
77+
});
78+
}
79+
}
80+
}
81+
82+
bug!("expected a `dyn Trait` ty, found {ty:?}")
83+
}
84+
5385
/// Creates a dynamic vtable for the given type and vtable origin.
5486
/// This is used only for objects.
5587
///

‎compiler/rustc_codegen_ssa/src/mir/block.rs‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
401401
args = &args[..1];
402402
(
403403
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
404-
.get_fn(&mut bx, vtable, &fn_abi),
404+
.get_fn(&mut bx, vtable, ty, &fn_abi),
405405
fn_abi,
406406
)
407407
}
@@ -819,17 +819,25 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
819819
// the data pointer as the first argument
820820
match op.val {
821821
Pair(data_ptr, meta) => {
822-
llfn = Some(
823-
meth::VirtualIndex::from_index(idx).get_fn(&mut bx, meta, &fn_abi),
824-
);
822+
llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
823+
&mut bx,
824+
meta,
825+
op.layout.ty,
826+
&fn_abi,
827+
));
825828
llargs.push(data_ptr);
826829
continue 'make_args;
827830
}
828831
other => bug!("expected a Pair, got {:?}", other),
829832
}
830833
} else if let Ref(data_ptr, Some(meta), _) = op.val {
831834
// by-value dynamic dispatch
832-
llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(&mut bx, meta, &fn_abi));
835+
llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
836+
&mut bx,
837+
meta,
838+
op.layout.ty,
839+
&fn_abi,
840+
));
833841
llargs.push(data_ptr);
834842
continue;
835843
} else {

‎compiler/rustc_codegen_ssa/src/traits/intrinsic.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,14 @@ pub trait IntrinsicCallMethods<'tcx>: BackendTypes {
2222
fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value;
2323
/// Trait method used to test whether a given pointer is associated with a type identifier.
2424
fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value;
25+
/// Trait method used to load a function while testing if it is associated with a type
26+
/// identifier.
27+
fn type_checked_load(
28+
&mut self,
29+
llvtable: Self::Value,
30+
vtable_byte_offset: u64,
31+
typeid: Self::Value,
32+
) -> Self::Value;
2533
/// Trait method used to inject `va_start` on the "spoofed" `VaListImpl` in
2634
/// Rust defined C-variadic functions.
2735
fn va_start(&mut self, val: Self::Value) -> Self::Value;

0 commit comments

Comments
 (0)