Skip to content

Commit 4c002fc

Browse files
Mark-SimulacrumMark-Simulacrum
committed
Omit non-needs_drop drop_in_place in vtables
This replaces the drop_in_place reference with null in vtables. On librustc_driver.so, this drops about ~17k dynamic relocations from the output, since many vtables can now be placed in read-only memory, rather than having a relocated pointer included. This makes a tradeoff by adding a null check at vtable call sites. That's hard to avoid without changing the vtable format (e.g., to use a pc-relative relocation instead of an absolute address, and avoid the dynamic relocation that way). But it seems likely that the check is cheap at runtime.
1 parent b0f8618 commit 4c002fc

File tree

7 files changed

+203
-116
lines changed

7 files changed

+203
-116
lines changed

‎compiler/rustc_codegen_cranelift/src/abi/mod.rs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -593,6 +593,7 @@ pub(crate) fn codegen_drop<'tcx>(
593593
fx: &mut FunctionCx<'_, '_, 'tcx>,
594594
source_info: mir::SourceInfo,
595595
drop_place: CPlace<'tcx>,
596+
target: BasicBlock,
596597
) {
597598
let ty = drop_place.layout().ty;
598599
let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx);
@@ -620,6 +621,12 @@ pub(crate) fn codegen_drop<'tcx>(
620621
let ptr = ptr.get_addr(fx);
621622
let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable);
622623

624+
let is_null = fx.bcx.ins().icmp_imm(IntCC::Equal, drop_fn, 0);
625+
let target_block = fx.get_block(target);
626+
let continued = fx.bcx.create_block();
627+
fx.bcx.ins().brif(is_null, target_block, &[], continued, &[]);
628+
fx.bcx.switch_to_block(continued);
629+
623630
// FIXME(eddyb) perhaps move some of this logic into
624631
// `Instance::resolve_drop_in_place`?
625632
let virtual_drop = Instance {
@@ -659,6 +666,12 @@ pub(crate) fn codegen_drop<'tcx>(
659666
let (data, vtable) = drop_place.to_cvalue(fx).dyn_star_force_data_on_stack(fx);
660667
let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable);
661668

669+
let is_null = fx.bcx.ins().icmp_imm(IntCC::Equal, drop_fn, 0);
670+
let target_block = fx.get_block(target);
671+
let continued = fx.bcx.create_block();
672+
fx.bcx.ins().brif(is_null, target_block, &[], continued, &[]);
673+
fx.bcx.switch_to_block(continued);
674+
662675
let virtual_drop = Instance {
663676
def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0),
664677
args: drop_instance.args,
@@ -697,4 +710,7 @@ pub(crate) fn codegen_drop<'tcx>(
697710
}
698711
}
699712
}
713+
714+
let target_block = fx.get_block(target);
715+
fx.bcx.ins().jump(target_block, &[]);
700716
}

‎compiler/rustc_codegen_cranelift/src/base.rs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -548,10 +548,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
548548
}
549549
TerminatorKind::Drop { place, target, unwind: _, replace: _ } => {
550550
let drop_place = codegen_place(fx, *place);
551-
crate::abi::codegen_drop(fx, source_info, drop_place);
552-
553-
let target_block = fx.get_block(*target);
554-
fx.bcx.ins().jump(target_block, &[]);
551+
crate::abi::codegen_drop(fx, source_info, drop_place, *target);
555552
}
556553
};
557554
}

‎compiler/rustc_codegen_ssa/src/meth.rs

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,13 @@ impl<'a, 'tcx> VirtualIndex {
1515
VirtualIndex(index as u64)
1616
}
1717

18-
pub fn get_fn<Bx: BuilderMethods<'a, 'tcx>>(
18+
fn get_fn_inner<Bx: BuilderMethods<'a, 'tcx>>(
1919
self,
2020
bx: &mut Bx,
2121
llvtable: Bx::Value,
2222
ty: Ty<'tcx>,
2323
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
24+
nonnull: bool,
2425
) -> Bx::Value {
2526
// Load the function pointer from the object.
2627
debug!("get_fn({llvtable:?}, {ty:?}, {self:?})");
@@ -41,13 +42,35 @@ impl<'a, 'tcx> VirtualIndex {
4142
} else {
4243
let gep = bx.inbounds_ptradd(llvtable, bx.const_usize(vtable_byte_offset));
4344
let ptr = bx.load(llty, gep, ptr_align);
44-
bx.nonnull_metadata(ptr);
4545
// VTable loads are invariant.
4646
bx.set_invariant_load(ptr);
47+
if nonnull {
48+
bx.nonnull_metadata(ptr);
49+
}
4750
ptr
4851
}
4952
}
5053

54+
pub fn get_optional_fn<Bx: BuilderMethods<'a, 'tcx>>(
55+
self,
56+
bx: &mut Bx,
57+
llvtable: Bx::Value,
58+
ty: Ty<'tcx>,
59+
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
60+
) -> Bx::Value {
61+
self.get_fn_inner(bx, llvtable, ty, fn_abi, false)
62+
}
63+
64+
pub fn get_fn<Bx: BuilderMethods<'a, 'tcx>>(
65+
self,
66+
bx: &mut Bx,
67+
llvtable: Bx::Value,
68+
ty: Ty<'tcx>,
69+
fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
70+
) -> Bx::Value {
71+
self.get_fn_inner(bx, llvtable, ty, fn_abi, true)
72+
}
73+
5174
pub fn get_usize<Bx: BuilderMethods<'a, 'tcx>>(
5275
self,
5376
bx: &mut Bx,

‎compiler/rustc_codegen_ssa/src/mir/block.rs

Lines changed: 112 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -500,6 +500,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
500500
&mut self,
501501
helper: TerminatorCodegenHelper<'tcx>,
502502
bx: &mut Bx,
503+
source_info: &mir::SourceInfo,
503504
location: mir::Place<'tcx>,
504505
target: mir::BasicBlock,
505506
unwind: mir::UnwindAction,
@@ -523,90 +524,106 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
523524
args1 = [place.val.llval];
524525
&args1[..]
525526
};
526-
let (drop_fn, fn_abi, drop_instance) =
527-
match ty.kind() {
528-
// FIXME(eddyb) perhaps move some of this logic into
529-
// `Instance::resolve_drop_in_place`?
530-
ty::Dynamic(_, _, ty::Dyn) => {
531-
// IN THIS ARM, WE HAVE:
532-
// ty = *mut (dyn Trait)
533-
// which is: exists<T> ( *mut T, Vtable<T: Trait> )
534-
// args[0] args[1]
535-
//
536-
// args = ( Data, Vtable )
537-
// |
538-
// v
539-
// /-------\
540-
// | ... |
541-
// \-------/
542-
//
543-
let virtual_drop = Instance {
544-
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function
545-
args: drop_fn.args,
546-
};
547-
debug!("ty = {:?}", ty);
548-
debug!("drop_fn = {:?}", drop_fn);
549-
debug!("args = {:?}", args);
550-
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
551-
let vtable = args[1];
552-
// Truncate vtable off of args list
553-
args = &args[..1];
554-
(
555-
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
556-
.get_fn(bx, vtable, ty, fn_abi),
557-
fn_abi,
558-
virtual_drop,
559-
)
560-
}
561-
ty::Dynamic(_, _, ty::DynStar) => {
562-
// IN THIS ARM, WE HAVE:
563-
// ty = *mut (dyn* Trait)
564-
// which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
565-
//
566-
// args = [ * ]
567-
// |
568-
// v
569-
// ( Data, Vtable )
570-
// |
571-
// v
572-
// /-------\
573-
// | ... |
574-
// \-------/
575-
//
576-
//
577-
// WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
578-
//
579-
// data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer)
580-
// vtable = (*args[0]).1 // loads the vtable out
581-
// (data, vtable) // an equivalent Rust `*mut dyn Trait`
582-
//
583-
// SO THEN WE CAN USE THE ABOVE CODE.
584-
let virtual_drop = Instance {
585-
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function
586-
args: drop_fn.args,
587-
};
588-
debug!("ty = {:?}", ty);
589-
debug!("drop_fn = {:?}", drop_fn);
590-
debug!("args = {:?}", args);
591-
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
592-
let meta_ptr = place.project_field(bx, 1);
593-
let meta = bx.load_operand(meta_ptr);
594-
// Truncate vtable off of args list
595-
args = &args[..1];
596-
debug!("args' = {:?}", args);
597-
(
598-
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
599-
.get_fn(bx, meta.immediate(), ty, fn_abi),
600-
fn_abi,
601-
virtual_drop,
602-
)
603-
}
604-
_ => (
605-
bx.get_fn_addr(drop_fn),
606-
bx.fn_abi_of_instance(drop_fn, ty::List::empty()),
607-
drop_fn,
608-
),
609-
};
527+
let (maybe_null, drop_fn, fn_abi, drop_instance) = match ty.kind() {
528+
// FIXME(eddyb) perhaps move some of this logic into
529+
// `Instance::resolve_drop_in_place`?
530+
ty::Dynamic(_, _, ty::Dyn) => {
531+
// IN THIS ARM, WE HAVE:
532+
// ty = *mut (dyn Trait)
533+
// which is: exists<T> ( *mut T, Vtable<T: Trait> )
534+
// args[0] args[1]
535+
//
536+
// args = ( Data, Vtable )
537+
// |
538+
// v
539+
// /-------\
540+
// | ... |
541+
// \-------/
542+
//
543+
let virtual_drop = Instance {
544+
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function
545+
args: drop_fn.args,
546+
};
547+
debug!("ty = {:?}", ty);
548+
debug!("drop_fn = {:?}", drop_fn);
549+
debug!("args = {:?}", args);
550+
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
551+
let vtable = args[1];
552+
// Truncate vtable off of args list
553+
args = &args[..1];
554+
(
555+
true,
556+
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
557+
.get_optional_fn(bx, vtable, ty, fn_abi),
558+
fn_abi,
559+
virtual_drop,
560+
)
561+
}
562+
ty::Dynamic(_, _, ty::DynStar) => {
563+
// IN THIS ARM, WE HAVE:
564+
// ty = *mut (dyn* Trait)
565+
// which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
566+
//
567+
// args = [ * ]
568+
// |
569+
// v
570+
// ( Data, Vtable )
571+
// |
572+
// v
573+
// /-------\
574+
// | ... |
575+
// \-------/
576+
//
577+
//
578+
// WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
579+
//
580+
// data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer)
581+
// vtable = (*args[0]).1 // loads the vtable out
582+
// (data, vtable) // an equivalent Rust `*mut dyn Trait`
583+
//
584+
// SO THEN WE CAN USE THE ABOVE CODE.
585+
let virtual_drop = Instance {
586+
def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0), // idx 0: the drop function
587+
args: drop_fn.args,
588+
};
589+
debug!("ty = {:?}", ty);
590+
debug!("drop_fn = {:?}", drop_fn);
591+
debug!("args = {:?}", args);
592+
let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
593+
let meta_ptr = place.project_field(bx, 1);
594+
let meta = bx.load_operand(meta_ptr);
595+
// Truncate vtable off of args list
596+
args = &args[..1];
597+
debug!("args' = {:?}", args);
598+
(
599+
true,
600+
meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
601+
.get_optional_fn(bx, meta.immediate(), ty, fn_abi),
602+
fn_abi,
603+
virtual_drop,
604+
)
605+
}
606+
_ => (
607+
false,
608+
bx.get_fn_addr(drop_fn),
609+
bx.fn_abi_of_instance(drop_fn, ty::List::empty()),
610+
drop_fn,
611+
),
612+
};
613+
614+
// We generate a null check for the drop_fn. This saves a bunch of relocations being
615+
// generated for no-op drops.
616+
if maybe_null {
617+
let is_not_null = bx.append_sibling_block("is_not_null");
618+
let llty = bx.fn_ptr_backend_type(fn_abi);
619+
let null = bx.const_null(llty);
620+
let non_null =
621+
bx.icmp(base::bin_op_to_icmp_predicate(mir::BinOp::Ne, false), drop_fn, null);
622+
bx.cond_br(non_null, is_not_null, helper.llbb_with_cleanup(self, target));
623+
bx.switch_to_block(is_not_null);
624+
self.set_debug_loc(bx, *source_info);
625+
}
626+
610627
helper.do_call(
611628
self,
612629
bx,
@@ -617,7 +634,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
617634
unwind,
618635
&[],
619636
Some(drop_instance),
620-
mergeable_succ,
637+
!maybe_null && mergeable_succ,
621638
)
622639
}
623640

@@ -1346,9 +1363,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
13461363
MergingSucc::False
13471364
}
13481365

1349-
mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => {
1350-
self.codegen_drop_terminator(helper, bx, place, target, unwind, mergeable_succ())
1351-
}
1366+
mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => self
1367+
.codegen_drop_terminator(
1368+
helper,
1369+
bx,
1370+
&terminator.source_info,
1371+
place,
1372+
target,
1373+
unwind,
1374+
mergeable_succ(),
1375+
),
13521376

13531377
mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, unwind } => self
13541378
.codegen_assert_terminator(

‎compiler/rustc_middle/src/ty/vtable.rs

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,10 +84,14 @@ pub(super) fn vtable_allocation_provider<'tcx>(
8484
let idx: u64 = u64::try_from(idx).unwrap();
8585
let scalar = match entry {
8686
VtblEntry::MetadataDropInPlace => {
87-
let instance = ty::Instance::resolve_drop_in_place(tcx, ty);
88-
let fn_alloc_id = tcx.reserve_and_set_fn_alloc(instance);
89-
let fn_ptr = Pointer::from(fn_alloc_id);
90-
Scalar::from_pointer(fn_ptr, &tcx)
87+
if ty.needs_drop(tcx, ty::ParamEnv::reveal_all()) {
88+
let instance = ty::Instance::resolve_drop_in_place(tcx, ty);
89+
let fn_alloc_id = tcx.reserve_and_set_fn_alloc(instance);
90+
let fn_ptr = Pointer::from(fn_alloc_id);
91+
Scalar::from_pointer(fn_ptr, &tcx)
92+
} else {
93+
Scalar::from_maybe_pointer(Pointer::null(), &tcx)
94+
}
9195
}
9296
VtblEntry::MetadataSize => Scalar::from_uint(size, ptr_size),
9397
VtblEntry::MetadataAlign => Scalar::from_uint(align, ptr_size),

0 commit comments

Comments
 (0)