Skip to content

Commit f8d5d0c

Browse files
eddybeddyb
committed
rustc_trans: compute better align/dereferenceable attributes from pointees.
1 parent ced5e04 commit f8d5d0c

File tree

5 files changed

+237
-123
lines changed

5 files changed

+237
-123
lines changed

‎src/librustc_llvm/ffi.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -730,7 +730,9 @@ extern "C" {
730730
FunctionTy: TypeRef)
731731
-> ValueRef;
732732
pub fn LLVMSetFunctionCallConv(Fn: ValueRef, CC: c_uint);
733+
pub fn LLVMRustAddAlignmentAttr(Fn: ValueRef, index: c_uint, bytes: u32);
733734
pub fn LLVMRustAddDereferenceableAttr(Fn: ValueRef, index: c_uint, bytes: u64);
735+
pub fn LLVMRustAddDereferenceableOrNullAttr(Fn: ValueRef, index: c_uint, bytes: u64);
734736
pub fn LLVMRustAddFunctionAttribute(Fn: ValueRef, index: c_uint, attr: Attribute);
735737
pub fn LLVMRustAddFunctionAttrStringValue(Fn: ValueRef,
736738
index: c_uint,
@@ -760,7 +762,11 @@ extern "C" {
760762
// Operations on call sites
761763
pub fn LLVMSetInstructionCallConv(Instr: ValueRef, CC: c_uint);
762764
pub fn LLVMRustAddCallSiteAttribute(Instr: ValueRef, index: c_uint, attr: Attribute);
765+
pub fn LLVMRustAddAlignmentCallSiteAttr(Instr: ValueRef, index: c_uint, bytes: u32);
763766
pub fn LLVMRustAddDereferenceableCallSiteAttr(Instr: ValueRef, index: c_uint, bytes: u64);
767+
pub fn LLVMRustAddDereferenceableOrNullCallSiteAttr(Instr: ValueRef,
768+
index: c_uint,
769+
bytes: u64);
764770

765771
// Operations on load/store instructions (only)
766772
pub fn LLVMSetVolatile(MemoryAccessInst: ValueRef, volatile: Bool);

‎src/librustc_trans/abi.rs

Lines changed: 167 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -96,20 +96,24 @@ impl ArgAttribute {
9696

9797
/// A compact representation of LLVM attributes (at least those relevant for this module)
9898
/// that can be manipulated without interacting with LLVM's Attribute machinery.
99-
#[derive(Copy, Clone, Debug, Default)]
99+
#[derive(Copy, Clone, Debug)]
100100
pub struct ArgAttributes {
101101
regular: ArgAttribute,
102-
dereferenceable_bytes: u64,
102+
pointee_size: Size,
103+
pointee_align: Option<Align>
103104
}
104105

105106
impl ArgAttributes {
106-
pub fn set(&mut self, attr: ArgAttribute) -> &mut Self {
107-
self.regular = self.regular | attr;
108-
self
107+
fn new() -> Self {
108+
ArgAttributes {
109+
regular: ArgAttribute::default(),
110+
pointee_size: Size::from_bytes(0),
111+
pointee_align: None,
112+
}
109113
}
110114

111-
pub fn set_dereferenceable(&mut self, size: Size) -> &mut Self {
112-
self.dereferenceable_bytes = size.bytes();
115+
pub fn set(&mut self, attr: ArgAttribute) -> &mut Self {
116+
self.regular = self.regular | attr;
113117
self
114118
}
115119

@@ -118,24 +122,52 @@ impl ArgAttributes {
118122
}
119123

120124
pub fn apply_llfn(&self, idx: AttributePlace, llfn: ValueRef) {
125+
let mut regular = self.regular;
121126
unsafe {
122-
self.regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn));
123-
if self.dereferenceable_bytes != 0 {
124-
llvm::LLVMRustAddDereferenceableAttr(llfn,
125-
idx.as_uint(),
126-
self.dereferenceable_bytes);
127+
let deref = self.pointee_size.bytes();
128+
if deref != 0 {
129+
if regular.contains(ArgAttribute::NonNull) {
130+
llvm::LLVMRustAddDereferenceableAttr(llfn,
131+
idx.as_uint(),
132+
deref);
133+
} else {
134+
llvm::LLVMRustAddDereferenceableOrNullAttr(llfn,
135+
idx.as_uint(),
136+
deref);
137+
}
138+
regular -= ArgAttribute::NonNull;
127139
}
140+
if let Some(align) = self.pointee_align {
141+
llvm::LLVMRustAddAlignmentAttr(llfn,
142+
idx.as_uint(),
143+
align.abi() as u32);
144+
}
145+
regular.for_each_kind(|attr| attr.apply_llfn(idx, llfn));
128146
}
129147
}
130148

131149
pub fn apply_callsite(&self, idx: AttributePlace, callsite: ValueRef) {
150+
let mut regular = self.regular;
132151
unsafe {
133-
self.regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite));
134-
if self.dereferenceable_bytes != 0 {
135-
llvm::LLVMRustAddDereferenceableCallSiteAttr(callsite,
136-
idx.as_uint(),
137-
self.dereferenceable_bytes);
152+
let deref = self.pointee_size.bytes();
153+
if deref != 0 {
154+
if regular.contains(ArgAttribute::NonNull) {
155+
llvm::LLVMRustAddDereferenceableCallSiteAttr(callsite,
156+
idx.as_uint(),
157+
deref);
158+
} else {
159+
llvm::LLVMRustAddDereferenceableOrNullCallSiteAttr(callsite,
160+
idx.as_uint(),
161+
deref);
162+
}
163+
regular -= ArgAttribute::NonNull;
164+
}
165+
if let Some(align) = self.pointee_align {
166+
llvm::LLVMRustAddAlignmentCallSiteAttr(callsite,
167+
idx.as_uint(),
168+
align.abi() as u32);
138169
}
170+
regular.for_each_kind(|attr| attr.apply_callsite(idx, callsite));
139171
}
140172
}
141173
}
@@ -439,12 +471,20 @@ pub struct ArgType<'tcx> {
439471

440472
impl<'a, 'tcx> ArgType<'tcx> {
441473
fn new(layout: TyLayout<'tcx>) -> ArgType<'tcx> {
474+
let mut attrs = ArgAttributes::new();
475+
476+
if let layout::Abi::Scalar(ref scalar) = layout.abi {
477+
if scalar.is_bool() {
478+
attrs.set(ArgAttribute::ZExt);
479+
}
480+
}
481+
442482
ArgType {
443483
kind: ArgKind::Direct,
444484
layout,
445485
cast: None,
446486
pad: None,
447-
attrs: ArgAttributes::default(),
487+
attrs,
448488
nested: vec![]
449489
}
450490
}
@@ -454,14 +494,16 @@ impl<'a, 'tcx> ArgType<'tcx> {
454494
assert_eq!(self.kind, ArgKind::Direct);
455495

456496
// Wipe old attributes, likely not valid through indirection.
457-
self.attrs = ArgAttributes::default();
497+
self.attrs = ArgAttributes::new();
458498

459499
// For non-immediate arguments the callee gets its own copy of
460500
// the value on the stack, so there are no aliases. It's also
461501
// program-invisible so can't possibly capture
462502
self.attrs.set(ArgAttribute::NoAlias)
463503
.set(ArgAttribute::NoCapture)
464-
.set_dereferenceable(self.layout.size);
504+
.set(ArgAttribute::NonNull);
505+
self.attrs.pointee_size = self.layout.size;
506+
self.attrs.pointee_align = Some(self.layout.align);
465507

466508
self.kind = ArgKind::Indirect;
467509
}
@@ -472,6 +514,22 @@ impl<'a, 'tcx> ArgType<'tcx> {
472514
self.kind = ArgKind::Ignore;
473515
}
474516

517+
fn safe_pointee(&mut self, layout: TyLayout) {
518+
match self.layout.abi {
519+
layout::Abi::Scalar(layout::Scalar {
520+
value: layout::Pointer,
521+
ref valid_range
522+
}) => {
523+
if valid_range.start > 0 {
524+
self.attrs.set(ArgAttribute::NonNull);
525+
}
526+
self.attrs.pointee_size = layout.size;
527+
self.attrs.pointee_align = Some(layout.align);
528+
}
529+
_ => bug!("ArgType::safe_pointee({:#?}): not a pointer", self.layout)
530+
}
531+
}
532+
475533
pub fn extend_integer_width_to(&mut self, bits: u64) {
476534
// Only integers have signedness
477535
if let layout::Abi::Scalar(ref scalar) = self.layout.abi {
@@ -694,123 +752,115 @@ impl<'a, 'tcx> FnType<'tcx> {
694752
_ => false
695753
};
696754

697-
let arg_of = |ty: Ty<'tcx>, is_return: bool| {
698-
let mut arg = ArgType::new(ccx.layout_of(ty));
699-
if let layout::Abi::Scalar(ref scalar) = arg.layout.abi {
700-
if scalar.is_bool() {
701-
arg.attrs.set(ArgAttribute::ZExt);
702-
}
703-
}
704-
if arg.layout.is_zst() {
705-
// For some forsaken reason, x86_64-pc-windows-gnu
706-
// doesn't ignore zero-sized struct arguments.
707-
// The same is true for s390x-unknown-linux-gnu.
708-
if is_return || rust_abi ||
709-
(!win_x64_gnu && !linux_s390x) {
710-
arg.ignore();
711-
}
712-
}
713-
arg
714-
};
715-
716-
let ret_ty = sig.output();
717-
let mut ret = arg_of(ret_ty, true);
718-
719-
if !type_is_fat_ptr(ccx, ret_ty) {
720-
// The `noalias` attribute on the return value is useful to a
721-
// function ptr caller.
722-
if ret_ty.is_box() {
723-
// `Box` pointer return values never alias because ownership
724-
// is transferred
725-
ret.attrs.set(ArgAttribute::NoAlias);
755+
// Handle safe Rust thin and fat pointers.
756+
let adjust_for_rust_type = |arg: &mut ArgType<'tcx>, is_return: bool| {
757+
// We only handle thin pointers here.
758+
match arg.layout.abi {
759+
layout::Abi::Scalar(layout::Scalar { value: layout::Pointer, .. }) => {}
760+
_ => return
726761
}
727762

728-
// We can also mark the return value as `dereferenceable` in certain cases
729-
match ret_ty.sty {
730-
// These are not really pointers but pairs, (pointer, len)
731-
ty::TyRef(_, ty::TypeAndMut { ty, .. }) => {
732-
ret.attrs.set_dereferenceable(ccx.size_of(ty));
733-
}
734-
ty::TyAdt(def, _) if def.is_box() => {
735-
ret.attrs.set_dereferenceable(ccx.size_of(ret_ty.boxed_ty()));
763+
let mut ty = arg.layout.ty;
764+
765+
// FIXME(eddyb) detect more nested cases than `Option<&T>` here.
766+
match arg.layout.variants {
767+
layout::Variants::NicheFilling { dataful_variant, .. } => {
768+
let variant = arg.layout.for_variant(ccx, dataful_variant);
769+
for i in 0..variant.fields.count() {
770+
let field = variant.field(ccx, i);
771+
match field.abi {
772+
layout::Abi::Scalar(layout::Scalar { value: layout::Pointer, .. }) => {
773+
// We found the pointer field, use its type.
774+
ty = field.ty;
775+
break;
776+
}
777+
_ => {}
778+
}
779+
}
736780
}
737781
_ => {}
738782
}
739-
}
740783

741-
let mut args = Vec::with_capacity(inputs.len() + extra_args.len());
784+
match ty.sty {
785+
// `Box` pointer parameters never alias because ownership is transferred
786+
ty::TyAdt(def, _) if def.is_box() => {
787+
arg.attrs.set(ArgAttribute::NoAlias);
742788

743-
// Handle safe Rust thin and fat pointers.
744-
let rust_ptr_attrs = |ty: Ty<'tcx>, arg: &mut ArgType| match ty.sty {
745-
// `Box` pointer parameters never alias because ownership is transferred
746-
ty::TyAdt(def, _) if def.is_box() => {
747-
arg.attrs.set(ArgAttribute::NoAlias);
748-
Some(ty.boxed_ty())
749-
}
789+
arg.safe_pointee(ccx.layout_of(ty.boxed_ty()));
790+
}
750791

751-
ty::TyRef(_, mt) => {
752-
// `&mut` pointer parameters never alias other parameters, or mutable global data
753-
//
754-
// `&T` where `T` contains no `UnsafeCell<U>` is immutable, and can be marked as
755-
// both `readonly` and `noalias`, as LLVM's definition of `noalias` is based solely
756-
// on memory dependencies rather than pointer equality
757-
let is_freeze = ccx.shared().type_is_freeze(mt.ty);
758-
759-
let no_alias_is_safe =
760-
if ccx.shared().tcx().sess.opts.debugging_opts.mutable_noalias ||
761-
ccx.shared().tcx().sess.panic_strategy() == PanicStrategy::Abort {
762-
// Mutable refrences or immutable shared references
763-
mt.mutbl == hir::MutMutable || is_freeze
764-
} else {
765-
// Only immutable shared references
766-
mt.mutbl != hir::MutMutable && is_freeze
767-
};
792+
ty::TyRef(_, mt) => {
793+
// `&mut` pointer parameters never alias other parameters,
794+
// or mutable global data
795+
//
796+
// `&T` where `T` contains no `UnsafeCell<U>` is immutable,
797+
// and can be marked as both `readonly` and `noalias`, as
798+
// LLVM's definition of `noalias` is based solely on memory
799+
// dependencies rather than pointer equality
800+
let is_freeze = ccx.shared().type_is_freeze(mt.ty);
801+
802+
let no_alias_is_safe =
803+
if ccx.shared().tcx().sess.opts.debugging_opts.mutable_noalias ||
804+
ccx.shared().tcx().sess.panic_strategy() == PanicStrategy::Abort {
805+
// Mutable refrences or immutable shared references
806+
mt.mutbl == hir::MutMutable || is_freeze
807+
} else {
808+
// Only immutable shared references
809+
mt.mutbl != hir::MutMutable && is_freeze
810+
};
768811

769-
if no_alias_is_safe {
770-
arg.attrs.set(ArgAttribute::NoAlias);
771-
}
812+
if no_alias_is_safe {
813+
arg.attrs.set(ArgAttribute::NoAlias);
814+
}
772815

773-
if mt.mutbl == hir::MutImmutable && is_freeze {
774-
arg.attrs.set(ArgAttribute::ReadOnly);
816+
if mt.mutbl == hir::MutImmutable && is_freeze && !is_return {
817+
arg.attrs.set(ArgAttribute::ReadOnly);
818+
}
819+
820+
arg.safe_pointee(ccx.layout_of(mt.ty));
775821
}
822+
_ => {}
823+
}
776824

777-
Some(mt.ty)
825+
// HACK(eddyb) LLVM inserts `llvm.assume` calls when inlining functions
826+
// with align attributes, and those calls later block optimizations.
827+
if !is_return {
828+
arg.attrs.pointee_align = None;
778829
}
779-
_ => None
780830
};
781831

782-
for ty in inputs.iter().chain(extra_args.iter()) {
783-
let mut arg = arg_of(ty, false);
784-
785-
if type_is_fat_ptr(ccx, ty) {
786-
let mut data = ArgType::new(arg.layout.field(ccx, 0));
787-
let mut info = ArgType::new(arg.layout.field(ccx, 1));
788-
789-
if let Some(inner) = rust_ptr_attrs(ty, &mut data) {
790-
data.attrs.set(ArgAttribute::NonNull);
791-
if ccx.tcx().struct_tail(inner).is_trait() {
792-
// vtables can be safely marked non-null, readonly
793-
// and noalias.
794-
info.attrs.set(ArgAttribute::NonNull);
795-
info.attrs.set(ArgAttribute::ReadOnly);
796-
info.attrs.set(ArgAttribute::NoAlias);
797-
}
798-
}
799-
// FIXME(eddyb) other ABIs don't have logic for nested.
800-
if rust_abi {
801-
arg.nested = vec![data, info];
832+
let arg_of = |ty: Ty<'tcx>, is_return: bool| {
833+
let mut arg = ArgType::new(ccx.layout_of(ty));
834+
if arg.layout.is_zst() {
835+
// For some forsaken reason, x86_64-pc-windows-gnu
836+
// doesn't ignore zero-sized struct arguments.
837+
// The same is true for s390x-unknown-linux-gnu.
838+
if is_return || rust_abi ||
839+
(!win_x64_gnu && !linux_s390x) {
840+
arg.ignore();
802841
}
842+
}
843+
844+
// FIXME(eddyb) other ABIs don't have logic for nested.
845+
if !is_return && type_is_fat_ptr(ccx, arg.layout.ty) && rust_abi {
846+
arg.nested = vec![
847+
ArgType::new(arg.layout.field(ccx, 0)),
848+
ArgType::new(arg.layout.field(ccx, 1))
849+
];
850+
adjust_for_rust_type(&mut arg.nested[0], false);
851+
adjust_for_rust_type(&mut arg.nested[1], false);
803852
} else {
804-
if let Some(inner) = rust_ptr_attrs(ty, &mut arg) {
805-
arg.attrs.set_dereferenceable(ccx.size_of(inner));
806-
}
853+
adjust_for_rust_type(&mut arg, is_return);
807854
}
808-
args.push(arg);
809-
}
855+
856+
arg
857+
};
810858

811859
FnType {
812-
args,
813-
ret,
860+
ret: arg_of(sig.output(), true),
861+
args: inputs.iter().chain(extra_args.iter()).map(|ty| {
862+
arg_of(ty, false)
863+
}).collect(),
814864
variadic: sig.variadic,
815865
cconv,
816866
}

0 commit comments

Comments
 (0)