Skip to content

Commit cdeb4b0

Browse files
eddybeddyb
committed
rustc: encode scalar pairs in layout ABI.
1 parent f1b7cd9 commit cdeb4b0

File tree

15 files changed

+266
-137
lines changed

15 files changed

+266
-137
lines changed

‎src/librustc/ty/layout.rs

Lines changed: 95 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -757,6 +757,7 @@ impl FieldPlacement {
757757
pub enum Abi {
758758
Uninhabited,
759759
Scalar(Scalar),
760+
ScalarPair(Scalar, Scalar),
760761
Vector,
761762
Aggregate {
762763
/// If true, the size is exact, otherwise it's only a lower bound.
@@ -769,15 +770,21 @@ impl Abi {
769770
/// Returns true if the layout corresponds to an unsized type.
770771
pub fn is_unsized(&self) -> bool {
771772
match *self {
772-
Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector => false,
773+
Abi::Uninhabited |
774+
Abi::Scalar(_) |
775+
Abi::ScalarPair(..) |
776+
Abi::Vector => false,
773777
Abi::Aggregate { sized, .. } => !sized
774778
}
775779
}
776780

777781
/// Returns true if the fields of the layout are packed.
778782
pub fn is_packed(&self) -> bool {
779783
match *self {
780-
Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector => false,
784+
Abi::Uninhabited |
785+
Abi::Scalar(_) |
786+
Abi::ScalarPair(..) |
787+
Abi::Vector => false,
781788
Abi::Aggregate { packed, .. } => packed
782789
}
783790
}
@@ -905,13 +912,32 @@ impl<'a, 'tcx> CachedLayout {
905912
-> Result<&'tcx Self, LayoutError<'tcx>> {
906913
let cx = (tcx, param_env);
907914
let dl = cx.data_layout();
908-
let scalar = |value: Primitive| {
915+
let scalar_unit = |value: Primitive| {
909916
let bits = value.size(dl).bits();
910917
assert!(bits <= 128);
911-
tcx.intern_layout(CachedLayout::scalar(cx, Scalar {
918+
Scalar {
912919
value,
913920
valid_range: 0..=(!0 >> (128 - bits))
914-
}))
921+
}
922+
};
923+
let scalar = |value: Primitive| {
924+
tcx.intern_layout(CachedLayout::scalar(cx, scalar_unit(value)))
925+
};
926+
let scalar_pair = |a: Scalar, b: Scalar| {
927+
let align = a.value.align(dl).max(b.value.align(dl)).max(dl.aggregate_align);
928+
let b_offset = a.value.size(dl).abi_align(b.value.align(dl));
929+
let size = (b_offset + b.value.size(dl)).abi_align(align);
930+
CachedLayout {
931+
variants: Variants::Single { index: 0 },
932+
fields: FieldPlacement::Arbitrary {
933+
offsets: vec![Size::from_bytes(0), b_offset],
934+
memory_index: vec![0, 1]
935+
},
936+
abi: Abi::ScalarPair(a, b),
937+
align,
938+
primitive_align: align,
939+
size
940+
}
915941
};
916942

917943
#[derive(Copy, Clone, Debug)]
@@ -1049,19 +1075,54 @@ impl<'a, 'tcx> CachedLayout {
10491075
memory_index = inverse_memory_index;
10501076
}
10511077

1078+
let size = min_size.abi_align(align);
1079+
let mut abi = Abi::Aggregate {
1080+
sized,
1081+
packed
1082+
};
1083+
1084+
// Look for a scalar pair, as an ABI optimization.
1085+
// FIXME(eddyb) ignore extra ZST fields and field ordering.
1086+
if sized && !packed && fields.len() == 2 {
1087+
match (&fields[0].abi, &fields[1].abi) {
1088+
(&Abi::Scalar(ref a), &Abi::Scalar(ref b)) => {
1089+
let pair = scalar_pair(a.clone(), b.clone());
1090+
let pair_offsets = match pair.fields {
1091+
FieldPlacement::Arbitrary {
1092+
ref offsets,
1093+
ref memory_index
1094+
} => {
1095+
assert_eq!(memory_index, &[0, 1]);
1096+
offsets
1097+
}
1098+
_ => bug!()
1099+
};
1100+
if offsets[0] == pair_offsets[0] &&
1101+
offsets[1] == pair_offsets[1] &&
1102+
memory_index[0] == 0 &&
1103+
memory_index[1] == 1 &&
1104+
align == pair.align &&
1105+
primitive_align == pair.primitive_align &&
1106+
size == pair.size {
1107+
// We can use `ScalarPair` only when it matches our
1108+
// already computed layout (including `#[repr(C)]`).
1109+
abi = pair.abi;
1110+
}
1111+
}
1112+
_ => {}
1113+
}
1114+
}
1115+
10521116
Ok(CachedLayout {
10531117
variants: Variants::Single { index: 0 },
10541118
fields: FieldPlacement::Arbitrary {
10551119
offsets,
10561120
memory_index
10571121
},
1058-
abi: Abi::Aggregate {
1059-
sized,
1060-
packed
1061-
},
1122+
abi,
10621123
align,
10631124
primitive_align,
1064-
size: min_size.abi_align(align)
1125+
size
10651126
})
10661127
};
10671128
let univariant = |fields: &[TyLayout], repr: &ReprOptions, kind| {
@@ -1070,45 +1131,34 @@ impl<'a, 'tcx> CachedLayout {
10701131
assert!(!ty.has_infer_types());
10711132

10721133
let ptr_layout = |pointee: Ty<'tcx>| {
1134+
let mut data_ptr = scalar_unit(Pointer);
1135+
if !ty.is_unsafe_ptr() {
1136+
data_ptr.valid_range.start = 1;
1137+
}
1138+
10731139
let pointee = tcx.normalize_associated_type_in_env(&pointee, param_env);
10741140
if pointee.is_sized(tcx, param_env, DUMMY_SP) {
1075-
let non_zero = !ty.is_unsafe_ptr();
1076-
let bits = Pointer.size(dl).bits();
1077-
return Ok(tcx.intern_layout(CachedLayout::scalar(cx, Scalar {
1078-
value: Pointer,
1079-
valid_range: (non_zero as u128)..=(!0 >> (128 - bits))
1080-
})));
1141+
return Ok(tcx.intern_layout(CachedLayout::scalar(cx, data_ptr)));
10811142
}
10821143

10831144
let unsized_part = tcx.struct_tail(pointee);
10841145
let metadata = match unsized_part.sty {
1085-
ty::TyForeign(..) => return Ok(scalar(Pointer)),
1146+
ty::TyForeign(..) => {
1147+
return Ok(tcx.intern_layout(CachedLayout::scalar(cx, data_ptr)));
1148+
}
10861149
ty::TySlice(_) | ty::TyStr => {
1087-
Int(dl.ptr_sized_integer(), false)
1150+
scalar_unit(Int(dl.ptr_sized_integer(), false))
1151+
}
1152+
ty::TyDynamic(..) => {
1153+
let mut vtable = scalar_unit(Pointer);
1154+
vtable.valid_range.start = 1;
1155+
vtable
10881156
}
1089-
ty::TyDynamic(..) => Pointer,
10901157
_ => return Err(LayoutError::Unknown(unsized_part))
10911158
};
10921159

10931160
// Effectively a (ptr, meta) tuple.
1094-
let align = Pointer.align(dl).max(metadata.align(dl));
1095-
let meta_offset = Pointer.size(dl);
1096-
assert_eq!(meta_offset, meta_offset.abi_align(metadata.align(dl)));
1097-
let fields = FieldPlacement::Arbitrary {
1098-
offsets: vec![Size::from_bytes(0), meta_offset],
1099-
memory_index: vec![0, 1]
1100-
};
1101-
Ok(tcx.intern_layout(CachedLayout {
1102-
variants: Variants::Single { index: 0 },
1103-
fields,
1104-
abi: Abi::Aggregate {
1105-
sized: true,
1106-
packed: false
1107-
},
1108-
align,
1109-
primitive_align: align,
1110-
size: (meta_offset + metadata.size(dl)).abi_align(align)
1111-
}))
1161+
Ok(tcx.intern_layout(scalar_pair(data_ptr, metadata)))
11121162
};
11131163

11141164
Ok(match ty.sty {
@@ -1134,11 +1184,9 @@ impl<'a, 'tcx> CachedLayout {
11341184
ty::TyFloat(FloatTy::F32) => scalar(F32),
11351185
ty::TyFloat(FloatTy::F64) => scalar(F64),
11361186
ty::TyFnPtr(_) => {
1137-
let bits = Pointer.size(dl).bits();
1138-
tcx.intern_layout(CachedLayout::scalar(cx, Scalar {
1139-
value: Pointer,
1140-
valid_range: 1..=(!0 >> (128 - bits))
1141-
}))
1187+
let mut ptr = scalar_unit(Pointer);
1188+
ptr.valid_range.start = 1;
1189+
tcx.intern_layout(CachedLayout::scalar(cx, ptr))
11421190
}
11431191

11441192
// The never type.
@@ -2194,7 +2242,7 @@ impl<'a, 'tcx> TyLayout<'tcx> {
21942242
pub fn is_zst(&self) -> bool {
21952243
match self.abi {
21962244
Abi::Uninhabited => true,
2197-
Abi::Scalar(_) => false,
2245+
Abi::Scalar(_) | Abi::ScalarPair(..) => false,
21982246
Abi::Vector => self.size.bytes() == 0,
21992247
Abi::Aggregate { sized, .. } => sized && self.size.bytes() == 0
22002248
}
@@ -2347,6 +2395,10 @@ impl<'gcx> HashStable<StableHashingContext<'gcx>> for Abi {
23472395
Scalar(ref value) => {
23482396
value.hash_stable(hcx, hasher);
23492397
}
2398+
ScalarPair(ref a, ref b) => {
2399+
a.hash_stable(hcx, hasher);
2400+
b.hash_stable(hcx, hasher);
2401+
}
23502402
Vector => {}
23512403
Aggregate { packed, sized } => {
23522404
packed.hash_stable(hcx, hasher);

‎src/librustc_trans/abi.rs

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,7 @@ impl<'tcx> LayoutExt<'tcx> for TyLayout<'tcx> {
311311
layout::Abi::Uninhabited |
312312
layout::Abi::Scalar(_) |
313313
layout::Abi::Vector => false,
314+
layout::Abi::ScalarPair(..) |
314315
layout::Abi::Aggregate { .. } => true
315316
}
316317
}
@@ -340,6 +341,7 @@ impl<'tcx> LayoutExt<'tcx> for TyLayout<'tcx> {
340341
})
341342
}
342343

344+
layout::Abi::ScalarPair(..) |
343345
layout::Abi::Aggregate { .. } => {
344346
let mut total = Size::from_bytes(0);
345347
let mut result = None;
@@ -745,10 +747,13 @@ impl<'a, 'tcx> FnType<'tcx> {
745747
arg.attrs.set(ArgAttribute::NonNull);
746748
}
747749
}
748-
_ => {}
750+
_ => {
751+
// Nothing to do for non-pointer types.
752+
return;
753+
}
749754
}
750755

751-
if let Some(pointee) = arg.layout.pointee_info(ccx) {
756+
if let Some(pointee) = arg.layout.pointee_info_at(ccx, Size::from_bytes(0)) {
752757
if let Some(kind) = pointee.safe {
753758
arg.attrs.pointee_size = pointee.size;
754759
arg.attrs.pointee_align = Some(pointee.align);

‎src/librustc_trans/cabi_x86_64.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ fn classify_arg<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, arg: &ArgType<'tcx>)
8888
}
8989
}
9090

91+
layout::Abi::ScalarPair(..) |
9192
layout::Abi::Aggregate { .. } => {
9293
match layout.variants {
9394
layout::Variants::Single { .. } => {

‎src/librustc_trans/cabi_x86_win64.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ pub fn compute_abi_info(fty: &mut FnType) {
1818
let fixup = |a: &mut ArgType| {
1919
match a.layout.abi {
2020
layout::Abi::Uninhabited => {}
21+
layout::Abi::ScalarPair(..) |
2122
layout::Abi::Aggregate { .. } => {
2223
match a.layout.size.bits() {
2324
8 => a.cast_to(Reg::i8()),

‎src/librustc_trans/common.rs

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -232,16 +232,9 @@ pub fn C_str_slice(cx: &CrateContext, s: InternedString) -> ValueRef {
232232
}
233233

234234
pub fn C_fat_ptr(cx: &CrateContext, ptr: ValueRef, meta: ValueRef) -> ValueRef {
235-
let empty = C_array(Type::i8(cx), &[]);
236235
assert_eq!(abi::FAT_PTR_ADDR, 0);
237236
assert_eq!(abi::FAT_PTR_EXTRA, 1);
238-
C_struct(cx, &[
239-
empty,
240-
ptr,
241-
empty,
242-
meta,
243-
empty
244-
], false)
237+
C_struct(cx, &[ptr, meta], false)
245238
}
246239

247240
pub fn C_struct(cx: &CrateContext, elts: &[ValueRef], packed: bool) -> ValueRef {

‎src/librustc_trans/context.rs

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ use rustc::middle::trans::Stats;
3131
use rustc_data_structures::stable_hasher::StableHashingContextProvider;
3232
use rustc::session::config::{self, NoDebugInfo};
3333
use rustc::session::Session;
34-
use rustc::ty::layout::{LayoutError, LayoutOf, TyLayout};
34+
use rustc::ty::layout::{LayoutError, LayoutOf, Size, TyLayout};
3535
use rustc::ty::{self, Ty, TyCtxt};
3636
use rustc::util::nodemap::FxHashMap;
3737
use rustc_trans_utils;
@@ -103,7 +103,7 @@ pub struct LocalCrateContext<'a, 'tcx: 'a> {
103103

104104
lltypes: RefCell<FxHashMap<(Ty<'tcx>, Option<usize>), Type>>,
105105
scalar_lltypes: RefCell<FxHashMap<Ty<'tcx>, Type>>,
106-
pointee_infos: RefCell<FxHashMap<Ty<'tcx>, Option<PointeeInfo>>>,
106+
pointee_infos: RefCell<FxHashMap<(Ty<'tcx>, Size), Option<PointeeInfo>>>,
107107
isize_ty: Type,
108108

109109
dbg_cx: Option<debuginfo::CrateDebugContext<'tcx>>,
@@ -516,7 +516,8 @@ impl<'b, 'tcx> CrateContext<'b, 'tcx> {
516516
&self.local().scalar_lltypes
517517
}
518518

519-
pub fn pointee_infos<'a>(&'a self) -> &'a RefCell<FxHashMap<Ty<'tcx>, Option<PointeeInfo>>> {
519+
pub fn pointee_infos<'a>(&'a self)
520+
-> &'a RefCell<FxHashMap<(Ty<'tcx>, Size), Option<PointeeInfo>>> {
520521
&self.local().pointee_infos
521522
}
522523

‎src/librustc_trans/mir/analyze.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ pub fn lvalue_locals<'a, 'tcx>(mircx: &MirContext<'a, 'tcx>) -> BitVector {
3535
if layout.is_llvm_immediate() {
3636
// These sorts of types are immediates that we can store
3737
// in an ValueRef without an alloca.
38-
} else if layout.is_llvm_scalar_pair(mircx.ccx) {
38+
} else if layout.is_llvm_scalar_pair() {
3939
// We allow pairs and uses of any of their 2 fields.
4040
} else {
4141
// These sorts of types require an alloca. Note that
@@ -146,7 +146,7 @@ impl<'mir, 'a, 'tcx> Visitor<'tcx> for LocalAnalyzer<'mir, 'a, 'tcx> {
146146

147147
let ty = self.cx.monomorphize(&ty.to_ty(self.cx.ccx.tcx()));
148148
let layout = self.cx.ccx.layout_of(ty);
149-
if layout.is_llvm_scalar_pair(self.cx.ccx) {
149+
if layout.is_llvm_scalar_pair() {
150150
return;
151151
}
152152
}

‎src/librustc_trans/mir/constant.rs

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,12 @@ impl<'a, 'tcx> Const<'tcx> {
117117
}
118118

119119
fn get_field(&self, ccx: &CrateContext<'a, 'tcx>, i: usize) -> ValueRef {
120-
const_get_elt(self.llval, ccx.layout_of(self.ty).llvm_field_index(i))
120+
let layout = ccx.layout_of(self.ty);
121+
if let layout::Abi::ScalarPair(..) = layout.abi {
122+
const_get_elt(self.llval, i as u64)
123+
} else {
124+
const_get_elt(self.llval, layout.llvm_field_index(i))
125+
}
121126
}
122127

123128
fn get_pair(&self, ccx: &CrateContext<'a, 'tcx>) -> (ValueRef, ValueRef) {
@@ -143,7 +148,7 @@ impl<'a, 'tcx> Const<'tcx> {
143148
let llty = layout.immediate_llvm_type(ccx);
144149
let llvalty = val_ty(self.llval);
145150

146-
let val = if llty == llvalty && layout.is_llvm_scalar_pair(ccx) {
151+
let val = if llty == llvalty && layout.is_llvm_scalar_pair() {
147152
let (a, b) = self.get_pair(ccx);
148153
OperandValue::Pair(a, b)
149154
} else if llty == llvalty && layout.is_llvm_immediate() {
@@ -1174,6 +1179,14 @@ fn build_const_struct<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>,
11741179
-> Const<'tcx> {
11751180
assert_eq!(vals.len(), layout.fields.count());
11761181

1182+
if let layout::Abi::ScalarPair(..) = layout.abi {
1183+
assert_eq!(vals.len(), 2);
1184+
return Const::new(C_struct(ccx, &[
1185+
vals[0].llval,
1186+
vals[1].llval,
1187+
], false), layout.ty);
1188+
}
1189+
11771190
// offset of current value
11781191
let mut offset = Size::from_bytes(0);
11791192
let mut cfields = Vec::new();

0 commit comments

Comments
 (0)