Skip to content

Commit b7cda37

Browse files
bjorn3bjorn3
committed
Pass Ty instead of TyAndLayout to the closure of various simd helpers
This reduces the total amount of llvm ir lines for simd related functions from 9604 to 9467.
1 parent 2633024 commit b7cda37

File tree

3 files changed

+75
-110
lines changed

3 files changed

+75
-110
lines changed

‎src/intrinsics/llvm.rs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,17 +73,17 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
7373
kind => unreachable!("kind {:?}", kind),
7474
};
7575

76-
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_layout, res_lane_layout, x_lane, y_lane| {
77-
let res_lane = match lane_layout.ty.kind() {
76+
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
77+
let res_lane = match lane_ty.kind() {
7878
ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
79-
_ => unreachable!("{:?}", lane_layout.ty),
79+
_ => unreachable!("{:?}", lane_ty),
8080
};
81-
bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
81+
bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane)
8282
});
8383
};
8484
"llvm.x86.sse2.psrli.d", (c a, o imm8) {
8585
let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
86-
simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _res_lane_layout, lane| {
86+
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
8787
match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
8888
imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
8989
_ => fx.bcx.ins().iconst(types::I32, 0),
@@ -92,7 +92,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
9292
};
9393
"llvm.x86.sse2.pslli.d", (c a, o imm8) {
9494
let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
95-
simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _res_lane_layout, lane| {
95+
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
9696
match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
9797
imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
9898
_ => fx.bcx.ins().iconst(types::I32, 0),

‎src/intrinsics/mod.rs

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -108,12 +108,7 @@ fn simd_for_each_lane<'tcx>(
108108
fx: &mut FunctionCx<'_, '_, 'tcx>,
109109
val: CValue<'tcx>,
110110
ret: CPlace<'tcx>,
111-
f: &dyn Fn(
112-
&mut FunctionCx<'_, '_, 'tcx>,
113-
TyAndLayout<'tcx>,
114-
TyAndLayout<'tcx>,
115-
Value,
116-
) -> Value,
111+
f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value) -> Value,
117112
) {
118113
let layout = val.layout();
119114

@@ -126,7 +121,7 @@ fn simd_for_each_lane<'tcx>(
126121
for lane_idx in 0..lane_count {
127122
let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
128123

129-
let res_lane = f(fx, lane_layout, ret_lane_layout, lane);
124+
let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, lane);
130125
let res_lane = CValue::by_val(res_lane, ret_lane_layout);
131126

132127
ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
@@ -138,13 +133,7 @@ fn simd_pair_for_each_lane<'tcx>(
138133
x: CValue<'tcx>,
139134
y: CValue<'tcx>,
140135
ret: CPlace<'tcx>,
141-
f: &dyn Fn(
142-
&mut FunctionCx<'_, '_, 'tcx>,
143-
TyAndLayout<'tcx>,
144-
TyAndLayout<'tcx>,
145-
Value,
146-
Value,
147-
) -> Value,
136+
f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value) -> Value,
148137
) {
149138
assert_eq!(x.layout(), y.layout());
150139
let layout = x.layout();
@@ -159,7 +148,7 @@ fn simd_pair_for_each_lane<'tcx>(
159148
let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx);
160149
let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx);
161150

162-
let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane);
151+
let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, x_lane, y_lane);
163152
let res_lane = CValue::by_val(res_lane, ret_lane_layout);
164153

165154
ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
@@ -171,7 +160,7 @@ fn simd_reduce<'tcx>(
171160
val: CValue<'tcx>,
172161
acc: Option<Value>,
173162
ret: CPlace<'tcx>,
174-
f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, TyAndLayout<'tcx>, Value, Value) -> Value,
163+
f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Value, Value) -> Value,
175164
) {
176165
let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
177166
let lane_layout = fx.layout_of(lane_ty);
@@ -181,7 +170,7 @@ fn simd_reduce<'tcx>(
181170
if let Some(acc) = acc { (acc, 0) } else { (val.value_lane(fx, 0).load_scalar(fx), 1) };
182171
for lane_idx in start_lane..lane_count {
183172
let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
184-
res_val = f(fx, lane_layout, res_val, lane);
173+
res_val = f(fx, lane_layout.ty, res_val, lane);
185174
}
186175
let res = CValue::by_val(res_val, lane_layout);
187176
ret.write_cvalue(fx, res);
@@ -215,10 +204,10 @@ fn simd_reduce_bool<'tcx>(
215204

216205
fn bool_to_zero_or_max_uint<'tcx>(
217206
fx: &mut FunctionCx<'_, '_, 'tcx>,
218-
layout: TyAndLayout<'tcx>,
207+
ty: Ty<'tcx>,
219208
val: Value,
220209
) -> Value {
221-
let ty = fx.clif_type(layout.ty).unwrap();
210+
let ty = fx.clif_type(ty).unwrap();
222211

223212
let int_ty = match ty {
224213
types::F32 => types::I32,

‎src/intrinsics/simd.rs

Lines changed: 61 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -17,76 +17,52 @@ fn validate_simd_type(fx: &mut FunctionCx<'_, '_, '_>, intrinsic: Symbol, span:
1717

1818
macro simd_cmp($fx:expr, $cc_u:ident|$cc_s:ident|$cc_f:ident($x:ident, $y:ident) -> $ret:ident) {
1919
// FIXME use vector instructions when possible
20-
simd_pair_for_each_lane(
21-
$fx,
22-
$x,
23-
$y,
24-
$ret,
25-
&|fx, lane_layout, res_lane_layout, x_lane, y_lane| {
26-
let res_lane = match lane_layout.ty.kind() {
27-
ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
28-
ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
29-
ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
30-
_ => unreachable!("{:?}", lane_layout.ty),
31-
};
20+
simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
21+
let res_lane = match lane_ty.kind() {
22+
ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
23+
ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
24+
ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::$cc_f, x_lane, y_lane),
25+
_ => unreachable!("{:?}", lane_ty),
26+
};
3227

33-
let ty = fx.clif_type(res_lane_layout.ty).unwrap();
28+
let ty = fx.clif_type(res_lane_ty).unwrap();
3429

35-
let res_lane = fx.bcx.ins().bint(ty, res_lane);
36-
fx.bcx.ins().ineg(res_lane)
37-
},
38-
);
30+
let res_lane = fx.bcx.ins().bint(ty, res_lane);
31+
fx.bcx.ins().ineg(res_lane)
32+
});
3933
}
4034

4135
macro simd_int_binop($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) {
4236
// FIXME use vector instructions when possible
43-
simd_pair_for_each_lane(
44-
$fx,
45-
$x,
46-
$y,
47-
$ret,
48-
&|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
49-
match lane_layout.ty.kind() {
50-
ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
51-
ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
52-
_ => unreachable!("{:?}", lane_layout.ty),
53-
}
54-
},
55-
);
37+
simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
38+
match lane_ty.kind() {
39+
ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
40+
ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
41+
_ => unreachable!("{:?}", lane_ty),
42+
}
43+
});
5644
}
5745

5846
macro simd_int_flt_binop($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) {
5947
// FIXME use vector instructions when possible
60-
simd_pair_for_each_lane(
61-
$fx,
62-
$x,
63-
$y,
64-
$ret,
65-
&|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
66-
match lane_layout.ty.kind() {
67-
ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
68-
ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
69-
ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
70-
_ => unreachable!("{:?}", lane_layout.ty),
71-
}
72-
},
73-
);
48+
simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
49+
match lane_ty.kind() {
50+
ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
51+
ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
52+
ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
53+
_ => unreachable!("{:?}", lane_ty),
54+
}
55+
});
7456
}
7557

7658
macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
7759
// FIXME use vector instructions when possible
78-
simd_pair_for_each_lane(
79-
$fx,
80-
$x,
81-
$y,
82-
$ret,
83-
&|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
84-
match lane_layout.ty.kind() {
85-
ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
86-
_ => unreachable!("{:?}", lane_layout.ty),
87-
}
88-
},
89-
);
60+
simd_pair_for_each_lane($fx, $x, $y, $ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
61+
match lane_ty.kind() {
62+
ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
63+
_ => unreachable!("{:?}", lane_ty),
64+
}
65+
});
9066
}
9167

9268
pub(super) fn codegen_simd_intrinsic_call<'tcx>(
@@ -105,13 +81,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
10581

10682
simd_cast, (c a) {
10783
validate_simd_type(fx, intrinsic, span, a.layout().ty);
108-
simd_for_each_lane(fx, a, ret, &|fx, lane_layout, ret_lane_layout, lane| {
109-
let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
84+
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
85+
let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
11086

111-
let from_signed = type_sign(lane_layout.ty);
112-
let to_signed = type_sign(ret_lane_layout.ty);
87+
let from_signed = type_sign(lane_ty);
88+
let to_signed = type_sign(ret_lane_ty);
11389

114-
clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed)
90+
clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
11591
});
11692
};
11793

@@ -277,8 +253,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
277253

278254
simd_neg, (c a) {
279255
validate_simd_type(fx, intrinsic, span, a.layout().ty);
280-
simd_for_each_lane(fx, a, ret, &|fx, lane_layout, _ret_lane_layout, lane| {
281-
match lane_layout.ty.kind() {
256+
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
257+
match lane_ty.kind() {
282258
ty::Int(_) => fx.bcx.ins().ineg(lane),
283259
ty::Float(_) => fx.bcx.ins().fneg(lane),
284260
_ => unreachable!(),
@@ -288,14 +264,14 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
288264

289265
simd_fabs, (c a) {
290266
validate_simd_type(fx, intrinsic, span, a.layout().ty);
291-
simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
267+
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
292268
fx.bcx.ins().fabs(lane)
293269
});
294270
};
295271

296272
simd_fsqrt, (c a) {
297273
validate_simd_type(fx, intrinsic, span, a.layout().ty);
298-
simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
274+
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
299275
fx.bcx.ins().sqrt(lane)
300276
});
301277
};
@@ -318,8 +294,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
318294
};
319295
simd_rem, (c x, c y) {
320296
validate_simd_type(fx, intrinsic, span, x.layout().ty);
321-
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_layout, _ret_lane_layout, x_lane, y_lane| {
322-
match lane_layout.ty.kind() {
297+
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
298+
match lane_ty.kind() {
323299
ty::Uint(_) => fx.bcx.ins().urem(x_lane, y_lane),
324300
ty::Int(_) => fx.bcx.ins().srem(x_lane, y_lane),
325301
ty::Float(FloatTy::F32) => fx.lib_call(
@@ -334,7 +310,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
334310
vec![AbiParam::new(types::F64)],
335311
&[x_lane, y_lane],
336312
)[0],
337-
_ => unreachable!("{:?}", lane_layout.ty),
313+
_ => unreachable!("{:?}", lane_ty),
338314
}
339315
});
340316
};
@@ -393,8 +369,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
393369

394370
simd_round, (c a) {
395371
validate_simd_type(fx, intrinsic, span, a.layout().ty);
396-
simd_for_each_lane(fx, a, ret, &|fx, lane_layout, _ret_lane_layout, lane| {
397-
match lane_layout.ty.kind() {
372+
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
373+
match lane_ty.kind() {
398374
ty::Float(FloatTy::F32) => fx.lib_call(
399375
"roundf",
400376
vec![AbiParam::new(types::F32)],
@@ -407,33 +383,33 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
407383
vec![AbiParam::new(types::F64)],
408384
&[lane],
409385
)[0],
410-
_ => unreachable!("{:?}", lane_layout.ty),
386+
_ => unreachable!("{:?}", lane_ty),
411387
}
412388
});
413389
};
414390
simd_ceil, (c a) {
415391
validate_simd_type(fx, intrinsic, span, a.layout().ty);
416-
simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
392+
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
417393
fx.bcx.ins().ceil(lane)
418394
});
419395
};
420396
simd_floor, (c a) {
421397
validate_simd_type(fx, intrinsic, span, a.layout().ty);
422-
simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
398+
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
423399
fx.bcx.ins().floor(lane)
424400
});
425401
};
426402
simd_trunc, (c a) {
427403
validate_simd_type(fx, intrinsic, span, a.layout().ty);
428-
simd_for_each_lane(fx, a, ret, &|fx, _lane_layout, _ret_lane_layout, lane| {
404+
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _ret_lane_ty, lane| {
429405
fx.bcx.ins().trunc(lane)
430406
});
431407
};
432408

433409
simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
434410
validate_simd_type(fx, intrinsic, span, v.layout().ty);
435-
simd_reduce(fx, v, Some(acc), ret, &|fx, lane_layout, a, b| {
436-
if lane_layout.ty.is_floating_point() {
411+
simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
412+
if lane_ty.is_floating_point() {
437413
fx.bcx.ins().fadd(a, b)
438414
} else {
439415
fx.bcx.ins().iadd(a, b)
@@ -443,8 +419,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
443419

444420
simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
445421
validate_simd_type(fx, intrinsic, span, v.layout().ty);
446-
simd_reduce(fx, v, Some(acc), ret, &|fx, lane_layout, a, b| {
447-
if lane_layout.ty.is_floating_point() {
422+
simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
423+
if lane_ty.is_floating_point() {
448424
fx.bcx.ins().fmul(a, b)
449425
} else {
450426
fx.bcx.ins().imul(a, b)
@@ -464,23 +440,23 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
464440

465441
simd_reduce_and, (c v) {
466442
validate_simd_type(fx, intrinsic, span, v.layout().ty);
467-
simd_reduce(fx, v, None, ret, &|fx, _layout, a, b| fx.bcx.ins().band(a, b));
443+
simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
468444
};
469445

470446
simd_reduce_or, (c v) {
471447
validate_simd_type(fx, intrinsic, span, v.layout().ty);
472-
simd_reduce(fx, v, None, ret, &|fx, _layout, a, b| fx.bcx.ins().bor(a, b));
448+
simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
473449
};
474450

475451
simd_reduce_xor, (c v) {
476452
validate_simd_type(fx, intrinsic, span, v.layout().ty);
477-
simd_reduce(fx, v, None, ret, &|fx, _layout, a, b| fx.bcx.ins().bxor(a, b));
453+
simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
478454
};
479455

480456
simd_reduce_min, (c v) {
481457
validate_simd_type(fx, intrinsic, span, v.layout().ty);
482-
simd_reduce(fx, v, None, ret, &|fx, layout, a, b| {
483-
let lt = match layout.ty.kind() {
458+
simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
459+
let lt = match ty.kind() {
484460
ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
485461
ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
486462
ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::LessThan, a, b),
@@ -492,8 +468,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
492468

493469
simd_reduce_max, (c v) {
494470
validate_simd_type(fx, intrinsic, span, v.layout().ty);
495-
simd_reduce(fx, v, None, ret, &|fx, layout, a, b| {
496-
let gt = match layout.ty.kind() {
471+
simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
472+
let gt = match ty.kind() {
497473
ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
498474
ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
499475
ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, a, b),

0 commit comments

Comments
 (0)