Skip to content

Commit fd975f7

Browse files
matthiaskrgrmatthiaskrgr
authored
Rollup merge of rust-lang#125266 - workingjubilee:stream-plastic-love, r=RalfJung,nikic
compiler: add simd_ctpop intrinsic Fairly straightforward addition. cc `@rust-lang/opsem` new (extremely boring) intrinsic
2 parents a8ee8d5 + 213351a commit fd975f7

File tree

10 files changed

+95
-40
lines changed

10 files changed

+95
-40
lines changed

‎compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
348348
| sym::simd_bswap
349349
| sym::simd_bitreverse
350350
| sym::simd_ctlz
351+
| sym::simd_ctpop
351352
| sym::simd_cttz => {
352353
intrinsic_args!(fx, args => (a); intrinsic);
353354

@@ -367,6 +368,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
367368
(ty::Uint(_) | ty::Int(_), sym::simd_bswap) => fx.bcx.ins().bswap(lane),
368369
(ty::Uint(_) | ty::Int(_), sym::simd_bitreverse) => fx.bcx.ins().bitrev(lane),
369370
(ty::Uint(_) | ty::Int(_), sym::simd_ctlz) => fx.bcx.ins().clz(lane),
371+
(ty::Uint(_) | ty::Int(_), sym::simd_ctpop) => fx.bcx.ins().popcnt(lane),
370372
(ty::Uint(_) | ty::Int(_), sym::simd_cttz) => fx.bcx.ins().ctz(lane),
371373

372374
_ => unreachable!(),

‎compiler/rustc_codegen_llvm/src/intrinsic.rs

Lines changed: 29 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2336,7 +2336,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
23362336
}
23372337

23382338
// Unary integer intrinsics
2339-
if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz) {
2339+
if matches!(
2340+
name,
2341+
sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop | sym::simd_cttz
2342+
) {
23402343
let vec_ty = bx.cx.type_vector(
23412344
match *in_elem.kind() {
23422345
ty::Int(i) => bx.cx.type_int_from_ty(i),
@@ -2354,31 +2357,38 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
23542357
sym::simd_bswap => "bswap",
23552358
sym::simd_bitreverse => "bitreverse",
23562359
sym::simd_ctlz => "ctlz",
2360+
sym::simd_ctpop => "ctpop",
23572361
sym::simd_cttz => "cttz",
23582362
_ => unreachable!(),
23592363
};
23602364
let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
23612365
let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,);
23622366

2363-
return if name == sym::simd_bswap && int_size == 8 {
2367+
return match name {
23642368
// byte swap is no-op for i8/u8
2365-
Ok(args[0].immediate())
2366-
} else if matches!(name, sym::simd_ctlz | sym::simd_cttz) {
2367-
let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty);
2368-
let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2369-
Ok(bx.call(
2370-
fn_ty,
2371-
None,
2372-
None,
2373-
f,
2374-
&[args[0].immediate(), bx.const_int(bx.type_i1(), 0)],
2375-
None,
2376-
None,
2377-
))
2378-
} else {
2379-
let fn_ty = bx.type_func(&[vec_ty], vec_ty);
2380-
let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2381-
Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None))
2369+
sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
2370+
sym::simd_ctlz | sym::simd_cttz => {
2371+
// for the (int, i1 immediate) pair, the second arg adds `(0, true) => poison`
2372+
let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty);
2373+
let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
2374+
let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2375+
Ok(bx.call(
2376+
fn_ty,
2377+
None,
2378+
None,
2379+
f,
2380+
&[args[0].immediate(), dont_poison_on_zero],
2381+
None,
2382+
None,
2383+
))
2384+
}
2385+
sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
2386+
// simple unary argument cases
2387+
let fn_ty = bx.type_func(&[vec_ty], vec_ty);
2388+
let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
2389+
Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None))
2390+
}
2391+
_ => unreachable!(),
23822392
};
23832393
}
23842394

‎compiler/rustc_hir_analysis/src/check/intrinsic.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -607,6 +607,7 @@ pub fn check_intrinsic_type(
607607
| sym::simd_bitreverse
608608
| sym::simd_ctlz
609609
| sym::simd_cttz
610+
| sym::simd_ctpop
610611
| sym::simd_fsqrt
611612
| sym::simd_fsin
612613
| sym::simd_fcos

‎compiler/rustc_span/src/symbol.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1682,6 +1682,7 @@ symbols! {
16821682
simd_cast_ptr,
16831683
simd_ceil,
16841684
simd_ctlz,
1685+
simd_ctpop,
16851686
simd_cttz,
16861687
simd_div,
16871688
simd_eq,

‎library/core/src/intrinsics/simd.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,13 @@ extern "rust-intrinsic" {
569569
#[rustc_nounwind]
570570
pub fn simd_ctlz<T>(x: T) -> T;
571571

572+
/// Count the number of ones in each element.
573+
///
574+
/// `T` must be a vector of integers.
575+
#[rustc_nounwind]
576+
#[cfg(not(bootstrap))]
577+
pub fn simd_ctpop<T>(x: T) -> T;
578+
572579
/// Count the trailing zeros of each element.
573580
///
574581
/// `T` must be a vector of integers.

‎src/tools/miri/src/intrinsics/simd.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
4242
| "flog2"
4343
| "flog10"
4444
| "ctlz"
45+
| "ctpop"
4546
| "cttz"
4647
| "bswap"
4748
| "bitreverse"
@@ -68,6 +69,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
6869
"round" => Op::Round(rustc_apfloat::Round::NearestTiesToAway),
6970
"trunc" => Op::Round(rustc_apfloat::Round::TowardZero),
7071
"ctlz" => Op::Numeric(sym::ctlz),
72+
"ctpop" => Op::Numeric(sym::ctpop),
7173
"cttz" => Op::Numeric(sym::cttz),
7274
"bswap" => Op::Numeric(sym::bswap),
7375
"bitreverse" => Op::Numeric(sym::bitreverse),

‎src/tools/miri/tests/pass/intrinsics/portable-simd.rs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -505,6 +505,21 @@ fn simd_intrinsics() {
505505
assert!(simd_reduce_all(i32x4::splat(-1)));
506506
assert!(!simd_reduce_all(i32x2::from_array([0, -1])));
507507

508+
assert_eq!(
509+
simd_ctlz(i32x4::from_array([0, i32::MAX, i32::MIN, -1_i32])),
510+
i32x4::from_array([32, 1, 0, 0])
511+
);
512+
513+
assert_eq!(
514+
simd_ctpop(i32x4::from_array([0, i32::MAX, i32::MIN, -1_i32])),
515+
i32x4::from_array([0, 31, 1, 32])
516+
);
517+
518+
assert_eq!(
519+
simd_cttz(i32x4::from_array([0, i32::MAX, i32::MIN, -1_i32])),
520+
i32x4::from_array([32, 0, 31, 0])
521+
);
522+
508523
assert_eq!(
509524
simd_select(i8x4::from_array([0, -1, -1, 0]), a, b),
510525
i32x4::from_array([1, 10, 10, 4])

‎tests/ui/simd/intrinsic/generic-arithmetic-2.rs

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ extern "rust-intrinsic" {
3030
fn simd_bswap<T>(x: T) -> T;
3131
fn simd_bitreverse<T>(x: T) -> T;
3232
fn simd_ctlz<T>(x: T) -> T;
33+
fn simd_ctpop<T>(x: T) -> T;
3334
fn simd_cttz<T>(x: T) -> T;
3435
}
3536

@@ -77,7 +78,6 @@ fn main() {
7778
simd_cttz(x);
7879
simd_cttz(y);
7980

80-
8181
simd_add(0, 0);
8282
//~^ ERROR expected SIMD input type, found non-SIMD `i32`
8383
simd_sub(0, 0);
@@ -108,24 +108,25 @@ fn main() {
108108
simd_cttz(0);
109109
//~^ ERROR expected SIMD input type, found non-SIMD `i32`
110110

111-
112111
simd_shl(z, z);
113-
//~^ ERROR unsupported operation on `f32x4` with element `f32`
112+
//~^ ERROR unsupported operation on `f32x4` with element `f32`
114113
simd_shr(z, z);
115-
//~^ ERROR unsupported operation on `f32x4` with element `f32`
114+
//~^ ERROR unsupported operation on `f32x4` with element `f32`
116115
simd_and(z, z);
117-
//~^ ERROR unsupported operation on `f32x4` with element `f32`
116+
//~^ ERROR unsupported operation on `f32x4` with element `f32`
118117
simd_or(z, z);
119-
//~^ ERROR unsupported operation on `f32x4` with element `f32`
118+
//~^ ERROR unsupported operation on `f32x4` with element `f32`
120119
simd_xor(z, z);
121-
//~^ ERROR unsupported operation on `f32x4` with element `f32`
120+
//~^ ERROR unsupported operation on `f32x4` with element `f32`
122121
simd_bswap(z);
123-
//~^ ERROR unsupported operation on `f32x4` with element `f32`
122+
//~^ ERROR unsupported operation on `f32x4` with element `f32`
124123
simd_bitreverse(z);
125-
//~^ ERROR unsupported operation on `f32x4` with element `f32`
124+
//~^ ERROR unsupported operation on `f32x4` with element `f32`
126125
simd_ctlz(z);
127-
//~^ ERROR unsupported operation on `f32x4` with element `f32`
126+
//~^ ERROR unsupported operation on `f32x4` with element `f32`
127+
simd_ctpop(z);
128+
//~^ ERROR unsupported operation on `f32x4` with element `f32`
128129
simd_cttz(z);
129-
//~^ ERROR unsupported operation on `f32x4` with element `f32`
130+
//~^ ERROR unsupported operation on `f32x4` with element `f32`
130131
}
131132
}

‎tests/ui/simd/intrinsic/generic-arithmetic-2.stderr

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -83,59 +83,65 @@ LL | simd_cttz(0);
8383
| ^^^^^^^^^^^^
8484

8585
error[E0511]: invalid monomorphization of `simd_shl` intrinsic: unsupported operation on `f32x4` with element `f32`
86-
--> $DIR/generic-arithmetic-2.rs:112:9
86+
--> $DIR/generic-arithmetic-2.rs:111:9
8787
|
8888
LL | simd_shl(z, z);
8989
| ^^^^^^^^^^^^^^
9090

9191
error[E0511]: invalid monomorphization of `simd_shr` intrinsic: unsupported operation on `f32x4` with element `f32`
92-
--> $DIR/generic-arithmetic-2.rs:114:9
92+
--> $DIR/generic-arithmetic-2.rs:113:9
9393
|
9494
LL | simd_shr(z, z);
9595
| ^^^^^^^^^^^^^^
9696

9797
error[E0511]: invalid monomorphization of `simd_and` intrinsic: unsupported operation on `f32x4` with element `f32`
98-
--> $DIR/generic-arithmetic-2.rs:116:9
98+
--> $DIR/generic-arithmetic-2.rs:115:9
9999
|
100100
LL | simd_and(z, z);
101101
| ^^^^^^^^^^^^^^
102102

103103
error[E0511]: invalid monomorphization of `simd_or` intrinsic: unsupported operation on `f32x4` with element `f32`
104-
--> $DIR/generic-arithmetic-2.rs:118:9
104+
--> $DIR/generic-arithmetic-2.rs:117:9
105105
|
106106
LL | simd_or(z, z);
107107
| ^^^^^^^^^^^^^
108108

109109
error[E0511]: invalid monomorphization of `simd_xor` intrinsic: unsupported operation on `f32x4` with element `f32`
110-
--> $DIR/generic-arithmetic-2.rs:120:9
110+
--> $DIR/generic-arithmetic-2.rs:119:9
111111
|
112112
LL | simd_xor(z, z);
113113
| ^^^^^^^^^^^^^^
114114

115115
error[E0511]: invalid monomorphization of `simd_bswap` intrinsic: unsupported operation on `f32x4` with element `f32`
116-
--> $DIR/generic-arithmetic-2.rs:122:9
116+
--> $DIR/generic-arithmetic-2.rs:121:9
117117
|
118118
LL | simd_bswap(z);
119119
| ^^^^^^^^^^^^^
120120

121121
error[E0511]: invalid monomorphization of `simd_bitreverse` intrinsic: unsupported operation on `f32x4` with element `f32`
122-
--> $DIR/generic-arithmetic-2.rs:124:9
122+
--> $DIR/generic-arithmetic-2.rs:123:9
123123
|
124124
LL | simd_bitreverse(z);
125125
| ^^^^^^^^^^^^^^^^^^
126126

127127
error[E0511]: invalid monomorphization of `simd_ctlz` intrinsic: unsupported operation on `f32x4` with element `f32`
128-
--> $DIR/generic-arithmetic-2.rs:126:9
128+
--> $DIR/generic-arithmetic-2.rs:125:9
129129
|
130130
LL | simd_ctlz(z);
131131
| ^^^^^^^^^^^^
132132

133+
error[E0511]: invalid monomorphization of `simd_ctpop` intrinsic: unsupported operation on `f32x4` with element `f32`
134+
--> $DIR/generic-arithmetic-2.rs:127:9
135+
|
136+
LL | simd_ctpop(z);
137+
| ^^^^^^^^^^^^^
138+
133139
error[E0511]: invalid monomorphization of `simd_cttz` intrinsic: unsupported operation on `f32x4` with element `f32`
134-
--> $DIR/generic-arithmetic-2.rs:128:9
140+
--> $DIR/generic-arithmetic-2.rs:129:9
135141
|
136142
LL | simd_cttz(z);
137143
| ^^^^^^^^^^^^
138144

139-
error: aborting due to 23 previous errors
145+
error: aborting due to 24 previous errors
140146

141147
For more information about this error, try `rustc --explain E0511`.

‎tests/ui/simd/intrinsic/generic-arithmetic-pass.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ extern "rust-intrinsic" {
4747
fn simd_bswap<T>(x: T) -> T;
4848
fn simd_bitreverse<T>(x: T) -> T;
4949
fn simd_ctlz<T>(x: T) -> T;
50+
fn simd_ctpop<T>(x: T) -> T;
5051
fn simd_cttz<T>(x: T) -> T;
5152
}
5253

@@ -57,6 +58,8 @@ fn main() {
5758
let x2 = i32x4(2, 3, 4, 5);
5859
let y2 = U32::<4>([2, 3, 4, 5]);
5960
let z2 = f32x4(2.0, 3.0, 4.0, 5.0);
61+
let x3 = i32x4(0, i32::MAX, i32::MIN, -1_i32);
62+
let y3 = U32::<4>([0, i32::MAX as _, i32::MIN as _, -1_i32 as _]);
6063

6164
unsafe {
6265
all_eq!(simd_add(x1, x2), i32x4(3, 5, 7, 9));
@@ -147,6 +150,13 @@ fn main() {
147150
all_eq!(simd_ctlz(x1), i32x4(31, 30, 30, 29));
148151
all_eq_!(simd_ctlz(y1), U32::<4>([31, 30, 30, 29]));
149152

153+
all_eq!(simd_ctpop(x1), i32x4(1, 1, 2, 1));
154+
all_eq_!(simd_ctpop(y1), U32::<4>([1, 1, 2, 1]));
155+
all_eq!(simd_ctpop(x2), i32x4(1, 2, 1, 2));
156+
all_eq_!(simd_ctpop(y2), U32::<4>([1, 2, 1, 2]));
157+
all_eq!(simd_ctpop(x3), i32x4(0, 31, 1, 32));
158+
all_eq_!(simd_ctpop(y3), U32::<4>([0, 31, 1, 32]));
159+
150160
all_eq!(simd_cttz(x1), i32x4(0, 1, 0, 2));
151161
all_eq_!(simd_cttz(y1), U32::<4>([0, 1, 0, 2]));
152162
}

0 commit comments

Comments
 (0)