Skip to content

Commit 2209d33

Browse files
ahmednoursphinxahmednoursphinx
authored
[CIR][X86] Add support for kunpck builtins (llvm#168757)
Part of llvm#167765
1 parent 5681c71 commit 2209d33

File tree

3 files changed

+128
-4
lines changed

3 files changed

+128
-4
lines changed

‎clang/lib/CIR/CodeGen/CIRGenBuiltinX86.cpp‎

Lines changed: 47 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,44 @@ static mlir::Value emitX86MaskAddLogic(CIRGenBuilderTy &builder,
100100
return builder.createBitcast(resVec, ops[0].getType());
101101
}
102102

103+
static mlir::Value emitX86MaskUnpack(CIRGenBuilderTy &builder,
104+
mlir::Location loc,
105+
const std::string &intrinsicName,
106+
SmallVectorImpl<mlir::Value> &ops) {
107+
unsigned numElems = cast<cir::IntType>(ops[0].getType()).getWidth();
108+
109+
// Convert both operands to mask vectors.
110+
mlir::Value lhs = getMaskVecValue(builder, loc, ops[0], numElems);
111+
mlir::Value rhs = getMaskVecValue(builder, loc, ops[1], numElems);
112+
113+
mlir::Type i32Ty = builder.getSInt32Ty();
114+
115+
// Create indices for extracting the first half of each vector.
116+
SmallVector<mlir::Attribute, 32> halfIndices;
117+
for (auto i : llvm::seq<unsigned>(0, numElems / 2))
118+
halfIndices.push_back(cir::IntAttr::get(i32Ty, i));
119+
120+
// Extract first half of each vector. This gives better codegen than
121+
// doing it in a single shuffle.
122+
mlir::Value lhsHalf = builder.createVecShuffle(loc, lhs, lhs, halfIndices);
123+
mlir::Value rhsHalf = builder.createVecShuffle(loc, rhs, rhs, halfIndices);
124+
125+
// Create indices for concatenating the vectors.
126+
// NOTE: Operands are swapped to match the intrinsic definition.
127+
// After the half extraction, both vectors have numElems/2 elements.
128+
// In createVecShuffle(rhsHalf, lhsHalf, indices), indices [0..numElems/2-1]
129+
// select from rhsHalf, and indices [numElems/2..numElems-1] select from
130+
// lhsHalf.
131+
SmallVector<mlir::Attribute, 64> concatIndices;
132+
for (auto i : llvm::seq<unsigned>(0, numElems))
133+
concatIndices.push_back(cir::IntAttr::get(i32Ty, i));
134+
135+
// Concat the vectors (RHS first, then LHS).
136+
mlir::Value res =
137+
builder.createVecShuffle(loc, rhsHalf, lhsHalf, concatIndices);
138+
return builder.createBitcast(res, ops[0].getType());
139+
}
140+
103141
static mlir::Value emitX86MaskLogic(CIRGenBuilderTy &builder,
104142
mlir::Location loc,
105143
cir::BinOpKind binOpKind,
@@ -257,7 +295,15 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
257295
return emitVecInsert(builder, getLoc(expr->getExprLoc()), ops[0], ops[1],
258296
ops[2]);
259297
}
260-
298+
case X86::BI__builtin_ia32_kunpckhi:
299+
return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
300+
"x86.avx512.kunpackb", ops);
301+
case X86::BI__builtin_ia32_kunpcksi:
302+
return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
303+
"x86.avx512.kunpackw", ops);
304+
case X86::BI__builtin_ia32_kunpckdi:
305+
return emitX86MaskUnpack(builder, getLoc(expr->getExprLoc()),
306+
"x86.avx512.kunpackd", ops);
261307
case X86::BI_mm_setcsr:
262308
case X86::BI__builtin_ia32_ldmxcsr: {
263309
mlir::Location loc = getLoc(expr->getExprLoc());
@@ -947,9 +993,6 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID,
947993
getMaskVecValue(builder, getLoc(expr->getExprLoc()), ops[0], numElts);
948994
return builder.createBitcast(resVec, ops[0].getType());
949995
}
950-
case X86::BI__builtin_ia32_kunpckdi:
951-
case X86::BI__builtin_ia32_kunpcksi:
952-
case X86::BI__builtin_ia32_kunpckhi:
953996
case X86::BI__builtin_ia32_sqrtsh_round_mask:
954997
case X86::BI__builtin_ia32_sqrtsd_round_mask:
955998
case X86::BI__builtin_ia32_sqrtss_round_mask:

‎clang/test/CIR/CodeGenBuiltins/X86/avx512bw-builtins.c‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -465,3 +465,57 @@ __mmask64 test_kmov_q(__mmask64 A) {
465465

466466
return __builtin_ia32_kmovq(A);
467467
}
468+
469+
__mmask32 test_mm512_kunpackw(__mmask32 A, __mmask32 B) {
470+
// CIR-LABEL: _mm512_kunpackw
471+
// CIR: cir.cast bitcast {{.*}} : !u32i -> !cir.vector<32 x !cir.int<u, 1>>
472+
// CIR: cir.cast bitcast {{.*}} : !u32i -> !cir.vector<32 x !cir.int<u, 1>>
473+
// CIR: cir.vec.shuffle
474+
// CIR: cir.vec.shuffle
475+
// CIR: cir.vec.shuffle
476+
// CIR: cir.cast bitcast {{.*}} : !cir.vector<32 x !cir.int<u, 1>> -> !u32i
477+
478+
// LLVM-LABEL: _mm512_kunpackw
479+
// LLVM: [[A_VEC:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
480+
// LLVM: [[B_VEC:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
481+
// LLVM: [[A_HALF:%.*]] = shufflevector <32 x i1> [[A_VEC]], <32 x i1> [[A_VEC]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
482+
// LLVM: [[B_HALF:%.*]] = shufflevector <32 x i1> [[B_VEC]], <32 x i1> [[B_VEC]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
483+
// LLVM: [[RES:%.*]] = shufflevector <16 x i1> [[B_HALF]], <16 x i1> [[A_HALF]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
484+
// LLVM: bitcast <32 x i1> [[RES]] to i32
485+
486+
// OGCG-LABEL: _mm512_kunpackw
487+
// OGCG: [[A_VEC:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
488+
// OGCG: [[B_VEC:%.*]] = bitcast i32 %{{.*}} to <32 x i1>
489+
// OGCG: [[A_HALF:%.*]] = shufflevector <32 x i1> [[A_VEC]], <32 x i1> [[A_VEC]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
490+
// OGCG: [[B_HALF:%.*]] = shufflevector <32 x i1> [[B_VEC]], <32 x i1> [[B_VEC]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
491+
// OGCG: [[RES:%.*]] = shufflevector <16 x i1> [[B_HALF]], <16 x i1> [[A_HALF]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
492+
// OGCG: bitcast <32 x i1> [[RES]] to i32
493+
return _mm512_kunpackw(A, B);
494+
}
495+
496+
__mmask64 test_mm512_kunpackd(__mmask64 A, __mmask64 B) {
497+
// CIR-LABEL: _mm512_kunpackd
498+
// CIR: cir.cast bitcast {{.*}} : !u64i -> !cir.vector<64 x !cir.int<u, 1>>
499+
// CIR: cir.cast bitcast {{.*}} : !u64i -> !cir.vector<64 x !cir.int<u, 1>>
500+
// CIR: cir.vec.shuffle
501+
// CIR: cir.vec.shuffle
502+
// CIR: cir.vec.shuffle
503+
// CIR: cir.cast bitcast {{.*}} : !cir.vector<64 x !cir.int<u, 1>> -> !u64i
504+
505+
// LLVM-LABEL: _mm512_kunpackd
506+
// LLVM: [[A_VEC:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
507+
// LLVM: [[B_VEC:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
508+
// LLVM: [[A_HALF:%.*]] = shufflevector <64 x i1> [[A_VEC]], <64 x i1> [[A_VEC]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
509+
// LLVM: [[B_HALF:%.*]] = shufflevector <64 x i1> [[B_VEC]], <64 x i1> [[B_VEC]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
510+
// LLVM: [[RES:%.*]] = shufflevector <32 x i1> [[B_HALF]], <32 x i1> [[A_HALF]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
511+
// LLVM: bitcast <64 x i1> [[RES]] to i64
512+
513+
// OGCG-LABEL: _mm512_kunpackd
514+
// OGCG: [[A_VEC:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
515+
// OGCG: [[B_VEC:%.*]] = bitcast i64 %{{.*}} to <64 x i1>
516+
// OGCG: [[A_HALF:%.*]] = shufflevector <64 x i1> [[A_VEC]], <64 x i1> [[A_VEC]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
517+
// OGCG: [[B_HALF:%.*]] = shufflevector <64 x i1> [[B_VEC]], <64 x i1> [[B_VEC]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
518+
// OGCG: [[RES:%.*]] = shufflevector <32 x i1> [[B_HALF]], <32 x i1> [[A_HALF]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
519+
// OGCG: bitcast <64 x i1> [[RES]] to i64
520+
return _mm512_kunpackd(A, B);
521+
}

‎clang/test/CIR/CodeGenBuiltins/X86/avx512f-builtins.c‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,33 @@ __mmask16 test_kmov_w(__mmask16 A) {
228228
// OGCG: bitcast <16 x i1> {{.*}} to i16
229229
return __builtin_ia32_kmovw(A);
230230
}
231+
232+
__mmask16 test_mm512_kunpackb(__mmask16 A, __mmask16 B) {
233+
// CIR-LABEL: _mm512_kunpackb
234+
// CIR: cir.cast bitcast {{.*}} : !u16i -> !cir.vector<16 x !cir.int<u, 1>>
235+
// CIR: cir.cast bitcast {{.*}} : !u16i -> !cir.vector<16 x !cir.int<u, 1>>
236+
// CIR: cir.vec.shuffle
237+
// CIR: cir.vec.shuffle
238+
// CIR: cir.vec.shuffle
239+
// CIR: cir.cast bitcast {{.*}} : !cir.vector<16 x !cir.int<u, 1>> -> !u16i
240+
241+
// LLVM-LABEL: _mm512_kunpackb
242+
// LLVM: [[A_VEC:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
243+
// LLVM: [[B_VEC:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
244+
// LLVM: [[A_HALF:%.*]] = shufflevector <16 x i1> [[A_VEC]], <16 x i1> [[A_VEC]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
245+
// LLVM: [[B_HALF:%.*]] = shufflevector <16 x i1> [[B_VEC]], <16 x i1> [[B_VEC]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
246+
// LLVM: [[RES:%.*]] = shufflevector <8 x i1> [[B_HALF]], <8 x i1> [[A_HALF]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
247+
// LLVM: bitcast <16 x i1> [[RES]] to i16
248+
249+
// OGCG-LABEL: _mm512_kunpackb
250+
// OGCG: [[A_VEC:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
251+
// OGCG: [[B_VEC:%.*]] = bitcast i16 %{{.*}} to <16 x i1>
252+
// OGCG: [[A_HALF:%.*]] = shufflevector <16 x i1> [[A_VEC]], <16 x i1> [[A_VEC]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
253+
// OGCG: [[B_HALF:%.*]] = shufflevector <16 x i1> [[B_VEC]], <16 x i1> [[B_VEC]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
254+
// OGCG: [[RES:%.*]] = shufflevector <8 x i1> [[B_HALF]], <8 x i1> [[A_HALF]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
255+
// OGCG: bitcast <16 x i1> [[RES]] to i16
256+
return _mm512_kunpackb(A, B);
257+
}
231258
__m256 test_mm512_i64gather_ps(__m512i __index, void const *__addr) {
232259
// CIR-LABEL: test_mm512_i64gather_ps
233260
// CIR: cir.call_llvm_intrinsic "x86.avx512.mask.gather.qps.512"

0 commit comments

Comments
 (0)