Skip to content

Commit 915c27d

Browse files
lukel97lukel97
committed
[IR] Add llvm.vector.(de)interleave4/6/8
1 parent fe56c8f commit 915c27d

File tree

9 files changed

+12995
-4055
lines changed

9 files changed

+12995
-4055
lines changed

‎llvm/docs/LangRef.rst‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20158,7 +20158,7 @@ Arguments:
2015820158

2015920159
The argument to this intrinsic must be a vector.
2016020160

20161-
'``llvm.vector.deinterleave2/3/5/7``' Intrinsic
20161+
'``llvm.vector.deinterleave2/3/4/5/6/7/8``' Intrinsic
2016220162
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2016320163

2016420164
Syntax:
@@ -20176,8 +20176,8 @@ This is an overloaded intrinsic.
2017620176
Overview:
2017720177
"""""""""
2017820178

20179-
The '``llvm.vector.deinterleave2/3/5/7``' intrinsics deinterleave adjacent lanes
20180-
into 2, 3, 5, and 7 separate vectors, respectively, and return them as the
20179+
The '``llvm.vector.deinterleave2/3/4/5/6/7/8``' intrinsics deinterleave adjacent lanes
20180+
into 2 through to 8 separate vectors, respectively, and return them as the
2018120181
result.
2018220182

2018320183
This intrinsic works for both fixed and scalable vectors. While this intrinsic
@@ -20199,7 +20199,7 @@ Arguments:
2019920199
The argument is a vector whose type corresponds to the logical concatenation of
2020020200
the aggregated result types.
2020120201

20202-
'``llvm.vector.interleave2/3/5/7``' Intrinsic
20202+
'``llvm.vector.interleave2/3/4/5/6/7/8``' Intrinsic
2020320203
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2020420204

2020520205
Syntax:
@@ -20217,7 +20217,7 @@ This is an overloaded intrinsic.
2021720217
Overview:
2021820218
"""""""""
2021920219

20220-
The '``llvm.vector.interleave2/3/5/7``' intrinsic constructs a vector
20220+
The '``llvm.vector.interleave2/3/4/5/6/7/8``' intrinsic constructs a vector
2022120221
by interleaving all the input vectors.
2022220222

2022320223
This intrinsic works for both fixed and scalable vectors. While this intrinsic

‎llvm/include/llvm/IR/Intrinsics.h‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,11 @@ namespace Intrinsic {
153153
TruncArgument,
154154
HalfVecArgument,
155155
OneThirdVecArgument,
156+
OneFourthVecArgument,
156157
OneFifthVecArgument,
158+
OneSixthVecArgument,
157159
OneSeventhVecArgument,
160+
OneEighthVecArgument,
158161
SameVecWidthArgument,
159162
VecOfAnyPtrsToElt,
160163
VecElementArgument,
@@ -167,8 +170,11 @@ namespace Intrinsic {
167170
} Kind;
168171

169172
// These three have to be contiguous.
170-
static_assert(OneFifthVecArgument == OneThirdVecArgument + 1 &&
171-
OneSeventhVecArgument == OneFifthVecArgument + 1);
173+
static_assert(OneFourthVecArgument == OneThirdVecArgument + 1 &&
174+
OneFifthVecArgument == OneFourthVecArgument + 1 &&
175+
OneSixthVecArgument == OneFifthVecArgument + 1 &&
176+
OneSeventhVecArgument == OneSixthVecArgument + 1 &&
177+
OneEighthVecArgument == OneSeventhVecArgument + 1);
172178
union {
173179
unsigned Integer_Width;
174180
unsigned Float_Width;
@@ -188,19 +194,23 @@ namespace Intrinsic {
188194
unsigned getArgumentNumber() const {
189195
assert(Kind == Argument || Kind == ExtendArgument ||
190196
Kind == TruncArgument || Kind == HalfVecArgument ||
191-
Kind == OneThirdVecArgument || Kind == OneFifthVecArgument ||
192-
Kind == OneSeventhVecArgument || Kind == SameVecWidthArgument ||
193-
Kind == VecElementArgument || Kind == Subdivide2Argument ||
194-
Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt);
197+
Kind == OneThirdVecArgument || Kind == OneFourthVecArgument ||
198+
Kind == OneFifthVecArgument || Kind == OneSixthVecArgument ||
199+
Kind == OneSeventhVecArgument || Kind == OneEighthVecArgument ||
200+
Kind == SameVecWidthArgument || Kind == VecElementArgument ||
201+
Kind == Subdivide2Argument || Kind == Subdivide4Argument ||
202+
Kind == VecOfBitcastsToInt);
195203
return Argument_Info >> 3;
196204
}
197205
ArgKind getArgumentKind() const {
198206
assert(Kind == Argument || Kind == ExtendArgument ||
199207
Kind == TruncArgument || Kind == HalfVecArgument ||
200-
Kind == OneThirdVecArgument || Kind == OneFifthVecArgument ||
201-
Kind == OneSeventhVecArgument || Kind == SameVecWidthArgument ||
202-
Kind == VecElementArgument || Kind == Subdivide2Argument ||
203-
Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt);
208+
Kind == OneThirdVecArgument || Kind == OneFourthVecArgument ||
209+
Kind == OneFifthVecArgument || Kind == OneSixthVecArgument ||
210+
Kind == OneSeventhVecArgument || Kind == OneEighthVecArgument ||
211+
Kind == SameVecWidthArgument || Kind == VecElementArgument ||
212+
Kind == Subdivide2Argument || Kind == Subdivide4Argument ||
213+
Kind == VecOfBitcastsToInt);
204214
return (ArgKind)(Argument_Info & 7);
205215
}
206216

‎llvm/include/llvm/IR/Intrinsics.td‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,9 @@ def IIT_ONE_FIFTH_VEC_ARG : IIT_Base<63>;
340340
def IIT_ONE_SEVENTH_VEC_ARG : IIT_Base<64>;
341341
def IIT_V2048: IIT_Vec<2048, 65>;
342342
def IIT_V4096: IIT_Vec<4096, 66>;
343+
def IIT_ONE_FOURTH_VEC_ARG : IIT_Base<67>;
344+
def IIT_ONE_SIXTH_VEC_ARG : IIT_Base<68>;
345+
def IIT_ONE_EIGHTH_VEC_ARG : IIT_Base<69>;
343346
}
344347

345348
defvar IIT_all_FixedTypes = !filter(iit, IIT_all,
@@ -483,12 +486,21 @@ class LLVMHalfElementsVectorType<int num>
483486
class LLVMOneThirdElementsVectorType<int num>
484487
: LLVMMatchType<num, IIT_ONE_THIRD_VEC_ARG>;
485488

489+
class LLVMOneFourthElementsVectorType<int num>
490+
: LLVMMatchType<num, IIT_ONE_FOURTH_VEC_ARG>;
491+
486492
class LLVMOneFifthElementsVectorType<int num>
487493
: LLVMMatchType<num, IIT_ONE_FIFTH_VEC_ARG>;
488494

495+
class LLVMOneSixthElementsVectorType<int num>
496+
: LLVMMatchType<num, IIT_ONE_SIXTH_VEC_ARG>;
497+
489498
class LLVMOneSeventhElementsVectorType<int num>
490499
: LLVMMatchType<num, IIT_ONE_SEVENTH_VEC_ARG>;
491500

501+
class LLVMOneEighthElementsVectorType<int num>
502+
: LLVMMatchType<num, IIT_ONE_EIGHTH_VEC_ARG>;
503+
492504
// Match the type of another intrinsic parameter that is expected to be a
493505
// vector type (i.e. <N x iM>) but with each element subdivided to
494506
// form a vector with more elements that are smaller than the original.
@@ -2776,6 +2788,20 @@ def int_vector_deinterleave3 : DefaultAttrsIntrinsic<[LLVMOneThirdElementsVector
27762788
[llvm_anyvector_ty],
27772789
[IntrNoMem]>;
27782790

2791+
def int_vector_interleave4 : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
2792+
[LLVMOneFourthElementsVectorType<0>,
2793+
LLVMOneFourthElementsVectorType<0>,
2794+
LLVMOneFourthElementsVectorType<0>,
2795+
LLVMOneFourthElementsVectorType<0>],
2796+
[IntrNoMem]>;
2797+
2798+
def int_vector_deinterleave4 : DefaultAttrsIntrinsic<[LLVMOneFourthElementsVectorType<0>,
2799+
LLVMOneFourthElementsVectorType<0>,
2800+
LLVMOneFourthElementsVectorType<0>,
2801+
LLVMOneFourthElementsVectorType<0>],
2802+
[llvm_anyvector_ty],
2803+
[IntrNoMem]>;
2804+
27792805
def int_vector_interleave5 : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
27802806
[LLVMOneFifthElementsVectorType<0>,
27812807
LLVMOneFifthElementsVectorType<0>,
@@ -2792,6 +2818,24 @@ def int_vector_deinterleave5 : DefaultAttrsIntrinsic<[LLVMOneFifthElementsVector
27922818
[llvm_anyvector_ty],
27932819
[IntrNoMem]>;
27942820

2821+
def int_vector_interleave6 : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
2822+
[LLVMOneSixthElementsVectorType<0>,
2823+
LLVMOneSixthElementsVectorType<0>,
2824+
LLVMOneSixthElementsVectorType<0>,
2825+
LLVMOneSixthElementsVectorType<0>,
2826+
LLVMOneSixthElementsVectorType<0>,
2827+
LLVMOneSixthElementsVectorType<0>],
2828+
[IntrNoMem]>;
2829+
2830+
def int_vector_deinterleave6 : DefaultAttrsIntrinsic<[LLVMOneSixthElementsVectorType<0>,
2831+
LLVMOneSixthElementsVectorType<0>,
2832+
LLVMOneSixthElementsVectorType<0>,
2833+
LLVMOneSixthElementsVectorType<0>,
2834+
LLVMOneSixthElementsVectorType<0>,
2835+
LLVMOneSixthElementsVectorType<0>],
2836+
[llvm_anyvector_ty],
2837+
[IntrNoMem]>;
2838+
27952839
def int_vector_interleave7 : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
27962840
[LLVMOneSeventhElementsVectorType<0>,
27972841
LLVMOneSeventhElementsVectorType<0>,
@@ -2812,6 +2856,28 @@ def int_vector_deinterleave7 : DefaultAttrsIntrinsic<[LLVMOneSeventhElementsVect
28122856
[llvm_anyvector_ty],
28132857
[IntrNoMem]>;
28142858

2859+
def int_vector_interleave8 : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
2860+
[LLVMOneEighthElementsVectorType<0>,
2861+
LLVMOneEighthElementsVectorType<0>,
2862+
LLVMOneEighthElementsVectorType<0>,
2863+
LLVMOneEighthElementsVectorType<0>,
2864+
LLVMOneEighthElementsVectorType<0>,
2865+
LLVMOneEighthElementsVectorType<0>,
2866+
LLVMOneEighthElementsVectorType<0>,
2867+
LLVMOneEighthElementsVectorType<0>],
2868+
[IntrNoMem]>;
2869+
2870+
def int_vector_deinterleave8 : DefaultAttrsIntrinsic<[LLVMOneEighthElementsVectorType<0>,
2871+
LLVMOneEighthElementsVectorType<0>,
2872+
LLVMOneEighthElementsVectorType<0>,
2873+
LLVMOneEighthElementsVectorType<0>,
2874+
LLVMOneEighthElementsVectorType<0>,
2875+
LLVMOneEighthElementsVectorType<0>,
2876+
LLVMOneEighthElementsVectorType<0>,
2877+
LLVMOneEighthElementsVectorType<0>],
2878+
[llvm_anyvector_ty],
2879+
[IntrNoMem]>;
2880+
28152881
//===-------------- Intrinsics to perform partial reduction ---------------===//
28162882

28172883
def int_experimental_vector_partial_reduce_add : DefaultAttrsIntrinsic<[LLVMMatchType<0>],

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8181,24 +8181,42 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
81818181
case Intrinsic::vector_interleave3:
81828182
visitVectorInterleave(I, 3);
81838183
return;
8184+
case Intrinsic::vector_interleave4:
8185+
visitVectorInterleave(I, 4);
8186+
return;
81848187
case Intrinsic::vector_interleave5:
81858188
visitVectorInterleave(I, 5);
81868189
return;
8190+
case Intrinsic::vector_interleave6:
8191+
visitVectorInterleave(I, 6);
8192+
return;
81878193
case Intrinsic::vector_interleave7:
81888194
visitVectorInterleave(I, 7);
81898195
return;
8196+
case Intrinsic::vector_interleave8:
8197+
visitVectorInterleave(I, 8);
8198+
return;
81908199
case Intrinsic::vector_deinterleave2:
81918200
visitVectorDeinterleave(I, 2);
81928201
return;
81938202
case Intrinsic::vector_deinterleave3:
81948203
visitVectorDeinterleave(I, 3);
81958204
return;
8205+
case Intrinsic::vector_deinterleave4:
8206+
visitVectorDeinterleave(I, 4);
8207+
return;
81968208
case Intrinsic::vector_deinterleave5:
81978209
visitVectorDeinterleave(I, 5);
81988210
return;
8211+
case Intrinsic::vector_deinterleave6:
8212+
visitVectorDeinterleave(I, 6);
8213+
return;
81998214
case Intrinsic::vector_deinterleave7:
82008215
visitVectorDeinterleave(I, 7);
82018216
return;
8217+
case Intrinsic::vector_deinterleave8:
8218+
visitVectorDeinterleave(I, 8);
8219+
return;
82028220
case Intrinsic::experimental_vector_compress:
82038221
setValue(&I, DAG.getNode(ISD::VECTOR_COMPRESS, sdl,
82048222
getValue(I.getArgOperand(0)).getValueType(),

‎llvm/lib/IR/Intrinsics.cpp‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -378,18 +378,36 @@ DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
378378
IITDescriptor::get(IITDescriptor::OneThirdVecArgument, ArgInfo));
379379
return;
380380
}
381+
case IIT_ONE_FOURTH_VEC_ARG: {
382+
unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
383+
OutputTable.push_back(
384+
IITDescriptor::get(IITDescriptor::OneFourthVecArgument, ArgInfo));
385+
return;
386+
}
381387
case IIT_ONE_FIFTH_VEC_ARG: {
382388
unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
383389
OutputTable.push_back(
384390
IITDescriptor::get(IITDescriptor::OneFifthVecArgument, ArgInfo));
385391
return;
386392
}
393+
case IIT_ONE_SIXTH_VEC_ARG: {
394+
unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
395+
OutputTable.push_back(
396+
IITDescriptor::get(IITDescriptor::OneSixthVecArgument, ArgInfo));
397+
return;
398+
}
387399
case IIT_ONE_SEVENTH_VEC_ARG: {
388400
unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
389401
OutputTable.push_back(
390402
IITDescriptor::get(IITDescriptor::OneSeventhVecArgument, ArgInfo));
391403
return;
392404
}
405+
case IIT_ONE_EIGHTH_VEC_ARG: {
406+
unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
407+
OutputTable.push_back(
408+
IITDescriptor::get(IITDescriptor::OneEighthVecArgument, ArgInfo));
409+
return;
410+
}
393411
case IIT_SAME_VEC_WIDTH_ARG: {
394412
unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
395413
OutputTable.push_back(
@@ -584,11 +602,14 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
584602
return VectorType::getHalfElementsVectorType(
585603
cast<VectorType>(Tys[D.getArgumentNumber()]));
586604
case IITDescriptor::OneThirdVecArgument:
605+
case IITDescriptor::OneFourthVecArgument:
587606
case IITDescriptor::OneFifthVecArgument:
607+
case IITDescriptor::OneSixthVecArgument:
588608
case IITDescriptor::OneSeventhVecArgument:
609+
case IITDescriptor::OneEighthVecArgument:
589610
return VectorType::getOneNthElementsVectorType(
590611
cast<VectorType>(Tys[D.getArgumentNumber()]),
591-
3 + (D.Kind - IITDescriptor::OneThirdVecArgument) * 2);
612+
3 + (D.Kind - IITDescriptor::OneThirdVecArgument));
592613
case IITDescriptor::SameVecWidthArgument: {
593614
Type *EltTy = DecodeFixedType(Infos, Tys, Context);
594615
Type *Ty = Tys[D.getArgumentNumber()];
@@ -974,15 +995,18 @@ matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor> &Infos,
974995
VectorType::getHalfElementsVectorType(
975996
cast<VectorType>(ArgTys[D.getArgumentNumber()])) != Ty;
976997
case IITDescriptor::OneThirdVecArgument:
998+
case IITDescriptor::OneFourthVecArgument:
977999
case IITDescriptor::OneFifthVecArgument:
1000+
case IITDescriptor::OneSixthVecArgument:
9781001
case IITDescriptor::OneSeventhVecArgument:
1002+
case IITDescriptor::OneEighthVecArgument:
9791003
// If this is a forward reference, defer the check for later.
9801004
if (D.getArgumentNumber() >= ArgTys.size())
9811005
return IsDeferredCheck || DeferCheck(Ty);
9821006
return !isa<VectorType>(ArgTys[D.getArgumentNumber()]) ||
9831007
VectorType::getOneNthElementsVectorType(
9841008
cast<VectorType>(ArgTys[D.getArgumentNumber()]),
985-
3 + (D.Kind - IITDescriptor::OneThirdVecArgument) * 2) != Ty;
1009+
3 + (D.Kind - IITDescriptor::OneThirdVecArgument)) != Ty;
9861010
case IITDescriptor::SameVecWidthArgument: {
9871011
if (D.getArgumentNumber() >= ArgTys.size()) {
9881012
// Defer check and subsequent check for the vector element type.

0 commit comments

Comments
 (0)