diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h index 5a810784a54bf..7aa6cbe93c132 100644 --- a/llvm/include/llvm/IR/Intrinsics.h +++ b/llvm/include/llvm/IR/Intrinsics.h @@ -151,13 +151,7 @@ namespace Intrinsic { Argument, ExtendArgument, TruncArgument, - HalfVecArgument, - OneThirdVecArgument, - OneFourthVecArgument, - OneFifthVecArgument, - OneSixthVecArgument, - OneSeventhVecArgument, - OneEighthVecArgument, + OneNthEltsVecArgument, SameVecWidthArgument, VecOfAnyPtrsToElt, VecElementArgument, @@ -169,12 +163,6 @@ namespace Intrinsic { AArch64Svcount, } Kind; - // These six have to be contiguous. - static_assert(OneFourthVecArgument == OneThirdVecArgument + 1 && - OneFifthVecArgument == OneFourthVecArgument + 1 && - OneSixthVecArgument == OneFifthVecArgument + 1 && - OneSeventhVecArgument == OneSixthVecArgument + 1 && - OneEighthVecArgument == OneSeventhVecArgument + 1); union { unsigned Integer_Width; unsigned Float_Width; @@ -193,20 +181,16 @@ namespace Intrinsic { unsigned getArgumentNumber() const { assert(Kind == Argument || Kind == ExtendArgument || - Kind == TruncArgument || Kind == HalfVecArgument || - (Kind >= OneThirdVecArgument && Kind <= OneEighthVecArgument) || - Kind == SameVecWidthArgument || Kind == VecElementArgument || - Kind == Subdivide2Argument || Kind == Subdivide4Argument || - Kind == VecOfBitcastsToInt); + Kind == TruncArgument || Kind == SameVecWidthArgument || + Kind == VecElementArgument || Kind == Subdivide2Argument || + Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt); return Argument_Info >> 3; } ArgKind getArgumentKind() const { assert(Kind == Argument || Kind == ExtendArgument || - Kind == TruncArgument || Kind == HalfVecArgument || - (Kind >= OneThirdVecArgument && Kind <= OneEighthVecArgument) || - Kind == SameVecWidthArgument || Kind == VecElementArgument || - Kind == Subdivide2Argument || Kind == Subdivide4Argument || - Kind == VecOfBitcastsToInt); + Kind == TruncArgument || Kind == SameVecWidthArgument || + Kind == VecElementArgument || Kind == Subdivide2Argument || + Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt); return (ArgKind)(Argument_Info & 7); } @@ -216,8 +200,14 @@ namespace Intrinsic { assert(Kind == VecOfAnyPtrsToElt); return Argument_Info >> 16; } + // OneNthEltsVecArguments uses both a divisor N and a reference argument for + // the full-width vector to match + unsigned getVectorDivisor() const { + assert(Kind == OneNthEltsVecArgument); + return Argument_Info >> 16; + } unsigned getRefArgNumber() const { - assert(Kind == VecOfAnyPtrsToElt); + assert(Kind == VecOfAnyPtrsToElt || Kind == OneNthEltsVecArgument); return Argument_Info & 0xFFFF; } diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td index 4eb2ff96fd9d1..d3899056bc240 100644 --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -306,7 +306,7 @@ def IIT_TRUNC_ARG : IIT_Base<26>; def IIT_ANYPTR : IIT_Base<27>; def IIT_V1 : IIT_Vec<1, 28>; def IIT_VARARG : IIT_VT; -def IIT_HALF_VEC_ARG : IIT_Base<30>; +def IIT_ONE_NTH_ELTS_VEC_ARG : IIT_Base<30>; def IIT_SAME_VEC_WIDTH_ARG : IIT_Base<31>; def IIT_VEC_OF_ANYPTRS_TO_ELT : IIT_Base<34>; def IIT_I128 : IIT_Int<128, 35>; @@ -335,14 +335,8 @@ def IIT_I4 : IIT_Int<4, 58>; def IIT_AARCH64_SVCOUNT : IIT_VT; def IIT_V6 : IIT_Vec<6, 60>; def IIT_V10 : IIT_Vec<10, 61>; -def IIT_ONE_THIRD_VEC_ARG : IIT_Base<62>; -def IIT_ONE_FIFTH_VEC_ARG : IIT_Base<63>; -def IIT_ONE_SEVENTH_VEC_ARG : IIT_Base<64>; -def IIT_V2048: IIT_Vec<2048, 65>; -def IIT_V4096: IIT_Vec<4096, 66>; -def IIT_ONE_FOURTH_VEC_ARG : IIT_Base<67>; -def IIT_ONE_SIXTH_VEC_ARG : IIT_Base<68>; -def IIT_ONE_EIGHTH_VEC_ARG : IIT_Base<69>; +def IIT_V2048: IIT_Vec<2048, 62>; +def IIT_V4096: IIT_Vec<4096, 63>; } defvar IIT_all_FixedTypes = !filter(iit, IIT_all, @@ -479,27 +473,15 @@ class LLVMVectorOfAnyPointersToElt class LLVMVectorElementType : LLVMMatchType; // Match the type of another intrinsic parameter that is expected to be a -// vector type, but change the element count to be half as many. -class LLVMHalfElementsVectorType - : LLVMMatchType; - -class LLVMOneThirdElementsVectorType - : LLVMMatchType; - -class LLVMOneFourthElementsVectorType - : LLVMMatchType; - -class LLVMOneFifthElementsVectorType - : LLVMMatchType; - -class LLVMOneSixthElementsVectorType - : LLVMMatchType; - -class LLVMOneSeventhElementsVectorType - : LLVMMatchType; - -class LLVMOneEighthElementsVectorType - : LLVMMatchType; +// vector type, but change the element count to be 1/n of it. +class LLVMOneNthElementsVectorType + : LLVMMatchType { + let Sig = [ + IIT_ONE_NTH_ELTS_VEC_ARG.Number, + EncNextArgN.ret, + n, + ]; +} // Match the type of another intrinsic parameter that is expected to be a // vector type (i.e. ) but with each element subdivided to @@ -2771,118 +2753,15 @@ def int_vector_extract : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i64_ty], [IntrNoMem, IntrSpeculatable, ImmArg>]>; +foreach n = 2...8 in { + def int_vector_interleave#n : DefaultAttrsIntrinsic<[llvm_anyvector_ty], + !listsplat(LLVMOneNthElementsVectorType<0, n>, n), + [IntrNoMem]>; -def int_vector_interleave2 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMHalfElementsVectorType<0>, - LLVMHalfElementsVectorType<0>], - [IntrNoMem]>; - -def int_vector_deinterleave2 : DefaultAttrsIntrinsic<[LLVMHalfElementsVectorType<0>, - LLVMHalfElementsVectorType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; - -def int_vector_interleave3 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMOneThirdElementsVectorType<0>, - LLVMOneThirdElementsVectorType<0>, - LLVMOneThirdElementsVectorType<0>], - [IntrNoMem]>; - -def int_vector_deinterleave3 : DefaultAttrsIntrinsic<[LLVMOneThirdElementsVectorType<0>, - LLVMOneThirdElementsVectorType<0>, - LLVMOneThirdElementsVectorType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; - -def int_vector_interleave4 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMOneFourthElementsVectorType<0>, - LLVMOneFourthElementsVectorType<0>, - LLVMOneFourthElementsVectorType<0>, - LLVMOneFourthElementsVectorType<0>], - [IntrNoMem]>; - -def int_vector_deinterleave4 : DefaultAttrsIntrinsic<[LLVMOneFourthElementsVectorType<0>, - LLVMOneFourthElementsVectorType<0>, - LLVMOneFourthElementsVectorType<0>, - LLVMOneFourthElementsVectorType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; - -def int_vector_interleave5 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMOneFifthElementsVectorType<0>, - LLVMOneFifthElementsVectorType<0>, - LLVMOneFifthElementsVectorType<0>, - LLVMOneFifthElementsVectorType<0>, - LLVMOneFifthElementsVectorType<0>], - [IntrNoMem]>; - -def int_vector_deinterleave5 : DefaultAttrsIntrinsic<[LLVMOneFifthElementsVectorType<0>, - LLVMOneFifthElementsVectorType<0>, - LLVMOneFifthElementsVectorType<0>, - LLVMOneFifthElementsVectorType<0>, - LLVMOneFifthElementsVectorType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; - -def int_vector_interleave6 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMOneSixthElementsVectorType<0>, - LLVMOneSixthElementsVectorType<0>, - LLVMOneSixthElementsVectorType<0>, - LLVMOneSixthElementsVectorType<0>, - LLVMOneSixthElementsVectorType<0>, - LLVMOneSixthElementsVectorType<0>], - [IntrNoMem]>; - -def int_vector_deinterleave6 : DefaultAttrsIntrinsic<[LLVMOneSixthElementsVectorType<0>, - LLVMOneSixthElementsVectorType<0>, - LLVMOneSixthElementsVectorType<0>, - LLVMOneSixthElementsVectorType<0>, - LLVMOneSixthElementsVectorType<0>, - LLVMOneSixthElementsVectorType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; - -def int_vector_interleave7 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>], - [IntrNoMem]>; - -def int_vector_deinterleave7 : DefaultAttrsIntrinsic<[LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>, - LLVMOneSeventhElementsVectorType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; - -def int_vector_interleave8 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>], - [IntrNoMem]>; - -def int_vector_deinterleave8 : DefaultAttrsIntrinsic<[LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>, - LLVMOneEighthElementsVectorType<0>], - [llvm_anyvector_ty], - [IntrNoMem]>; + def int_vector_deinterleave#n : DefaultAttrsIntrinsic, n), + [llvm_anyvector_ty], + [IntrNoMem]>; +} //===-------------- Intrinsics to perform partial reduction ---------------===// diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td index 67fd91f0896eb..0ec5f5163118e 100644 --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -186,7 +186,7 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". [IntrNoMem]>; class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty], + [LLVMOneNthElementsVectorType<0, 2>, llvm_anyvector_ty], [IntrNoMem]>; class AdvSIMD_2VectorArg_Lane_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyint_ty], @@ -207,11 +207,11 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". [IntrNoMem]>; class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, + [LLVMOneNthElementsVectorType<0, 2>, llvm_anyvector_ty, LLVMMatchType<1>], [IntrNoMem]>; class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyvector_ty], - [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty], + [LLVMOneNthElementsVectorType<0, 2>, llvm_anyvector_ty, llvm_i32_ty], [IntrNoMem]>; class AdvSIMD_CvtFxToFP_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty], @@ -1330,7 +1330,7 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". [IntrNoMem, ImmArg>]>; class AdvSIMD_SVE_PUNPKHI_Intrinsic - : DefaultAttrsIntrinsic<[LLVMHalfElementsVectorType<0>], + : DefaultAttrsIntrinsic<[LLVMOneNthElementsVectorType<0, 2>], [llvm_anyvector_ty], [IntrNoMem]>; @@ -4229,4 +4229,4 @@ let TargetPrefix = "aarch64" in { def int_aarch64_sme_fp8_fvdot_lane_za16_vg1x2 : SME_FP8_ZA_LANE_VGx2_Intrinsic; def int_aarch64_sme_fp8_fvdotb_lane_za32_vg1x4 : SME_FP8_ZA_LANE_VGx2_Intrinsic; def int_aarch64_sme_fp8_fvdott_lane_za32_vg1x4 : SME_FP8_ZA_LANE_VGx2_Intrinsic; -} \ No newline at end of file +} diff --git a/llvm/lib/IR/Intrinsics.cpp b/llvm/lib/IR/Intrinsics.cpp index 28f7523476774..e631419d5e1c2 100644 --- a/llvm/lib/IR/Intrinsics.cpp +++ b/llvm/lib/IR/Intrinsics.cpp @@ -366,46 +366,11 @@ DecodeIITType(unsigned &NextElt, ArrayRef Infos, IITDescriptor::get(IITDescriptor::TruncArgument, ArgInfo)); return; } - case IIT_HALF_VEC_ARG: { - unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); - OutputTable.push_back( - IITDescriptor::get(IITDescriptor::HalfVecArgument, ArgInfo)); - return; - } - case IIT_ONE_THIRD_VEC_ARG: { - unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); - OutputTable.push_back( - IITDescriptor::get(IITDescriptor::OneThirdVecArgument, ArgInfo)); - return; - } - case IIT_ONE_FOURTH_VEC_ARG: { - unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); - OutputTable.push_back( - IITDescriptor::get(IITDescriptor::OneFourthVecArgument, ArgInfo)); - return; - } - case IIT_ONE_FIFTH_VEC_ARG: { - unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); - OutputTable.push_back( - IITDescriptor::get(IITDescriptor::OneFifthVecArgument, ArgInfo)); - return; - } - case IIT_ONE_SIXTH_VEC_ARG: { - unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); - OutputTable.push_back( - IITDescriptor::get(IITDescriptor::OneSixthVecArgument, ArgInfo)); - return; - } - case IIT_ONE_SEVENTH_VEC_ARG: { - unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); - OutputTable.push_back( - IITDescriptor::get(IITDescriptor::OneSeventhVecArgument, ArgInfo)); - return; - } - case IIT_ONE_EIGHTH_VEC_ARG: { - unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); + case IIT_ONE_NTH_ELTS_VEC_ARG: { + unsigned short ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); + unsigned short N = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); OutputTable.push_back( - IITDescriptor::get(IITDescriptor::OneEighthVecArgument, ArgInfo)); + IITDescriptor::get(IITDescriptor::OneNthEltsVecArgument, N, ArgNo)); return; } case IIT_SAME_VEC_WIDTH_ARG: { @@ -598,18 +563,9 @@ static Type *DecodeFixedType(ArrayRef &Infos, int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2; return VectorType::getSubdividedVectorType(VTy, SubDivs); } - case IITDescriptor::HalfVecArgument: - return VectorType::getHalfElementsVectorType( - cast(Tys[D.getArgumentNumber()])); - case IITDescriptor::OneThirdVecArgument: - case IITDescriptor::OneFourthVecArgument: - case IITDescriptor::OneFifthVecArgument: - case IITDescriptor::OneSixthVecArgument: - case IITDescriptor::OneSeventhVecArgument: - case IITDescriptor::OneEighthVecArgument: + case IITDescriptor::OneNthEltsVecArgument: return VectorType::getOneNthElementsVectorType( - cast(Tys[D.getArgumentNumber()]), - 3 + (D.Kind - IITDescriptor::OneThirdVecArgument)); + cast(Tys[D.getRefArgNumber()]), D.getVectorDivisor()); case IITDescriptor::SameVecWidthArgument: { Type *EltTy = DecodeFixedType(Infos, Tys, Context); Type *Ty = Tys[D.getArgumentNumber()]; @@ -987,26 +943,14 @@ matchIntrinsicType(Type *Ty, ArrayRef &Infos, return Ty != NewTy; } - case IITDescriptor::HalfVecArgument: + case IITDescriptor::OneNthEltsVecArgument: // If this is a forward reference, defer the check for later. - if (D.getArgumentNumber() >= ArgTys.size()) - return IsDeferredCheck || DeferCheck(Ty); - return !isa(ArgTys[D.getArgumentNumber()]) || - VectorType::getHalfElementsVectorType( - cast(ArgTys[D.getArgumentNumber()])) != Ty; - case IITDescriptor::OneThirdVecArgument: - case IITDescriptor::OneFourthVecArgument: - case IITDescriptor::OneFifthVecArgument: - case IITDescriptor::OneSixthVecArgument: - case IITDescriptor::OneSeventhVecArgument: - case IITDescriptor::OneEighthVecArgument: - // If this is a forward reference, defer the check for later. - if (D.getArgumentNumber() >= ArgTys.size()) + if (D.getRefArgNumber() >= ArgTys.size()) return IsDeferredCheck || DeferCheck(Ty); - return !isa(ArgTys[D.getArgumentNumber()]) || + return !isa(ArgTys[D.getRefArgNumber()]) || VectorType::getOneNthElementsVectorType( - cast(ArgTys[D.getArgumentNumber()]), - 3 + (D.Kind - IITDescriptor::OneThirdVecArgument)) != Ty; + cast(ArgTys[D.getRefArgNumber()]), + D.getVectorDivisor()) != Ty; case IITDescriptor::SameVecWidthArgument: { if (D.getArgumentNumber() >= ArgTys.size()) { // Defer check and subsequent check for the vector element type.