diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index 6d71c150c8da6..6811b37767cb2 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -273,6 +273,11 @@ inline LegalityPredicate typeIsNot(unsigned TypeIdx, LLT Type) { LegalityPredicate typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, std::initializer_list> TypesInit); +/// True iff the given types for the given tuple of type indexes is one of the +/// specified type tuple. +LegalityPredicate +typeTupleInSet(unsigned TypeIdx0, unsigned TypeIdx1, unsigned TypeIdx2, + std::initializer_list> TypesInit); /// True iff the given types for the given pair of type indexes is one of the /// specified type pairs. LegalityPredicate typePairAndMemDescInSet( @@ -504,6 +509,15 @@ class LegalizeRuleSet { using namespace LegalityPredicates; return actionIf(Action, typePairInSet(typeIdx(0), typeIdx(1), Types)); } + + LegalizeRuleSet & + actionFor(LegalizeAction Action, + std::initializer_list> Types) { + using namespace LegalityPredicates; + return actionIf(Action, + typeTupleInSet(typeIdx(0), typeIdx(1), typeIdx(2), Types)); + } + /// Use the given action when type indexes 0 and 1 is any type pair in the /// given list. /// Action should be an action that requires mutation. @@ -615,6 +629,12 @@ class LegalizeRuleSet { return *this; return actionFor(LegalizeAction::Legal, Types); } + LegalizeRuleSet & + legalFor(bool Pred, std::initializer_list> Types) { + if (!Pred) + return *this; + return actionFor(LegalizeAction::Legal, Types); + } /// The instruction is legal when type index 0 is any type in the given list /// and imm index 0 is anything. LegalizeRuleSet &legalForTypeWithAnyImm(std::initializer_list Types) { diff --git a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp index 8fe48195c610b..dc7ed6cbe8b7d 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp @@ -49,6 +49,17 @@ LegalityPredicate LegalityPredicates::typePairInSet( }; } +LegalityPredicate LegalityPredicates::typeTupleInSet( + unsigned TypeIdx0, unsigned TypeIdx1, unsigned TypeIdx2, + std::initializer_list> TypesInit) { + SmallVector, 4> Types = TypesInit; + return [=](const LegalityQuery &Query) { + std::tuple Match = { + Query.Types[TypeIdx0], Query.Types[TypeIdx1], Query.Types[TypeIdx2]}; + return llvm::is_contained(Types, Match); + }; +} + LegalityPredicate LegalityPredicates::typePairAndMemDescInSet( unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, std::initializer_list TypesAndMemDescInit) { diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 6024027afaf6c..7beda0e92a75b 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -978,6 +978,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT) .legalIf( typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64, v2p0})) + .legalFor(HasSVE, {{nxv16s8, s32, s64}, + {nxv8s16, s32, s64}, + {nxv4s32, s32, s64}, + {nxv2s64, s64, s64}}) .moreElementsToNextPow2(0) .widenVectorEltsToVectorMinSize(0, 64) .clampNumElements(0, v8s8, v16s8) diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp index b40fe55fdfaf6..0bf0a4bf27c44 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerLowering.cpp @@ -161,6 +161,8 @@ bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI, Register Dst = MI.getOperand(0).getReg(); Register Src = MI.getOperand(1).getReg(); LLT Ty = MRI.getType(Dst); + if (Ty.isScalableVector()) + return false; unsigned EltSize = Ty.getScalarSizeInBits(); // Element size for a rev cannot be 64. @@ -196,7 +198,10 @@ bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI, unsigned WhichResult; ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); Register Dst = MI.getOperand(0).getReg(); - unsigned NumElts = MRI.getType(Dst).getNumElements(); + LLT DstTy = MRI.getType(Dst); + if (DstTy.isScalableVector()) + return false; + unsigned NumElts = DstTy.getNumElements(); if (!isTRNMask(ShuffleMask, NumElts, WhichResult)) return false; unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2; @@ -217,7 +222,10 @@ bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI, unsigned WhichResult; ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); Register Dst = MI.getOperand(0).getReg(); - unsigned NumElts = MRI.getType(Dst).getNumElements(); + LLT DstTy = MRI.getType(Dst); + if (DstTy.isScalableVector()) + return false; + unsigned NumElts = DstTy.getNumElements(); if (!isUZPMask(ShuffleMask, NumElts, WhichResult)) return false; unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2; @@ -233,7 +241,10 @@ bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI, unsigned WhichResult; ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); Register Dst = MI.getOperand(0).getReg(); - unsigned NumElts = MRI.getType(Dst).getNumElements(); + LLT DstTy = MRI.getType(Dst); + if (DstTy.isScalableVector()) + return false; + unsigned NumElts = DstTy.getNumElements(); if (!isZIPMask(ShuffleMask, NumElts, WhichResult)) return false; unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2; @@ -288,7 +299,10 @@ bool matchDupFromBuildVector(int Lane, MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo) { assert(Lane >= 0 && "Expected positive lane?"); - int NumElements = MRI.getType(MI.getOperand(1).getReg()).getNumElements(); + LLT Op1Ty = MRI.getType(MI.getOperand(1).getReg()); + if (Op1Ty.isScalableVector()) + return false; + int NumElements = Op1Ty.getNumElements(); // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the // lane's definition directly. auto *BuildVecMI = @@ -326,6 +340,8 @@ bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI, // Check if an EXT instruction can handle the shuffle mask when the vector // sources of the shuffle are the same. bool isSingletonExtMask(ArrayRef M, LLT Ty) { + if (Ty.isScalableVector()) + return false; unsigned NumElts = Ty.getNumElements(); // Assume that the first shuffle index is not UNDEF. Fail if it is. @@ -357,12 +373,17 @@ bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI, assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); Register Dst = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(Dst); + if (DstTy.isScalableVector()) + return false; Register V1 = MI.getOperand(1).getReg(); Register V2 = MI.getOperand(2).getReg(); auto Mask = MI.getOperand(3).getShuffleMask(); uint64_t Imm; auto ExtInfo = getExtMask(Mask, DstTy.getNumElements()); - uint64_t ExtFactor = MRI.getType(V1).getScalarSizeInBits() / 8; + LLT V1Ty = MRI.getType(V1); + if (V1Ty.isScalableVector()) + return false; + uint64_t ExtFactor = V1Ty.getScalarSizeInBits() / 8; if (!ExtInfo) { if (!getOpcodeDef(V2, MRI) || @@ -423,6 +444,8 @@ void applyNonConstInsert(MachineInstr &MI, MachineRegisterInfo &MRI, Register Offset = Insert.getIndexReg(); LLT VecTy = MRI.getType(Insert.getReg(0)); + if (VecTy.isScalableVector()) + return; LLT EltTy = MRI.getType(Insert.getElementReg()); LLT IdxTy = MRI.getType(Insert.getIndexReg()); @@ -473,7 +496,10 @@ bool matchINS(MachineInstr &MI, MachineRegisterInfo &MRI, assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR); ArrayRef ShuffleMask = MI.getOperand(3).getShuffleMask(); Register Dst = MI.getOperand(0).getReg(); - int NumElts = MRI.getType(Dst).getNumElements(); + LLT DstTy = MRI.getType(Dst); + if (DstTy.isScalableVector()) + return false; + int NumElts = DstTy.getNumElements(); auto DstIsLeftAndDstLane = isINSMask(ShuffleMask, NumElts); if (!DstIsLeftAndDstLane) return false; @@ -522,6 +548,8 @@ bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty, if (!Cst) return false; Cnt = *Cst; + if (Ty.isScalableVector()) + return false; int64_t ElementBits = Ty.getScalarSizeInBits(); return Cnt >= 1 && Cnt <= ElementBits; } @@ -698,6 +726,8 @@ bool matchDupLane(MachineInstr &MI, MachineRegisterInfo &MRI, Register Src1Reg = MI.getOperand(1).getReg(); const LLT SrcTy = MRI.getType(Src1Reg); const LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); + if (SrcTy.isScalableVector()) + return false; auto LaneIdx = getSplatIndex(MI); if (!LaneIdx) @@ -774,6 +804,8 @@ bool matchScalarizeVectorUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI) { auto &Unmerge = cast(MI); Register Src1Reg = Unmerge.getReg(Unmerge.getNumOperands() - 1); const LLT SrcTy = MRI.getType(Src1Reg); + if (SrcTy.isScalableVector()) + return false; if (SrcTy.getSizeInBits() != 128 && SrcTy.getSizeInBits() != 64) return false; return SrcTy.isVector() && !SrcTy.isScalable() && @@ -987,7 +1019,10 @@ bool matchLowerVectorFCMP(MachineInstr &MI, MachineRegisterInfo &MRI, if (!DstTy.isVector() || !ST.hasNEON()) return false; Register LHS = MI.getOperand(2).getReg(); - unsigned EltSize = MRI.getType(LHS).getScalarSizeInBits(); + LLT LHSTy = MRI.getType(LHS); + if (LHSTy.isScalableVector()) + return false; + unsigned EltSize = LHSTy.getScalarSizeInBits(); if (EltSize == 16 && !ST.hasFullFP16()) return false; if (EltSize != 16 && EltSize != 32 && EltSize != 64) @@ -1183,7 +1218,7 @@ bool matchExtMulToMULL(MachineInstr &MI, MachineRegisterInfo &MRI) { MachineInstr *I1 = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI); MachineInstr *I2 = getDefIgnoringCopies(MI.getOperand(2).getReg(), MRI); - if (DstTy.isVector()) { + if (DstTy.isFixedVector()) { // If the source operands were EXTENDED before, then {U/S}MULL can be used unsigned I1Opc = I1->getOpcode(); unsigned I2Opc = I2->getOpcode(); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir new file mode 100644 index 0000000000000..6d24478cbfb3d --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-insert-elt.mir @@ -0,0 +1,423 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -O0 -mtriple=aarch64-apple-ios -mattr=+sve -aarch64-enable-gisel-sve=1 -global-isel -start-before=legalizer -stop-after=instruction-select %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SELECT +# RUN: llc -O0 -mtriple=aarch64-apple-ios -mattr=+sve -aarch64-enable-gisel-sve=1 -global-isel -start-before=legalizer -stop-after=regbankselect %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-REGBANK +# RUN: llc -O0 -mtriple=aarch64-apple-ios -mattr=+sve -aarch64-enable-gisel-sve=1 -global-isel -run-pass=legalizer %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-LEGAL + +--- +name: test_insert_vector_elt_nxv_16_s8_idx_0 +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_16_s8_idx_0 + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0 + ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $xzr + ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32 + ; CHECK-SELECT-NEXT: [[DUP_ZR_B:%[0-9]+]]:zpr = DUP_ZR_B [[COPY]] + ; CHECK-SELECT-NEXT: [[INDEX_II_B:%[0-9]+]]:zpr = INDEX_II_B 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_B:%[0-9]+]]:ppr_3b = PTRUE_B 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_B:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_B [[PTRUE_B]], [[INDEX_II_B]], [[DUP_ZR_B]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_B %vec, [[CMPEQ_PPzZZ_B]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_16_s8_idx_0 + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_16_s8_idx_0 + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s32) = COPY $w0 + %idx:_(s64) = G_CONSTANT i64 0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s32), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_16_s8_constant +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_16_s8_constant + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr32common = MOVi32imm 5 + ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0 + ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32 + ; CHECK-SELECT-NEXT: [[DUP_ZR_B:%[0-9]+]]:zpr = DUP_ZR_B [[COPY]] + ; CHECK-SELECT-NEXT: [[INDEX_II_B:%[0-9]+]]:zpr = INDEX_II_B 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_B:%[0-9]+]]:ppr_3b = PTRUE_B 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_B:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_B [[PTRUE_B]], [[INDEX_II_B]], [[DUP_ZR_B]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_B %vec, [[CMPEQ_PPzZZ_B]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_16_s8_constant + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = G_CONSTANT i32 5 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_16_s8_constant + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s32) = G_CONSTANT i32 5 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s32) = G_CONSTANT i32 5 + %idx:_(s64) = COPY $x0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s32), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_16_s8 +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_16_s8 + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0 + ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0 + ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32 + ; CHECK-SELECT-NEXT: [[DUP_ZR_B:%[0-9]+]]:zpr = DUP_ZR_B [[COPY]] + ; CHECK-SELECT-NEXT: [[INDEX_II_B:%[0-9]+]]:zpr = INDEX_II_B 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_B:%[0-9]+]]:ppr_3b = PTRUE_B 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_B:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_B [[PTRUE_B]], [[INDEX_II_B]], [[DUP_ZR_B]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_B %vec, [[CMPEQ_PPzZZ_B]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_16_s8 + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_16_s8 + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s32) = COPY $w0 + %idx:_(s64) = COPY $x0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s32), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_8_s16_idx_0 +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_8_s16_idx_0 + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0 + ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $xzr + ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32 + ; CHECK-SELECT-NEXT: [[DUP_ZR_H:%[0-9]+]]:zpr = DUP_ZR_H [[COPY]] + ; CHECK-SELECT-NEXT: [[INDEX_II_H:%[0-9]+]]:zpr = INDEX_II_H 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_H:%[0-9]+]]:ppr_3b = PTRUE_H 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_H:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_H [[PTRUE_H]], [[INDEX_II_H]], [[DUP_ZR_H]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_H %vec, [[CMPEQ_PPzZZ_H]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_8_s16_idx_0 + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_8_s16_idx_0 + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s32) = COPY $w0 + %idx:_(s64) = G_CONSTANT i64 0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s32), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_8_s16_constant +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_8_s16_constant + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr32common = MOVi32imm 5 + ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0 + ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32 + ; CHECK-SELECT-NEXT: [[DUP_ZR_H:%[0-9]+]]:zpr = DUP_ZR_H [[COPY]] + ; CHECK-SELECT-NEXT: [[INDEX_II_H:%[0-9]+]]:zpr = INDEX_II_H 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_H:%[0-9]+]]:ppr_3b = PTRUE_H 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_H:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_H [[PTRUE_H]], [[INDEX_II_H]], [[DUP_ZR_H]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_H %vec, [[CMPEQ_PPzZZ_H]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_8_s16_constant + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = G_CONSTANT i32 5 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_8_s16_constant + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s32) = G_CONSTANT i32 5 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s32) = G_CONSTANT i32 5 + %idx:_(s64) = COPY $x0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s32), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_8_s16 +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_8_s16 + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0 + ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0 + ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32 + ; CHECK-SELECT-NEXT: [[DUP_ZR_H:%[0-9]+]]:zpr = DUP_ZR_H [[COPY]] + ; CHECK-SELECT-NEXT: [[INDEX_II_H:%[0-9]+]]:zpr = INDEX_II_H 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_H:%[0-9]+]]:ppr_3b = PTRUE_H 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_H:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_H [[PTRUE_H]], [[INDEX_II_H]], [[DUP_ZR_H]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_H %vec, [[CMPEQ_PPzZZ_H]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_8_s16 + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_8_s16 + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s32) = COPY $w0 + %idx:_(s64) = COPY $x0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s32), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_4_s32_idx_0 +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_4_s32_idx_0 + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0 + ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $xzr + ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32 + ; CHECK-SELECT-NEXT: [[DUP_ZR_S:%[0-9]+]]:zpr = DUP_ZR_S [[COPY]] + ; CHECK-SELECT-NEXT: [[INDEX_II_S:%[0-9]+]]:zpr = INDEX_II_S 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_S:%[0-9]+]]:ppr_3b = PTRUE_S 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_S:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_S [[PTRUE_S]], [[INDEX_II_S]], [[DUP_ZR_S]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_S %vec, [[CMPEQ_PPzZZ_S]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_4_s32_idx_0 + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_4_s32_idx_0 + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s32) = COPY $w0 + %idx:_(s64) = G_CONSTANT i64 0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s32), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_4_s32_constant +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_4_s32_constant + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr32common = MOVi32imm 5 + ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0 + ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32 + ; CHECK-SELECT-NEXT: [[DUP_ZR_S:%[0-9]+]]:zpr = DUP_ZR_S [[COPY]] + ; CHECK-SELECT-NEXT: [[INDEX_II_S:%[0-9]+]]:zpr = INDEX_II_S 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_S:%[0-9]+]]:ppr_3b = PTRUE_S 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_S:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_S [[PTRUE_S]], [[INDEX_II_S]], [[DUP_ZR_S]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_S %vec, [[CMPEQ_PPzZZ_S]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_4_s32_constant + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = G_CONSTANT i32 5 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_4_s32_constant + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s32) = G_CONSTANT i32 5 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s32) = G_CONSTANT i32 5 + %idx:_(s64) = COPY $x0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s32), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_4_s32 +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_4_s32 + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr32sp = COPY $w0 + ; CHECK-SELECT-NEXT: %idx:gpr64 = COPY $x0 + ; CHECK-SELECT-NEXT: [[COPY:%[0-9]+]]:gpr32common = COPY %idx.sub_32 + ; CHECK-SELECT-NEXT: [[DUP_ZR_S:%[0-9]+]]:zpr = DUP_ZR_S [[COPY]] + ; CHECK-SELECT-NEXT: [[INDEX_II_S:%[0-9]+]]:zpr = INDEX_II_S 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_S:%[0-9]+]]:ppr_3b = PTRUE_S 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_S:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_S [[PTRUE_S]], [[INDEX_II_S]], [[DUP_ZR_S]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_S %vec, [[CMPEQ_PPzZZ_S]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_4_s32 + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s32) = COPY $w0 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_4_s32 + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s32) = COPY $w0 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s32), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s32) = COPY $w0 + %idx:_(s64) = COPY $x0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s32), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_2s64_idx_0 +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_2s64_idx_0 + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr64sp = COPY $x0 + ; CHECK-SELECT-NEXT: %idx:gpr64common = COPY $xzr + ; CHECK-SELECT-NEXT: [[DUP_ZR_D:%[0-9]+]]:zpr = DUP_ZR_D %idx + ; CHECK-SELECT-NEXT: [[INDEX_II_D:%[0-9]+]]:zpr = INDEX_II_D 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_D:%[0-9]+]]:ppr_3b = PTRUE_D 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_D:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_D [[PTRUE_D]], [[INDEX_II_D]], [[DUP_ZR_D]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_D %vec, [[CMPEQ_PPzZZ_D]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_2s64_idx_0 + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s64) = COPY $x0 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = G_CONSTANT i64 0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_2s64_idx_0 + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s64) = COPY $x0 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = G_CONSTANT i64 0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s64) = COPY $x0 + %idx:_(s64) = G_CONSTANT i64 0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s64), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_2_s64_constant +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_2_s64_constant + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 5 + ; CHECK-SELECT-NEXT: %elt:gpr64sp = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32 + ; CHECK-SELECT-NEXT: %idx:gpr64sp = COPY $x0 + ; CHECK-SELECT-NEXT: [[DUP_ZR_D:%[0-9]+]]:zpr = DUP_ZR_D %idx + ; CHECK-SELECT-NEXT: [[INDEX_II_D:%[0-9]+]]:zpr = INDEX_II_D 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_D:%[0-9]+]]:ppr_3b = PTRUE_D 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_D:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_D [[PTRUE_D]], [[INDEX_II_D]], [[DUP_ZR_D]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_D %vec, [[CMPEQ_PPzZZ_D]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_2_s64_constant + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s64) = G_CONSTANT i64 5 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_2_s64_constant + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s64) = G_CONSTANT i64 5 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s64) = G_CONSTANT i64 5 + %idx:_(s64) = COPY $x0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s64), %idx(s64) + $z0 = COPY %result() +... +--- +name: test_insert_vector_elt_nxv_2_s64 +body: | + bb.1: + ; CHECK-SELECT-LABEL: name: test_insert_vector_elt_nxv_2_s64 + ; CHECK-SELECT: %vec:zpr = COPY $z0 + ; CHECK-SELECT-NEXT: %elt:gpr64sp = COPY $x0 + ; CHECK-SELECT-NEXT: %idx:gpr64sp = COPY $x0 + ; CHECK-SELECT-NEXT: [[DUP_ZR_D:%[0-9]+]]:zpr = DUP_ZR_D %idx + ; CHECK-SELECT-NEXT: [[INDEX_II_D:%[0-9]+]]:zpr = INDEX_II_D 0, 1, implicit $vg + ; CHECK-SELECT-NEXT: [[PTRUE_D:%[0-9]+]]:ppr_3b = PTRUE_D 31, implicit $vg + ; CHECK-SELECT-NEXT: [[CMPEQ_PPzZZ_D:%[0-9]+]]:ppr_3b = CMPEQ_PPzZZ_D [[PTRUE_D]], [[INDEX_II_D]], [[DUP_ZR_D]], implicit-def dead $nzcv + ; CHECK-SELECT-NEXT: %result:zpr = CPY_ZPmR_D %vec, [[CMPEQ_PPzZZ_D]], %elt + ; CHECK-SELECT-NEXT: $z0 = COPY %result + ; + ; CHECK-REGBANK-LABEL: name: test_insert_vector_elt_nxv_2_s64 + ; CHECK-REGBANK: %vec:fpr() = COPY $z0 + ; CHECK-REGBANK-NEXT: %elt:gpr(s64) = COPY $x0 + ; CHECK-REGBANK-NEXT: %idx:gpr(s64) = COPY $x0 + ; CHECK-REGBANK-NEXT: %result:fpr() = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64) + ; CHECK-REGBANK-NEXT: $z0 = COPY %result() + ; + ; CHECK-LEGAL-LABEL: name: test_insert_vector_elt_nxv_2_s64 + ; CHECK-LEGAL: %vec:_() = COPY $z0 + ; CHECK-LEGAL-NEXT: %elt:_(s64) = COPY $x0 + ; CHECK-LEGAL-NEXT: %idx:_(s64) = COPY $x0 + ; CHECK-LEGAL-NEXT: %result:_() = G_INSERT_VECTOR_ELT %vec, %elt(s64), %idx(s64) + ; CHECK-LEGAL-NEXT: $z0 = COPY %result() + %vec:_() = COPY $z0 + %elt:_(s64) = COPY $x0 + %idx:_(s64) = COPY $x0 + %result:_() = G_INSERT_VECTOR_ELT %vec(), %elt(s64), %idx(s64) + $z0 = COPY %result() +...