diff --git a/llvm/lib/Target/RISCV/CMakeLists.txt b/llvm/lib/Target/RISCV/CMakeLists.txt index 98d3615ebab58..9b23a5ab521c8 100644 --- a/llvm/lib/Target/RISCV/CMakeLists.txt +++ b/llvm/lib/Target/RISCV/CMakeLists.txt @@ -63,6 +63,7 @@ add_llvm_target(RISCVCodeGen RISCVVectorMaskDAGMutation.cpp RISCVVectorPeephole.cpp RISCVVLOptimizer.cpp + RISCVVMV0Elimination.cpp RISCVZacasABIFix.cpp GISel/RISCVCallLowering.cpp GISel/RISCVInstructionSelector.cpp diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h index b1aee98739e85..851eea1352852 100644 --- a/llvm/lib/Target/RISCV/RISCV.h +++ b/llvm/lib/Target/RISCV/RISCV.h @@ -107,6 +107,9 @@ void initializeRISCVPreLegalizerCombinerPass(PassRegistry &); FunctionPass *createRISCVVLOptimizerPass(); void initializeRISCVVLOptimizerPass(PassRegistry &); + +FunctionPass *createRISCVVMV0EliminationPass(); +void initializeRISCVVMV0EliminationPass(PassRegistry &); } // namespace llvm #endif diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index ec2e8f1d50264..fb2c5c62ef871 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -254,7 +254,6 @@ void RISCVDAGToDAGISel::addVectorLoadStoreOperands( bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl &Operands, bool IsLoad, MVT *IndexVT) { SDValue Chain = Node->getOperand(0); - SDValue Glue; Operands.push_back(Node->getOperand(CurOp++)); // Base pointer. @@ -265,11 +264,8 @@ void RISCVDAGToDAGISel::addVectorLoadStoreOperands( } if (IsMasked) { - // Mask needs to be copied to V0. SDValue Mask = Node->getOperand(CurOp++); - Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue()); - Glue = Chain.getValue(1); - Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType())); + Operands.push_back(Mask); } SDValue VL; selectVLOp(Node->getOperand(CurOp++), VL); @@ -291,8 +287,6 @@ void RISCVDAGToDAGISel::addVectorLoadStoreOperands( } Operands.push_back(Chain); // Chain. - if (Glue) - Operands.push_back(Glue); } void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned NF, bool IsMasked, @@ -1844,19 +1838,13 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { return; } - // Mask needs to be copied to V0. - SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL, - RISCV::V0, Mask, SDValue()); - SDValue Glue = Chain.getValue(1); - SDValue V0 = CurDAG->getRegister(RISCV::V0, VT); - if (IsCmpConstant) { SDValue Imm = selectImm(CurDAG, SDLoc(Src2), XLenVT, CVal - 1, *Subtarget); ReplaceNode(Node, CurDAG->getMachineNode( VMSGTMaskOpcode, DL, VT, - {MaskedOff, Src1, Imm, V0, VL, SEW, Glue})); + {MaskedOff, Src1, Imm, Mask, VL, SEW})); return; } @@ -1867,7 +1855,7 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { // the agnostic result can be either undisturbed or all 1. SDValue Cmp = SDValue( CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT, - {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}), + {MaskedOff, Src1, Src2, Mask, VL, SEW}), 0); // vmxor.mm vd, vd, v0 is used to update active value. ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT, @@ -3287,12 +3275,10 @@ static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo, return false; assert(RISCVII::hasVLOp(TSFlags)); - bool HasGlueOp = User->getGluedNode() != nullptr; - unsigned ChainOpIdx = User->getNumOperands() - HasGlueOp - 1; + unsigned ChainOpIdx = User->getNumOperands() - 1; bool HasChainOp = User->getOperand(ChainOpIdx).getValueType() == MVT::Other; bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TSFlags); - unsigned VLIdx = - User->getNumOperands() - HasVecPolicyOp - HasChainOp - HasGlueOp - 2; + unsigned VLIdx = User->getNumOperands() - HasVecPolicyOp - HasChainOp - 2; const unsigned Log2SEW = User->getConstantOperandVal(VLIdx + 1); if (UserOpNo == VLIdx) @@ -3759,43 +3745,7 @@ bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) { return false; } -// After ISel, a vector pseudo's mask will be copied to V0 via a CopyToReg -// that's glued to the pseudo. This tries to look up the value that was copied -// to V0. -static SDValue getMaskSetter(SDValue MaskOp, SDValue GlueOp) { - // Check that we're using V0 as a mask register. - if (!isa(MaskOp) || - cast(MaskOp)->getReg() != RISCV::V0) - return SDValue(); - - // The glued user defines V0. - const auto *Glued = GlueOp.getNode(); - - if (!Glued || Glued->getOpcode() != ISD::CopyToReg) - return SDValue(); - - // Check that we're defining V0 as a mask register. - if (!isa(Glued->getOperand(1)) || - cast(Glued->getOperand(1))->getReg() != RISCV::V0) - return SDValue(); - - SDValue MaskSetter = Glued->getOperand(2); - - // Sometimes the VMSET is wrapped in a COPY_TO_REGCLASS, e.g. if the mask came - // from an extract_subvector or insert_subvector. - if (MaskSetter->isMachineOpcode() && - MaskSetter->getMachineOpcode() == RISCV::COPY_TO_REGCLASS) - MaskSetter = MaskSetter->getOperand(0); - - return MaskSetter; -} - -static bool usesAllOnesMask(SDValue MaskOp, SDValue GlueOp) { - // Check the instruction defining V0; it needs to be a VMSET pseudo. - SDValue MaskSetter = getMaskSetter(MaskOp, GlueOp); - if (!MaskSetter) - return false; - +static bool usesAllOnesMask(SDValue MaskOp) { const auto IsVMSet = [](unsigned Opc) { return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 || Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 || @@ -3806,14 +3756,7 @@ static bool usesAllOnesMask(SDValue MaskOp, SDValue GlueOp) { // TODO: Check that the VMSET is the expected bitwidth? The pseudo has // undefined behaviour if it's the wrong bitwidth, so we could choose to // assume that it's all-ones? Same applies to its VL. - return MaskSetter->isMachineOpcode() && - IsVMSet(MaskSetter.getMachineOpcode()); -} - -// Return true if we can make sure mask of N is all-ones mask. -static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx) { - return usesAllOnesMask(N->getOperand(MaskOpIdx), - N->getOperand(N->getNumOperands() - 1)); + return MaskOp->isMachineOpcode() && IsVMSet(MaskOp.getMachineOpcode()); } static bool isImplicitDef(SDValue V) { @@ -3829,9 +3772,7 @@ static bool isImplicitDef(SDValue V) { } // Optimize masked RVV pseudo instructions with a known all-ones mask to their -// corresponding "unmasked" pseudo versions. The mask we're interested in will -// take the form of a V0 physical register operand, with a glued -// register-setting instruction. +// corresponding "unmasked" pseudo versions. bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) { const RISCV::RISCVMaskedPseudoInfo *I = RISCV::getMaskedPseudoInfo(N->getMachineOpcode()); @@ -3839,7 +3780,7 @@ bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) { return false; unsigned MaskOpIdx = I->MaskOpIdx; - if (!usesAllOnesMask(N, MaskOpIdx)) + if (!usesAllOnesMask(N->getOperand(MaskOpIdx))) return false; // There are two classes of pseudos in the table - compares and @@ -3863,18 +3804,13 @@ bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) { // Skip the passthru operand at index 0 if the unmasked don't have one. bool ShouldSkip = !HasPassthru && MaskedHasPassthru; for (unsigned I = ShouldSkip, E = N->getNumOperands(); I != E; I++) { - // Skip the mask, and the Glue. + // Skip the mask SDValue Op = N->getOperand(I); - if (I == MaskOpIdx || Op.getValueType() == MVT::Glue) + if (I == MaskOpIdx) continue; Ops.push_back(Op); } - // Transitively apply any node glued to our new node. - const auto *Glued = N->getGluedNode(); - if (auto *TGlued = Glued->getGluedNode()) - Ops.push_back(SDValue(TGlued, TGlued->getNumValues() - 1)); - MachineSDNode *Result = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops); @@ -3910,17 +3846,13 @@ static bool IsVMerge(SDNode *N) { // The resulting policy is the effective policy the vmerge would have had, // i.e. whether or not it's passthru operand was implicit-def. bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) { - SDValue Passthru, False, True, VL, Mask, Glue; + SDValue Passthru, False, True, VL, Mask; assert(IsVMerge(N)); Passthru = N->getOperand(0); False = N->getOperand(1); True = N->getOperand(2); Mask = N->getOperand(3); VL = N->getOperand(4); - // We always have a glue node for the mask at v0. - Glue = N->getOperand(N->getNumOperands() - 1); - assert(cast(Mask)->getReg() == RISCV::V0); - assert(Glue.getValueType() == MVT::Glue); // If the EEW of True is different from vmerge's SEW, then we can't fold. if (True.getSimpleValueType() != N->getSimpleValueType(0)) @@ -3963,12 +3895,7 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) { if (TII->get(TrueOpc).hasUnmodeledSideEffects()) return false; - // The last operand of a masked instruction may be glued. - bool HasGlueOp = True->getGluedNode() != nullptr; - - // The chain operand may exist either before the glued operands or in the last - // position. - unsigned TrueChainOpIdx = True.getNumOperands() - HasGlueOp - 1; + unsigned TrueChainOpIdx = True.getNumOperands() - 1; bool HasChainOp = True.getOperand(TrueChainOpIdx).getValueType() == MVT::Other; @@ -3980,7 +3907,6 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) { LoopWorklist.push_back(False.getNode()); LoopWorklist.push_back(Mask.getNode()); LoopWorklist.push_back(VL.getNode()); - LoopWorklist.push_back(Glue.getNode()); if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist)) return false; } @@ -3988,7 +3914,7 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) { // The vector policy operand may be present for masked intrinsics bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TrueTSFlags); unsigned TrueVLIndex = - True.getNumOperands() - HasVecPolicyOp - HasChainOp - HasGlueOp - 2; + True.getNumOperands() - HasVecPolicyOp - HasChainOp - 2; SDValue TrueVL = True.getOperand(TrueVLIndex); SDValue SEW = True.getOperand(TrueVLIndex + 1); @@ -4020,7 +3946,7 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) { if (RISCVII::elementsDependOnVL(TrueBaseMCID.TSFlags) && (TrueVL != VL)) return false; if (RISCVII::elementsDependOnMask(TrueBaseMCID.TSFlags) && - (Mask && !usesAllOnesMask(Mask, Glue))) + (Mask && !usesAllOnesMask(Mask))) return false; // Make sure it doesn't raise any observable fp exceptions, since changing the @@ -4077,9 +4003,6 @@ bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) { if (HasChainOp) Ops.push_back(True.getOperand(TrueChainOpIdx)); - // Add the glue for the CopyToReg of mask->v0. - Ops.push_back(Glue); - MachineSDNode *Result = CurDAG->getMachineNode(MaskedOpc, DL, True->getVTList(), Ops); Result->setFlags(True->getFlags()); diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 268bfe70673a2..46cf27838d1ce 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -3945,7 +3945,7 @@ class VPatUnaryMask(intrinsic_name#"_mask") (result_type result_reg_class:$passthru), (op2_type op2_reg_class:$rs2), - (mask_type V0), + (mask_type VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast( !if(isSEWAware, @@ -3953,7 +3953,7 @@ class VPatUnaryMask; + (mask_type VMV0:$vm), GPR:$vl, log2sew, (XLenVT timm:$policy))>; class VPatUnaryMaskRoundingMode(intrinsic_name#"_mask") (result_type result_reg_class:$passthru), (op2_type op2_reg_class:$rs2), - (mask_type V0), + (mask_type VMV0:$vm), (XLenVT timm:$round), VLOpFrag, (XLenVT timm:$policy))), (!cast( @@ -3978,7 +3978,7 @@ class VPatUnaryMaskRoundingMode; @@ -3996,7 +3996,7 @@ class VPatUnaryMaskRTZ(intrinsic_name#"_mask") (result_type result_reg_class:$passthru), (op2_type op2_reg_class:$rs2), - (mask_type V0), + (mask_type VMV0:$vm), (XLenVT 0b001), VLOpFrag, (XLenVT timm:$policy))), (!cast( @@ -4005,7 +4005,7 @@ class VPatUnaryMaskRTZ; class VPatMaskUnaryNoMask(intrinsic_name#"_mask") (mti.Mask VR:$passthru), (mti.Mask VR:$rs2), - (mti.Mask V0), + (mti.Mask VMV0:$vm), VLOpFrag)), (!cast(inst#"_M_"#mti.BX#"_MASK") (mti.Mask VR:$passthru), (mti.Mask VR:$rs2), - (mti.Mask V0), GPR:$vl, mti.Log2SEW, TU_MU)>; + (mti.Mask VMV0:$vm), GPR:$vl, mti.Log2SEW, TU_MU)>; class VPatUnaryAnyMask(inst#"_MASK") (result_type result_reg_class:$passthru), (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), GPR:$vl, sew)>; + (mask_type VMV0:$vm), GPR:$vl, sew)>; class VPatBinaryMaskPolicy(inst#"_MASK") (result_type result_reg_class:$passthru), (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), GPR:$vl, sew, (XLenVT timm:$policy))>; + (mask_type VMV0:$vm), GPR:$vl, sew, (XLenVT timm:$policy))>; class VPatBinaryMaskPolicyRoundingMode(inst#"_MASK") (result_type result_reg_class:$passthru), (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), + (mask_type VMV0:$vm), (XLenVT timm:$round), GPR:$vl, sew, (XLenVT timm:$policy))>; @@ -4214,13 +4214,13 @@ class VPatBinaryMaskSwapped(inst#"_MASK") (result_type result_reg_class:$passthru), (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), GPR:$vl, sew)>; + (mask_type VMV0:$vm), GPR:$vl, sew)>; class VPatTiedBinaryNoMask(inst#"_MASK_TIED") (result_type result_reg_class:$passthru), (op2_type op2_kind:$rs2), - (mask_type V0), GPR:$vl, sew, (XLenVT timm:$policy))>; + (mask_type VMV0:$vm), GPR:$vl, sew, (XLenVT timm:$policy))>; class VPatTiedBinaryMaskRoundingMode(inst#"_MASK_TIED") (result_type result_reg_class:$passthru), (op2_type op2_kind:$rs2), - (mask_type V0), + (mask_type VMV0:$vm), (XLenVT timm:$round), GPR:$vl, sew, (XLenVT timm:$policy))>; @@ -4447,13 +4447,13 @@ class VPatTernaryMaskPolicy(inst#"_"#kind#"_"#vlmul.MX # "_MASK") result_reg_class:$rs3, (op1_type op1_reg_class:$rs1), op2_kind:$rs2, - (mask_type V0), + (mask_type VMV0:$vm), GPR:$vl, sew, (XLenVT timm:$policy))>; class VPatTernaryMaskPolicyRoundingMode(!if(isSEWAware, @@ -4482,7 +4482,7 @@ class VPatTernaryMaskPolicyRoundingMode; @@ -4502,13 +4502,13 @@ class VPatTernaryMaskTU(inst#"_"#kind#"_"#vlmul.MX#"_E"#!shl(1, log2sew)# "_MASK") result_reg_class:$rs3, (op1_type op1_reg_class:$rs1), op2_kind:$rs2, - (mask_type V0), + (mask_type VMV0:$vm), GPR:$vl, log2sew, TU_MU)>; class VPatTernaryMaskTURoundingMode(inst#"_"#kind#"_"#vlmul.MX#"_E"#!shl(1, log2sew)# "_MASK") result_reg_class:$rs3, (op1_type op1_reg_class:$rs1), op2_kind:$rs2, - (mask_type V0), + (mask_type VMV0:$vm), (XLenVT timm:$round), GPR:$vl, log2sew, TU_MU)>; @@ -4546,9 +4546,9 @@ multiclass VPatUnaryS_M(inst#"_M_"#mti.BX) $rs1, GPR:$vl, mti.Log2SEW)>; def : Pat<(XLenVT (!cast(intrinsic_name # "_mask") - (mti.Mask VR:$rs1), (mti.Mask V0), VLOpFrag)), + (mti.Mask VR:$rs1), (mti.Mask VMV0:$vm), VLOpFrag)), (!cast(inst#"_M_"#mti.BX#"_MASK") $rs1, - (mti.Mask V0), GPR:$vl, mti.Log2SEW)>; + (mti.Mask VMV0:$vm), GPR:$vl, mti.Log2SEW)>; } } @@ -4636,9 +4636,9 @@ multiclass VPatNullaryV { vti.RegClass:$passthru, GPR:$vl, vti.Log2SEW, TU_MU)>; def : Pat<(vti.Vector (!cast(intrinsic # "_mask") (vti.Vector vti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag, (XLenVT timm:$policy))), + (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast(instruction#"_V_" # vti.LMul.MX # "_MASK") - vti.RegClass:$passthru, (vti.Mask V0), + vti.RegClass:$passthru, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>; } } @@ -4736,13 +4736,13 @@ multiclass VPatBinaryCarryInTAIL(inst#"_"#kind#"_"#vlmul.MX) (result_type result_reg_class:$passthru), (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), GPR:$vl, sew)>; + (mask_type VMV0:$vm), GPR:$vl, sew)>; } multiclass VPatBinaryCarryIn(intrinsic) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), + (mask_type VMV0:$vm), VLOpFrag)), (!cast(inst#"_"#kind#"_"#vlmul.MX) (op1_type op1_reg_class:$rs1), (op2_type op2_kind:$rs2), - (mask_type V0), GPR:$vl, sew)>; + (mask_type VMV0:$vm), GPR:$vl, sew)>; } multiclass VPatBinaryMaskOut; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; } } @@ -6174,14 +6174,14 @@ foreach vti = AllIntegerVectors in { def : Pat<(vti.Vector (int_riscv_vrsub_mask (vti.Vector vti.RegClass:$passthru), (vti.Vector vti.RegClass:$rs2), (vti.Vector vti.RegClass:$rs1), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast("PseudoVSUB_VV_"#vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>; @@ -6200,14 +6200,14 @@ foreach vti = AllIntegerVectors in { def : Pat<(vti.Vector (int_riscv_vsub_mask (vti.Vector vti.RegClass:$passthru), (vti.Vector vti.RegClass:$rs1), (vti.Scalar simm5_plus1:$rs2), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast("PseudoVADD_VI_"#vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs1, (NegImm simm5_plus1:$rs2), - (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>; @@ -6844,14 +6844,14 @@ foreach vti = AllIntegerVectors in { def : Pat<(vti.Vector (int_riscv_vsll_mask (vti.Vector vti.RegClass:$passthru), (vti.Vector vti.RegClass:$rs1), (XLenVT 1), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast("PseudoVADD_VV_"#vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs1, vti.RegClass:$rs1, - (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>; @@ -7194,9 +7194,9 @@ foreach fvti = AllFloatVectors in { def : Pat<(fvti.Vector (int_riscv_vfmerge (fvti.Vector fvti.RegClass:$passthru), (fvti.Vector fvti.RegClass:$rs2), (fvti.Scalar (fpimm0)), - (fvti.Mask V0), VLOpFrag)), + (fvti.Mask VMV0:$vm), VLOpFrag)), (instr fvti.RegClass:$passthru, fvti.RegClass:$rs2, 0, - (fvti.Mask V0), GPR:$vl, fvti.Log2SEW)>; + (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW)>; } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td index 8f77b2ce34d1f..629e5ccf2f27d 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td @@ -928,16 +928,16 @@ foreach vtiToWti = AllWidenableIntVectors in { (!cast("PseudoVWADDU_VV_"#vti.LMul.MX) (wti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1, vti.RegClass:$rs1, vti.AVL, vti.Log2SEW, TA_MA)>; - def : Pat<(shl (wti.Vector (riscv_sext_vl_oneuse (vti.Vector vti.RegClass:$rs1), (vti.Mask V0), VLOpFrag)), + def : Pat<(shl (wti.Vector (riscv_sext_vl_oneuse (vti.Vector vti.RegClass:$rs1), (vti.Mask VMV0:$vm), VLOpFrag)), (wti.Vector (riscv_vmv_v_x_vl (wti.Vector undef), 1, (XLenVT srcvalue)))), (!cast("PseudoVWADD_VV_"#vti.LMul.MX#"_MASK") (wti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; - def : Pat<(shl (wti.Vector (riscv_zext_vl_oneuse (vti.Vector vti.RegClass:$rs1), (vti.Mask V0), VLOpFrag)), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + def : Pat<(shl (wti.Vector (riscv_zext_vl_oneuse (vti.Vector vti.RegClass:$rs1), (vti.Mask VMV0:$vm), VLOpFrag)), (wti.Vector (riscv_vmv_v_x_vl (wti.Vector undef), 1, (XLenVT srcvalue)))), (!cast("PseudoVWADDU_VV_"#vti.LMul.MX#"_MASK") (wti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } @@ -1081,24 +1081,24 @@ defm : VPatWidenMulAddSDNode_VX; // 11.15. Vector Integer Merge Instructions foreach vti = AllIntegerVectors in { let Predicates = GetVTypePredicates.Predicates in { - def : Pat<(vti.Vector (vselect (vti.Mask V0), vti.RegClass:$rs1, + def : Pat<(vti.Vector (vselect (vti.Mask VMV0:$vm), vti.RegClass:$rs1, vti.RegClass:$rs2)), (!cast("PseudoVMERGE_VVM_"#vti.LMul.MX) (vti.Vector (IMPLICIT_DEF)), - vti.RegClass:$rs2, vti.RegClass:$rs1, (vti.Mask V0), + vti.RegClass:$rs2, vti.RegClass:$rs1, (vti.Mask VMV0:$vm), vti.AVL, vti.Log2SEW)>; - def : Pat<(vti.Vector (vselect (vti.Mask V0), (SplatPat XLenVT:$rs1), + def : Pat<(vti.Vector (vselect (vti.Mask VMV0:$vm), (SplatPat XLenVT:$rs1), vti.RegClass:$rs2)), (!cast("PseudoVMERGE_VXM_"#vti.LMul.MX) (vti.Vector (IMPLICIT_DEF)), - vti.RegClass:$rs2, GPR:$rs1, (vti.Mask V0), vti.AVL, vti.Log2SEW)>; + vti.RegClass:$rs2, GPR:$rs1, (vti.Mask VMV0:$vm), vti.AVL, vti.Log2SEW)>; - def : Pat<(vti.Vector (vselect (vti.Mask V0), (SplatPat_simm5 simm5:$rs1), + def : Pat<(vti.Vector (vselect (vti.Mask VMV0:$vm), (SplatPat_simm5 simm5:$rs1), vti.RegClass:$rs2)), (!cast("PseudoVMERGE_VIM_"#vti.LMul.MX) (vti.Vector (IMPLICIT_DEF)), - vti.RegClass:$rs2, simm5:$rs1, (vti.Mask V0), vti.AVL, vti.Log2SEW)>; + vti.RegClass:$rs2, simm5:$rs1, (vti.Mask VMV0:$vm), vti.AVL, vti.Log2SEW)>; } } @@ -1348,39 +1348,39 @@ defm : VPatFPSetCCSDNode_VV_VF_FV; foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { defvar ivti = GetIntVTypeInfo.Vti; let Predicates = GetVTypePredicates.Predicates in { - def : Pat<(fvti.Vector (vselect (fvti.Mask V0), fvti.RegClass:$rs1, + def : Pat<(fvti.Vector (vselect (fvti.Mask VMV0:$vm), fvti.RegClass:$rs1, fvti.RegClass:$rs2)), (!cast("PseudoVMERGE_VVM_"#fvti.LMul.MX) (fvti.Vector (IMPLICIT_DEF)), - fvti.RegClass:$rs2, fvti.RegClass:$rs1, (fvti.Mask V0), + fvti.RegClass:$rs2, fvti.RegClass:$rs1, (fvti.Mask VMV0:$vm), fvti.AVL, fvti.Log2SEW)>; - def : Pat<(fvti.Vector (vselect (fvti.Mask V0), + def : Pat<(fvti.Vector (vselect (fvti.Mask VMV0:$vm), (SplatFPOp (SelectScalarFPAsInt (XLenVT GPR:$imm))), fvti.RegClass:$rs2)), (!cast("PseudoVMERGE_VXM_"#fvti.LMul.MX) (fvti.Vector (IMPLICIT_DEF)), - fvti.RegClass:$rs2, GPR:$imm, (fvti.Mask V0), fvti.AVL, fvti.Log2SEW)>; + fvti.RegClass:$rs2, GPR:$imm, (fvti.Mask VMV0:$vm), fvti.AVL, fvti.Log2SEW)>; - def : Pat<(fvti.Vector (vselect (fvti.Mask V0), + def : Pat<(fvti.Vector (vselect (fvti.Mask VMV0:$vm), (SplatFPOp (fvti.Scalar fpimm0)), fvti.RegClass:$rs2)), (!cast("PseudoVMERGE_VIM_"#fvti.LMul.MX) (fvti.Vector (IMPLICIT_DEF)), - fvti.RegClass:$rs2, 0, (fvti.Mask V0), fvti.AVL, fvti.Log2SEW)>; + fvti.RegClass:$rs2, 0, (fvti.Mask VMV0:$vm), fvti.AVL, fvti.Log2SEW)>; } } foreach fvti = AllFloatVectors in { let Predicates = GetVTypePredicates.Predicates in - def : Pat<(fvti.Vector (vselect (fvti.Mask V0), + def : Pat<(fvti.Vector (vselect (fvti.Mask VMV0:$vm), (SplatFPOp fvti.ScalarRegClass:$rs1), fvti.RegClass:$rs2)), (!cast("PseudoVFMERGE_V"#fvti.ScalarSuffix#"M_"#fvti.LMul.MX) (fvti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs2, (fvti.Scalar fvti.ScalarRegClass:$rs1), - (fvti.Mask V0), fvti.AVL, fvti.Log2SEW)>; + (fvti.Mask VMV0:$vm), fvti.AVL, fvti.Log2SEW)>; } // 13.17. Vector Single-Width Floating-Point/Integer Type-Convert Instructions diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td index 333ae52534681..9ba0745ce5745 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td @@ -628,7 +628,7 @@ class VPatBinaryVL_V( !if(isSEWAware, @@ -637,7 +637,7 @@ class VPatBinaryVL_V; + (mask_type VMV0:$vm), GPR:$vl, log2sew, TAIL_AGNOSTIC)>; class VPatBinaryVL_V_RM( !if(isSEWAware, @@ -665,7 +665,7 @@ class VPatBinaryVL_V_RM(instruction_name#"_"#suffix#"_"# vlmul.MX#"_MASK_TIED") result_reg_class:$rs1, op2_reg_class:$rs2, - (mask_type V0), GPR:$vl, sew, TU_MU)>; + (mask_type VMV0:$vm), GPR:$vl, sew, TU_MU)>; multiclass VPatTiedBinaryNoMaskVL_V_RM( !if(isSEWAware, @@ -797,7 +797,7 @@ class VPatBinaryVL_XI; + (mask_type VMV0:$vm), GPR:$vl, log2sew, TAIL_AGNOSTIC)>; multiclass VPatBinaryVL_VV_VX vtilist = AllIntegerVectors, @@ -889,7 +889,7 @@ class VPatBinaryVL_VF( !if(isSEWAware, @@ -898,7 +898,7 @@ class VPatBinaryVL_VF; + (mask_type VMV0:$vm), GPR:$vl, log2sew, TAIL_AGNOSTIC)>; class VPatBinaryVL_VF_RM( !if(isSEWAware, @@ -924,7 +924,7 @@ class VPatBinaryVL_VF_RM( !if(isSEWAware, @@ -977,7 +977,7 @@ multiclass VPatBinaryFPVL_R_VF; + (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW, TAIL_AGNOSTIC)>; } } @@ -988,7 +988,7 @@ multiclass VPatBinaryFPVL_R_VF_RM( !if(isSEWAware, @@ -996,7 +996,7 @@ multiclass VPatBinaryFPVL_R_VF_RM(instruction_name#"_VV_"#vti.LMul.MX#"_MASK") VR:$passthru, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; } // Inherits from VPatIntegerSetCCVL_VV and adds a pattern with operands swapped. @@ -1025,11 +1025,11 @@ multiclass VPatIntegerSetCCVL_VV_Swappable(instruction_name#"_VV_"#vti.LMul.MX#"_MASK") VR:$passthru, vti.RegClass:$rs1, - vti.RegClass:$rs2, (vti.Mask V0), GPR:$vl, vti.Log2SEW)>; + vti.RegClass:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; } multiclass VPatIntegerSetCCVL_VX_Swappable; + GPR:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; def : Pat<(vti.Mask (riscv_setcc_vl (SplatPat (XLenVT GPR:$rs2)), (vti.Vector vti.RegClass:$rs1), invcc, VR:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (instruction_masked VR:$passthru, vti.RegClass:$rs1, - GPR:$rs2, (vti.Mask V0), GPR:$vl, vti.Log2SEW)>; + GPR:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; } multiclass VPatIntegerSetCCVL_VI_Swappable; // FIXME: Can do some canonicalization to remove these patterns. def : Pat<(vti.Mask (riscv_setcc_vl (splatpat_kind simm5:$rs2), (vti.Vector vti.RegClass:$rs1), invcc, VR:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (instruction_masked VR:$passthru, vti.RegClass:$rs1, - simm5:$rs2, (vti.Mask V0), GPR:$vl, + simm5:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; } @@ -1084,31 +1084,31 @@ multiclass VPatFPSetCCVL_VV_VF_FV(inst_name#"_VV_"#fvti.LMul.MX#"_MASK") VR:$passthru, fvti.RegClass:$rs1, - fvti.RegClass:$rs2, (fvti.Mask V0), + fvti.RegClass:$rs2, (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW)>; def : Pat<(fvti.Mask (vop (fvti.Vector fvti.RegClass:$rs1), (SplatFPOp fvti.ScalarRegClass:$rs2), cc, VR:$passthru, - (fvti.Mask V0), + (fvti.Mask VMV0:$vm), VLOpFrag)), (!cast(inst_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_MASK") VR:$passthru, fvti.RegClass:$rs1, - fvti.ScalarRegClass:$rs2, (fvti.Mask V0), + fvti.ScalarRegClass:$rs2, (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW)>; def : Pat<(fvti.Mask (vop (SplatFPOp fvti.ScalarRegClass:$rs2), (fvti.Vector fvti.RegClass:$rs1), cc, VR:$passthru, - (fvti.Mask V0), + (fvti.Mask VMV0:$vm), VLOpFrag)), (!cast(swapped_op_inst_name#"_V"#fvti.ScalarSuffix#"_"#fvti.LMul.MX#"_MASK") VR:$passthru, fvti.RegClass:$rs1, - fvti.ScalarRegClass:$rs2, (fvti.Mask V0), + fvti.ScalarRegClass:$rs2, (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW)>; } } @@ -1122,11 +1122,11 @@ multiclass VPatExtendVL_V.Predicates, GetVTypePredicates.Predicates) in def : Pat<(vti.Vector (vop (fti.Vector fti.RegClass:$rs2), - (fti.Mask V0), VLOpFrag)), + (fti.Mask VMV0:$vm), VLOpFrag)), (!cast(inst_name#"_"#suffix#"_"#vti.LMul.MX#"_MASK") (vti.Vector (IMPLICIT_DEF)), fti.RegClass:$rs2, - (fti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; + (fti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; } } @@ -1138,11 +1138,11 @@ multiclass VPatConvertFP2IVL_V { let Predicates = !listconcat(GetVTypePredicates.Predicates, GetVTypePredicates.Predicates) in def : Pat<(ivti.Vector (vop (fvti.Vector fvti.RegClass:$rs1), - (fvti.Mask V0), + (fvti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_"#ivti.LMul.MX#"_MASK") (ivti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1, - (fvti.Mask V0), GPR:$vl, ivti.Log2SEW, TA_MA)>; + (fvti.Mask VMV0:$vm), GPR:$vl, ivti.Log2SEW, TA_MA)>; } } @@ -1153,11 +1153,11 @@ multiclass VPatConvertFP2I_RM_VL_V.Predicates, GetVTypePredicates.Predicates) in def : Pat<(ivti.Vector (vop (fvti.Vector fvti.RegClass:$rs1), - (fvti.Mask V0), (XLenVT timm:$frm), + (fvti.Mask VMV0:$vm), (XLenVT timm:$frm), VLOpFrag)), (!cast(instruction_name#"_"#ivti.LMul.MX#"_MASK") (ivti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1, - (fvti.Mask V0), timm:$frm, GPR:$vl, ivti.Log2SEW, + (fvti.Mask VMV0:$vm), timm:$frm, GPR:$vl, ivti.Log2SEW, TA_MA)>; } } @@ -1168,11 +1168,11 @@ multiclass VPatConvertI2FPVL_V_RM.Predicates, GetVTypePredicates.Predicates) in def : Pat<(fvti.Vector (vop (ivti.Vector ivti.RegClass:$rs1), - (ivti.Mask V0), + (ivti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), ivti.RegClass:$rs1, - (ivti.Mask V0), + (ivti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, @@ -1186,11 +1186,11 @@ multiclass VPatConvertI2FP_RM_VL_V { let Predicates = !listconcat(GetVTypePredicates.Predicates, GetVTypePredicates.Predicates) in def : Pat<(fvti.Vector (vop (ivti.Vector ivti.RegClass:$rs1), - (ivti.Mask V0), (XLenVT timm:$frm), + (ivti.Mask VMV0:$vm), (XLenVT timm:$frm), VLOpFrag)), (!cast(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), ivti.RegClass:$rs1, - (ivti.Mask V0), timm:$frm, GPR:$vl, fvti.Log2SEW, TA_MA)>; + (ivti.Mask VMV0:$vm), timm:$frm, GPR:$vl, fvti.Log2SEW, TA_MA)>; } } @@ -1203,11 +1203,11 @@ multiclass VPatWConvertFP2IVL_V let Predicates = !listconcat(GetVTypePredicates.Predicates, GetVTypePredicates.Predicates) in def : Pat<(iwti.Vector (vop (fvti.Vector fvti.RegClass:$rs1), - (fvti.Mask V0), + (fvti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_"#fvti.LMul.MX#"_MASK") (iwti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1, - (fvti.Mask V0), GPR:$vl, fvti.Log2SEW, TA_MA)>; + (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW, TA_MA)>; } } @@ -1219,11 +1219,11 @@ multiclass VPatWConvertFP2I_RM_VL_V { let Predicates = !listconcat(GetVTypePredicates.Predicates, GetVTypePredicates.Predicates) in def : Pat<(iwti.Vector (vop (fvti.Vector fvti.RegClass:$rs1), - (fvti.Mask V0), (XLenVT timm:$frm), + (fvti.Mask VMV0:$vm), (XLenVT timm:$frm), VLOpFrag)), (!cast(instruction_name#"_"#fvti.LMul.MX#"_MASK") (iwti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1, - (fvti.Mask V0), timm:$frm, GPR:$vl, fvti.Log2SEW, TA_MA)>; + (fvti.Mask VMV0:$vm), timm:$frm, GPR:$vl, fvti.Log2SEW, TA_MA)>; } } @@ -1235,11 +1235,11 @@ multiclass VPatWConvertI2FPVL_V.Predicates, GetVTypePredicates.Predicates) in def : Pat<(fwti.Vector (vop (ivti.Vector ivti.RegClass:$rs1), - (ivti.Mask V0), + (ivti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_"#ivti.LMul.MX#"_E"#ivti.SEW#"_MASK") (fwti.Vector (IMPLICIT_DEF)), ivti.RegClass:$rs1, - (ivti.Mask V0), + (ivti.Mask VMV0:$vm), GPR:$vl, ivti.Log2SEW, TA_MA)>; } } @@ -1256,11 +1256,11 @@ multiclass VPatNConvertFP2IVL_W.Predicates, GetVTypePredicates.Predicates) in def : Pat<(vti.Vector (vop (fwti.Vector fwti.RegClass:$rs1), - (fwti.Mask V0), + (fwti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_"#vti.LMul.MX#"_MASK") (vti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1, - (fwti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; + (fwti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; } } @@ -1271,11 +1271,11 @@ multiclass VPatNConvertFP2I_RM_VL_W { let Predicates = !listconcat(GetVTypePredicates.Predicates, GetVTypePredicates.Predicates) in def : Pat<(vti.Vector (vop (fwti.Vector fwti.RegClass:$rs1), - (fwti.Mask V0), (XLenVT timm:$frm), + (fwti.Mask VMV0:$vm), (XLenVT timm:$frm), VLOpFrag)), (!cast(instruction_name#"_"#vti.LMul.MX#"_MASK") (vti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1, - (fwti.Mask V0), timm:$frm, GPR:$vl, vti.Log2SEW, TA_MA)>; + (fwti.Mask VMV0:$vm), timm:$frm, GPR:$vl, vti.Log2SEW, TA_MA)>; } } @@ -1287,11 +1287,11 @@ multiclass VPatNConvertI2FPVL_W_RM.Predicates, GetVTypePredicates.Predicates) in def : Pat<(fvti.Vector (vop (iwti.Vector iwti.RegClass:$rs1), - (iwti.Mask V0), + (iwti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), iwti.RegClass:$rs1, - (iwti.Mask V0), + (iwti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, @@ -1306,11 +1306,11 @@ multiclass VPatNConvertI2FP_RM_VL_W { let Predicates = !listconcat(GetVTypePredicates.Predicates, GetVTypePredicates.Predicates) in def : Pat<(fvti.Vector (vop (iwti.Vector iwti.RegClass:$rs1), - (iwti.Mask V0), (XLenVT timm:$frm), + (iwti.Mask VMV0:$vm), (XLenVT timm:$frm), VLOpFrag)), (!cast(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), iwti.RegClass:$rs1, - (iwti.Mask V0), timm:$frm, GPR:$vl, fvti.Log2SEW, TA_MA)>; + (iwti.Mask VMV0:$vm), timm:$frm, GPR:$vl, fvti.Log2SEW, TA_MA)>; } } @@ -1320,13 +1320,13 @@ multiclass VPatReductionVL { let Predicates = GetVTypePredicates.Predicates in { def: Pat<(vti_m1.Vector (vop (vti_m1.Vector VR:$passthru), (vti.Vector vti.RegClass:$rs1), VR:$rs2, - (vti.Mask V0), VLOpFrag, + (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast(instruction_name#"_VS_"#vti.LMul.MX#"_E"#vti.SEW#"_MASK") (vti_m1.Vector VR:$passthru), (vti.Vector vti.RegClass:$rs1), (vti_m1.Vector VR:$rs2), - (vti.Mask V0), GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>; } } } @@ -1337,13 +1337,13 @@ multiclass VPatReductionVL_RM let Predicates = GetVTypePredicates.Predicates in { def: Pat<(vti_m1.Vector (vop (vti_m1.Vector VR:$passthru), (vti.Vector vti.RegClass:$rs1), VR:$rs2, - (vti.Mask V0), VLOpFrag, + (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast(instruction_name#"_VS_"#vti.LMul.MX#"_E"#vti.SEW#"_MASK") (vti_m1.Vector VR:$passthru), (vti.Vector vti.RegClass:$rs1), (vti_m1.Vector VR:$rs2), - (vti.Mask V0), + (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, @@ -1402,11 +1402,11 @@ multiclass VPatWidenReductionVL.Predicates) in { def: Pat<(wti_m1.Vector (vop (wti_m1.Vector VR:$passthru), (wti.Vector (extop (vti.Vector vti.RegClass:$rs1))), - VR:$rs2, (vti.Mask V0), VLOpFrag, + VR:$rs2, (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast(instruction_name#"_VS_"#vti.LMul.MX#"_E"#vti.SEW#"_MASK") (wti_m1.Vector VR:$passthru), (vti.Vector vti.RegClass:$rs1), - (wti_m1.Vector VR:$rs2), (vti.Mask V0), GPR:$vl, vti.Log2SEW, + (wti_m1.Vector VR:$rs2), (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>; } } @@ -1421,11 +1421,11 @@ multiclass VPatWidenReductionVL_Ext_VL.Predicates) in { def: Pat<(wti_m1.Vector (vop (wti_m1.Vector VR:$passthru), (wti.Vector (extop (vti.Vector vti.RegClass:$rs1), (vti.Mask true_mask), (XLenVT srcvalue))), - VR:$rs2, (vti.Mask V0), VLOpFrag, + VR:$rs2, (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast(instruction_name#"_VS_"#vti.LMul.MX#"_E"#vti.SEW#"_MASK") (wti_m1.Vector VR:$passthru), (vti.Vector vti.RegClass:$rs1), - (wti_m1.Vector VR:$rs2), (vti.Mask V0), GPR:$vl, vti.Log2SEW, + (wti_m1.Vector VR:$rs2), (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>; } } @@ -1440,11 +1440,11 @@ multiclass VPatWidenReductionVL_Ext_VL_RM.Predicates) in { def: Pat<(wti_m1.Vector (vop (wti_m1.Vector VR:$passthru), (wti.Vector (extop (vti.Vector vti.RegClass:$rs1), (vti.Mask true_mask), (XLenVT srcvalue))), - VR:$rs2, (vti.Mask V0), VLOpFrag, + VR:$rs2, (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast(instruction_name#"_VS_"#vti.LMul.MX#"_E"#vti.SEW#"_MASK") (wti_m1.Vector VR:$passthru), (vti.Vector vti.RegClass:$rs1), - (wti_m1.Vector VR:$rs2), (vti.Mask V0), + (wti_m1.Vector VR:$rs2), (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, @@ -1570,10 +1570,10 @@ multiclass VPatNarrowShiftExtVL_WV(instruction_name#"_WV_"#vti.LMul.MX#"_MASK") (vti.Vector (IMPLICIT_DEF)), wti.RegClass:$rs2, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; } } @@ -1616,7 +1616,7 @@ multiclass VPatMultiplyAccVL_VV_VX { foreach vti = AllIntegerVectors in { defvar suffix = vti.LMul.MX; let Predicates = GetVTypePredicates.Predicates in { - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (op vti.RegClass:$rd, (riscv_mul_vl_oneuse vti.RegClass:$rs1, vti.RegClass:$rs2, srcvalue, (vti.Mask true_mask), VLOpFrag), @@ -1624,8 +1624,8 @@ multiclass VPatMultiplyAccVL_VV_VX { vti.RegClass:$rd, vti.RegClass:$rd, VLOpFrag), (!cast(instruction_name#"_VV_"# suffix #"_MASK") vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TU_MU)>; - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TU_MU)>; + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (op vti.RegClass:$rd, (riscv_mul_vl_oneuse (SplatPat XLenVT:$rs1), vti.RegClass:$rs2, srcvalue, (vti.Mask true_mask), VLOpFrag), @@ -1633,8 +1633,8 @@ multiclass VPatMultiplyAccVL_VV_VX { vti.RegClass:$rd, vti.RegClass:$rd, VLOpFrag), (!cast(instruction_name#"_VX_"# suffix #"_MASK") vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TU_MU)>; - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TU_MU)>; + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (op vti.RegClass:$rd, (riscv_mul_vl_oneuse vti.RegClass:$rs1, vti.RegClass:$rs2, srcvalue, (vti.Mask true_mask), VLOpFrag), @@ -1642,8 +1642,8 @@ multiclass VPatMultiplyAccVL_VV_VX { vti.RegClass:$rd, undef, VLOpFrag), (!cast(instruction_name#"_VV_"# suffix #"_MASK") vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (op vti.RegClass:$rd, (riscv_mul_vl_oneuse (SplatPat XLenVT:$rs1), vti.RegClass:$rs2, srcvalue, (vti.Mask true_mask), VLOpFrag), @@ -1651,7 +1651,7 @@ multiclass VPatMultiplyAccVL_VV_VX { vti.RegClass:$rd, undef, VLOpFrag), (!cast(instruction_name#"_VX_"# suffix #"_MASK") vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } } @@ -1665,17 +1665,17 @@ multiclass VPatWidenMultiplyAddVL_VV_VX { def : Pat<(vwmacc_op (vti.Vector vti.RegClass:$rs1), (vti.Vector vti.RegClass:$rs2), (wti.Vector wti.RegClass:$rd), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast(instr_name#"_VV_"#vti.LMul.MX#"_MASK") wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vwmacc_op (SplatPat XLenVT:$rs1), (vti.Vector vti.RegClass:$rs2), (wti.Vector wti.RegClass:$rd), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast(instr_name#"_VX_"#vti.LMul.MX#"_MASK") wti.RegClass:$rd, vti.ScalarRegClass:$rs1, - vti.RegClass:$rs2, (vti.Mask V0), GPR:$vl, vti.Log2SEW, + vti.RegClass:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } @@ -1708,19 +1708,19 @@ multiclass VPatFPMulAddVL_VV_VF defvar suffix = vti.LMul.MX; let Predicates = GetVTypePredicates.Predicates in { def : Pat<(vti.Vector (vop vti.RegClass:$rs1, vti.RegClass:$rd, - vti.RegClass:$rs2, (vti.Mask V0), + vti.RegClass:$rs2, (vti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_VV_"# suffix #"_MASK") vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; def : Pat<(vti.Vector (vop (SplatFPOp vti.ScalarRegClass:$rs1), vti.RegClass:$rd, vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_V" # vti.ScalarSuffix # "_" # suffix # "_MASK") vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; } } } @@ -1730,11 +1730,11 @@ multiclass VPatFPMulAddVL_VV_VF_RM.Predicates in { def : Pat<(vti.Vector (vop vti.RegClass:$rs1, vti.RegClass:$rd, - vti.RegClass:$rs2, (vti.Mask V0), + vti.RegClass:$rs2, (vti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_VV_"# suffix #"_MASK") vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, @@ -1742,11 +1742,11 @@ multiclass VPatFPMulAddVL_VV_VF_RM(instruction_name#"_V" # vti.ScalarSuffix # "_" # suffix # "_MASK") vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, @@ -1759,34 +1759,34 @@ multiclass VPatFPMulAccVL_VV_VF { foreach vti = AllFloatVectors in { defvar suffix = vti.LMul.MX; let Predicates = GetVTypePredicates.Predicates in { - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (vop vti.RegClass:$rs1, vti.RegClass:$rs2, vti.RegClass:$rd, (vti.Mask true_mask), VLOpFrag)), vti.RegClass:$rd, vti.RegClass:$rd, VLOpFrag), (!cast(instruction_name#"_VV_"# suffix #"_MASK") vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TU_MU)>; - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TU_MU)>; + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (vop (SplatFPOp vti.ScalarRegClass:$rs1), vti.RegClass:$rs2, vti.RegClass:$rd, (vti.Mask true_mask), VLOpFrag)), vti.RegClass:$rd, vti.RegClass:$rd, VLOpFrag), (!cast(instruction_name#"_V" # vti.ScalarSuffix # "_" # suffix # "_MASK") vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TU_MU)>; - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TU_MU)>; + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (vop vti.RegClass:$rs1, vti.RegClass:$rs2, vti.RegClass:$rd, (vti.Mask true_mask), VLOpFrag)), vti.RegClass:$rd, undef, VLOpFrag), (!cast(instruction_name#"_VV_"# suffix #"_MASK") vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (vop (SplatFPOp vti.ScalarRegClass:$rs1), vti.RegClass:$rs2, vti.RegClass:$rd, (vti.Mask true_mask), VLOpFrag)), vti.RegClass:$rd, undef, VLOpFrag), (!cast(instruction_name#"_V" # vti.ScalarSuffix # "_" # suffix # "_MASK") vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } } @@ -1795,46 +1795,46 @@ multiclass VPatFPMulAccVL_VV_VF_RM { foreach vti = AllFloatVectors in { defvar suffix = vti.LMul.MX # "_E" # vti.SEW; let Predicates = GetVTypePredicates.Predicates in { - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (vop vti.RegClass:$rs1, vti.RegClass:$rs2, vti.RegClass:$rd, (vti.Mask true_mask), VLOpFrag)), vti.RegClass:$rd, vti.RegClass:$rd, VLOpFrag), (!cast(instruction_name#"_VV_"# suffix #"_MASK") vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, GPR:$vl, vti.Log2SEW, TU_MU)>; - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (vop (SplatFPOp vti.ScalarRegClass:$rs1), vti.RegClass:$rs2, vti.RegClass:$rd, (vti.Mask true_mask), VLOpFrag)), vti.RegClass:$rd, vti.RegClass:$rd, VLOpFrag), (!cast(instruction_name#"_V" # vti.ScalarSuffix # "_" # suffix # "_MASK") vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, GPR:$vl, vti.Log2SEW, TU_MU)>; - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (vop vti.RegClass:$rs1, vti.RegClass:$rs2, vti.RegClass:$rd, (vti.Mask true_mask), VLOpFrag)), vti.RegClass:$rd, undef, VLOpFrag), (!cast(instruction_name#"_VV_"# suffix #"_MASK") vti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; - def : Pat<(riscv_vmerge_vl (vti.Mask V0), + def : Pat<(riscv_vmerge_vl (vti.Mask VMV0:$vm), (vti.Vector (vop (SplatFPOp vti.ScalarRegClass:$rs1), vti.RegClass:$rs2, vti.RegClass:$rd, (vti.Mask true_mask), VLOpFrag)), vti.RegClass:$rd, undef, VLOpFrag), (!cast(instruction_name#"_V" # vti.ScalarSuffix # "_" # suffix # "_MASK") vti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, @@ -1851,18 +1851,18 @@ multiclass VPatWidenFPMulAccVL_VV_VF { GetVTypePredicates.Predicates) in { def : Pat<(vop (vti.Vector vti.RegClass:$rs1), (vti.Vector vti.RegClass:$rs2), - (wti.Vector wti.RegClass:$rd), (vti.Mask V0), + (wti.Vector wti.RegClass:$rd), (vti.Mask VMV0:$vm), VLOpFrag), (!cast(instruction_name#"_VV_"#vti.LMul.MX #"_MASK") wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; def : Pat<(vop (vti.Vector (SplatFPOp vti.ScalarRegClass:$rs1)), (vti.Vector vti.RegClass:$rs2), - (wti.Vector wti.RegClass:$rd), (vti.Mask V0), + (wti.Vector wti.RegClass:$rd), (vti.Mask VMV0:$vm), VLOpFrag), (!cast(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX #"_MASK") wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; } } } @@ -1881,22 +1881,22 @@ multiclass VPatWidenFPMulAccVL_VV_VF_RM(instruction_name#"_VV_"#suffix#"_MASK") wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, GPR:$vl, vti.Log2SEW, TA_MA)>; def : Pat<(vop (vti.Vector (SplatFPOp vti.ScalarRegClass:$rs1)), (vti.Vector vti.RegClass:$rs2), - (wti.Vector wti.RegClass:$rd), (vti.Mask V0), + (wti.Vector wti.RegClass:$rd), (vti.Mask VMV0:$vm), VLOpFrag), (!cast(instruction_name#"_V"#vti.ScalarSuffix#"_"#suffix#"_MASK") wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, @@ -1911,20 +1911,20 @@ multiclass VPatSlideVL_VX_VI { let Predicates = GetVTypePredicates.Predicates in { def : Pat<(vti.Vector (vop (vti.Vector vti.RegClass:$rd), (vti.Vector vti.RegClass:$rs1), - uimm5:$rs2, (vti.Mask V0), + uimm5:$rs2, (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast(instruction_name#"_VI_"#vti.LMul.MX#"_MASK") vti.RegClass:$rd, vti.RegClass:$rs1, uimm5:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>; def : Pat<(vti.Vector (vop (vti.Vector vti.RegClass:$rd), (vti.Vector vti.RegClass:$rs1), - GPR:$rs2, (vti.Mask V0), + GPR:$rs2, (vti.Mask VMV0:$vm), VLOpFrag, (XLenVT timm:$policy))), (!cast(instruction_name#"_VX_"#vti.LMul.MX#"_MASK") vti.RegClass:$rd, vti.RegClass:$rs1, GPR:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, (XLenVT timm:$policy))>; } } @@ -1935,10 +1935,10 @@ multiclass VPatSlide1VL_VX { let Predicates = GetVTypePredicates.Predicates in { def : Pat<(vti.Vector (vop (vti.Vector vti.RegClass:$rs3), (vti.Vector vti.RegClass:$rs1), - GPR:$rs2, (vti.Mask V0), VLOpFrag)), + GPR:$rs2, (vti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_VX_"#vti.LMul.MX#"_MASK") vti.RegClass:$rs3, vti.RegClass:$rs1, GPR:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TU_MU)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TU_MU)>; } } } @@ -1948,10 +1948,10 @@ multiclass VPatSlide1VL_VF { let Predicates = GetVTypePredicates.Predicates in { def : Pat<(vti.Vector (vop (vti.Vector vti.RegClass:$rs3), (vti.Vector vti.RegClass:$rs1), - vti.Scalar:$rs2, (vti.Mask V0), VLOpFrag)), + vti.Scalar:$rs2, (vti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX#"_MASK") vti.RegClass:$rs3, vti.RegClass:$rs1, vti.Scalar:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TU_MU)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TU_MU)>; } } } @@ -1961,16 +1961,16 @@ multiclass VPatAVGADDVL_VV_VX_RM { let Predicates = GetVTypePredicates.Predicates in { def : Pat<(vop (vti.Vector vti.RegClass:$rs1), (vti.Vector vti.RegClass:$rs2), - vti.RegClass:$passthru, (vti.Mask V0), VLOpFrag), + vti.RegClass:$passthru, (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVAADD"#suffix#"_VV_"#vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs1, vti.RegClass:$rs2, - (vti.Mask V0), vxrm, GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), vxrm, GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vop (vti.Vector vti.RegClass:$rs1), (vti.Vector (SplatPat (XLenVT GPR:$rs2))), - vti.RegClass:$passthru, (vti.Mask V0), VLOpFrag), + vti.RegClass:$passthru, (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVAADD"#suffix#"_VX_"#vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs1, GPR:$rs2, - (vti.Mask V0), vxrm, GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), vxrm, GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } } @@ -1990,16 +1990,16 @@ foreach vti = AllIntegerVectors in { let Predicates = GetVTypePredicates.Predicates in { def : Pat<(riscv_sub_vl (vti.Vector (SplatPat (XLenVT GPR:$rs2))), (vti.Vector vti.RegClass:$rs1), - vti.RegClass:$passthru, (vti.Mask V0), VLOpFrag), + vti.RegClass:$passthru, (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVRSUB_VX_"# vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs1, GPR:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_sub_vl (vti.Vector (SplatPat_simm5 simm5:$rs2)), (vti.Vector vti.RegClass:$rs1), - vti.RegClass:$passthru, (vti.Mask V0), VLOpFrag), + vti.RegClass:$passthru, (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVRSUB_VI_"# vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs1, simm5:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } @@ -2017,22 +2017,22 @@ foreach vtiToWti = AllWidenableIntVectors in { GetVTypePredicates.Predicates) in { def : Pat<(riscv_shl_vl (wti.Vector (riscv_sext_vl_oneuse (vti.Vector vti.RegClass:$rs1), - (vti.Mask V0), VLOpFrag)), + (vti.Mask VMV0:$vm), VLOpFrag)), (wti.Vector (riscv_vmv_v_x_vl (wti.Vector undef), 1, VLOpFrag)), - wti.RegClass:$passthru, (vti.Mask V0), VLOpFrag), + wti.RegClass:$passthru, (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWADD_VV_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs1, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_shl_vl (wti.Vector (riscv_zext_vl_oneuse (vti.Vector vti.RegClass:$rs1), - (vti.Mask V0), VLOpFrag)), + (vti.Mask VMV0:$vm), VLOpFrag)), (wti.Vector (riscv_vmv_v_x_vl (wti.Vector undef), 1, VLOpFrag)), - wti.RegClass:$passthru, (vti.Mask V0), VLOpFrag), + wti.RegClass:$passthru, (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWADDU_VV_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs1, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } @@ -2091,11 +2091,11 @@ foreach vtiTowti = AllWidenableIntVectors in { let Predicates = !listconcat(GetVTypePredicates.Predicates, GetVTypePredicates.Predicates) in def : Pat<(vti.Vector (riscv_trunc_vector_vl (wti.Vector wti.RegClass:$rs1), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVNSRL_WI_"#vti.LMul.MX#"_MASK") (vti.Vector (IMPLICIT_DEF)), wti.RegClass:$rs1, 0, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; } // 11.8. Vector Integer Comparison Instructions @@ -2182,41 +2182,41 @@ foreach vtiTowti = AllWidenableIntVectors in { def : Pat<(riscv_vwmaccsu_vl (vti.Vector vti.RegClass:$rs1), (SplatPat XLenVT:$rs2), (wti.Vector wti.RegClass:$rd), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWMACCUS_VX_"#vti.LMul.MX#"_MASK") wti.RegClass:$rd, vti.ScalarRegClass:$rs2, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } // 11.15. Vector Integer Merge Instructions foreach vti = AllIntegerVectors in { let Predicates = GetVTypePredicates.Predicates in { - def : Pat<(vti.Vector (riscv_vmerge_vl (vti.Mask V0), + def : Pat<(vti.Vector (riscv_vmerge_vl (vti.Mask VMV0:$vm), vti.RegClass:$rs1, vti.RegClass:$rs2, vti.RegClass:$passthru, VLOpFrag)), (!cast("PseudoVMERGE_VVM_"#vti.LMul.MX) vti.RegClass:$passthru, vti.RegClass:$rs2, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; - def : Pat<(vti.Vector (riscv_vmerge_vl (vti.Mask V0), + def : Pat<(vti.Vector (riscv_vmerge_vl (vti.Mask VMV0:$vm), (SplatPat XLenVT:$rs1), vti.RegClass:$rs2, vti.RegClass:$passthru, VLOpFrag)), (!cast("PseudoVMERGE_VXM_"#vti.LMul.MX) vti.RegClass:$passthru, vti.RegClass:$rs2, GPR:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; - def : Pat<(vti.Vector (riscv_vmerge_vl (vti.Mask V0), + def : Pat<(vti.Vector (riscv_vmerge_vl (vti.Mask VMV0:$vm), (SplatPat_simm5 simm5:$rs1), vti.RegClass:$rs2, vti.RegClass:$passthru, VLOpFrag)), (!cast("PseudoVMERGE_VIM_"#vti.LMul.MX) vti.RegClass:$passthru, vti.RegClass:$rs2, simm5:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW)>; } } @@ -2266,17 +2266,17 @@ foreach vtiTowti = AllWidenableIntVectors in { GetVTypePredicates.Predicates) in { // Rounding mode here is arbitrary since we aren't shifting out any bits. def : Pat<(vti.Vector (riscv_trunc_vector_vl_ssat (wti.Vector wti.RegClass:$rs1), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVNCLIP_WI_"#vti.LMul.MX#"_MASK") (vti.Vector (IMPLICIT_DEF)), wti.RegClass:$rs1, 0, - (vti.Mask V0), /*RNU*/0, GPR:$vl, vti.Log2SEW, TA_MA)>; + (vti.Mask VMV0:$vm), /*RNU*/0, GPR:$vl, vti.Log2SEW, TA_MA)>; def : Pat<(vti.Vector (riscv_trunc_vector_vl_usat (wti.Vector wti.RegClass:$rs1), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVNCLIPU_WI_"#vti.LMul.MX#"_MASK") (vti.Vector (IMPLICIT_DEF)), wti.RegClass:$rs1, 0, - (vti.Mask V0), /*RNU*/0, GPR:$vl, vti.Log2SEW, TA_MA)>; + (vti.Mask VMV0:$vm), /*RNU*/0, GPR:$vl, vti.Log2SEW, TA_MA)>; } } @@ -2344,39 +2344,39 @@ defm : VPatFPSetCCVL_VV_VF_FV.Predicates in { // 13.8. Vector Floating-Point Square-Root Instruction - def : Pat<(any_riscv_fsqrt_vl (vti.Vector vti.RegClass:$rs2), (vti.Mask V0), + def : Pat<(any_riscv_fsqrt_vl (vti.Vector vti.RegClass:$rs2), (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVFSQRT_V_"# vti.LMul.MX # "_E" # vti.SEW # "_MASK") (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2, - (vti.Mask V0), + (vti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, GPR:$vl, vti.Log2SEW, TA_MA)>; // 13.12. Vector Floating-Point Sign-Injection Instructions - def : Pat<(riscv_fabs_vl (vti.Vector vti.RegClass:$rs), (vti.Mask V0), + def : Pat<(riscv_fabs_vl (vti.Vector vti.RegClass:$rs), (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVFSGNJX_VV_"# vti.LMul.MX #"_E"#vti.SEW#"_MASK") (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs, - vti.RegClass:$rs, (vti.Mask V0), GPR:$vl, vti.Log2SEW, + vti.RegClass:$rs, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; // Handle fneg with VFSGNJN using the same input for both operands. - def : Pat<(riscv_fneg_vl (vti.Vector vti.RegClass:$rs), (vti.Mask V0), + def : Pat<(riscv_fneg_vl (vti.Vector vti.RegClass:$rs), (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVFSGNJN_VV_"# vti.LMul.MX#"_E"#vti.SEW #"_MASK") (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs, - vti.RegClass:$rs, (vti.Mask V0), GPR:$vl, vti.Log2SEW, + vti.RegClass:$rs, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; def : Pat<(riscv_fcopysign_vl (vti.Vector vti.RegClass:$rs1), (vti.Vector vti.RegClass:$rs2), vti.RegClass:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVFSGNJ_VV_"# vti.LMul.MX#"_E"#vti.SEW#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs1, - vti.RegClass:$rs2, (vti.Mask V0), GPR:$vl, vti.Log2SEW, + vti.RegClass:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_fcopysign_vl (vti.Vector vti.RegClass:$rs1), @@ -2393,26 +2393,26 @@ foreach vti = AllFloatVectors in { def : Pat<(riscv_fcopysign_vl (vti.Vector vti.RegClass:$rs1), (SplatFPOp vti.ScalarRegClass:$rs2), vti.RegClass:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVFSGNJ_V"#vti.ScalarSuffix#"_"# vti.LMul.MX#"_E"#vti.SEW#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs1, - vti.ScalarRegClass:$rs2, (vti.Mask V0), GPR:$vl, vti.Log2SEW, + vti.ScalarRegClass:$rs2, (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; // Rounding without exception to implement nearbyint. def : Pat<(any_riscv_vfround_noexcept_vl (vti.Vector vti.RegClass:$rs1), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVFROUND_NOEXCEPT_V_" # vti.LMul.MX #"_MASK") (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; // 14.14. Vector Floating-Point Classify Instruction def : Pat<(riscv_fclass_vl (vti.Vector vti.RegClass:$rs2), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVFCLASS_V_"# vti.LMul.MX #"_MASK") (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TA_MA)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TA_MA)>; } } @@ -2422,39 +2422,39 @@ foreach fvti = !listconcat(AllFloatVectors, AllBFloatVectors) in { // 13.15. Vector Floating-Point Merge Instruction defvar ivti = GetIntVTypeInfo.Vti; let Predicates = GetVTypePredicates.Predicates in { - def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask V0), + def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask VMV0:$vm), fvti.RegClass:$rs1, fvti.RegClass:$rs2, fvti.RegClass:$passthru, VLOpFrag)), (!cast("PseudoVMERGE_VVM_"#fvti.LMul.MX) - fvti.RegClass:$passthru, fvti.RegClass:$rs2, fvti.RegClass:$rs1, (fvti.Mask V0), + fvti.RegClass:$passthru, fvti.RegClass:$rs2, fvti.RegClass:$rs1, (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW)>; - def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask V0), + def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask VMV0:$vm), (SplatFPOp (SelectScalarFPAsInt (XLenVT GPR:$imm))), fvti.RegClass:$rs2, fvti.RegClass:$passthru, VLOpFrag)), (!cast("PseudoVMERGE_VXM_"#fvti.LMul.MX) - fvti.RegClass:$passthru, fvti.RegClass:$rs2, GPR:$imm, (fvti.Mask V0), + fvti.RegClass:$passthru, fvti.RegClass:$rs2, GPR:$imm, (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW)>; - def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask V0), + def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask VMV0:$vm), (SplatFPOp (fvti.Scalar fpimm0)), fvti.RegClass:$rs2, fvti.RegClass:$passthru, VLOpFrag)), (!cast("PseudoVMERGE_VIM_"#fvti.LMul.MX) - fvti.RegClass:$passthru, fvti.RegClass:$rs2, 0, (fvti.Mask V0), + fvti.RegClass:$passthru, fvti.RegClass:$rs2, 0, (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW)>; } } foreach fvti = AllFloatVectors in { let Predicates = GetVTypePredicates.Predicates in { - def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask V0), + def : Pat<(fvti.Vector (riscv_vmerge_vl (fvti.Mask VMV0:$vm), (SplatFPOp fvti.ScalarRegClass:$rs1), fvti.RegClass:$rs2, fvti.RegClass:$passthru, @@ -2462,7 +2462,7 @@ foreach fvti = AllFloatVectors in { (!cast("PseudoVFMERGE_V"#fvti.ScalarSuffix#"M_"#fvti.LMul.MX) fvti.RegClass:$passthru, fvti.RegClass:$rs2, (fvti.Scalar fvti.ScalarRegClass:$rs1), - (fvti.Mask V0), GPR:$vl, fvti.Log2SEW)>; + (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW)>; } } @@ -2524,11 +2524,11 @@ foreach fvtiToFWti = AllWidenableFloatVectors in { GetVTypePredicates.Predicates)) in def : Pat<(fwti.Vector (any_riscv_fpextend_vl (fvti.Vector fvti.RegClass:$rs1), - (fvti.Mask V0), + (fvti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVFWCVT_F_F_V_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fwti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1, - (fvti.Mask V0), + (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW, TA_MA)>; } @@ -2538,11 +2538,11 @@ foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in { let Predicates = [HasVInstructionsBF16Minimal] in def : Pat<(fwti.Vector (any_riscv_fpextend_vl (fvti.Vector fvti.RegClass:$rs1), - (fvti.Mask V0), + (fvti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVFWCVTBF16_F_F_V_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fwti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1, - (fvti.Mask V0), + (fvti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW, TA_MA)>; } @@ -2568,10 +2568,10 @@ foreach fvtiToFWti = AllWidenableFloatVectors in { GetVTypePredicates.Predicates)) in { def : Pat<(fvti.Vector (any_riscv_fpround_vl (fwti.Vector fwti.RegClass:$rs1), - (fwti.Mask V0), VLOpFrag)), + (fwti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVFNCVT_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1, - (fwti.Mask V0), + (fwti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, @@ -2581,10 +2581,10 @@ foreach fvtiToFWti = AllWidenableFloatVectors in { GetVTypePredicates.Predicates) in def : Pat<(fvti.Vector (any_riscv_fncvt_rod_vl (fwti.Vector fwti.RegClass:$rs1), - (fwti.Mask V0), VLOpFrag)), + (fwti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVFNCVT_ROD_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1, - (fwti.Mask V0), GPR:$vl, fvti.Log2SEW, TA_MA)>; + (fwti.Mask VMV0:$vm), GPR:$vl, fvti.Log2SEW, TA_MA)>; } } @@ -2594,10 +2594,10 @@ foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in { let Predicates = [HasVInstructionsBF16Minimal] in def : Pat<(fvti.Vector (any_riscv_fpround_vl (fwti.Vector fwti.RegClass:$rs1), - (fwti.Mask V0), VLOpFrag)), + (fwti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVFNCVTBF16_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") (fvti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1, - (fwti.Mask V0), + (fwti.Mask VMV0:$vm), // Value to indicate no rounding mode change in // RISCVInsertReadWriteCSR FRM_DYN, @@ -2700,20 +2700,20 @@ foreach mti = AllMasks in { VLOpFrag)), (!cast("PseudoVCPOP_M_" # mti.BX) VR:$rs2, GPR:$vl, mti.Log2SEW)>; - def : Pat<(XLenVT (riscv_vcpop_vl (mti.Mask VR:$rs2), (mti.Mask V0), + def : Pat<(XLenVT (riscv_vcpop_vl (mti.Mask VR:$rs2), (mti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVCPOP_M_" # mti.BX # "_MASK") - VR:$rs2, (mti.Mask V0), GPR:$vl, mti.Log2SEW)>; + VR:$rs2, (mti.Mask VMV0:$vm), GPR:$vl, mti.Log2SEW)>; // 15.3 vfirst find-first-set mask bit def : Pat<(XLenVT (riscv_vfirst_vl (mti.Mask VR:$rs2), (mti.Mask true_mask), VLOpFrag)), (!cast("PseudoVFIRST_M_" # mti.BX) VR:$rs2, GPR:$vl, mti.Log2SEW)>; - def : Pat<(XLenVT (riscv_vfirst_vl (mti.Mask VR:$rs2), (mti.Mask V0), + def : Pat<(XLenVT (riscv_vfirst_vl (mti.Mask VR:$rs2), (mti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVFIRST_M_" # mti.BX # "_MASK") - VR:$rs2, (mti.Mask V0), GPR:$vl, mti.Log2SEW)>; + VR:$rs2, (mti.Mask VMV0:$vm), GPR:$vl, mti.Log2SEW)>; } } @@ -2736,26 +2736,26 @@ foreach vti = AllIntegerVectors in { def : Pat<(vti.Vector (riscv_vrgather_vv_vl vti.RegClass:$rs2, vti.RegClass:$rs1, vti.RegClass:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVRGATHER_VV_"# vti.LMul.MX#"_E"# vti.SEW#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, GPR:$rs1, vti.RegClass:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVRGATHER_VX_"# vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, GPR:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, uimm5:$imm, vti.RegClass:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVRGATHER_VI_"# vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, uimm5:$imm, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } // emul = lmul * 16 / sew @@ -2771,11 +2771,11 @@ foreach vti = AllIntegerVectors in { (riscv_vrgatherei16_vv_vl vti.RegClass:$rs2, (ivti.Vector ivti.RegClass:$rs1), vti.RegClass:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast(inst#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, ivti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } @@ -2806,27 +2806,27 @@ foreach vti = !listconcat(AllFloatVectors, AllBFloatVectors) in { (riscv_vrgather_vv_vl vti.RegClass:$rs2, (ivti.Vector vti.RegClass:$rs1), vti.RegClass:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVRGATHER_VV_"# vti.LMul.MX#"_E"# vti.SEW#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, GPR:$rs1, vti.RegClass:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVRGATHER_VX_"# vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, GPR:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(vti.Vector (riscv_vrgather_vx_vl vti.RegClass:$rs2, uimm5:$imm, vti.RegClass:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVRGATHER_VI_"# vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, uimm5:$imm, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } defvar vlmul = vti.LMul; @@ -2842,11 +2842,11 @@ foreach vti = !listconcat(AllFloatVectors, AllBFloatVectors) in { (riscv_vrgatherei16_vv_vl vti.RegClass:$rs2, (ivti.Vector ivti.RegClass:$rs1), vti.RegClass:$passthru, - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast(inst#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, ivti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } @@ -2883,10 +2883,10 @@ def riscv_fslide1down_vl : SDNode<"RISCVISD::VFSLIDE1DOWN_VL", SDTRVVFSlide1, [ foreach vti = AllIntegerVectors in { let Predicates = GetVTypePredicates.Predicates in { - def : Pat<(vti.Vector (riscv_vid_vl (vti.Mask V0), + def : Pat<(vti.Vector (riscv_vid_vl (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVID_V_"#vti.LMul.MX#"_MASK") - (vti.Vector (IMPLICIT_DEF)), (vti.Mask V0), GPR:$vl, vti.Log2SEW, + (vti.Vector (IMPLICIT_DEF)), (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td index 430d75e5cec5b..470555769d493 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td @@ -716,12 +716,12 @@ multiclass VPatUnaryVL_V.Predicates) in { def : Pat<(vti.Vector (op (vti.Vector vti.RegClass:$rs1), (vti.Vector vti.RegClass:$passthru), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast(instruction_name#"_V_"#vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs1, - (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; @@ -736,17 +736,17 @@ foreach vti = AllIntegerVectors in { (vti.Vector vti.RegClass:$rs1), (riscv_splat_vector -1), (vti.Vector vti.RegClass:$passthru), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag), (vti.Vector vti.RegClass:$rs2), (vti.Vector vti.RegClass:$passthru), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVANDN_VV_"#vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, vti.RegClass:$rs1, - (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; @@ -755,13 +755,13 @@ foreach vti = AllIntegerVectors in { (not vti.ScalarRegClass:$rs1)), (vti.Vector vti.RegClass:$rs2), (vti.Vector vti.RegClass:$passthru), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVANDN_VX_"#vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, vti.ScalarRegClass:$rs1, - (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; @@ -769,13 +769,13 @@ foreach vti = AllIntegerVectors in { def : Pat<(vti.Vector (riscv_and_vl (riscv_splat_vector invLogicImm:$rs1), (vti.Vector vti.RegClass:$rs2), (vti.Vector vti.RegClass:$passthru), - (vti.Mask V0), + (vti.Mask VMV0:$vm), VLOpFrag)), (!cast("PseudoVANDN_VX_"#vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, invLogicImm:$rs1, - (vti.Mask V0), + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; @@ -797,12 +797,12 @@ foreach vti = AllIntegerVectors in { def : Pat<(riscv_rotl_vl vti.RegClass:$rs2, (vti.Vector (SplatPat_uimm6 uimm6:$rs1)), (vti.Vector vti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVROR_VI_"#vti.LMul.MX#"_MASK") vti.RegClass:$passthru, vti.RegClass:$rs2, (!cast("InvRot" # vti.SEW # "Imm") uimm6:$rs1), - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } defm : VPatBinaryVL_VV_VX_VI; @@ -817,90 +817,90 @@ foreach vtiToWti = AllWidenableIntVectors in { (wti.Vector (zext_oneuse (vti.Vector vti.RegClass:$rs2))), (wti.Vector (ext_oneuse (vti.Vector vti.RegClass:$rs1))), (wti.Vector wti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWSLL_VV_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs2, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_shl_vl (wti.Vector (riscv_zext_vl_oneuse (vti.Vector vti.RegClass:$rs2), - (vti.Mask V0), VLOpFrag)), + (vti.Mask VMV0:$vm), VLOpFrag)), (wti.Vector (riscv_ext_vl_oneuse (vti.Vector vti.RegClass:$rs1), - (vti.Mask V0), VLOpFrag)), + (vti.Mask VMV0:$vm), VLOpFrag)), (wti.Vector wti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWSLL_VV_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs2, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_shl_vl (wti.Vector (zext_oneuse (vti.Vector vti.RegClass:$rs2))), (wti.Vector (Low8BitsSplatPat (XLenVT GPR:$rs1))), (wti.Vector wti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWSLL_VX_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs2, GPR:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_shl_vl (wti.Vector (riscv_zext_vl_oneuse (vti.Vector vti.RegClass:$rs2), - (vti.Mask V0), VLOpFrag)), + (vti.Mask VMV0:$vm), VLOpFrag)), (wti.Vector (Low8BitsSplatPat (XLenVT GPR:$rs1))), (wti.Vector wti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWSLL_VX_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs2, GPR:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_shl_vl (wti.Vector (zext_oneuse (vti.Vector vti.RegClass:$rs2))), (wti.Vector (SplatPat_uimm5 uimm5:$rs1)), (wti.Vector wti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWSLL_VI_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs2, uimm5:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_shl_vl (wti.Vector (riscv_zext_vl_oneuse (vti.Vector vti.RegClass:$rs2), - (vti.Mask V0), VLOpFrag)), + (vti.Mask VMV0:$vm), VLOpFrag)), (wti.Vector (SplatPat_uimm5 uimm5:$rs1)), (wti.Vector wti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWSLL_VI_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs2, uimm5:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_vwsll_vl (vti.Vector vti.RegClass:$rs2), (vti.Vector vti.RegClass:$rs1), (wti.Vector wti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWSLL_VV_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs2, vti.RegClass:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_vwsll_vl (vti.Vector vti.RegClass:$rs2), (vti.Vector (Low8BitsSplatPat (XLenVT GPR:$rs1))), (wti.Vector wti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWSLL_VX_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs2, GPR:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; def : Pat<(riscv_vwsll_vl (vti.Vector vti.RegClass:$rs2), (vti.Vector (SplatPat_uimm5 uimm5:$rs1)), (wti.Vector wti.RegClass:$passthru), - (vti.Mask V0), VLOpFrag), + (vti.Mask VMV0:$vm), VLOpFrag), (!cast("PseudoVWSLL_VI_"#vti.LMul.MX#"_MASK") wti.RegClass:$passthru, vti.RegClass:$rs2, uimm5:$rs1, - (vti.Mask V0), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; + (vti.Mask VMV0:$vm), GPR:$vl, vti.Log2SEW, TAIL_AGNOSTIC)>; } } @@ -1046,12 +1046,12 @@ multiclass VPatBinaryV_VI_VROL; } } diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp index 4a69bdeb76161..167dbb53c5950 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp @@ -137,6 +137,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() { initializeRISCVExpandPseudoPass(*PR); initializeRISCVVectorPeepholePass(*PR); initializeRISCVVLOptimizerPass(*PR); + initializeRISCVVMV0EliminationPass(*PR); initializeRISCVInsertVSETVLIPass(*PR); initializeRISCVInsertReadWriteCSRPass(*PR); initializeRISCVInsertWriteVXRMPass(*PR); @@ -587,6 +588,8 @@ void RISCVPassConfig::addPreEmitPass2() { void RISCVPassConfig::addMachineSSAOptimization() { addPass(createRISCVVectorPeepholePass()); + // TODO: Move this to pre regalloc + addPass(createRISCVVMV0EliminationPass()); TargetPassConfig::addMachineSSAOptimization(); @@ -599,6 +602,10 @@ void RISCVPassConfig::addMachineSSAOptimization() { } void RISCVPassConfig::addPreRegAlloc() { + // TODO: Move this as late as possible before regalloc + if (TM->getOptLevel() == CodeGenOptLevel::None) + addPass(createRISCVVMV0EliminationPass()); + addPass(createRISCVPreRAExpandPseudoPass()); if (TM->getOptLevel() != CodeGenOptLevel::None) { addPass(createRISCVMergeBaseOffsetOptPass()); diff --git a/llvm/lib/Target/RISCV/RISCVVMV0Elimination.cpp b/llvm/lib/Target/RISCV/RISCVVMV0Elimination.cpp new file mode 100644 index 0000000000000..a2daa19614d0a --- /dev/null +++ b/llvm/lib/Target/RISCV/RISCVVMV0Elimination.cpp @@ -0,0 +1,172 @@ +//===- RISCVVMV0Elimination.cpp - VMV0 Elimination -----------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===---------------------------------------------------------------------===// +// +// Mask operands in vector pseudos have to be in v0. We select them as a virtual +// register in the singleton vmv0 register class instead of copying them to $v0 +// straight away, to make optimizing masks easier. +// +// However register coalescing may end up coleascing copies into vmv0, resulting +// in instructions with multiple uses of vmv0 that the register allocator can't +// allocate: +// +// %x:vrnov0 = PseudoVADD_VV_M1_MASK %0:vrnov0, %1:vr, %2:vmv0, %3:vmv0, ... +// +// To avoid this, this pass replaces any uses* of vmv0 with copies to $v0 before +// register coalescing and allocation: +// +// %x:vrnov0 = PseudoVADD_VV_M1_MASK %0:vrnov0, %1:vr, %2:vr, %3:vmv0, ... +// -> +// $v0 = COPY %3:vr +// %x:vrnov0 = PseudoVADD_VV_M1_MASK %0:vrnov0, %1:vr, %2:vr, $0, ... +// +// * The only uses of vmv0 left behind are when used for inline asm with the vm +// constraint. +// +//===---------------------------------------------------------------------===// + +#include "RISCV.h" +#include "RISCVSubtarget.h" +#ifndef NDEBUG +#include "llvm/ADT/PostOrderIterator.h" +#endif +#include "llvm/CodeGen/MachineFunctionPass.h" + +using namespace llvm; + +#define DEBUG_TYPE "riscv-vmv0-elimination" + +namespace { + +class RISCVVMV0Elimination : public MachineFunctionPass { +public: + static char ID; + RISCVVMV0Elimination() : MachineFunctionPass(ID) {} + + bool runOnMachineFunction(MachineFunction &MF) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + MachineFunctionProperties getRequiredProperties() const override { + // TODO: We could move this closer to regalloc, out of SSA, which would + // allow scheduling past mask operands. We would need to preserve live + // intervals. + return MachineFunctionProperties().set( + MachineFunctionProperties::Property::IsSSA); + } +}; + +} // namespace + +char RISCVVMV0Elimination::ID = 0; + +INITIALIZE_PASS(RISCVVMV0Elimination, DEBUG_TYPE, "RISC-V VMV0 Elimination", + false, false) + +FunctionPass *llvm::createRISCVVMV0EliminationPass() { + return new RISCVVMV0Elimination(); +} + +static bool isVMV0(const MCOperandInfo &MCOI) { + return MCOI.RegClass == RISCV::VMV0RegClassID; +} + +bool RISCVVMV0Elimination::runOnMachineFunction(MachineFunction &MF) { + if (skipFunction(MF.getFunction())) + return false; + + // Skip if the vector extension is not enabled. + const RISCVSubtarget *ST = &MF.getSubtarget(); + if (!ST->hasVInstructions()) + return false; + + MachineRegisterInfo &MRI = MF.getRegInfo(); + const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo(); + const TargetInstrInfo *TII = ST->getInstrInfo(); + +#ifndef NDEBUG + // Assert that we won't clobber any existing reads of v0 where we need to + // insert copies. + ReversePostOrderTraversal RPOT(&*MF.begin()); + for (MachineBasicBlock *MBB : RPOT) { + bool V0Clobbered = false; + for (MachineInstr &MI : *MBB) { + assert(!(MI.readsRegister(RISCV::V0, TRI) && V0Clobbered) && + "Inserting a copy to v0 would clobber a read"); + if (MI.modifiesRegister(RISCV::V0, TRI)) + V0Clobbered = false; + + if (any_of(MI.getDesc().operands(), isVMV0)) + V0Clobbered = true; + } + + assert(!(V0Clobbered && + any_of(MBB->successors(), + [](auto *Succ) { return Succ->isLiveIn(RISCV::V0); })) && + "Clobbered a v0 used in a successor"); + } +#endif + + bool MadeChange = false; + + // For any instruction with a vmv0 operand, replace it with a copy to v0. + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &MI : MBB) { + assert(count_if(MI.getDesc().operands(), isVMV0) < 2 && + "Expected only one or zero vmv0 operands"); + + for (auto [OpNo, MCOI] : enumerate(MI.getDesc().operands())) { + if (isVMV0(MCOI)) { + MachineOperand &MO = MI.getOperand(OpNo); + Register Src = MO.getReg(); + assert(MO.isUse() && MO.getSubReg() == RISCV::NoSubRegister && + Src.isVirtual() && "vmv0 use in unexpected form"); + + // Peek through a single copy to match what isel does. + if (MachineInstr *SrcMI = MRI.getVRegDef(Src); + SrcMI->isCopy() && SrcMI->getOperand(1).getReg().isVirtual()) { + assert(SrcMI->getOperand(1).getSubReg() == RISCV::NoSubRegister); + Src = SrcMI->getOperand(1).getReg(); + } + + BuildMI(MBB, MI, MI.getDebugLoc(), TII->get(RISCV::COPY), RISCV::V0) + .addReg(Src); + + MO.setReg(RISCV::V0); + MadeChange = true; + break; + } + } + } + } + + if (!MadeChange) + return false; + + // Now that any constraints requiring vmv0 are gone, eliminate any uses of + // vmv0 by recomputing the reg class. + // The only remaining uses should be around inline asm. + for (MachineBasicBlock &MBB : MF) { + for (MachineInstr &MI : MBB) { + for (MachineOperand &MO : MI.uses()) { + if (MO.isReg() && MO.getReg().isVirtual() && + MRI.getRegClass(MO.getReg()) == &RISCV::VMV0RegClass) { + MRI.recomputeRegClass(MO.getReg()); + assert(MRI.getRegClass(MO.getReg()) != &RISCV::VMV0RegClass || + MI.isInlineAsm() || + MRI.getVRegDef(MO.getReg())->isInlineAsm() && + "Non-inline-asm use of vmv0 left behind"); + } + } + } + } + + return true; +} diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp index bb2d1717c3b1e..a4e7219c39f37 100644 --- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp +++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp @@ -73,9 +73,7 @@ class RISCVVectorPeephole : public MachineFunctionPass { bool isAllOnesMask(const MachineInstr *MaskDef) const; std::optional getConstant(const MachineOperand &VL) const; bool ensureDominates(const MachineOperand &Use, MachineInstr &Src) const; - - /// Maps uses of V0 to the corresponding def of V0. - DenseMap V0Defs; + bool isKnownSameDefs(const MachineOperand &A, const MachineOperand &B) const; }; } // namespace @@ -268,14 +266,8 @@ bool RISCVVectorPeephole::convertToVLMAX(MachineInstr &MI) const { } bool RISCVVectorPeephole::isAllOnesMask(const MachineInstr *MaskDef) const { - assert(MaskDef && MaskDef->isCopy() && - MaskDef->getOperand(0).getReg() == RISCV::V0); - Register SrcReg = TRI->lookThruCopyLike(MaskDef->getOperand(1).getReg(), MRI); - if (!SrcReg.isVirtual()) - return false; - MaskDef = MRI->getVRegDef(SrcReg); - if (!MaskDef) - return false; + while (MaskDef->isCopy() && MaskDef->getOperand(1).getReg().isVirtual()) + MaskDef = MRI->getVRegDef(MaskDef->getOperand(1).getReg()); // TODO: Check that the VMSET is the expected bitwidth? The pseudo has // undefined behaviour if it's the wrong bitwidth, so we could choose to @@ -372,8 +364,7 @@ bool RISCVVectorPeephole::convertAllOnesVMergeToVMv(MachineInstr &MI) const { unsigned NewOpc = getVMV_V_VOpcodeForVMERGE_VVM(MI); if (!NewOpc) return false; - assert(MI.getOperand(4).isReg() && MI.getOperand(4).getReg() == RISCV::V0); - if (!isAllOnesMask(V0Defs.lookup(&MI))) + if (!isAllOnesMask(MRI->getVRegDef(MI.getOperand(4).getReg()))) return false; MI.setDesc(TII->get(NewOpc)); @@ -390,6 +381,15 @@ bool RISCVVectorPeephole::convertAllOnesVMergeToVMv(MachineInstr &MI) const { return true; } +bool RISCVVectorPeephole::isKnownSameDefs(const MachineOperand &A, + const MachineOperand &B) const { + if (A.getReg().isPhysical() || B.getReg().isPhysical()) + return false; + + return TRI->lookThruCopyLike(A.getReg(), MRI) == + TRI->lookThruCopyLike(B.getReg(), MRI); +} + /// If a PseudoVMERGE_VVM's true operand is a masked pseudo and both have the /// same mask, and the masked pseudo's passthru is the same as the false /// operand, we can convert the PseudoVMERGE_VVM to a PseudoVMV_V_V. @@ -404,14 +404,18 @@ bool RISCVVectorPeephole::convertSameMaskVMergeToVMv(MachineInstr &MI) { if (!NewOpc) return false; MachineInstr *True = MRI->getVRegDef(MI.getOperand(3).getReg()); - if (!True || True->getParent() != MI.getParent() || - !RISCV::getMaskedPseudoInfo(True->getOpcode()) || !hasSameEEW(MI, *True)) + + if (!True || True->getParent() != MI.getParent()) return false; - const MachineInstr *TrueV0Def = V0Defs.lookup(True); - const MachineInstr *MIV0Def = V0Defs.lookup(&MI); - assert(TrueV0Def && TrueV0Def->isCopy() && MIV0Def && MIV0Def->isCopy()); - if (TrueV0Def->getOperand(1).getReg() != MIV0Def->getOperand(1).getReg()) + auto *TrueMaskedInfo = RISCV::getMaskedPseudoInfo(True->getOpcode()); + if (!TrueMaskedInfo || !hasSameEEW(MI, *True)) + return false; + + const MachineOperand &TrueMask = + True->getOperand(TrueMaskedInfo->MaskOpIdx + True->getNumExplicitDefs()); + const MachineOperand &MIMask = MI.getOperand(4); + if (!isKnownSameDefs(TrueMask, MIMask)) return false; // True's passthru needs to be equivalent to False @@ -450,7 +454,8 @@ bool RISCVVectorPeephole::convertToUnmasked(MachineInstr &MI) const { if (!I) return false; - if (!isAllOnesMask(V0Defs.lookup(&MI))) + if (!isAllOnesMask(MRI->getVRegDef( + MI.getOperand(I->MaskOpIdx + MI.getNumExplicitDefs()).getReg()))) return false; // There are two classes of pseudos in the table - compares and @@ -575,7 +580,6 @@ bool RISCVVectorPeephole::foldUndefPassthruVMV_V_V(MachineInstr &MI) { MRI->replaceRegWith(MI.getOperand(0).getReg(), MI.getOperand(2).getReg()); MI.eraseFromParent(); - V0Defs.erase(&MI); return true; } @@ -645,7 +649,6 @@ bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) { MRI->replaceRegWith(MI.getOperand(0).getReg(), Src->getOperand(0).getReg()); MI.eraseFromParent(); - V0Defs.erase(&MI); return true; } @@ -665,24 +668,6 @@ bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) { bool Changed = false; - // Masked pseudos coming out of isel will have their mask operand in the form: - // - // $v0:vr = COPY %mask:vr - // %x:vr = Pseudo_MASK %a:vr, %b:br, $v0:vr - // - // Because $v0 isn't in SSA, keep track of its definition at each use so we - // can check mask operands. - for (const MachineBasicBlock &MBB : MF) { - const MachineInstr *CurrentV0Def = nullptr; - for (const MachineInstr &MI : MBB) { - if (MI.readsRegister(RISCV::V0, TRI)) - V0Defs[&MI] = CurrentV0Def; - - if (MI.definesRegister(RISCV::V0, TRI)) - CurrentV0Def = &MI; - } - } - for (MachineBasicBlock &MBB : MF) { for (MachineInstr &MI : make_early_inc_range(MBB)) { Changed |= convertToVLMAX(MI); diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/select.mir b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/select.mir index 42bf321228705..f8061462c6220 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/select.mir +++ b/llvm/test/CodeGen/RISCV/GlobalISel/instruction-select/rvv/select.mir @@ -10,20 +10,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv1i8 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 3 /* e8 */ ; RV32I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF4_]] ; RV32I-NEXT: PseudoRET implicit $v8 ; ; RV64I-LABEL: name: select_nxv1i8 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 3 /* e8 */ ; RV64I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF4_]] ; RV64I-NEXT: PseudoRET implicit $v8 %0:vrb() = G_IMPLICIT_DEF @@ -41,20 +39,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv4i8 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 3 /* e8 */ ; RV32I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_M1_]] ; RV32I-NEXT: PseudoRET implicit $v8 ; ; RV64I-LABEL: name: select_nxv4i8 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 3 /* e8 */ ; RV64I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_M1_]] ; RV64I-NEXT: PseudoRET implicit $v8 %0:vrb() = G_IMPLICIT_DEF @@ -72,20 +68,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv16i8 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm4 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm4nov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 3 /* e8 */ ; RV32I-NEXT: $v8m4 = COPY [[PseudoVMERGE_VVM_M4_]] ; RV32I-NEXT: PseudoRET implicit $v8m4 ; ; RV64I-LABEL: name: select_nxv16i8 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm4 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm4nov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 3 /* e8 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 3 /* e8 */ ; RV64I-NEXT: $v8m4 = COPY [[PseudoVMERGE_VVM_M4_]] ; RV64I-NEXT: PseudoRET implicit $v8m4 %0:vrb() = G_IMPLICIT_DEF @@ -103,20 +97,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv64i8 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 4 /* e16 */ ; RV32I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF4_]] ; RV32I-NEXT: PseudoRET implicit $v8 ; ; RV64I-LABEL: name: select_nxv64i8 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_MF4_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF4 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 4 /* e16 */ ; RV64I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF4_]] ; RV64I-NEXT: PseudoRET implicit $v8 %0:vrb() = G_IMPLICIT_DEF @@ -134,20 +126,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv2i16 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 4 /* e16 */ ; RV32I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_M1_]] ; RV32I-NEXT: PseudoRET implicit $v8 ; ; RV64I-LABEL: name: select_nxv2i16 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 4 /* e16 */ ; RV64I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_M1_]] ; RV64I-NEXT: PseudoRET implicit $v8 %0:vrb() = G_IMPLICIT_DEF @@ -165,20 +155,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv8i16 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm4 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm4nov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 4 /* e16 */ ; RV32I-NEXT: $v8m4 = COPY [[PseudoVMERGE_VVM_M4_]] ; RV32I-NEXT: PseudoRET implicit $v8m4 ; ; RV64I-LABEL: name: select_nxv8i16 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm4 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm4nov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 4 /* e16 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_M4_:%[0-9]+]]:vrm4nov0 = PseudoVMERGE_VVM_M4 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 4 /* e16 */ ; RV64I-NEXT: $v8m4 = COPY [[PseudoVMERGE_VVM_M4_]] ; RV64I-NEXT: PseudoRET implicit $v8m4 %0:vrb() = G_IMPLICIT_DEF @@ -196,20 +184,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv32i16 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_MF2_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_MF2_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF2 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 5 /* e32 */ ; RV32I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF2_]] ; RV32I-NEXT: PseudoRET implicit $v8 ; ; RV64I-LABEL: name: select_nxv32i16 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vr = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrnov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_MF2_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_MF2_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_MF2 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 5 /* e32 */ ; RV64I-NEXT: $v8 = COPY [[PseudoVMERGE_VVM_MF2_]] ; RV64I-NEXT: PseudoRET implicit $v8 %0:vrb() = G_IMPLICIT_DEF @@ -227,20 +213,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv2i32 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm2 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm2nov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 5 /* e32 */ ; RV32I-NEXT: $v8m2 = COPY [[PseudoVMERGE_VVM_M2_]] ; RV32I-NEXT: PseudoRET implicit $v8m2 ; ; RV64I-LABEL: name: select_nxv2i32 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm2 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm2nov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 5 /* e32 */ ; RV64I-NEXT: $v8m2 = COPY [[PseudoVMERGE_VVM_M2_]] ; RV64I-NEXT: PseudoRET implicit $v8m2 %0:vrb() = G_IMPLICIT_DEF @@ -258,20 +242,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv8i32 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm8 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm8nov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 5 /* e32 */ ; RV32I-NEXT: $v8m8 = COPY [[PseudoVMERGE_VVM_M8_]] ; RV32I-NEXT: PseudoRET implicit $v8m8 ; ; RV64I-LABEL: name: select_nxv8i32 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm8 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm8nov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 5 /* e32 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 5 /* e32 */ ; RV64I-NEXT: $v8m8 = COPY [[PseudoVMERGE_VVM_M8_]] ; RV64I-NEXT: PseudoRET implicit $v8m8 %0:vrb() = G_IMPLICIT_DEF @@ -289,20 +271,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv1i64 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm2 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm2nov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 6 /* e64 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 6 /* e64 */ ; RV32I-NEXT: $v8m2 = COPY [[PseudoVMERGE_VVM_M2_]] ; RV32I-NEXT: PseudoRET implicit $v8m2 ; ; RV64I-LABEL: name: select_nxv1i64 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm2 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm2nov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 6 /* e64 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_M2_:%[0-9]+]]:vrm2nov0 = PseudoVMERGE_VVM_M2 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 6 /* e64 */ ; RV64I-NEXT: $v8m2 = COPY [[PseudoVMERGE_VVM_M2_]] ; RV64I-NEXT: PseudoRET implicit $v8m2 %0:vrb() = G_IMPLICIT_DEF @@ -320,20 +300,18 @@ tracksRegLiveness: true body: | bb.0.entry: ; RV32I-LABEL: name: select_nxv4i64 - ; RV32I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV32I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF1:%[0-9]+]]:vrm8 = IMPLICIT_DEF ; RV32I-NEXT: [[DEF2:%[0-9]+]]:vrm8nov0 = IMPLICIT_DEF - ; RV32I-NEXT: $v0 = COPY [[DEF]] - ; RV32I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 6 /* e64 */ + ; RV32I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 6 /* e64 */ ; RV32I-NEXT: $v8m8 = COPY [[PseudoVMERGE_VVM_M8_]] ; RV32I-NEXT: PseudoRET implicit $v8m8 ; ; RV64I-LABEL: name: select_nxv4i64 - ; RV64I: [[DEF:%[0-9]+]]:vr = IMPLICIT_DEF + ; RV64I: [[DEF:%[0-9]+]]:vmv0 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF1:%[0-9]+]]:vrm8 = IMPLICIT_DEF ; RV64I-NEXT: [[DEF2:%[0-9]+]]:vrm8nov0 = IMPLICIT_DEF - ; RV64I-NEXT: $v0 = COPY [[DEF]] - ; RV64I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], $v0, -1, 6 /* e64 */ + ; RV64I-NEXT: [[PseudoVMERGE_VVM_M8_:%[0-9]+]]:vrm8nov0 = PseudoVMERGE_VVM_M8 [[DEF2]], [[DEF1]], [[DEF1]], [[DEF]], -1, 6 /* e64 */ ; RV64I-NEXT: $v8m8 = COPY [[PseudoVMERGE_VVM_M8_]] ; RV64I-NEXT: PseudoRET implicit $v8m8 %0:vrb() = G_IMPLICIT_DEF diff --git a/llvm/test/CodeGen/RISCV/O0-pipeline.ll b/llvm/test/CodeGen/RISCV/O0-pipeline.ll index f60def9d546f8..a50c303819f23 100644 --- a/llvm/test/CodeGen/RISCV/O0-pipeline.ll +++ b/llvm/test/CodeGen/RISCV/O0-pipeline.ll @@ -39,6 +39,7 @@ ; CHECK-NEXT: RISC-V DAG->DAG Pattern Instruction Selection ; CHECK-NEXT: Finalize ISel and expand pseudo-instructions ; CHECK-NEXT: Local Stack Slot Allocation +; CHECK-NEXT: RISC-V VMV0 Elimination ; CHECK-NEXT: RISC-V Pre-RA pseudo instruction expansion pass ; CHECK-NEXT: RISC-V Insert Read/Write CSR Pass ; CHECK-NEXT: RISC-V Insert Write VXRM Pass diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll index 668c734612447..2646dfeca4eb6 100644 --- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll +++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll @@ -97,6 +97,7 @@ ; CHECK-NEXT: RISC-V DAG->DAG Pattern Instruction Selection ; CHECK-NEXT: Finalize ISel and expand pseudo-instructions ; CHECK-NEXT: RISC-V Vector Peephole Optimization +; CHECK-NEXT: RISC-V VMV0 Elimination ; CHECK-NEXT: Lazy Machine Block Frequency Analysis ; CHECK-NEXT: Early Tail Duplication ; CHECK-NEXT: Optimize machine instruction PHIs diff --git a/llvm/test/CodeGen/RISCV/rvv/copyprop.mir b/llvm/test/CodeGen/RISCV/rvv/copyprop.mir index a9da6c305aac3..0b905b57f92b8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/copyprop.mir +++ b/llvm/test/CodeGen/RISCV/rvv/copyprop.mir @@ -45,8 +45,8 @@ body: | %3:vr = COPY $v8 %17:vr = PseudoVSLL_VI_M1 undef $noreg, %3, 5, 1, 6 /* e64 */, 0 %22:vr = PseudoVMSNE_VI_M1 %3, 0, 1, 6 /* e64 */ - $v0 = COPY %22 - %25:vrnov0 = PseudoVMERGE_VIM_M1 undef $noreg, %17, -1, $v0, 1, 6 /* e64 */ + %23:vmv0 = COPY %22 + %25:vrnov0 = PseudoVMERGE_VIM_M1 undef $noreg, %17, -1, %23, 1, 6 /* e64 */ %29:vr = PseudoVC_V_X_SE_M1 3, 31, %2, 1, 6 /* e64 */, implicit-def dead $sf_vcix_state, implicit $sf_vcix_state %30:vr = PseudoVMV_V_I_M1 undef $noreg, 0, 1, 6 /* e64 */, 0 BGEU %1, $x0, %bb.2 diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll index 80a9143d1ad8b..fe65da0d330f1 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-nearbyint-vp.ll @@ -569,59 +569,69 @@ declare <32 x double> @llvm.vp.nearbyint.v32f64(<32 x double>, <32 x i1>, i32) define <32 x double> @vp_nearbyint_v32f64(<32 x double> %va, <32 x i1> %m, i32 zeroext %evl) { ; CHECK-LABEL: vp_nearbyint_v32f64: ; CHECK: # %bb.0: +; CHECK-NEXT: addi sp, sp, -16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: csrr a1, vlenb +; CHECK-NEXT: slli a1, a1, 4 +; CHECK-NEXT: sub sp, sp, a1 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb ; CHECK-NEXT: vsetivli zero, 2, e8, mf4, ta, ma -; CHECK-NEXT: vmv1r.v v6, v0 +; CHECK-NEXT: vmv1r.v v25, v0 +; CHECK-NEXT: csrr a1, vlenb +; CHECK-NEXT: slli a1, a1, 3 +; CHECK-NEXT: add a1, sp, a1 +; CHECK-NEXT: addi a1, a1, 16 +; CHECK-NEXT: vs8r.v v16, (a1) # Unknown-size Folded Spill ; CHECK-NEXT: li a2, 16 -; CHECK-NEXT: vslidedown.vi v7, v0, 2 +; CHECK-NEXT: vslidedown.vi v24, v0, 2 ; CHECK-NEXT: mv a1, a0 ; CHECK-NEXT: bltu a0, a2, .LBB26_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: li a1, 16 ; CHECK-NEXT: .LBB26_2: -; CHECK-NEXT: addi sp, sp, -16 -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: csrr a2, vlenb -; CHECK-NEXT: slli a2, a2, 3 -; CHECK-NEXT: sub sp, sp, a2 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb -; CHECK-NEXT: vmv1r.v v0, v6 -; CHECK-NEXT: lui a2, %hi(.LCPI26_0) -; CHECK-NEXT: fld fa5, %lo(.LCPI26_0)(a2) +; CHECK-NEXT: vmv1r.v v0, v25 ; CHECK-NEXT: vsetvli zero, a1, e64, m8, ta, ma -; CHECK-NEXT: vfabs.v v24, v8, v0.t +; CHECK-NEXT: vfabs.v v16, v8, v0.t +; CHECK-NEXT: lui a1, %hi(.LCPI26_0) +; CHECK-NEXT: fld fa5, %lo(.LCPI26_0)(a1) +; CHECK-NEXT: addi a1, a0, -16 +; CHECK-NEXT: sltu a0, a0, a1 +; CHECK-NEXT: addi a0, a0, -1 +; CHECK-NEXT: and a0, a0, a1 ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, mu -; CHECK-NEXT: vmflt.vf v6, v24, fa5, v0.t +; CHECK-NEXT: vmflt.vf v25, v16, fa5, v0.t ; CHECK-NEXT: frflags a1 -; CHECK-NEXT: vmv1r.v v0, v6 +; CHECK-NEXT: vmv1r.v v0, v25 ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vfcvt.x.f.v v24, v8, v0.t -; CHECK-NEXT: addi a2, sp, 16 -; CHECK-NEXT: vs8r.v v24, (a2) # Unknown-size Folded Spill -; CHECK-NEXT: addi a2, a0, -16 -; CHECK-NEXT: sltu a0, a0, a2 -; CHECK-NEXT: addi a0, a0, -1 -; CHECK-NEXT: and a0, a0, a2 -; CHECK-NEXT: addi a2, sp, 16 -; CHECK-NEXT: vl8r.v v24, (a2) # Unknown-size Folded Reload -; CHECK-NEXT: vfcvt.f.x.v v24, v24, v0.t +; CHECK-NEXT: vfcvt.x.f.v v16, v8, v0.t +; CHECK-NEXT: vfcvt.f.x.v v16, v16, v0.t ; CHECK-NEXT: fsflags a1 ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, mu -; CHECK-NEXT: vfsgnj.vv v8, v24, v8, v0.t -; CHECK-NEXT: vmv1r.v v0, v7 +; CHECK-NEXT: vfsgnj.vv v8, v16, v8, v0.t +; CHECK-NEXT: addi a1, sp, 16 +; CHECK-NEXT: vs8r.v v8, (a1) # Unknown-size Folded Spill +; CHECK-NEXT: vmv1r.v v0, v24 +; CHECK-NEXT: csrr a1, vlenb +; CHECK-NEXT: slli a1, a1, 3 +; CHECK-NEXT: add a1, sp, a1 +; CHECK-NEXT: addi a1, a1, 16 +; CHECK-NEXT: vl8r.v v16, (a1) # Unknown-size Folded Reload ; CHECK-NEXT: vsetvli zero, a0, e64, m8, ta, ma -; CHECK-NEXT: vfabs.v v24, v16, v0.t +; CHECK-NEXT: vfabs.v v8, v16, v0.t ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, mu -; CHECK-NEXT: vmflt.vf v7, v24, fa5, v0.t +; CHECK-NEXT: vmflt.vf v24, v8, fa5, v0.t ; CHECK-NEXT: frflags a0 -; CHECK-NEXT: vmv1r.v v0, v7 +; CHECK-NEXT: vmv1r.v v0, v24 ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vfcvt.x.f.v v24, v16, v0.t -; CHECK-NEXT: vfcvt.f.x.v v24, v24, v0.t +; CHECK-NEXT: vfcvt.x.f.v v8, v16, v0.t +; CHECK-NEXT: vfcvt.f.x.v v8, v8, v0.t ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, mu -; CHECK-NEXT: vfsgnj.vv v16, v24, v16, v0.t +; CHECK-NEXT: vfsgnj.vv v16, v8, v16, v0.t ; CHECK-NEXT: fsflags a0 +; CHECK-NEXT: addi a0, sp, 16 +; CHECK-NEXT: vl8r.v v8, (a0) # Unknown-size Folded Reload ; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 +; CHECK-NEXT: slli a0, a0, 4 ; CHECK-NEXT: add sp, sp, a0 ; CHECK-NEXT: .cfi_def_cfa sp, 16 ; CHECK-NEXT: addi sp, sp, 16 diff --git a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll index 807a3e460b153..4ea3269cec0b1 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fnearbyint-sdnode.ll @@ -130,50 +130,35 @@ define @nearbyint_nxv16bf16( %x) { define @nearbyint_nxv32bf16( %x) { ; CHECK-LABEL: nearbyint_nxv32bf16: ; CHECK: # %bb.0: -; CHECK-NEXT: addi sp, sp, -16 -; CHECK-NEXT: .cfi_def_cfa_offset 16 -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: sub sp, sp, a0 -; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb ; CHECK-NEXT: vsetvli a0, zero, e16, m4, ta, ma ; CHECK-NEXT: vfwcvtbf16.f.f.v v16, v8 ; CHECK-NEXT: lui a0, 307200 +; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; CHECK-NEXT: vfabs.v v24, v16 ; CHECK-NEXT: fmv.w.x fa5, a0 +; CHECK-NEXT: vmflt.vf v0, v24, fa5 ; CHECK-NEXT: frflags a0 +; CHECK-NEXT: vfcvt.x.f.v v24, v16, v0.t +; CHECK-NEXT: vfcvt.f.x.v v24, v24, v0.t +; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, mu +; CHECK-NEXT: vfsgnj.vv v16, v24, v16, v0.t +; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; CHECK-NEXT: vfwcvtbf16.f.f.v v24, v12 ; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfabs.v v8, v16 -; CHECK-NEXT: vmflt.vf v0, v8, fa5 ; CHECK-NEXT: vfabs.v v8, v24 -; CHECK-NEXT: vmflt.vf v7, v8, fa5 -; CHECK-NEXT: vfcvt.x.f.v v8, v16, v0.t -; CHECK-NEXT: vfcvt.f.x.v v8, v8, v0.t +; CHECK-NEXT: vmflt.vf v0, v8, fa5 ; CHECK-NEXT: fsflags a0 -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, mu -; CHECK-NEXT: vfsgnj.vv v16, v8, v16, v0.t ; CHECK-NEXT: frflags a0 -; CHECK-NEXT: vmv1r.v v0, v7 -; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; CHECK-NEXT: vfcvt.x.f.v v8, v24, v0.t -; CHECK-NEXT: addi a1, sp, 16 -; CHECK-NEXT: vs8r.v v8, (a1) # Unknown-size Folded Spill ; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; CHECK-NEXT: vfncvtbf16.f.f.w v8, v16 -; CHECK-NEXT: vl8r.v v16, (a1) # Unknown-size Folded Reload ; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; CHECK-NEXT: vfcvt.x.f.v v16, v24, v0.t ; CHECK-NEXT: vfcvt.f.x.v v16, v16, v0.t ; CHECK-NEXT: vsetvli zero, zero, e32, m8, ta, mu ; CHECK-NEXT: vfsgnj.vv v24, v16, v24, v0.t ; CHECK-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; CHECK-NEXT: vfncvtbf16.f.f.w v12, v24 ; CHECK-NEXT: fsflags a0 -; CHECK-NEXT: csrr a0, vlenb -; CHECK-NEXT: slli a0, a0, 3 -; CHECK-NEXT: add sp, sp, a0 -; CHECK-NEXT: .cfi_def_cfa sp, 16 -; CHECK-NEXT: addi sp, sp, 16 -; CHECK-NEXT: .cfi_def_cfa_offset 0 ; CHECK-NEXT: ret %a = call @llvm.nearbyint.nxv32bf16( %x) ret %a @@ -392,50 +377,35 @@ define @nearbyint_nxv32f16( %x) { ; ; ZVFHMIN-LABEL: nearbyint_nxv32f16: ; ZVFHMIN: # %bb.0: -; ZVFHMIN-NEXT: addi sp, sp, -16 -; ZVFHMIN-NEXT: .cfi_def_cfa_offset 16 -; ZVFHMIN-NEXT: csrr a0, vlenb -; ZVFHMIN-NEXT: slli a0, a0, 3 -; ZVFHMIN-NEXT: sub sp, sp, a0 -; ZVFHMIN-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb ; ZVFHMIN-NEXT: vsetvli a0, zero, e16, m4, ta, ma ; ZVFHMIN-NEXT: vfwcvt.f.f.v v16, v8 ; ZVFHMIN-NEXT: lui a0, 307200 +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfabs.v v24, v16 ; ZVFHMIN-NEXT: fmv.w.x fa5, a0 +; ZVFHMIN-NEXT: vmflt.vf v0, v24, fa5 ; ZVFHMIN-NEXT: frflags a0 +; ZVFHMIN-NEXT: vfcvt.x.f.v v24, v16, v0.t +; ZVFHMIN-NEXT: vfcvt.f.x.v v24, v24, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, mu +; ZVFHMIN-NEXT: vfsgnj.vv v16, v24, v16, v0.t +; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; ZVFHMIN-NEXT: vfwcvt.f.f.v v24, v12 ; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; ZVFHMIN-NEXT: vfabs.v v8, v16 -; ZVFHMIN-NEXT: vmflt.vf v0, v8, fa5 ; ZVFHMIN-NEXT: vfabs.v v8, v24 -; ZVFHMIN-NEXT: vmflt.vf v7, v8, fa5 -; ZVFHMIN-NEXT: vfcvt.x.f.v v8, v16, v0.t -; ZVFHMIN-NEXT: vfcvt.f.x.v v8, v8, v0.t +; ZVFHMIN-NEXT: vmflt.vf v0, v8, fa5 ; ZVFHMIN-NEXT: fsflags a0 -; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, mu -; ZVFHMIN-NEXT: vfsgnj.vv v16, v8, v16, v0.t ; ZVFHMIN-NEXT: frflags a0 -; ZVFHMIN-NEXT: vmv1r.v v0, v7 -; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma -; ZVFHMIN-NEXT: vfcvt.x.f.v v8, v24, v0.t -; ZVFHMIN-NEXT: addi a1, sp, 16 -; ZVFHMIN-NEXT: vs8r.v v8, (a1) # Unknown-size Folded Spill ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v8, v16 -; ZVFHMIN-NEXT: vl8r.v v16, (a1) # Unknown-size Folded Reload ; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, ma +; ZVFHMIN-NEXT: vfcvt.x.f.v v16, v24, v0.t ; ZVFHMIN-NEXT: vfcvt.f.x.v v16, v16, v0.t ; ZVFHMIN-NEXT: vsetvli zero, zero, e32, m8, ta, mu ; ZVFHMIN-NEXT: vfsgnj.vv v24, v16, v24, v0.t ; ZVFHMIN-NEXT: vsetvli zero, zero, e16, m4, ta, ma ; ZVFHMIN-NEXT: vfncvt.f.f.w v12, v24 ; ZVFHMIN-NEXT: fsflags a0 -; ZVFHMIN-NEXT: csrr a0, vlenb -; ZVFHMIN-NEXT: slli a0, a0, 3 -; ZVFHMIN-NEXT: add sp, sp, a0 -; ZVFHMIN-NEXT: .cfi_def_cfa sp, 16 -; ZVFHMIN-NEXT: addi sp, sp, 16 -; ZVFHMIN-NEXT: .cfi_def_cfa_offset 0 ; ZVFHMIN-NEXT: ret %a = call @llvm.nearbyint.nxv32f16( %x) ret %a diff --git a/llvm/test/CodeGen/RISCV/rvv/implicit-def-copy.ll b/llvm/test/CodeGen/RISCV/rvv/implicit-def-copy.ll index 292f1deb2cce8..9475989d46343 100644 --- a/llvm/test/CodeGen/RISCV/rvv/implicit-def-copy.ll +++ b/llvm/test/CodeGen/RISCV/rvv/implicit-def-copy.ll @@ -11,8 +11,8 @@ define @vpload_nxv8i64(ptr %ptr, %m, i32 ze ; CHECK-NEXT: [[COPY:%[0-9]+]]:gprnox0 = COPY $x11 ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY $v0 ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x10 - ; CHECK-NEXT: $v0 = COPY [[COPY1]] - ; CHECK-NEXT: [[PseudoVLE64_V_M8_MASK:%[0-9]+]]:vrm8nov0 = PseudoVLE64_V_M8_MASK $noreg, [[COPY2]], $v0, [[COPY]], 6 /* e64 */, 1 /* ta, mu */ :: (load unknown-size from %ir.ptr, align 64) + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vmv0 = COPY [[COPY1]] + ; CHECK-NEXT: [[PseudoVLE64_V_M8_MASK:%[0-9]+]]:vrm8nov0 = PseudoVLE64_V_M8_MASK $noreg, [[COPY2]], [[COPY3]], [[COPY]], 6 /* e64 */, 1 /* ta, mu */ :: (load unknown-size from %ir.ptr, align 64) ; CHECK-NEXT: $v8m8 = COPY [[PseudoVLE64_V_M8_MASK]] ; CHECK-NEXT: PseudoRET implicit $v8m8 %load = call @llvm.vp.load.nxv8i64.p0(ptr %ptr, %m, i32 %evl) diff --git a/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir b/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir index 6fe228f44a1c8..2d49b4e4f493f 100644 --- a/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir +++ b/llvm/test/CodeGen/RISCV/rvv/mask-reg-alloc.mir @@ -17,9 +17,9 @@ body: | ; CHECK: liveins: $v0, $v1, $v2, $v3 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: dead $x0 = PseudoVSETIVLI 1, 192 /* e8, m1, ta, ma */, implicit-def $vl, implicit-def $vtype - ; CHECK-NEXT: renamable $v8 = PseudoVMERGE_VIM_M1 undef renamable $v8, killed renamable $v2, 1, killed renamable $v0, 1, 3 /* e8 */, implicit $vl, implicit $vtype - ; CHECK-NEXT: renamable $v0 = COPY killed renamable $v1, implicit $vtype - ; CHECK-NEXT: renamable $v9 = PseudoVMERGE_VIM_M1 undef renamable $v9, killed renamable $v3, 1, killed renamable $v0, 1, 3 /* e8 */, implicit $vl, implicit $vtype + ; CHECK-NEXT: renamable $v8 = PseudoVMERGE_VIM_M1 undef renamable $v8, killed renamable $v2, 1, $v0, 1, 3 /* e8 */, implicit $vl, implicit $vtype + ; CHECK-NEXT: $v0 = COPY killed renamable $v1, implicit $vtype + ; CHECK-NEXT: renamable $v9 = PseudoVMERGE_VIM_M1 undef renamable $v9, killed renamable $v3, 1, $v0, 1, 3 /* e8 */, implicit $vl, implicit $vtype ; CHECK-NEXT: renamable $v0 = PseudoVADD_VV_M1 undef renamable $v0, killed renamable $v8, killed renamable $v9, 1, 3 /* e8 */, 0 /* tu, mu */, implicit $vl, implicit $vtype ; CHECK-NEXT: PseudoRET implicit $v0 %0:vr = COPY $v0 diff --git a/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll b/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll index 7d3700492ea7b..937b3e6636df8 100644 --- a/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/nearbyint-vp.ll @@ -1495,6 +1495,12 @@ declare @llvm.vp.nearbyint.nxv16f64( @vp_nearbyint_nxv16f64( %va, %m, i32 zeroext %evl) { ; CHECK-LABEL: vp_nearbyint_nxv16f64: ; CHECK: # %bb.0: +; CHECK-NEXT: addi sp, sp, -16 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: csrr a1, vlenb +; CHECK-NEXT: slli a1, a1, 3 +; CHECK-NEXT: sub sp, sp, a1 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 8 * vlenb ; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, ma ; CHECK-NEXT: vmv1r.v v7, v0 ; CHECK-NEXT: csrr a1, vlenb @@ -1516,26 +1522,39 @@ define @vp_nearbyint_nxv16f64( %va, ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma ; CHECK-NEXT: vfcvt.x.f.v v24, v16, v0.t ; CHECK-NEXT: vfcvt.f.x.v v24, v24, v0.t +; CHECK-NEXT: addi a3, sp, 16 +; CHECK-NEXT: vs8r.v v24, (a3) # Unknown-size Folded Spill ; CHECK-NEXT: fsflags a2 +; CHECK-NEXT: addi a2, sp, 16 +; CHECK-NEXT: vl8r.v v24, (a2) # Unknown-size Folded Reload ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, mu ; CHECK-NEXT: vfsgnj.vv v16, v24, v16, v0.t +; CHECK-NEXT: vs8r.v v16, (a2) # Unknown-size Folded Spill ; CHECK-NEXT: bltu a0, a1, .LBB44_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: mv a0, a1 ; CHECK-NEXT: .LBB44_2: ; CHECK-NEXT: vmv1r.v v0, v7 ; CHECK-NEXT: vsetvli zero, a0, e64, m8, ta, ma -; CHECK-NEXT: vfabs.v v24, v8, v0.t +; CHECK-NEXT: vfabs.v v16, v8, v0.t ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, mu -; CHECK-NEXT: vmflt.vf v7, v24, fa5, v0.t +; CHECK-NEXT: vmflt.vf v7, v16, fa5, v0.t ; CHECK-NEXT: frflags a0 ; CHECK-NEXT: vmv1r.v v0, v7 ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, ma -; CHECK-NEXT: vfcvt.x.f.v v24, v8, v0.t -; CHECK-NEXT: vfcvt.f.x.v v24, v24, v0.t +; CHECK-NEXT: vfcvt.x.f.v v16, v8, v0.t +; CHECK-NEXT: vfcvt.f.x.v v16, v16, v0.t ; CHECK-NEXT: vsetvli zero, zero, e64, m8, ta, mu -; CHECK-NEXT: vfsgnj.vv v8, v24, v8, v0.t +; CHECK-NEXT: vfsgnj.vv v8, v16, v8, v0.t ; CHECK-NEXT: fsflags a0 +; CHECK-NEXT: addi a0, sp, 16 +; CHECK-NEXT: vl8r.v v16, (a0) # Unknown-size Folded Reload +; CHECK-NEXT: csrr a0, vlenb +; CHECK-NEXT: slli a0, a0, 3 +; CHECK-NEXT: add sp, sp, a0 +; CHECK-NEXT: .cfi_def_cfa sp, 16 +; CHECK-NEXT: addi sp, sp, 16 +; CHECK-NEXT: .cfi_def_cfa_offset 0 ; CHECK-NEXT: ret %v = call @llvm.vp.nearbyint.nxv16f64( %va, %m, i32 %evl) ret %v diff --git a/llvm/test/CodeGen/RISCV/rvv/pass-fast-math-flags-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/pass-fast-math-flags-sdnode.ll index 8457f3d2c149c..c6662e092aa5a 100644 --- a/llvm/test/CodeGen/RISCV/rvv/pass-fast-math-flags-sdnode.ll +++ b/llvm/test/CodeGen/RISCV/rvv/pass-fast-math-flags-sdnode.ll @@ -14,8 +14,8 @@ define @foo( %x, @llvm.vp.fmul.nxv1f64( %x, %y, %m, i32 %vl) diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir index 936fa21763eba..a050034c63168 100644 --- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir +++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir @@ -58,15 +58,13 @@ body: | ; CHECK-NEXT: %true:vr = COPY $v9 ; CHECK-NEXT: %avl:gprnox0 = COPY $x1 ; CHECK-NEXT: %mask:vmv0 = PseudoVMSET_M_B8 %avl, 0 /* e8 */ - ; CHECK-NEXT: $v0 = COPY %mask ; CHECK-NEXT: %x:vr = PseudoVMV_V_V_M1 %pt, %true, %avl, 5 /* e32 */, 0 /* tu, mu */ %false:vr = COPY $v8 %pt:vrnov0 = COPY $v8 %true:vr = COPY $v9 %avl:gprnox0 = COPY $x1 %mask:vmv0 = PseudoVMSET_M_B8 %avl, 0 - $v0 = COPY %mask - %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, $v0, %avl, 5 + %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, %mask, %avl, 5 ... --- name: same_mask @@ -78,18 +76,14 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: %pt:vr = COPY $v8 ; CHECK-NEXT: %false:vrnov0 = COPY $v9 - ; CHECK-NEXT: %mask:vr = COPY $v0 - ; CHECK-NEXT: $v0 = COPY %mask - ; CHECK-NEXT: %true:vrnov0 = PseudoVADD_VV_M1_MASK %false, $noreg, $noreg, $v0, 4, 5 /* e32 */, 0 /* tu, mu */ - ; CHECK-NEXT: $v0 = COPY %mask + ; CHECK-NEXT: %mask:vmv0 = COPY $v0 + ; CHECK-NEXT: %true:vrnov0 = PseudoVADD_VV_M1_MASK %false, $noreg, $noreg, %mask, 4, 5 /* e32 */, 0 /* tu, mu */ ; CHECK-NEXT: %x:vr = PseudoVMV_V_V_M1 %pt, %true, 8, 5 /* e32 */, 0 /* tu, mu */ %pt:vrnov0 = COPY $v8 %false:vrnov0 = COPY $v9 - %mask:vr = COPY $v0 - $v0 = COPY %mask - %true:vrnov0 = PseudoVADD_VV_M1_MASK %false, $noreg, $noreg, $v0, 4, 5 /* e32 */, 0 /* tu, mu */ - $v0 = COPY %mask - %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, $v0, 8, 5 /* e32 */ + %mask:vmv0 = COPY $v0 + %true:vrnov0 = PseudoVADD_VV_M1_MASK %false, $noreg, $noreg, %mask, 4, 5 /* e32 */, 0 /* tu, mu */ + %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, %mask, 8, 5 /* e32 */ ... --- # Shouldn't be converted because false operands are different @@ -102,18 +96,14 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: %pt:vrnov0 = COPY $v8 ; CHECK-NEXT: %false:vrnov0 = COPY $v9 - ; CHECK-NEXT: %mask:vr = COPY $v0 - ; CHECK-NEXT: $v0 = COPY %mask - ; CHECK-NEXT: %true:vrnov0 = PseudoVADD_VV_M1_MASK %pt, $noreg, $noreg, $v0, 4, 5 /* e32 */, 0 /* tu, mu */ - ; CHECK-NEXT: $v0 = COPY %mask - ; CHECK-NEXT: %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, $v0, 8, 5 /* e32 */ + ; CHECK-NEXT: %mask:vmv0 = COPY $v0 + ; CHECK-NEXT: %true:vrnov0 = PseudoVADD_VV_M1_MASK %pt, $noreg, $noreg, %mask, 4, 5 /* e32 */, 0 /* tu, mu */ + ; CHECK-NEXT: %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, %mask, 8, 5 /* e32 */ %pt:vrnov0 = COPY $v8 %false:vrnov0 = COPY $v9 - %mask:vr = COPY $v0 - $v0 = COPY %mask - %true:vrnov0 = PseudoVADD_VV_M1_MASK %pt, $noreg, $noreg, $v0, 4, 5 /* e32 */, 0 /* tu, mu */ - $v0 = COPY %mask - %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, $v0, 8, 5 /* e32 */ + %mask:vmv0 = COPY $v0 + %true:vrnov0 = PseudoVADD_VV_M1_MASK %pt, $noreg, $noreg, %mask, 4, 5 /* e32 */, 0 /* tu, mu */ + %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, %mask, 8, 5 /* e32 */ ... --- # Shouldn't be converted because EEWs are different @@ -126,18 +116,14 @@ body: | ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: %pt:vrnov0 = COPY $v8 ; CHECK-NEXT: %false:vrnov0 = COPY $v9 - ; CHECK-NEXT: %mask:vr = COPY $v0 - ; CHECK-NEXT: $v0 = COPY %mask - ; CHECK-NEXT: %true:vrnov0 = PseudoVADD_VV_M1_MASK %false, $noreg, $noreg, $v0, 4, 4 /* e16 */, 0 /* tu, mu */ - ; CHECK-NEXT: $v0 = COPY %mask - ; CHECK-NEXT: %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, $v0, 8, 5 /* e32 */ + ; CHECK-NEXT: %mask:vmv0 = COPY $v0 + ; CHECK-NEXT: %true:vrnov0 = PseudoVADD_VV_M1_MASK %false, $noreg, $noreg, %mask, 4, 4 /* e16 */, 0 /* tu, mu */ + ; CHECK-NEXT: %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, %mask, 8, 5 /* e32 */ %pt:vrnov0 = COPY $v8 %false:vrnov0 = COPY $v9 - %mask:vr = COPY $v0 - $v0 = COPY %mask - %true:vrnov0 = PseudoVADD_VV_M1_MASK %false, $noreg, $noreg, $v0, 4, 4 /* e16 */, 0 /* tu, mu */ - $v0 = COPY %mask - %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, $v0, 8, 5 /* e32 */ + %mask:vmv0 = COPY $v0 + %true:vrnov0 = PseudoVADD_VV_M1_MASK %false, $noreg, $noreg, %mask, 4, 4 /* e16 */, 0 /* tu, mu */ + %x:vrnov0 = PseudoVMERGE_VVM_M1 %pt, %false, %true, %mask, 8, 5 /* e32 */ ... --- name: same_mask_undef_truepassthru @@ -148,16 +134,12 @@ body: | ; CHECK: liveins: $v8, $v0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: %false:vrnov0 = COPY $v8 - ; CHECK-NEXT: %mask:vr = COPY $v0 - ; CHECK-NEXT: $v0 = COPY %mask - ; CHECK-NEXT: %true:vrnov0 = PseudoVADD_VV_M1_MASK %false, $noreg, $noreg, $v0, 4, 5 /* e32 */, 1 /* ta, mu */ - ; CHECK-NEXT: $v0 = COPY %mask + ; CHECK-NEXT: %mask:vmv0 = COPY $v0 + ; CHECK-NEXT: %true:vrnov0 = PseudoVADD_VV_M1_MASK %false, $noreg, $noreg, %mask, 4, 5 /* e32 */, 1 /* ta, mu */ %false:vr = COPY $v8 - %mask:vr = COPY $v0 - $v0 = COPY %mask - %true:vrnov0 = PseudoVADD_VV_M1_MASK $noreg, $noreg, $noreg, $v0, 4, 5 /* e32 */, 0 /* tu, mu */ - $v0 = COPY %mask - %x:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %true, $v0, 4, 5 /* e32 */ + %mask:vmv0 = COPY $v0 + %true:vrnov0 = PseudoVADD_VV_M1_MASK $noreg, $noreg, $noreg, %mask, 4, 5 /* e32 */, 0 /* tu, mu */ + %x:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %true, %mask, 4, 5 /* e32 */ ... --- # Shouldn't be converted because true is in a different block @@ -169,19 +151,15 @@ body: | ; CHECK-NEXT: liveins: $v8, $v0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: %false:vr = COPY $v8 - ; CHECK-NEXT: %mask:vr = COPY $v0 - ; CHECK-NEXT: $v0 = COPY %mask - ; CHECK-NEXT: %true:vrnov0 = PseudoVADD_VV_M1_MASK $noreg, $noreg, $noreg, $v0, 4, 5 /* e32 */, 0 /* tu, mu */ + ; CHECK-NEXT: %mask:vmv0 = COPY $v0 + ; CHECK-NEXT: %true:vrnov0 = PseudoVADD_VV_M1_MASK $noreg, $noreg, $noreg, %mask, 4, 5 /* e32 */, 0 /* tu, mu */ ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: bb.1: - ; CHECK-NEXT: $v0 = COPY %mask - ; CHECK-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %true, $v0, 4, 5 /* e32 */ + ; CHECK-NEXT: [[PseudoVMERGE_VVM_M1_:%[0-9]+]]:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %true, %mask, 4, 5 /* e32 */ bb.0: liveins: $v8, $v0 %false:vr = COPY $v8 - %mask:vr = COPY $v0 - $v0 = COPY %mask - %true:vrnov0 = PseudoVADD_VV_M1_MASK $noreg, $noreg, $noreg, $v0, 4, 5 /* e32 */, 0 /* tu, mu */ + %mask:vmv0 = COPY $v0 + %true:vrnov0 = PseudoVADD_VV_M1_MASK $noreg, $noreg, $noreg, %mask, 4, 5 /* e32 */, 0 /* tu, mu */ bb.1: - $v0 = COPY %mask - %5:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %true, $v0, 4, 5 /* e32 */ + %5:vrnov0 = PseudoVMERGE_VVM_M1 $noreg, %false, %true, %mask, 4, 5 /* e32 */ diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll index c01cbf49483b7..0c058b562f53d 100644 --- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll +++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-vops-mir.ll @@ -15,8 +15,8 @@ define void @vpmerge_vpload_store( %passthru, ptr %p, ) into %ir.p) ; CHECK-NEXT: PseudoRET %a = call @llvm.vp.load.nxv2i32.p0(ptr %p, splat (i1 -1), i32 %vl) @@ -34,8 +34,8 @@ define void @vpselect_vpload_store( %passthru, ptr %p, ) into %ir.p) ; CHECK-NEXT: PseudoRET %a = call @llvm.vp.load.nxv2i32.p0(ptr %p, splat (i1 -1), i32 %vl) diff --git a/llvm/test/CodeGen/RISCV/rvv/strided-vpload-vpstore-output.ll b/llvm/test/CodeGen/RISCV/rvv/strided-vpload-vpstore-output.ll index a8934bb25571c..081afcfab8dae 100644 --- a/llvm/test/CodeGen/RISCV/rvv/strided-vpload-vpstore-output.ll +++ b/llvm/test/CodeGen/RISCV/rvv/strided-vpload-vpstore-output.ll @@ -16,8 +16,8 @@ define @strided_vpload_nxv1i8_i8(ptr %ptr, i8 signext %stride, ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr = COPY $v0 ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x11 ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr = COPY $x10 - ; CHECK-NEXT: $v0 = COPY [[COPY1]] - ; CHECK-NEXT: [[PseudoVLSE8_V_MF8_MASK:%[0-9]+]]:vrnov0 = PseudoVLSE8_V_MF8_MASK $noreg, [[COPY3]], [[COPY2]], $v0, [[COPY]], 3 /* e8 */, 1 /* ta, mu */ :: (load unknown-size, align 1) + ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vmv0 = COPY [[COPY1]] + ; CHECK-NEXT: [[PseudoVLSE8_V_MF8_MASK:%[0-9]+]]:vrnov0 = PseudoVLSE8_V_MF8_MASK $noreg, [[COPY3]], [[COPY2]], [[COPY4]], [[COPY]], 3 /* e8 */, 1 /* ta, mu */ :: (load unknown-size, align 1) ; CHECK-NEXT: $v8 = COPY [[PseudoVLSE8_V_MF8_MASK]] ; CHECK-NEXT: PseudoRET implicit $v8 %load = call @llvm.experimental.vp.strided.load.nxv1i8.p0.i8(ptr %ptr, i8 %stride, %m, i32 %evl) @@ -36,8 +36,8 @@ define void @strided_vpstore_nxv1i8_i8( %val, ptr %ptr, i8 sign ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr = COPY $x11 ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr = COPY $x10 ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vr = COPY $v8 - ; CHECK-NEXT: $v0 = COPY [[COPY1]] - ; CHECK-NEXT: PseudoVSSE8_V_MF8_MASK [[COPY4]], [[COPY3]], [[COPY2]], $v0, [[COPY]], 3 /* e8 */ :: (store unknown-size, align 1) + ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vmv0 = COPY [[COPY1]] + ; CHECK-NEXT: PseudoVSSE8_V_MF8_MASK [[COPY4]], [[COPY3]], [[COPY2]], [[COPY5]], [[COPY]], 3 /* e8 */ :: (store unknown-size, align 1) ; CHECK-NEXT: PseudoRET call void @llvm.experimental.vp.strided.store.nxv1i8.p0.i8( %val, ptr %ptr, i8 %stride, %m, i32 %evl) ret void diff --git a/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll b/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll index 737ef6bae4e42..e4235d03cda31 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vleff-vlseg2ff-output.ll @@ -49,8 +49,8 @@ define i64 @test_vleff_nxv8i8_mask( %maskedoff, ptr %p, @between_inline_asm( %a, %b, %mask, ptr %p) { +; CHECK-LABEL: between_inline_asm: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 1, e8, m1, ta, ma +; CHECK-NEXT: vmv1r.v v10, v0 +; CHECK-NEXT: #APP +; CHECK-NEXT: vadd.vv v0, v8, v9 +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: vsetvli a1, zero, e64, m1, ta, ma +; CHECK-NEXT: vmv1r.v v11, v0 +; CHECK-NEXT: vmv1r.v v0, v10 +; CHECK-NEXT: vadd.vv v9, v8, v9, v0.t +; CHECK-NEXT: vmv1r.v v0, v11 +; CHECK-NEXT: #APP +; CHECK-NEXT: vadd.vv v8, v8, v0 +; CHECK-NEXT: #NO_APP +; CHECK-NEXT: vs1r.v v9, (a0) +; CHECK-NEXT: ret + %asm1 = tail call asm "vadd.vv $0, $1, $2", "={v0},^vr,^vr"( %a, %b) + %x = call @llvm.riscv.vadd.mask( poison, %a, %b, %mask, i64 -1, i64 0) + store %x, ptr %p + %asm2 = tail call asm "vadd.vv $0, $1, $2", "=^vr,^vr,{v0}"( %a, %asm1) + ret %asm2 +}