@@ -1745,13 +1745,6 @@ bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
17451745 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
17461746 MachineMemOperand::MOVolatile;
17471747 return true;
1748- case Intrinsic::riscv_seg2_load:
1749- case Intrinsic::riscv_seg3_load:
1750- case Intrinsic::riscv_seg4_load:
1751- case Intrinsic::riscv_seg5_load:
1752- case Intrinsic::riscv_seg6_load:
1753- case Intrinsic::riscv_seg7_load:
1754- case Intrinsic::riscv_seg8_load:
17551748 case Intrinsic::riscv_seg2_load_mask:
17561749 case Intrinsic::riscv_seg3_load_mask:
17571750 case Intrinsic::riscv_seg4_load_mask:
@@ -1761,17 +1754,6 @@ bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
17611754 case Intrinsic::riscv_seg8_load_mask:
17621755 return SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
17631756 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
1764- case Intrinsic::riscv_seg2_store:
1765- case Intrinsic::riscv_seg3_store:
1766- case Intrinsic::riscv_seg4_store:
1767- case Intrinsic::riscv_seg5_store:
1768- case Intrinsic::riscv_seg6_store:
1769- case Intrinsic::riscv_seg7_store:
1770- case Intrinsic::riscv_seg8_store:
1771- // Operands are (vec, ..., vec, ptr, vl)
1772- return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 2,
1773- /*IsStore*/ true,
1774- /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
17751757 case Intrinsic::riscv_seg2_store_mask:
17761758 case Intrinsic::riscv_seg3_store_mask:
17771759 case Intrinsic::riscv_seg4_store_mask:
@@ -10573,13 +10555,6 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
1057310555 switch (IntNo) {
1057410556 default:
1057510557 break;
10576- case Intrinsic::riscv_seg2_load:
10577- case Intrinsic::riscv_seg3_load:
10578- case Intrinsic::riscv_seg4_load:
10579- case Intrinsic::riscv_seg5_load:
10580- case Intrinsic::riscv_seg6_load:
10581- case Intrinsic::riscv_seg7_load:
10582- case Intrinsic::riscv_seg8_load:
1058310558 case Intrinsic::riscv_seg2_load_mask:
1058410559 case Intrinsic::riscv_seg3_load_mask:
1058510560 case Intrinsic::riscv_seg4_load_mask:
@@ -10602,18 +10577,13 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
1060210577 ContainerVT.getScalarSizeInBits();
1060310578 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
1060410579
10605- // Masked: (pointer, mask, vl)
10606- // Non-masked: (pointer, vl)
10607- bool IsMasked = Op.getNumOperands() > 4;
10580+ // Operands: (chain, int_id, pointer, mask, vl)
1060810581 SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
10609- SDValue Mask =
10610- IsMasked ? Op.getOperand(3) : getAllOnesMask(ContainerVT, VL, DL, DAG);
10582+ SDValue Mask = Op.getOperand(3);
1061110583 MVT MaskVT = Mask.getSimpleValueType();
10612- if (MaskVT.isFixedLengthVector()) {
10613- MVT MaskContainerVT =
10614- ::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
10615- Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
10616- }
10584+ MVT MaskContainerVT =
10585+ ::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
10586+ Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
1061710587
1061810588 SDValue IntID = DAG.getTargetConstant(VlsegInts[NF - 2], DL, XLenVT);
1061910589 auto *Load = cast<MemIntrinsicSDNode>(Op);
@@ -10681,13 +10651,6 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
1068110651 switch (IntNo) {
1068210652 default:
1068310653 break;
10684- case Intrinsic::riscv_seg2_store:
10685- case Intrinsic::riscv_seg3_store:
10686- case Intrinsic::riscv_seg4_store:
10687- case Intrinsic::riscv_seg5_store:
10688- case Intrinsic::riscv_seg6_store:
10689- case Intrinsic::riscv_seg7_store:
10690- case Intrinsic::riscv_seg8_store:
1069110654 case Intrinsic::riscv_seg2_store_mask:
1069210655 case Intrinsic::riscv_seg3_store_mask:
1069310656 case Intrinsic::riscv_seg4_store_mask:
@@ -10702,24 +10665,8 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
1070210665 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
1070310666 Intrinsic::riscv_vsseg8_mask};
1070410667
10705- bool IsMasked = false;
10706- switch (IntNo) {
10707- case Intrinsic::riscv_seg2_store_mask:
10708- case Intrinsic::riscv_seg3_store_mask:
10709- case Intrinsic::riscv_seg4_store_mask:
10710- case Intrinsic::riscv_seg5_store_mask:
10711- case Intrinsic::riscv_seg6_store_mask:
10712- case Intrinsic::riscv_seg7_store_mask:
10713- case Intrinsic::riscv_seg8_store_mask:
10714- IsMasked = true;
10715- break;
10716- default:
10717- break;
10718- }
10719-
10720- // Non-masked: (chain, int_id, vec*, ptr, vl)
10721- // Masked: (chain, int_id, vec*, ptr, mask, vl)
10722- unsigned NF = Op->getNumOperands() - (IsMasked ? 5 : 4);
10668+ // Operands: (chain, int_id, vec*, ptr, mask, vl)
10669+ unsigned NF = Op->getNumOperands() - 5;
1072310670 assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
1072410671 MVT XLenVT = Subtarget.getXLenVT();
1072510672 MVT VT = Op->getOperand(2).getSimpleValueType();
@@ -10729,14 +10676,11 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
1072910676 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
1073010677
1073110678 SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
10732- SDValue Mask = IsMasked ? Op.getOperand(Op.getNumOperands() - 2)
10733- : getAllOnesMask(ContainerVT, VL, DL, DAG);
10679+ SDValue Mask = Op.getOperand(Op.getNumOperands() - 2);
1073410680 MVT MaskVT = Mask.getSimpleValueType();
10735- if (MaskVT.isFixedLengthVector()) {
10736- MVT MaskContainerVT =
10737- ::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
10738- Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
10739- }
10681+ MVT MaskContainerVT =
10682+ ::getContainerForFixedLengthVector(DAG, MaskVT, Subtarget);
10683+ Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
1074010684
1074110685 SDValue IntID = DAG.getTargetConstant(VssegInts[NF - 2], DL, XLenVT);
1074210686 SDValue Ptr = Op->getOperand(NF + 2);
@@ -23781,10 +23725,10 @@ bool RISCVTargetLowering::isLegalStridedLoadStore(EVT DataType,
2378123725}
2378223726
2378323727static const Intrinsic::ID FixedVlsegIntrIds[] = {
23784- Intrinsic::riscv_seg2_load , Intrinsic::riscv_seg3_load ,
23785- Intrinsic::riscv_seg4_load , Intrinsic::riscv_seg5_load ,
23786- Intrinsic::riscv_seg6_load , Intrinsic::riscv_seg7_load ,
23787- Intrinsic::riscv_seg8_load };
23728+ Intrinsic::riscv_seg2_load_mask , Intrinsic::riscv_seg3_load_mask ,
23729+ Intrinsic::riscv_seg4_load_mask , Intrinsic::riscv_seg5_load_mask ,
23730+ Intrinsic::riscv_seg6_load_mask , Intrinsic::riscv_seg7_load_mask ,
23731+ Intrinsic::riscv_seg8_load_mask };
2378823732
2378923733/// Lower an interleaved load into a vlsegN intrinsic.
2379023734///
@@ -23835,10 +23779,10 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
2383523779 };
2383623780
2383723781 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
23838-
23839- CallInst *VlsegN = Builder.CreateIntrinsic(
23840- FixedVlsegIntrIds[Factor - 2], {VTy, LI->getPointerOperandType() , XLenTy},
23841- {LI->getPointerOperand(), VL});
23782+ Value *Mask = Builder.getAllOnesMask(VTy->getElementCount());
23783+ CallInst *VlsegN =
23784+ Builder.CreateIntrinsic( FixedVlsegIntrIds[Factor - 2], {VTy, XLenTy},
23785+ {LI->getPointerOperand(), Mask , VL});
2384223786
2384323787 for (unsigned i = 0; i < Shuffles.size(); i++) {
2384423788 Value *SubVec = Builder.CreateExtractValue(VlsegN, Indices[i]);
@@ -23849,10 +23793,10 @@ bool RISCVTargetLowering::lowerInterleavedLoad(
2384923793}
2385023794
2385123795static const Intrinsic::ID FixedVssegIntrIds[] = {
23852- Intrinsic::riscv_seg2_store , Intrinsic::riscv_seg3_store ,
23853- Intrinsic::riscv_seg4_store , Intrinsic::riscv_seg5_store ,
23854- Intrinsic::riscv_seg6_store , Intrinsic::riscv_seg7_store ,
23855- Intrinsic::riscv_seg8_store };
23796+ Intrinsic::riscv_seg2_store_mask , Intrinsic::riscv_seg3_store_mask ,
23797+ Intrinsic::riscv_seg4_store_mask , Intrinsic::riscv_seg5_store_mask ,
23798+ Intrinsic::riscv_seg6_store_mask , Intrinsic::riscv_seg7_store_mask ,
23799+ Intrinsic::riscv_seg8_store_mask };
2385623800
2385723801/// Lower an interleaved store into a vssegN intrinsic.
2385823802///
@@ -23912,8 +23856,7 @@ bool RISCVTargetLowering::lowerInterleavedStore(StoreInst *SI,
2391223856 }
2391323857
2391423858 Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
23915- SI->getModule(), FixedVssegIntrIds[Factor - 2],
23916- {VTy, SI->getPointerOperandType(), XLenTy});
23859+ SI->getModule(), FixedVssegIntrIds[Factor - 2], {VTy, XLenTy});
2391723860
2391823861 SmallVector<Value *, 10> Ops;
2391923862 SmallVector<int, 16> NewShuffleMask;
@@ -23933,7 +23876,8 @@ bool RISCVTargetLowering::lowerInterleavedStore(StoreInst *SI,
2393323876 // potentially under larger LMULs) because we checked that the fixed vector
2393423877 // type fits in isLegalInterleavedAccessType
2393523878 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
23936- Ops.append({SI->getPointerOperand(), VL});
23879+ Value *StoreMask = Builder.getAllOnesMask(VTy->getElementCount());
23880+ Ops.append({SI->getPointerOperand(), StoreMask, VL});
2393723881
2393823882 Builder.CreateCall(VssegNFunc, Ops);
2393923883
@@ -23962,10 +23906,10 @@ bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
2396223906
2396323907 if (auto *FVTy = dyn_cast<FixedVectorType>(ResVTy)) {
2396423908 Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
23909+ Value *Mask = Builder.getAllOnesMask(FVTy->getElementCount());
2396523910 Return =
23966- Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2],
23967- {ResVTy, LI->getPointerOperandType(), XLenTy},
23968- {LI->getPointerOperand(), VL});
23911+ Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2], {ResVTy, XLenTy},
23912+ {LI->getPointerOperand(), Mask, VL});
2396923913 } else {
2397023914 static const Intrinsic::ID IntrIds[] = {
2397123915 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
@@ -24029,12 +23973,12 @@ bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
2402923973
2403023974 if (auto *FVTy = dyn_cast<FixedVectorType>(InVTy)) {
2403123975 Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
24032- SI->getModule(), FixedVssegIntrIds[Factor - 2],
24033- {InVTy, SI->getPointerOperandType(), XLenTy});
23976+ SI->getModule(), FixedVssegIntrIds[Factor - 2], {InVTy, XLenTy});
2403423977
2403523978 SmallVector<Value *, 10> Ops(InterleaveValues);
2403623979 Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
24037- Ops.append({SI->getPointerOperand(), VL});
23980+ Value *Mask = Builder.getAllOnesMask(FVTy->getElementCount());
23981+ Ops.append({SI->getPointerOperand(), Mask, VL});
2403823982
2403923983 Builder.CreateCall(VssegNFunc, Ops);
2404023984 } else {
@@ -24156,15 +24100,9 @@ bool RISCVTargetLowering::lowerInterleavedVPLoad(
2415624100
2415724101 Value *Return = nullptr;
2415824102 if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
24159- static const Intrinsic::ID FixedMaskedVlsegIntrIds[] = {
24160- Intrinsic::riscv_seg2_load_mask, Intrinsic::riscv_seg3_load_mask,
24161- Intrinsic::riscv_seg4_load_mask, Intrinsic::riscv_seg5_load_mask,
24162- Intrinsic::riscv_seg6_load_mask, Intrinsic::riscv_seg7_load_mask,
24163- Intrinsic::riscv_seg8_load_mask};
24164-
24165- Return = Builder.CreateIntrinsic(FixedMaskedVlsegIntrIds[Factor - 2],
24166- {FVTy, XLenTy},
24167- {Load->getArgOperand(0), Mask, EVL});
24103+ Return =
24104+ Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2], {FVTy, XLenTy},
24105+ {Load->getArgOperand(0), Mask, EVL});
2416824106 } else {
2416924107 static const Intrinsic::ID IntrMaskIds[] = {
2417024108 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
@@ -24276,15 +24214,9 @@ bool RISCVTargetLowering::lowerInterleavedVPStore(
2427624214 XLenTy);
2427724215
2427824216 if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
24279- static const Intrinsic::ID FixedMaskedVssegIntrIds[] = {
24280- Intrinsic::riscv_seg2_store_mask, Intrinsic::riscv_seg3_store_mask,
24281- Intrinsic::riscv_seg4_store_mask, Intrinsic::riscv_seg5_store_mask,
24282- Intrinsic::riscv_seg6_store_mask, Intrinsic::riscv_seg7_store_mask,
24283- Intrinsic::riscv_seg8_store_mask};
24284-
2428524217 SmallVector<Value *, 8> Operands(InterleaveOperands);
2428624218 Operands.append({Store->getArgOperand(1), Mask, EVL});
24287- Builder.CreateIntrinsic(FixedMaskedVssegIntrIds [Factor - 2], {FVTy, XLenTy},
24219+ Builder.CreateIntrinsic(FixedVssegIntrIds [Factor - 2], {FVTy, XLenTy},
2428824220 Operands);
2428924221 return true;
2429024222 }
0 commit comments