Skip to content

Commit b69932b

Browse files
HsiangkaiHsiangkai
committed
[RISCV] Implement vlsegff intrinsics.
Differential Revision: https://reviews.llvm.org/D95303
1 parent c85b6bf commit b69932b

File tree

10 files changed

+11303
-6
lines changed

10 files changed

+11303
-6
lines changed

‎llvm/include/llvm/IR/IntrinsicsRISCV.td

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -524,6 +524,30 @@ let TargetPrefix = "riscv" in {
524524
llvm_anyint_ty]),
525525
[NoCapture<ArgIndex<nf>>, IntrReadMem]>, RISCVVIntrinsic;
526526

527+
// For unit stride fault-only-first segment load
528+
// Input: (pointer, vl)
529+
// Output: (data, vl)
530+
// NOTE: We model this with default memory properties since we model writing
531+
// VL as a side effect. IntrReadMem, IntrHasSideEffects does not work.
532+
class RISCVUSSegLoadFF<int nf>
533+
: Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>,
534+
!add(nf, -1)), [llvm_anyint_ty]),
535+
[LLVMPointerToElt<0>, LLVMMatchType<1>],
536+
[NoCapture<ArgIndex<0>>]>, RISCVVIntrinsic;
537+
// For unit stride fault-only-first segment load with mask
538+
// Input: (maskedoff, pointer, mask, vl)
539+
// Output: (data, vl)
540+
// NOTE: We model this with default memory properties since we model writing
541+
// VL as a side effect. IntrReadMem, IntrHasSideEffects does not work.
542+
class RISCVUSSegLoadFFMask<int nf>
543+
: Intrinsic<!listconcat([llvm_anyvector_ty], !listsplat(LLVMMatchType<0>,
544+
!add(nf, -1)), [llvm_anyint_ty]),
545+
!listconcat(!listsplat(LLVMMatchType<0>, nf),
546+
[LLVMPointerToElt<0>,
547+
LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
548+
LLVMMatchType<1>]),
549+
[NoCapture<ArgIndex<nf>>]>, RISCVVIntrinsic;
550+
527551
// For stride segment load
528552
// Input: (pointer, offset, vl)
529553
class RISCVSSegLoad<int nf>
@@ -731,6 +755,10 @@ let TargetPrefix = "riscv" in {
731755
def "int_riscv_" # NAME : RISCVUSSegLoad<nf>;
732756
def "int_riscv_" # NAME # "_mask" : RISCVUSSegLoadMask<nf>;
733757
}
758+
multiclass RISCVUSSegLoadFF<int nf> {
759+
def "int_riscv_" # NAME : RISCVUSSegLoadFF<nf>;
760+
def "int_riscv_" # NAME # "_mask" : RISCVUSSegLoadFFMask<nf>;
761+
}
734762
multiclass RISCVSSegLoad<int nf> {
735763
def "int_riscv_" # NAME : RISCVSSegLoad<nf>;
736764
def "int_riscv_" # NAME # "_mask" : RISCVSSegLoadMask<nf>;
@@ -1048,6 +1076,7 @@ let TargetPrefix = "riscv" in {
10481076

10491077
foreach nf = [2, 3, 4, 5, 6, 7, 8] in {
10501078
defm vlseg # nf : RISCVUSSegLoad<nf>;
1079+
defm vlseg # nf # ff : RISCVUSSegLoadFF<nf>;
10511080
defm vlsseg # nf : RISCVSSegLoad<nf>;
10521081
defm vloxseg # nf : RISCVISegLoad<nf>;
10531082
defm vluxseg # nf : RISCVISegLoad<nf>;

‎llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,70 @@ void RISCVDAGToDAGISel::selectVLSEGMask(SDNode *Node, unsigned IntNo,
222222
CurDAG->RemoveDeadNode(Node);
223223
}
224224

225+
void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node) {
226+
SDLoc DL(Node);
227+
unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
228+
unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue.
229+
EVT VT = Node->getValueType(0);
230+
unsigned ScalarSize = VT.getScalarSizeInBits();
231+
MVT XLenVT = Subtarget->getXLenVT();
232+
RISCVVLMUL LMUL = getLMUL(VT);
233+
SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
234+
SmallVector<SDValue, 5> Operands;
235+
Operands.push_back(Node->getOperand(2)); // Base pointer.
236+
Operands.push_back(Node->getOperand(3)); // VL.
237+
Operands.push_back(SEW);
238+
Operands.push_back(Node->getOperand(0)); // Chain.
239+
const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
240+
IntNo, ScalarSize, static_cast<unsigned>(LMUL),
241+
static_cast<unsigned>(RISCVVLMUL::LMUL_1));
242+
SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
243+
MVT::Glue, Operands);
244+
SDValue SuperReg = SDValue(Load, 0);
245+
for (unsigned I = 0; I < NF; ++I)
246+
ReplaceUses(SDValue(Node, I),
247+
CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
248+
VT, SuperReg));
249+
250+
ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // Chain.
251+
ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue.
252+
CurDAG->RemoveDeadNode(Node);
253+
}
254+
255+
void RISCVDAGToDAGISel::selectVLSEGFFMask(SDNode *Node) {
256+
SDLoc DL(Node);
257+
unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
258+
unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue.
259+
EVT VT = Node->getValueType(0);
260+
unsigned ScalarSize = VT.getScalarSizeInBits();
261+
MVT XLenVT = Subtarget->getXLenVT();
262+
RISCVVLMUL LMUL = getLMUL(VT);
263+
SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
264+
SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
265+
SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
266+
SmallVector<SDValue, 7> Operands;
267+
Operands.push_back(MaskedOff);
268+
Operands.push_back(Node->getOperand(NF + 2)); // Base pointer.
269+
Operands.push_back(Node->getOperand(NF + 3)); // Mask.
270+
Operands.push_back(Node->getOperand(NF + 4)); // VL.
271+
Operands.push_back(SEW);
272+
Operands.push_back(Node->getOperand(0)); /// Chain.
273+
const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
274+
IntNo, ScalarSize, static_cast<unsigned>(LMUL),
275+
static_cast<unsigned>(RISCVVLMUL::LMUL_1));
276+
SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
277+
MVT::Glue, Operands);
278+
SDValue SuperReg = SDValue(Load, 0);
279+
for (unsigned I = 0; I < NF; ++I)
280+
ReplaceUses(SDValue(Node, I),
281+
CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
282+
VT, SuperReg));
283+
284+
ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // Chain.
285+
ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue.
286+
CurDAG->RemoveDeadNode(Node);
287+
}
288+
225289
void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned IntNo) {
226290
SDLoc DL(Node);
227291
unsigned NF = Node->getNumValues() - 1;
@@ -699,6 +763,14 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
699763
}
700764
break;
701765
}
766+
case RISCVISD::VLSEGFF: {
767+
selectVLSEGFF(Node);
768+
return;
769+
}
770+
case RISCVISD::VLSEGFF_MASK: {
771+
selectVLSEGFFMask(Node);
772+
return;
773+
}
702774
}
703775

704776
// Select the default instruction.

‎llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
5757

5858
void selectVLSEG(SDNode *Node, unsigned IntNo, bool IsStrided);
5959
void selectVLSEGMask(SDNode *Node, unsigned IntNo, bool IsStrided);
60+
void selectVLSEGFF(SDNode *Node);
61+
void selectVLSEGFFMask(SDNode *Node);
6062
void selectVLXSEG(SDNode *Node, unsigned IntNo);
6163
void selectVLXSEGMask(SDNode *Node, unsigned IntNo);
6264
void selectVSSEG(SDNode *Node, unsigned IntNo, bool IsStrided);

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1469,6 +1469,7 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
14691469
}
14701470
}
14711471

1472+
unsigned NF = 1;
14721473
switch (IntNo) {
14731474
default:
14741475
return SDValue(); // Don't custom lower most intrinsics.
@@ -1491,6 +1492,88 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
14911492
SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs, Load.getValue(2));
14921493
return DAG.getMergeValues({Load, ReadVL, Load.getValue(1)}, DL);
14931494
}
1495+
case Intrinsic::riscv_vlseg8ff:
1496+
NF++;
1497+
LLVM_FALLTHROUGH;
1498+
case Intrinsic::riscv_vlseg7ff:
1499+
NF++;
1500+
LLVM_FALLTHROUGH;
1501+
case Intrinsic::riscv_vlseg6ff:
1502+
NF++;
1503+
LLVM_FALLTHROUGH;
1504+
case Intrinsic::riscv_vlseg5ff:
1505+
NF++;
1506+
LLVM_FALLTHROUGH;
1507+
case Intrinsic::riscv_vlseg4ff:
1508+
NF++;
1509+
LLVM_FALLTHROUGH;
1510+
case Intrinsic::riscv_vlseg3ff:
1511+
NF++;
1512+
LLVM_FALLTHROUGH;
1513+
case Intrinsic::riscv_vlseg2ff: {
1514+
NF++;
1515+
SDLoc DL(Op);
1516+
SmallVector<EVT, 8> EVTs(NF, Op.getValueType());
1517+
EVTs.push_back(MVT::Other);
1518+
EVTs.push_back(MVT::Glue);
1519+
SDVTList VTs = DAG.getVTList(EVTs);
1520+
SDValue Load =
1521+
DAG.getNode(RISCVISD::VLSEGFF, DL, VTs, Op.getOperand(0),
1522+
Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
1523+
VTs = DAG.getVTList(Op->getValueType(NF), MVT::Other);
1524+
SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs,
1525+
/*Glue*/ Load.getValue(NF + 1));
1526+
SmallVector<SDValue, 8> Results;
1527+
for (unsigned i = 0; i < NF; ++i)
1528+
Results.push_back(Load.getValue(i));
1529+
Results.push_back(ReadVL);
1530+
Results.push_back(Load.getValue(NF)); // Chain.
1531+
return DAG.getMergeValues(Results, DL);
1532+
}
1533+
case Intrinsic::riscv_vlseg8ff_mask:
1534+
NF++;
1535+
LLVM_FALLTHROUGH;
1536+
case Intrinsic::riscv_vlseg7ff_mask:
1537+
NF++;
1538+
LLVM_FALLTHROUGH;
1539+
case Intrinsic::riscv_vlseg6ff_mask:
1540+
NF++;
1541+
LLVM_FALLTHROUGH;
1542+
case Intrinsic::riscv_vlseg5ff_mask:
1543+
NF++;
1544+
LLVM_FALLTHROUGH;
1545+
case Intrinsic::riscv_vlseg4ff_mask:
1546+
NF++;
1547+
LLVM_FALLTHROUGH;
1548+
case Intrinsic::riscv_vlseg3ff_mask:
1549+
NF++;
1550+
LLVM_FALLTHROUGH;
1551+
case Intrinsic::riscv_vlseg2ff_mask: {
1552+
NF++;
1553+
SDLoc DL(Op);
1554+
SmallVector<EVT, 8> EVTs(NF, Op.getValueType());
1555+
EVTs.push_back(MVT::Other);
1556+
EVTs.push_back(MVT::Glue);
1557+
SDVTList VTs = DAG.getVTList(EVTs);
1558+
SmallVector<SDValue, 13> LoadOps;
1559+
LoadOps.push_back(Op.getOperand(0)); // Chain.
1560+
LoadOps.push_back(Op.getOperand(1)); // Intrinsic ID.
1561+
for (unsigned i = 0; i < NF; ++i)
1562+
LoadOps.push_back(Op.getOperand(2 + i)); // MaskedOff.
1563+
LoadOps.push_back(Op.getOperand(2 + NF)); // Base.
1564+
LoadOps.push_back(Op.getOperand(3 + NF)); // Mask.
1565+
LoadOps.push_back(Op.getOperand(4 + NF)); // VL.
1566+
SDValue Load = DAG.getNode(RISCVISD::VLSEGFF_MASK, DL, VTs, LoadOps);
1567+
VTs = DAG.getVTList(Op->getValueType(NF), MVT::Other);
1568+
SDValue ReadVL = DAG.getNode(RISCVISD::READ_VL, DL, VTs,
1569+
/*Glue*/ Load.getValue(NF + 1));
1570+
SmallVector<SDValue, 8> Results;
1571+
for (unsigned i = 0; i < NF; ++i)
1572+
Results.push_back(Load.getValue(i));
1573+
Results.push_back(ReadVL);
1574+
Results.push_back(Load.getValue(NF)); // Chain.
1575+
return DAG.getMergeValues(Results, DL);
1576+
}
14941577
}
14951578
}
14961579

@@ -4019,6 +4102,8 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
40194102
NODE_NAME_CASE(TRUNCATE_VECTOR)
40204103
NODE_NAME_CASE(VLEFF)
40214104
NODE_NAME_CASE(VLEFF_MASK)
4105+
NODE_NAME_CASE(VLSEGFF)
4106+
NODE_NAME_CASE(VLSEGFF_MASK)
40224107
NODE_NAME_CASE(READ_VL)
40234108
NODE_NAME_CASE(VSLIDEUP)
40244109
NODE_NAME_CASE(VSLIDEDOWN)

‎llvm/lib/Target/RISCV/RISCVISelLowering.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,9 @@ enum NodeType : unsigned {
9999
// Unit-stride fault-only-first load
100100
VLEFF,
101101
VLEFF_MASK,
102+
// Unit-stride fault-only-first segment load
103+
VLSEGFF,
104+
VLSEGFF_MASK,
102105
// read vl CSR
103106
READ_VL,
104107
// Matches the semantics of vslideup/vslidedown. The first operand is the

‎llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -456,14 +456,15 @@ class PseudoToVInst<string PseudoInst> {
456456
}
457457

458458
class ToLowerCase<string Upper> {
459-
string L = !subst("VLSEG", "vlseg",
459+
string L = !subst("FF", "ff",
460+
!subst("VLSEG", "vlseg",
460461
!subst("VLSSEG", "vlsseg",
461462
!subst("VSSEG", "vsseg",
462463
!subst("VSSSEG", "vssseg",
463464
!subst("VLOXSEG", "vloxseg",
464465
!subst("VLUXSEG", "vluxseg",
465466
!subst("VSOXSEG", "vsoxseg",
466-
!subst("VSUXSEG", "vsuxseg", Upper))))))));
467+
!subst("VSUXSEG", "vsuxseg", Upper)))))))));
467468
}
468469

469470
// Example: PseudoVLSEG2E32_V_M2 -> int_riscv_vlseg2
@@ -1744,15 +1745,18 @@ multiclass VPseudoConversionV_W {
17441745
defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint>;
17451746
}
17461747

1747-
multiclass VPseudoUSSegLoad {
1748+
multiclass VPseudoUSSegLoad<bit isFF> {
17481749
foreach eew = EEWList in {
17491750
foreach lmul = MxSet<eew>.m in {
17501751
defvar LInfo = lmul.MX;
17511752
let VLMul = lmul.value in {
17521753
foreach nf = NFSet<lmul>.L in {
17531754
defvar vreg = SegRegClass<lmul, nf>.RC;
1754-
def nf # "E" # eew # "_V_" # LInfo : VPseudoUSSegLoadNoMask<vreg, eew>;
1755-
def nf # "E" # eew # "_V_" # LInfo # "_MASK" : VPseudoUSSegLoadMask<vreg, eew>;
1755+
defvar FFStr = !if(isFF, "FF", "");
1756+
def nf # "E" # eew # FFStr # "_V_" # LInfo :
1757+
VPseudoUSSegLoadNoMask<vreg, eew>;
1758+
def nf # "E" # eew # FFStr # "_V_" # LInfo # "_MASK" :
1759+
VPseudoUSSegLoadMask<vreg, eew>;
17561760
}
17571761
}
17581762
}
@@ -3086,7 +3090,7 @@ foreach eew = EEWList in {
30863090
//===----------------------------------------------------------------------===//
30873091
// 7.8. Vector Load/Store Segment Instructions
30883092
//===----------------------------------------------------------------------===//
3089-
defm PseudoVLSEG : VPseudoUSSegLoad;
3093+
defm PseudoVLSEG : VPseudoUSSegLoad</*fault-only-first*/false>;
30903094
defm PseudoVLSSEG : VPseudoSSegLoad;
30913095
defm PseudoVLOXSEG : VPseudoISegLoad;
30923096
defm PseudoVLUXSEG : VPseudoISegLoad;
@@ -3095,6 +3099,10 @@ defm PseudoVSSSEG : VPseudoSSegStore;
30953099
defm PseudoVSOXSEG : VPseudoISegStore;
30963100
defm PseudoVSUXSEG : VPseudoISegStore;
30973101

3102+
// vlseg<nf>e<eew>ff.v may update VL register
3103+
let hasSideEffects = 1, Defs = [VL] in
3104+
defm PseudoVLSEG : VPseudoUSSegLoad</*fault-only-first*/true>;
3105+
30983106
//===----------------------------------------------------------------------===//
30993107
// 8. Vector AMO Operations
31003108
//===----------------------------------------------------------------------===//

0 commit comments

Comments
 (0)