Skip to content

Commit b41b408

Browse files
fhossein-quicSergei LarinSergeiYLarinSyllo
authored and
dvbuka
committed
Add HVX vgather/vscatter Support (llvm#164421)
This patch adds HVX vgather/vscatter genertion for i16, i32, and i8. It also adds a flag to control generation of scatter/gather instructions for HVX. Default to "disable". Co-authored-by: Sergei Larin <[email protected]> Co-authored-by: Sergei Larin <[email protected]> Co-authored-by: Maxime Schmitt <[email protected]>
1 parent 5d3b95e commit b41b408

15 files changed

+1374
-77
lines changed

‎llvm/include/llvm/IR/IntrinsicsHexagon.td‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,3 +447,15 @@ def int_hexagon_instrprof_custom
447447

448448

449449
include "llvm/IR/IntrinsicsHexagonDep.td"
450+
451+
class Hexagon__ptri32i32v64i16_Intrinsic<string GCCIntSuffix,
452+
list<IntrinsicProperty> intr_properties = [IntrNoMem]>
453+
: Hexagon_Intrinsic<GCCIntSuffix,
454+
[], [llvm_ptr_ty,llvm_i32_ty,llvm_i32_ty,llvm_v64i16_ty],
455+
intr_properties>;
456+
457+
def int_hexagon_V6_vgather_vscattermh :
458+
Hexagon__ptri32i32v64i16_Intrinsic<"HEXAGON_V6_vgather_vscattermh", [IntrArgMemOnly]>;
459+
460+
def int_hexagon_V6_vgather_vscattermh_128B :
461+
Hexagon__ptri32i32v32i32_Intrinsic<"HEXAGON_V6_vgather_vscattermh_128B", [IntrArgMemOnly]>;

‎llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -654,7 +654,9 @@ void HexagonDAGToDAGISel::SelectIntrinsicWChain(SDNode *N) {
654654
IntNo == Intrinsic::hexagon_V6_vgathermh ||
655655
IntNo == Intrinsic::hexagon_V6_vgathermh_128B ||
656656
IntNo == Intrinsic::hexagon_V6_vgathermhw ||
657-
IntNo == Intrinsic::hexagon_V6_vgathermhw_128B) {
657+
IntNo == Intrinsic::hexagon_V6_vgathermhw_128B ||
658+
IntNo == Intrinsic::hexagon_V6_vgather_vscattermh ||
659+
IntNo == Intrinsic::hexagon_V6_vgather_vscattermh_128B) {
658660
SelectV65Gather(N);
659661
return;
660662
}

‎llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2953,6 +2953,10 @@ void HexagonDAGToDAGISel::SelectV65Gather(SDNode *N) {
29532953
case Intrinsic::hexagon_V6_vgathermhw_128B:
29542954
Opcode = Hexagon::V6_vgathermhw_pseudo;
29552955
break;
2956+
case Intrinsic::hexagon_V6_vgather_vscattermh:
2957+
case Intrinsic::hexagon_V6_vgather_vscattermh_128B:
2958+
Opcode = Hexagon::V6_vgather_vscatter_mh_pseudo;
2959+
break;
29562960
}
29572961

29582962
SDVTList VTs = CurDAG->getVTList(MVT::Other);

‎llvm/lib/Target/Hexagon/HexagonISelLowering.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2145,7 +2145,9 @@ bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
21452145
case Intrinsic::hexagon_V6_vgathermhq:
21462146
case Intrinsic::hexagon_V6_vgathermhq_128B:
21472147
case Intrinsic::hexagon_V6_vgathermhwq:
2148-
case Intrinsic::hexagon_V6_vgathermhwq_128B: {
2148+
case Intrinsic::hexagon_V6_vgathermhwq_128B:
2149+
case Intrinsic::hexagon_V6_vgather_vscattermh:
2150+
case Intrinsic::hexagon_V6_vgather_vscattermh_128B: {
21492151
const Module &M = *I.getParent()->getParent()->getParent();
21502152
Info.opc = ISD::INTRINSIC_W_CHAIN;
21512153
Type *VecTy = I.getArgOperand(1)->getType();

‎llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp‎

Lines changed: 88 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -1554,80 +1554,93 @@ HexagonInstrInfo::expandVGatherPseudo(MachineInstr &MI) const {
15541554
MachineBasicBlock::iterator First;
15551555

15561556
switch (Opc) {
1557-
case Hexagon::V6_vgathermh_pseudo:
1558-
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermh))
1559-
.add(MI.getOperand(2))
1560-
.add(MI.getOperand(3))
1561-
.add(MI.getOperand(4));
1562-
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1563-
.add(MI.getOperand(0))
1564-
.addImm(MI.getOperand(1).getImm())
1565-
.addReg(Hexagon::VTMP);
1566-
MBB.erase(MI);
1567-
return First.getInstrIterator();
1568-
1569-
case Hexagon::V6_vgathermw_pseudo:
1570-
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermw))
1571-
.add(MI.getOperand(2))
1572-
.add(MI.getOperand(3))
1573-
.add(MI.getOperand(4));
1574-
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1575-
.add(MI.getOperand(0))
1576-
.addImm(MI.getOperand(1).getImm())
1577-
.addReg(Hexagon::VTMP);
1578-
MBB.erase(MI);
1579-
return First.getInstrIterator();
1580-
1581-
case Hexagon::V6_vgathermhw_pseudo:
1582-
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhw))
1583-
.add(MI.getOperand(2))
1584-
.add(MI.getOperand(3))
1585-
.add(MI.getOperand(4));
1586-
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1587-
.add(MI.getOperand(0))
1588-
.addImm(MI.getOperand(1).getImm())
1589-
.addReg(Hexagon::VTMP);
1590-
MBB.erase(MI);
1591-
return First.getInstrIterator();
1592-
1593-
case Hexagon::V6_vgathermhq_pseudo:
1594-
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhq))
1595-
.add(MI.getOperand(2))
1596-
.add(MI.getOperand(3))
1597-
.add(MI.getOperand(4))
1598-
.add(MI.getOperand(5));
1599-
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1600-
.add(MI.getOperand(0))
1601-
.addImm(MI.getOperand(1).getImm())
1602-
.addReg(Hexagon::VTMP);
1603-
MBB.erase(MI);
1604-
return First.getInstrIterator();
1605-
1606-
case Hexagon::V6_vgathermwq_pseudo:
1607-
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermwq))
1608-
.add(MI.getOperand(2))
1609-
.add(MI.getOperand(3))
1610-
.add(MI.getOperand(4))
1611-
.add(MI.getOperand(5));
1612-
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1613-
.add(MI.getOperand(0))
1614-
.addImm(MI.getOperand(1).getImm())
1615-
.addReg(Hexagon::VTMP);
1616-
MBB.erase(MI);
1617-
return First.getInstrIterator();
1618-
1619-
case Hexagon::V6_vgathermhwq_pseudo:
1620-
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhwq))
1621-
.add(MI.getOperand(2))
1622-
.add(MI.getOperand(3))
1623-
.add(MI.getOperand(4))
1624-
.add(MI.getOperand(5));
1625-
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1626-
.add(MI.getOperand(0))
1627-
.addImm(MI.getOperand(1).getImm())
1628-
.addReg(Hexagon::VTMP);
1629-
MBB.erase(MI);
1630-
return First.getInstrIterator();
1557+
case Hexagon::V6_vgather_vscatter_mh_pseudo:
1558+
// This is mainly a place holder. It will be extended.
1559+
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermh))
1560+
.add(MI.getOperand(2))
1561+
.add(MI.getOperand(3))
1562+
.add(MI.getOperand(4));
1563+
BuildMI(MBB, MI, DL, get(Hexagon::V6_vscattermh))
1564+
.add(MI.getOperand(2))
1565+
.add(MI.getOperand(3))
1566+
.add(MI.getOperand(4))
1567+
.addReg(Hexagon::VTMP);
1568+
MBB.erase(MI);
1569+
return First.getInstrIterator();
1570+
case Hexagon::V6_vgathermh_pseudo:
1571+
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermh))
1572+
.add(MI.getOperand(2))
1573+
.add(MI.getOperand(3))
1574+
.add(MI.getOperand(4));
1575+
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1576+
.add(MI.getOperand(0))
1577+
.addImm(MI.getOperand(1).getImm())
1578+
.addReg(Hexagon::VTMP);
1579+
MBB.erase(MI);
1580+
return First.getInstrIterator();
1581+
1582+
case Hexagon::V6_vgathermw_pseudo:
1583+
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermw))
1584+
.add(MI.getOperand(2))
1585+
.add(MI.getOperand(3))
1586+
.add(MI.getOperand(4));
1587+
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1588+
.add(MI.getOperand(0))
1589+
.addImm(MI.getOperand(1).getImm())
1590+
.addReg(Hexagon::VTMP);
1591+
MBB.erase(MI);
1592+
return First.getInstrIterator();
1593+
1594+
case Hexagon::V6_vgathermhw_pseudo:
1595+
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhw))
1596+
.add(MI.getOperand(2))
1597+
.add(MI.getOperand(3))
1598+
.add(MI.getOperand(4));
1599+
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1600+
.add(MI.getOperand(0))
1601+
.addImm(MI.getOperand(1).getImm())
1602+
.addReg(Hexagon::VTMP);
1603+
MBB.erase(MI);
1604+
return First.getInstrIterator();
1605+
1606+
case Hexagon::V6_vgathermhq_pseudo:
1607+
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhq))
1608+
.add(MI.getOperand(2))
1609+
.add(MI.getOperand(3))
1610+
.add(MI.getOperand(4))
1611+
.add(MI.getOperand(5));
1612+
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1613+
.add(MI.getOperand(0))
1614+
.addImm(MI.getOperand(1).getImm())
1615+
.addReg(Hexagon::VTMP);
1616+
MBB.erase(MI);
1617+
return First.getInstrIterator();
1618+
1619+
case Hexagon::V6_vgathermwq_pseudo:
1620+
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermwq))
1621+
.add(MI.getOperand(2))
1622+
.add(MI.getOperand(3))
1623+
.add(MI.getOperand(4))
1624+
.add(MI.getOperand(5));
1625+
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1626+
.add(MI.getOperand(0))
1627+
.addImm(MI.getOperand(1).getImm())
1628+
.addReg(Hexagon::VTMP);
1629+
MBB.erase(MI);
1630+
return First.getInstrIterator();
1631+
1632+
case Hexagon::V6_vgathermhwq_pseudo:
1633+
First = BuildMI(MBB, MI, DL, get(Hexagon::V6_vgathermhwq))
1634+
.add(MI.getOperand(2))
1635+
.add(MI.getOperand(3))
1636+
.add(MI.getOperand(4))
1637+
.add(MI.getOperand(5));
1638+
BuildMI(MBB, MI, DL, get(Hexagon::V6_vS32b_new_ai))
1639+
.add(MI.getOperand(0))
1640+
.addImm(MI.getOperand(1).getImm())
1641+
.addReg(Hexagon::VTMP);
1642+
MBB.erase(MI);
1643+
return First.getInstrIterator();
16311644
}
16321645

16331646
return MI.getIterator();
@@ -2806,6 +2819,7 @@ bool HexagonInstrInfo::isValidOffset(unsigned Opcode, int Offset,
28062819
case Hexagon::V6_vL32b_nt_tmp_npred_ai:
28072820
case Hexagon::V6_vS32Ub_npred_ai:
28082821
case Hexagon::V6_vgathermh_pseudo:
2822+
case Hexagon::V6_vgather_vscatter_mh_pseudo:
28092823
case Hexagon::V6_vgathermw_pseudo:
28102824
case Hexagon::V6_vgathermhw_pseudo:
28112825
case Hexagon::V6_vgathermhq_pseudo:

‎llvm/lib/Target/Hexagon/HexagonPatternsV65.td‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,19 @@ defm V6_vgathermh_pseudo : vgathermh<HvxVR>;
4040
defm V6_vgathermw_pseudo : vgathermw<HvxVR>;
4141
defm V6_vgathermhw_pseudo : vgathermhw<HvxWR>;
4242

43+
44+
multiclass vgather_scatter_mh<RegisterClass RC> {
45+
let isCodeGenOnly = 1, isPseudo = 1, mayLoad = 1,
46+
mayStore = 1, addrMode = BaseImmOffset, accessSize = HalfWordAccess in
47+
def NAME : CVI_GATHER_TMP_LD_Resource_NoOpcode<(outs ),
48+
(ins IntRegs:$_dst_, s4_0Imm:$Ii,
49+
IntRegs:$Rt, ModRegs:$Mu, RC:$Vv),
50+
".error \"should not emit\" ",
51+
[]>;
52+
}
53+
54+
defm V6_vgather_vscatter_mh_pseudo : vgather_scatter_mh<HvxVR>;
55+
4356
multiclass vgathermhq<RegisterClass RC1, RegisterClass RC2> {
4457
let isCodeGenOnly = 1, isPseudo = 1, mayLoad = 1,
4558
mayStore = 1, addrMode = BaseImmOffset, accessSize = HalfWordAccess in

‎llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ using namespace llvm;
3131
static cl::opt<bool> HexagonAutoHVX("hexagon-autohvx", cl::init(false),
3232
cl::Hidden, cl::desc("Enable loop vectorizer for HVX"));
3333

34+
cl::opt<bool> HexagonAllowScatterGatherHVX(
35+
"hexagon-allow-scatter-gather-hvx", cl::init(false), cl::Hidden,
36+
cl::desc("Allow auto-generation of HVX scatter-gather"));
37+
3438
static cl::opt<bool> EnableV68FloatAutoHVX(
3539
"force-hvx-float", cl::Hidden,
3640
cl::desc("Enable auto-vectorization of floatint point types on v68."));
@@ -354,6 +358,61 @@ bool HexagonTTIImpl::isLegalMaskedLoad(Type *DataType, Align /*Alignment*/,
354358
return HexagonMaskedVMem && ST.isTypeForHVX(DataType);
355359
}
356360

361+
bool HexagonTTIImpl::isLegalMaskedGather(Type *Ty, Align Alignment) const {
362+
// For now assume we can not deal with all HVX datatypes.
363+
if (!Ty->isVectorTy() || !ST.isTypeForHVX(Ty) ||
364+
!HexagonAllowScatterGatherHVX)
365+
return false;
366+
// This must be in sync with HexagonVectorCombine pass.
367+
switch (Ty->getScalarSizeInBits()) {
368+
case 8:
369+
return (getTypeNumElements(Ty) == 128);
370+
case 16:
371+
if (getTypeNumElements(Ty) == 64 || getTypeNumElements(Ty) == 32)
372+
return (Alignment >= 2);
373+
break;
374+
case 32:
375+
if (getTypeNumElements(Ty) == 32)
376+
return (Alignment >= 4);
377+
break;
378+
default:
379+
break;
380+
}
381+
return false;
382+
}
383+
384+
bool HexagonTTIImpl::isLegalMaskedScatter(Type *Ty, Align Alignment) const {
385+
if (!Ty->isVectorTy() || !ST.isTypeForHVX(Ty) ||
386+
!HexagonAllowScatterGatherHVX)
387+
return false;
388+
// This must be in sync with HexagonVectorCombine pass.
389+
switch (Ty->getScalarSizeInBits()) {
390+
case 8:
391+
return (getTypeNumElements(Ty) == 128);
392+
case 16:
393+
if (getTypeNumElements(Ty) == 64)
394+
return (Alignment >= 2);
395+
break;
396+
case 32:
397+
if (getTypeNumElements(Ty) == 32)
398+
return (Alignment >= 4);
399+
break;
400+
default:
401+
break;
402+
}
403+
return false;
404+
}
405+
406+
bool HexagonTTIImpl::forceScalarizeMaskedGather(VectorType *VTy,
407+
Align Alignment) const {
408+
return !isLegalMaskedGather(VTy, Alignment);
409+
}
410+
411+
bool HexagonTTIImpl::forceScalarizeMaskedScatter(VectorType *VTy,
412+
Align Alignment) const {
413+
return !isLegalMaskedScatter(VTy, Alignment);
414+
}
415+
357416
/// --- Vector TTI end ---
358417

359418
unsigned HexagonTTIImpl::getPrefetchDistance() const {

‎llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,12 @@ class HexagonTTIImpl final : public BasicTTIImplBase<HexagonTTIImpl> {
169169
unsigned AddressSpace) const override;
170170
bool isLegalMaskedLoad(Type *DataType, Align Alignment,
171171
unsigned AddressSpace) const override;
172+
bool isLegalMaskedGather(Type *Ty, Align Alignment) const override;
173+
bool isLegalMaskedScatter(Type *Ty, Align Alignment) const override;
174+
bool forceScalarizeMaskedGather(VectorType *VTy,
175+
Align Alignment) const override;
176+
bool forceScalarizeMaskedScatter(VectorType *VTy,
177+
Align Alignment) const override;
172178

173179
/// @}
174180

0 commit comments

Comments
 (0)