Skip to content

Commit b53c1e4

Browse files
john-brawn-armjohn-brawn-arm
authored
[AArch64] Add ISel for postindex ld1/st1 in big-endian (#144387)
When big-endian we need to use ld1/st1 for vector loads and stores so that we get the elements in the correct order, but this prevents postindex addressing from being used. Fix this by adding the appropriate ISel patterns, plus the relevant changes in ISelLowering and ISelDAGToDAG to cause postindex addressing to be used.
1 parent e4c3b03 commit b53c1e4

File tree

5 files changed

+2231
-55
lines changed

5 files changed

+2231
-55
lines changed

‎llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1583,6 +1583,8 @@ bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
15831583
EVT DstVT = N->getValueType(0);
15841584
ISD::MemIndexedMode AM = LD->getAddressingMode();
15851585
bool IsPre = AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
1586+
ConstantSDNode *OffsetOp = cast<ConstantSDNode>(LD->getOffset());
1587+
int OffsetVal = (int)OffsetOp->getZExtValue();
15861588

15871589
// We're not doing validity checking here. That was done when checking
15881590
// if we should mark the load as indexed or not. We're just selecting
@@ -1637,18 +1639,58 @@ bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
16371639
Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
16381640
} else if (VT == MVT::f32) {
16391641
Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1640-
} else if (VT == MVT::f64 || VT.is64BitVector()) {
1642+
} else if (VT == MVT::f64 ||
1643+
(VT.is64BitVector() && Subtarget->isLittleEndian())) {
16411644
Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1642-
} else if (VT.is128BitVector()) {
1645+
} else if (VT.is128BitVector() && Subtarget->isLittleEndian()) {
16431646
Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1647+
} else if (VT.is64BitVector()) {
1648+
if (IsPre || OffsetVal != 8)
1649+
return false;
1650+
switch (VT.getScalarSizeInBits()) {
1651+
case 8:
1652+
Opcode = AArch64::LD1Onev8b_POST;
1653+
break;
1654+
case 16:
1655+
Opcode = AArch64::LD1Onev4h_POST;
1656+
break;
1657+
case 32:
1658+
Opcode = AArch64::LD1Onev2s_POST;
1659+
break;
1660+
case 64:
1661+
Opcode = AArch64::LD1Onev1d_POST;
1662+
break;
1663+
default:
1664+
llvm_unreachable("Expected vector element to be a power of 2");
1665+
}
1666+
} else if (VT.is128BitVector()) {
1667+
if (IsPre || OffsetVal != 16)
1668+
return false;
1669+
switch (VT.getScalarSizeInBits()) {
1670+
case 8:
1671+
Opcode = AArch64::LD1Onev16b_POST;
1672+
break;
1673+
case 16:
1674+
Opcode = AArch64::LD1Onev8h_POST;
1675+
break;
1676+
case 32:
1677+
Opcode = AArch64::LD1Onev4s_POST;
1678+
break;
1679+
case 64:
1680+
Opcode = AArch64::LD1Onev2d_POST;
1681+
break;
1682+
default:
1683+
llvm_unreachable("Expected vector element to be a power of 2");
1684+
}
16441685
} else
16451686
return false;
16461687
SDValue Chain = LD->getChain();
16471688
SDValue Base = LD->getBasePtr();
1648-
ConstantSDNode *OffsetOp = cast<ConstantSDNode>(LD->getOffset());
1649-
int OffsetVal = (int)OffsetOp->getZExtValue();
16501689
SDLoc dl(N);
1651-
SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1690+
// LD1 encodes an immediate offset by using XZR as the offset register.
1691+
SDValue Offset = (VT.isVector() && !Subtarget->isLittleEndian())
1692+
? CurDAG->getRegister(AArch64::XZR, MVT::i64)
1693+
: CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
16521694
SDValue Ops[] = { Base, Offset, Chain };
16531695
SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
16541696
MVT::Other, Ops);

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2087,12 +2087,18 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
20872087
setOperationAction(ISD::STRICT_FSETCC, VT, Expand);
20882088
setOperationAction(ISD::STRICT_FSETCCS, VT, Expand);
20892089

2090+
// When little-endian we can use ordinary d and q register loads/stores for
2091+
// vector types, but when big-endian we need to use structure load/store which
2092+
// only allow post-index addressing.
20902093
if (Subtarget->isLittleEndian()) {
20912094
for (unsigned im = (unsigned)ISD::PRE_INC;
20922095
im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
20932096
setIndexedLoadAction(im, VT, Legal);
20942097
setIndexedStoreAction(im, VT, Legal);
20952098
}
2099+
} else {
2100+
setIndexedLoadAction(ISD::POST_INC, VT, Legal);
2101+
setIndexedStoreAction(ISD::POST_INC, VT, Legal);
20962102
}
20972103

20982104
if (Subtarget->hasD128()) {
@@ -27047,6 +27053,12 @@ bool AArch64TargetLowering::getIndexedAddressParts(SDNode *N, SDNode *Op,
2704727053
RHSC = -(uint64_t)RHSC;
2704827054
if (!isInt<9>(RHSC))
2704927055
return false;
27056+
// When big-endian VLD1/VST1 are used for vector load and store, and these
27057+
// only allow an offset that's equal to the store size.
27058+
EVT MemType = cast<MemSDNode>(N)->getMemoryVT();
27059+
if (!Subtarget->isLittleEndian() && MemType.isVector() &&
27060+
RHSC != MemType.getStoreSize())
27061+
return false;
2705027062
// Always emit pre-inc/post-inc addressing mode. Use negated constant offset
2705127063
// when dealing with subtraction.
2705227064
Offset = DAG.getConstant(RHSC, SDLoc(N), RHS->getValueType(0));

‎llvm/lib/Target/AArch64/AArch64InstrInfo.td‎

Lines changed: 51 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -4942,39 +4942,42 @@ def : Pat<(post_truncsti8 GPR64:$Rt, GPR64sp:$addr, simm9:$off),
49424942
def : Pat<(post_store (bf16 FPR16:$Rt), GPR64sp:$addr, simm9:$off),
49434943
(STRHpost FPR16:$Rt, GPR64sp:$addr, simm9:$off)>;
49444944

4945-
def : Pat<(post_store (v8i8 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4946-
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4947-
def : Pat<(post_store (v4i16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4948-
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4949-
def : Pat<(post_store (v2i32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4950-
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4951-
def : Pat<(post_store (v2f32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4952-
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4953-
def : Pat<(post_store (v1i64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4954-
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4955-
def : Pat<(post_store (v1f64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4956-
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4957-
def : Pat<(post_store (v4f16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4958-
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4959-
def : Pat<(post_store (v4bf16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4960-
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4961-
4962-
def : Pat<(post_store (v16i8 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4963-
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4964-
def : Pat<(post_store (v8i16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4965-
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4966-
def : Pat<(post_store (v4i32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4967-
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4968-
def : Pat<(post_store (v4f32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4969-
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4970-
def : Pat<(post_store (v2i64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4971-
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4972-
def : Pat<(post_store (v2f64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4973-
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4974-
def : Pat<(post_store (v8f16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4975-
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4976-
def : Pat<(post_store (v8bf16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4977-
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4945+
let Predicates = [IsLE] in {
4946+
// We must use ST1 to store vectors in big-endian.
4947+
def : Pat<(post_store(v8i8 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4948+
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4949+
def : Pat<(post_store(v4i16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4950+
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4951+
def : Pat<(post_store(v2i32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4952+
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4953+
def : Pat<(post_store(v2f32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4954+
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4955+
def : Pat<(post_store(v1i64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4956+
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4957+
def : Pat<(post_store(v1f64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4958+
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4959+
def : Pat<(post_store(v4f16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4960+
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4961+
def : Pat<(post_store(v4bf16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
4962+
(STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
4963+
4964+
def : Pat<(post_store(v16i8 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4965+
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4966+
def : Pat<(post_store(v8i16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4967+
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4968+
def : Pat<(post_store(v4i32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4969+
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4970+
def : Pat<(post_store(v4f32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4971+
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4972+
def : Pat<(post_store(v2i64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4973+
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4974+
def : Pat<(post_store(v2f64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4975+
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4976+
def : Pat<(post_store(v8f16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4977+
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4978+
def : Pat<(post_store(v8bf16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
4979+
(STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
4980+
}
49784981

49794982
//===----------------------------------------------------------------------===//
49804983
// Load/store exclusive instructions.
@@ -8925,6 +8928,21 @@ def : St1Pat<v4i16, ST1Onev4h>;
89258928
def : St1Pat<v2i32, ST1Onev2s>;
89268929
def : St1Pat<v1i64, ST1Onev1d>;
89278930

8931+
class St1PostPat<ValueType ty, Instruction INST, int off>
8932+
: Pat<(post_store ty:$Vt, GPR64sp:$Rn, (i64 off)),
8933+
(INST ty:$Vt, GPR64sp:$Rn, XZR)>;
8934+
8935+
let Predicates = [IsBE] in {
8936+
def : St1PostPat<v16i8, ST1Onev16b_POST, 16>;
8937+
def : St1PostPat<v8i16, ST1Onev8h_POST, 16>;
8938+
def : St1PostPat<v4i32, ST1Onev4s_POST, 16>;
8939+
def : St1PostPat<v2i64, ST1Onev2d_POST, 16>;
8940+
def : St1PostPat<v8i8, ST1Onev8b_POST, 8>;
8941+
def : St1PostPat<v4i16, ST1Onev4h_POST, 8>;
8942+
def : St1PostPat<v2i32, ST1Onev2s_POST, 8>;
8943+
def : St1PostPat<v1i64, ST1Onev1d_POST, 8>;
8944+
}
8945+
89288946
//---
89298947
// Single-element
89308948
//---

0 commit comments

Comments
 (0)