Skip to content

Commit b0b64af

Browse files
Prince781Prince781
committed
[NVPTX] handle more cases for loads and stores
Split unaligned stores and loads of v2f32. Add DAGCombiner rules for: - target-independent stores that store a v2f32 BUILD_VECTOR. We scalarize the value and rewrite the store Fix test cases.
1 parent 20a2f0b commit b0b64af

File tree

5 files changed

+52
-25
lines changed

5 files changed

+52
-25
lines changed

‎llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp‎

Lines changed: 42 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -833,7 +833,7 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
833833
setTargetDAGCombine({ISD::ADD, ISD::AND, ISD::EXTRACT_VECTOR_ELT, ISD::FADD,
834834
ISD::MUL, ISD::SHL, ISD::SREM, ISD::UREM, ISD::VSELECT,
835835
ISD::BUILD_VECTOR, ISD::ADDRSPACECAST, ISD::FP_ROUND,
836-
ISD::TRUNCATE, ISD::LOAD, ISD::BITCAST});
836+
ISD::TRUNCATE, ISD::LOAD, ISD::STORE, ISD::BITCAST});
837837

838838
// setcc for f16x2 and bf16x2 needs special handling to prevent
839839
// legalizer's attempt to scalarize it due to v2i1 not being legal.
@@ -3091,10 +3091,10 @@ SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
30913091
if (Op.getValueType() == MVT::i1)
30923092
return LowerLOADi1(Op, DAG);
30933093

3094-
// v2f16/v2bf16/v2i16/v4i8 are legal, so we can't rely on legalizer to handle
3095-
// unaligned loads and have to handle it here.
3094+
// v2f16/v2bf16/v2i16/v4i8/v2f32 are legal, so we can't rely on legalizer to
3095+
// handle unaligned loads and have to handle it here.
30963096
EVT VT = Op.getValueType();
3097-
if (Isv2x16VT(VT) || VT == MVT::v4i8) {
3097+
if (Isv2x16VT(VT) || VT == MVT::v4i8 || VT == MVT::v2f32) {
30983098
LoadSDNode *Load = cast<LoadSDNode>(Op);
30993099
EVT MemVT = Load->getMemoryVT();
31003100
if (!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
@@ -3138,15 +3138,15 @@ SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
31383138
if (VT == MVT::i1)
31393139
return LowerSTOREi1(Op, DAG);
31403140

3141-
// v2f16 is legal, so we can't rely on legalizer to handle unaligned
3142-
// stores and have to handle it here.
3143-
if ((Isv2x16VT(VT) || VT == MVT::v4i8) &&
3141+
// v2f16/v2bf16/v2i16/v4i8/v2f32 are legal, so we can't rely on legalizer to
3142+
// handle unaligned stores and have to handle it here.
3143+
if ((Isv2x16VT(VT) || VT == MVT::v4i8 || VT == MVT::v2f32) &&
31443144
!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
31453145
VT, *Store->getMemOperand()))
31463146
return expandUnalignedStore(Store, DAG);
31473147

3148-
// v2f16, v2bf16 and v2i16 don't need special handling.
3149-
if (Isv2x16VT(VT) || VT == MVT::v4i8)
3148+
// v2f16/v2bf16/v2i16/v4i8/v2f32 don't need special handling.
3149+
if (Isv2x16VT(VT) || VT == MVT::v4i8 || VT == MVT::v2f32)
31503150
return SDValue();
31513151

31523152
if (VT.isVector())
@@ -3155,8 +3155,8 @@ SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
31553155
return SDValue();
31563156
}
31573157

3158-
SDValue
3159-
NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
3158+
static SDValue convertVectorStore(SDValue Op, SelectionDAG &DAG,
3159+
const SmallVectorImpl<SDValue> &Elements) {
31603160
SDNode *N = Op.getNode();
31613161
SDValue Val = N->getOperand(1);
31623162
SDLoc DL(N);
@@ -3223,6 +3223,8 @@ NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
32233223
SDValue SubVector = DAG.getBuildVector(EltVT, DL, SubVectorElts);
32243224
Ops.push_back(SubVector);
32253225
}
3226+
} else if (!Elements.empty()) {
3227+
Ops.insert(Ops.end(), Elements.begin(), Elements.end());
32263228
} else {
32273229
for (unsigned i = 0; i < NumElts; ++i) {
32283230
SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Val,
@@ -3240,10 +3242,19 @@ NVPTXTargetLowering::LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const {
32403242
DAG.getMemIntrinsicNode(Opcode, DL, DAG.getVTList(MVT::Other), Ops,
32413243
MemSD->getMemoryVT(), MemSD->getMemOperand());
32423244

3243-
// return DCI.CombineTo(N, NewSt, true);
32443245
return NewSt;
32453246
}
32463247

3248+
// Default variant where we don't pass in elements.
3249+
static SDValue convertVectorStore(SDValue Op, SelectionDAG &DAG) {
3250+
return convertVectorStore(Op, DAG, SmallVector<SDValue>{});
3251+
}
3252+
3253+
SDValue NVPTXTargetLowering::LowerSTOREVector(SDValue Op,
3254+
SelectionDAG &DAG) const {
3255+
return convertVectorStore(Op, DAG);
3256+
}
3257+
32473258
// st i1 v, addr
32483259
// =>
32493260
// v1 = zxt v to i16
@@ -5402,6 +5413,9 @@ static SDValue PerformStoreCombineHelper(SDNode *N,
54025413
// -->
54035414
// StoreRetvalV2 {a, b}
54045415
// likewise for V2 -> V4 case
5416+
//
5417+
// We also handle target-independent stores, which require us to first
5418+
// convert to StoreV2.
54055419

54065420
std::optional<NVPTXISD::NodeType> NewOpcode;
54075421
switch (N->getOpcode()) {
@@ -5427,8 +5441,8 @@ static SDValue PerformStoreCombineHelper(SDNode *N,
54275441
SDValue CurrentOp = N->getOperand(I);
54285442
if (CurrentOp->getOpcode() == ISD::BUILD_VECTOR) {
54295443
assert(CurrentOp.getValueType() == MVT::v2f32);
5430-
NewOps.push_back(CurrentOp.getNode()->getOperand(0));
5431-
NewOps.push_back(CurrentOp.getNode()->getOperand(1));
5444+
NewOps.push_back(CurrentOp.getOperand(0));
5445+
NewOps.push_back(CurrentOp.getOperand(1));
54325446
} else {
54335447
NewOps.clear();
54345448
break;
@@ -6199,6 +6213,18 @@ static SDValue PerformBITCASTCombine(SDNode *N,
61996213
return SDValue();
62006214
}
62016215

6216+
static SDValue PerformStoreCombine(SDNode *N,
6217+
TargetLowering::DAGCombinerInfo &DCI) {
6218+
// check if the store'd value can be scalarized
6219+
SDValue StoredVal = N->getOperand(1);
6220+
if (StoredVal.getValueType() == MVT::v2f32 &&
6221+
StoredVal.getOpcode() == ISD::BUILD_VECTOR) {
6222+
SmallVector<SDValue> Elements(StoredVal->op_values());
6223+
return convertVectorStore(SDValue(N, 0), DCI.DAG, Elements);
6224+
}
6225+
return SDValue();
6226+
}
6227+
62026228
SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
62036229
DAGCombinerInfo &DCI) const {
62046230
CodeGenOptLevel OptLevel = getTargetMachine().getOptLevel();
@@ -6228,6 +6254,8 @@ SDValue NVPTXTargetLowering::PerformDAGCombine(SDNode *N,
62286254
case NVPTXISD::LoadParam:
62296255
case NVPTXISD::LoadParamV2:
62306256
return PerformLoadCombine(N, DCI);
6257+
case ISD::STORE:
6258+
return PerformStoreCombine(N, DCI);
62316259
case NVPTXISD::StoreParam:
62326260
case NVPTXISD::StoreParamV2:
62336261
case NVPTXISD::StoreParamV4:

‎llvm/test/CodeGen/NVPTX/aggregate-return.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ define void @test_v2f32(<2 x float> %input, ptr %output) {
1010
; CHECK-LABEL: @test_v2f32
1111
%call = tail call <2 x float> @barv(<2 x float> %input)
1212
; CHECK: .param .align 8 .b8 retval0[8];
13-
; CHECK: ld.param.v2.f32 {[[E0:%f[0-9]+]], [[E1:%f[0-9]+]]}, [retval0];
13+
; CHECK: ld.param.b64 [[E0_1:%rd[0-9]+]], [retval0];
1414
store <2 x float> %call, ptr %output, align 8
15-
; CHECK: st.v2.f32 [{{%rd[0-9]+}}], {[[E0]], [[E1]]}
15+
; CHECK: st.b64 [{{%rd[0-9]+}}], [[E0_1]]
1616
ret void
1717
}
1818

‎llvm/test/CodeGen/NVPTX/f32x2-instructions.ll‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -512,14 +512,13 @@ define <2 x float> @test_frem_ftz(<2 x float> %a, <2 x float> %b) #2 {
512512
define void @test_ldst_v2f32(ptr %a, ptr %b) #0 {
513513
; CHECK-LABEL: test_ldst_v2f32(
514514
; CHECK: {
515-
; CHECK-NEXT: .reg .f32 %f<3>;
516-
; CHECK-NEXT: .reg .b64 %rd<3>;
515+
; CHECK-NEXT: .reg .b64 %rd<4>;
517516
; CHECK-EMPTY:
518517
; CHECK-NEXT: // %bb.0:
519518
; CHECK-NEXT: ld.param.u64 %rd2, [test_ldst_v2f32_param_1];
520519
; CHECK-NEXT: ld.param.u64 %rd1, [test_ldst_v2f32_param_0];
521-
; CHECK-NEXT: ld.v2.f32 {%f1, %f2}, [%rd1];
522-
; CHECK-NEXT: st.v2.f32 [%rd2], {%f1, %f2};
520+
; CHECK-NEXT: ld.b64 %rd3, [%rd1];
521+
; CHECK-NEXT: st.b64 [%rd2], %rd3;
523522
; CHECK-NEXT: ret;
524523
%t1 = load <2 x float>, ptr %a
525524
store <2 x float> %t1, ptr %b, align 32

‎llvm/test/CodeGen/NVPTX/load-with-non-coherent-cache.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,9 @@ define ptx_kernel void @foo10(ptr noalias readonly %from, ptr %to) {
110110
}
111111

112112
; SM20-LABEL: .visible .entry foo11(
113-
; SM20: ld.global.v2.f32
113+
; SM20: ld.global.b64
114114
; SM35-LABEL: .visible .entry foo11(
115-
; SM35: ld.global.nc.v2.f32
115+
; SM35: ld.global.nc.b64
116116
define ptx_kernel void @foo11(ptr noalias readonly %from, ptr %to) {
117117
%1 = load <2 x float>, ptr %from
118118
store <2 x float> %1, ptr %to

‎llvm/test/CodeGen/NVPTX/misaligned-vector-ldst.ll‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,15 @@ define <4 x float> @t1(ptr %p1) {
1818
define <4 x float> @t2(ptr %p1) {
1919
; CHECK-NOT: ld.v4
2020
; CHECK-NOT: ld.v2
21-
; CHECK: ld.f32
21+
; CHECK: ld.u32
2222
%r = load <4 x float>, ptr %p1, align 4
2323
ret <4 x float> %r
2424
}
2525

2626
; CHECK-LABEL: t3
2727
define <4 x float> @t3(ptr %p1) {
2828
; CHECK-NOT: ld.v4
29-
; CHECK: ld.v2
29+
; CHECK: ld.b64
3030
%r = load <4 x float>, ptr %p1, align 8
3131
ret <4 x float> %r
3232
}
@@ -111,7 +111,7 @@ define void @s1(ptr %p1, <4 x float> %v) {
111111
define void @s2(ptr %p1, <4 x float> %v) {
112112
; CHECK-NOT: st.v4
113113
; CHECK-NOT: st.v2
114-
; CHECK: st.f32
114+
; CHECK: st.u32
115115
store <4 x float> %v, ptr %p1, align 4
116116
ret void
117117
}

0 commit comments

Comments
 (0)