Skip to content

Commit 658a931

Browse files
author
daniel-trujillo-bsc
authored
[CodeGen][RISCV] Add support of RISCV nontemporal to vector predication instructions. (#153033)
This PR adds support for VP intrinsics to be aware of the nontemporal metadata information.
1 parent e7c9f2d commit 658a931

File tree

6 files changed

+44748
-16
lines changed

6 files changed

+44748
-16
lines changed

‎llvm/include/llvm/CodeGen/TargetLowering.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,8 @@ class LLVM_ABI TargetLoweringBase {
472472
const DataLayout &DL) const;
473473
MachineMemOperand::Flags getAtomicMemOperandFlags(const Instruction &AI,
474474
const DataLayout &DL) const;
475+
MachineMemOperand::Flags
476+
getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const;
475477

476478
virtual bool isSelectSupported(SelectSupportKind /*kind*/) const {
477479
return true;

‎llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2517,10 +2517,10 @@ void DAGTypeLegalizer::SplitVecRes_Gather(MemSDNode *N, SDValue &Lo,
25172517
else
25182518
std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, dl);
25192519

2520+
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
25202521
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2521-
N->getPointerInfo(), MachineMemOperand::MOLoad,
2522-
LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
2523-
N->getRanges());
2522+
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
2523+
Alignment, N->getAAInfo(), N->getRanges());
25242524

25252525
if (auto *MGT = dyn_cast<MaskedGatherSDNode>(N)) {
25262526
SDValue PassThru = MGT->getPassThru();
@@ -4321,10 +4321,10 @@ SDValue DAGTypeLegalizer::SplitVecOp_Scatter(MemSDNode *N, unsigned OpNo) {
43214321
std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index, DL);
43224322

43234323
SDValue Lo;
4324+
MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
43244325
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
4325-
N->getPointerInfo(), MachineMemOperand::MOStore,
4326-
LocationSize::beforeOrAfterPointer(), Alignment, N->getAAInfo(),
4327-
N->getRanges());
4326+
N->getPointerInfo(), MMOFlags, LocationSize::beforeOrAfterPointer(),
4327+
Alignment, N->getAAInfo(), N->getRanges());
43284328

43294329
if (auto *MSC = dyn_cast<MaskedScatterSDNode>(N)) {
43304330
SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Ops.Scale};

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8456,8 +8456,11 @@ void SelectionDAGBuilder::visitVPLoad(
84568456
MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo);
84578457
bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
84588458
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
8459+
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8460+
MachineMemOperand::Flags MMOFlags =
8461+
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
84598462
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8460-
MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad,
8463+
MachinePointerInfo(PtrOperand), MMOFlags,
84618464
LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
84628465
LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2],
84638466
MMO, false /*IsExpanding */);
@@ -8508,9 +8511,11 @@ void SelectionDAGBuilder::visitVPGather(
85088511
Alignment = DAG.getEVTAlign(VT.getScalarType());
85098512
unsigned AS =
85108513
PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8514+
MachineMemOperand::Flags MMOFlags =
8515+
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
85118516
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8512-
MachinePointerInfo(AS), MachineMemOperand::MOLoad,
8513-
LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8517+
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
8518+
*Alignment, AAInfo, Ranges);
85148519
SDValue Base, Index, Scale;
85158520
bool UniformBase =
85168521
getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
@@ -8546,8 +8551,11 @@ void SelectionDAGBuilder::visitVPStore(
85468551
Alignment = DAG.getEVTAlign(VT);
85478552
SDValue Ptr = OpValues[1];
85488553
SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
8554+
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8555+
MachineMemOperand::Flags MMOFlags =
8556+
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
85498557
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8550-
MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore,
8558+
MachinePointerInfo(PtrOperand), MMOFlags,
85518559
LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
85528560
ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset,
85538561
OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED,
@@ -8569,9 +8577,11 @@ void SelectionDAGBuilder::visitVPScatter(
85698577
Alignment = DAG.getEVTAlign(VT.getScalarType());
85708578
unsigned AS =
85718579
PtrOperand->getType()->getScalarType()->getPointerAddressSpace();
8580+
MachineMemOperand::Flags MMOFlags =
8581+
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
85728582
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8573-
MachinePointerInfo(AS), MachineMemOperand::MOStore,
8574-
LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8583+
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
8584+
*Alignment, AAInfo);
85758585
SDValue Base, Index, Scale;
85768586
bool UniformBase =
85778587
getUniformBase(PtrOperand, Base, Index, Scale, this, VPIntrin.getParent(),
@@ -8609,9 +8619,12 @@ void SelectionDAGBuilder::visitVPStridedLoad(
86098619
bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML);
86108620
SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
86118621
unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8622+
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8623+
MachineMemOperand::Flags MMOFlags =
8624+
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
86128625
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8613-
MachinePointerInfo(AS), MachineMemOperand::MOLoad,
8614-
LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo, Ranges);
8626+
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
8627+
*Alignment, AAInfo, Ranges);
86158628

86168629
SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1],
86178630
OpValues[2], OpValues[3], MMO,
@@ -8632,9 +8645,12 @@ void SelectionDAGBuilder::visitVPStridedStore(
86328645
Alignment = DAG.getEVTAlign(VT.getScalarType());
86338646
AAMDNodes AAInfo = VPIntrin.getAAMetadata();
86348647
unsigned AS = PtrOperand->getType()->getPointerAddressSpace();
8648+
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8649+
MachineMemOperand::Flags MMOFlags =
8650+
TLI.getVPIntrinsicMemOperandFlags(VPIntrin);
86358651
MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
8636-
MachinePointerInfo(AS), MachineMemOperand::MOStore,
8637-
LocationSize::beforeOrAfterPointer(), *Alignment, AAInfo);
8652+
MachinePointerInfo(AS), MMOFlags, LocationSize::beforeOrAfterPointer(),
8653+
*Alignment, AAInfo);
86388654

86398655
SDValue ST = DAG.getStridedStoreVP(
86408656
getMemoryRoot(), DL, OpValues[0], OpValues[1],

‎llvm/lib/CodeGen/TargetLoweringBase.cpp‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2406,6 +2406,34 @@ TargetLoweringBase::getAtomicMemOperandFlags(const Instruction &AI,
24062406
return Flags;
24072407
}
24082408

2409+
MachineMemOperand::Flags TargetLoweringBase::getVPIntrinsicMemOperandFlags(
2410+
const VPIntrinsic &VPIntrin) const {
2411+
MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
2412+
Intrinsic::ID IntrinID = VPIntrin.getIntrinsicID();
2413+
2414+
switch (IntrinID) {
2415+
default:
2416+
llvm_unreachable("unexpected intrinsic. Existing code may be appropriate "
2417+
"for it, but support must be explicitly enabled");
2418+
case Intrinsic::vp_load:
2419+
case Intrinsic::vp_gather:
2420+
case Intrinsic::experimental_vp_strided_load:
2421+
Flags = MachineMemOperand::MOLoad;
2422+
break;
2423+
case Intrinsic::vp_store:
2424+
case Intrinsic::vp_scatter:
2425+
case Intrinsic::experimental_vp_strided_store:
2426+
Flags = MachineMemOperand::MOStore;
2427+
break;
2428+
}
2429+
2430+
if (VPIntrin.hasMetadata(LLVMContext::MD_nontemporal))
2431+
Flags |= MachineMemOperand::MONonTemporal;
2432+
2433+
Flags |= getTargetMMOFlags(VPIntrin);
2434+
return Flags;
2435+
}
2436+
24092437
Instruction *TargetLoweringBase::emitLeadingFence(IRBuilderBase &Builder,
24102438
Instruction *Inst,
24112439
AtomicOrdering Ord) const {

0 commit comments

Comments
 (0)