Skip to content

[RISCV] Split f64 loads/stores for RV32+Zdinx during isel instead of post-RA. #139840

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
topperc merged 2 commits into from
May 15, 2025
Merged

[RISCV] Split f64 loads/stores for RV32+Zdinx during isel instead of post-RA. #139840

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
83f0620
[RISCV] Split f64 loads/stores for RV32+Zdinx during isel instead of …
topperc May 14, 2025
0b899a3
Merge remote-tracking branch 'origin/main' into pr/zdinx-early-split
topperc May 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 11 additions & 43 deletions llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2531,8 +2531,7 @@ bool RISCVDAGToDAGISel::SelectAddrFrameIndex(SDValue Addr, SDValue &Base,
static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
const MVT VT, const RISCVSubtarget *Subtarget,
SDValue Addr, SDValue &Base, SDValue &Offset,
bool IsPrefetch = false,
bool IsRV32Zdinx = false) {
bool IsPrefetch = false) {
if (!isa<ConstantSDNode>(Addr))
return false;

Expand All @@ -2546,9 +2545,6 @@ static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
if (IsPrefetch && (Lo12 & 0b11111) != 0)
return false;
if (IsRV32Zdinx && !isInt<12>(Lo12 + 4))
return false;

if (Hi) {
int64_t Hi20 = (Hi >> 12) & 0xfffff;
Base = SDValue(
Expand All @@ -2572,8 +2568,6 @@ static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
Lo12 = Seq.back().getImm();
if (IsPrefetch && (Lo12 & 0b11111) != 0)
return false;
if (IsRV32Zdinx && !isInt<12>(Lo12 + 4))
return false;

// Drop the last instruction.
Seq.pop_back();
Expand Down Expand Up @@ -2665,47 +2659,22 @@ bool RISCVDAGToDAGISel::SelectAddrRegRegScale(SDValue Addr,
}

bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
SDValue &Offset, bool IsRV32Zdinx) {
SDValue &Offset) {
if (SelectAddrFrameIndex(Addr, Base, Offset))
return true;

SDLoc DL(Addr);
MVT VT = Addr.getSimpleValueType();

if (Addr.getOpcode() == RISCVISD::ADD_LO) {
// If this is non RV32Zdinx we can always fold.
if (!IsRV32Zdinx) {
Base = Addr.getOperand(0);
Offset = Addr.getOperand(1);
return true;
}

// For RV32Zdinx we need to have more than 4 byte alignment so we can add 4
// to the offset when we expand in RISCVExpandPseudoInsts.
if (auto *GA = dyn_cast<GlobalAddressSDNode>(Addr.getOperand(1))) {
const DataLayout &DL = CurDAG->getDataLayout();
Align Alignment = commonAlignment(
GA->getGlobal()->getPointerAlignment(DL), GA->getOffset());
if (Alignment > 4) {
Base = Addr.getOperand(0);
Offset = Addr.getOperand(1);
return true;
}
}
if (auto *CP = dyn_cast<ConstantPoolSDNode>(Addr.getOperand(1))) {
Align Alignment = commonAlignment(CP->getAlign(), CP->getOffset());
if (Alignment > 4) {
Base = Addr.getOperand(0);
Offset = Addr.getOperand(1);
return true;
}
}
Base = Addr.getOperand(0);
Offset = Addr.getOperand(1);
return true;
}

int64_t RV32ZdinxRange = IsRV32Zdinx ? 4 : 0;
if (CurDAG->isBaseWithConstantOffset(Addr)) {
int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
if (isInt<12>(CVal) && isInt<12>(CVal + RV32ZdinxRange)) {
if (isInt<12>(CVal) && isInt<12>(CVal)) {
Base = Addr.getOperand(0);
if (Base.getOpcode() == RISCVISD::ADD_LO) {
SDValue LoOperand = Base.getOperand(1);
Expand All @@ -2718,8 +2687,7 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
const DataLayout &DL = CurDAG->getDataLayout();
Align Alignment = commonAlignment(
GA->getGlobal()->getPointerAlignment(DL), GA->getOffset());
if ((CVal == 0 || Alignment > CVal) &&
(!IsRV32Zdinx || commonAlignment(Alignment, CVal) > 4)) {
if ((CVal == 0 || Alignment > CVal)) {
int64_t CombinedOffset = CVal + GA->getOffset();
Base = Base.getOperand(0);
Offset = CurDAG->getTargetGlobalAddress(
Expand All @@ -2740,13 +2708,13 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
// Handle ADD with large immediates.
if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1))) {
int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
assert(!(isInt<12>(CVal) && isInt<12>(CVal + RV32ZdinxRange)) &&
assert(!(isInt<12>(CVal) && isInt<12>(CVal)) &&
"simm12 not already handled?");

// Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
// an ADDI for part of the offset and fold the rest into the load/store.
// This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
if (CVal >= -4096 && CVal <= (4094 - RV32ZdinxRange)) {
if (CVal >= -4096 && CVal <= 4094) {
int64_t Adj = CVal < 0 ? -2048 : 2047;
Base = SDValue(
CurDAG->getMachineNode(RISCV::ADDI, DL, VT, Addr.getOperand(0),
Expand All @@ -2764,7 +2732,7 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
// instructions.
if (isWorthFoldingAdd(Addr) &&
selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr.getOperand(1), Base,
Offset, /*IsPrefetch=*/false, RV32ZdinxRange)) {
Offset, /*IsPrefetch=*/false)) {
// Insert an ADD instruction with the materialized Hi52 bits.
Base = SDValue(
CurDAG->getMachineNode(RISCV::ADD, DL, VT, Addr.getOperand(0), Base),
Expand All @@ -2774,7 +2742,7 @@ bool RISCVDAGToDAGISel::SelectAddrRegImm(SDValue Addr, SDValue &Base,
}

if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
/*IsPrefetch=*/false, RV32ZdinxRange))
/*IsPrefetch=*/false))
return true;

Base = Addr;
Expand Down
6 changes: 1 addition & 5 deletions llvm/lib/Target/RISCV/RISCVISelDAGToDAG.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,7 @@ class RISCVDAGToDAGISel : public SelectionDAGISel {
std::vector<SDValue> &OutOps) override;

bool SelectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset);
bool SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset,
bool IsRV32Zdinx = false);
bool SelectAddrRegImmRV32Zdinx(SDValue Addr, SDValue &Base, SDValue &Offset) {
return SelectAddrRegImm(Addr, Base, Offset, true);
}
bool SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset);
bool SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base, SDValue &Offset);

bool SelectAddrRegRegScale(SDValue Addr, unsigned MaxShiftAmount,
Expand Down
69 changes: 59 additions & 10 deletions llvm/lib/Target/RISCV/RISCVISelLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,11 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
if (!Subtarget.is64Bit())
setOperationAction(ISD::BITCAST, MVT::i64, Custom);

if (Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit()) {
setOperationAction(ISD::LOAD, MVT::f64, Custom);
setOperationAction(ISD::STORE, MVT::f64, Custom);
}

if (Subtarget.hasStdExtZfa()) {
setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
setOperationAction(FPRndMode, MVT::f64, Legal);
Expand Down Expand Up @@ -7705,19 +7710,42 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
}
case ISD::LOAD: {
auto *Load = cast<LoadSDNode>(Op);
EVT VecTy = Load->getMemoryVT();
EVT VT = Load->getValueType(0);
if (VT == MVT::f64) {
assert(Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit() &&
"Unexpected custom legalisation");

// Replace a double precision load with two i32 loads and a BuildPairF64.
SDLoc DL(Op);
SDValue BasePtr = Load->getBasePtr();
SDValue Chain = Load->getChain();

SDValue Lo = DAG.getLoad(MVT::i32, DL, Chain, BasePtr,
Load->getPointerInfo(), Load->getOriginalAlign(),
Load->getMemOperand()->getFlags());
BasePtr = DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(4));
SDValue Hi = DAG.getLoad(
MVT::i32, DL, Chain, BasePtr, Load->getPointerInfo().getWithOffset(4),
Load->getOriginalAlign(), Load->getMemOperand()->getFlags());
Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
Hi.getValue(1));

SDValue Pair = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
return DAG.getMergeValues({Pair, Chain}, DL);
}

// Handle normal vector tuple load.
if (VecTy.isRISCVVectorTuple()) {
if (VT.isRISCVVectorTuple()) {
SDLoc DL(Op);
MVT XLenVT = Subtarget.getXLenVT();
unsigned NF = VecTy.getRISCVVectorTupleNumFields();
unsigned Sz = VecTy.getSizeInBits().getKnownMinValue();
unsigned NF = VT.getRISCVVectorTupleNumFields();
unsigned Sz = VT.getSizeInBits().getKnownMinValue();
unsigned NumElts = Sz / (NF * 8);
int Log2LMUL = Log2_64(NumElts) - 3;

auto Flag = SDNodeFlags();
Flag.setNoUnsignedWrap(true);
SDValue Ret = DAG.getUNDEF(VecTy);
SDValue Ret = DAG.getUNDEF(VT);
SDValue BasePtr = Load->getBasePtr();
SDValue VROffset = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT);
VROffset =
Expand All @@ -7731,7 +7759,7 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
MVT::getScalableVectorVT(MVT::i8, NumElts), DL, Load->getChain(),
BasePtr, MachinePointerInfo(Load->getAddressSpace()), Align(8));
OutChains.push_back(LoadVal.getValue(1));
Ret = DAG.getNode(RISCVISD::TUPLE_INSERT, DL, VecTy, Ret, LoadVal,
Ret = DAG.getNode(RISCVISD::TUPLE_INSERT, DL, VT, Ret, LoadVal,
DAG.getVectorIdxConstant(i, DL));
BasePtr = DAG.getNode(ISD::ADD, DL, XLenVT, BasePtr, VROffset, Flag);
}
Expand All @@ -7748,13 +7776,34 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
case ISD::STORE: {
auto *Store = cast<StoreSDNode>(Op);
SDValue StoredVal = Store->getValue();
EVT VecTy = StoredVal.getValueType();
EVT VT = StoredVal.getValueType();
if (VT == MVT::f64) {
assert(Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit() &&
"Unexpected custom legalisation");

// Replace a double precision store with a SplitF64 and i32 stores.
SDValue DL(Op);
SDValue BasePtr = Store->getBasePtr();
SDValue Chain = Store->getChain();
SDValue Split = DAG.getNode(RISCVISD::SplitF64, DL,
DAG.getVTList(MVT::i32, MVT::i32), StoredVal);

SDValue Lo = DAG.getStore(
Chain, DL, Split.getValue(0), BasePtr, Store->getPointerInfo(),
Store->getOriginalAlign(), Store->getMemOperand()->getFlags());
BasePtr = DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(4));
SDValue Hi = DAG.getStore(Chain, DL, Split.getValue(1), BasePtr,
Store->getPointerInfo().getWithOffset(4),
Store->getOriginalAlign(),
Store->getMemOperand()->getFlags());
return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
}
// Handle normal vector tuple store.
if (VecTy.isRISCVVectorTuple()) {
if (VT.isRISCVVectorTuple()) {
SDLoc DL(Op);
MVT XLenVT = Subtarget.getXLenVT();
unsigned NF = VecTy.getRISCVVectorTupleNumFields();
unsigned Sz = VecTy.getSizeInBits().getKnownMinValue();
unsigned NF = VT.getRISCVVectorTupleNumFields();
unsigned Sz = VT.getSizeInBits().getKnownMinValue();
unsigned NumElts = Sz / (NF * 8);
int Log2LMUL = Log2_64(NumElts) - 3;

Expand Down
10 changes: 2 additions & 8 deletions llvm/lib/Target/RISCV/RISCVInstrInfoD.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ def : GINodeEquiv<G_MERGE_VALUES, RISCVBuildPairF64>;
def RISCVSplitF64 : RVSDNode<"SplitF64", SDT_RISCVSplitF64>;
def : GINodeEquiv<G_UNMERGE_VALUES, RISCVSplitF64>;

def AddrRegImmINX : ComplexPattern<iPTR, 2, "SelectAddrRegImmRV32Zdinx">;

//===----------------------------------------------------------------------===//
// Operand and SDNode transformation definitions.
//===----------------------------------------------------------------------===//
Expand Down Expand Up @@ -529,16 +527,12 @@ defm Select_FPR64IN32X : SelectCC_GPR_rrirr<FPR64IN32X, f64>;
def PseudoFROUND_D_IN32X : PseudoFROUND<FPR64IN32X, f64>;

/// Loads
let isCall = 0, mayLoad = 1, mayStore = 0, Size = 8, isCodeGenOnly = 1 in
let hasSideEffects = 0, mayLoad = 1, mayStore = 0, Size = 8, isCodeGenOnly = 1 in
def PseudoRV32ZdinxLD : Pseudo<(outs GPRPair:$dst), (ins GPR:$rs1, simm12:$imm12), []>;
def : Pat<(f64 (load (AddrRegImmINX (XLenVT GPR:$rs1), simm12:$imm12))),
(PseudoRV32ZdinxLD GPR:$rs1, simm12:$imm12)>;

/// Stores
let isCall = 0, mayLoad = 0, mayStore = 1, Size = 8, isCodeGenOnly = 1 in
let hasSideEffects = 0, mayLoad = 0, mayStore = 1, Size = 8, isCodeGenOnly = 1 in
def PseudoRV32ZdinxSD : Pseudo<(outs), (ins GPRPair:$rs2, GPRNoX0:$rs1, simm12:$imm12), []>;
def : Pat<(store (f64 GPRPair:$rs2), (AddrRegImmINX (XLenVT GPR:$rs1), simm12:$imm12)),
(PseudoRV32ZdinxSD GPRPair:$rs2, GPR:$rs1, simm12:$imm12)>;
} // Predicates = [HasStdExtZdinx, IsRV32]

let Predicates = [HasStdExtD, IsRV32] in {
Expand Down
4 changes: 2 additions & 2 deletions llvm/test/CodeGen/RISCV/double-calling-conv.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,10 +165,10 @@ define double @callee_double_stack(i64 %a, i64 %b, i64 %c, i64 %d, double %e, do
;
; RV32IZFINXZDINX-LABEL: callee_double_stack:
; RV32IZFINXZDINX: # %bb.0:
; RV32IZFINXZDINX-NEXT: lw a0, 8(sp)
; RV32IZFINXZDINX-NEXT: lw a1, 12(sp)
; RV32IZFINXZDINX-NEXT: lw a2, 0(sp)
; RV32IZFINXZDINX-NEXT: lw a3, 4(sp)
; RV32IZFINXZDINX-NEXT: lw a0, 8(sp)
; RV32IZFINXZDINX-NEXT: lw a2, 0(sp)
; RV32IZFINXZDINX-NEXT: fadd.d a0, a2, a0
; RV32IZFINXZDINX-NEXT: ret
%1 = fadd double %e, %f
Expand Down
Loading