diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp index 5025122db3681..0be5d8e731b60 100644 --- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp @@ -1817,52 +1817,77 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) { case RISCVISD::LD_RV32: { assert(Subtarget->hasStdExtZilsd() && "LD_RV32 is only used with Zilsd"); + auto *MemNode = cast(Node); + SDValue Base, Offset; - SDValue Chain = Node->getOperand(0); - SDValue Addr = Node->getOperand(1); + SDValue Chain = MemNode->getChain(); + SDValue Addr = MemNode->getBasePtr(); SelectAddrRegImm(Addr, Base, Offset); SDValue Ops[] = {Base, Offset, Chain}; - MachineSDNode *New = CurDAG->getMachineNode( - RISCV::LD_RV32, DL, {MVT::Untyped, MVT::Other}, Ops); - SDValue Lo = CurDAG->getTargetExtractSubreg(RISCV::sub_gpr_even, DL, - MVT::i32, SDValue(New, 0)); - SDValue Hi = CurDAG->getTargetExtractSubreg(RISCV::sub_gpr_odd, DL, - MVT::i32, SDValue(New, 0)); - CurDAG->setNodeMemRefs(New, {cast(Node)->getMemOperand()}); + MachineSDNode *New; + SDValue Lo, Hi, OutChain; + if (MemNode->isVolatile()) { + New = CurDAG->getMachineNode(RISCV::LD_RV32, DL, + {MVT::Untyped, MVT::Other}, Ops); + + Lo = CurDAG->getTargetExtractSubreg(RISCV::sub_gpr_even, DL, MVT::i32, + SDValue(New, 0)); + Hi = CurDAG->getTargetExtractSubreg(RISCV::sub_gpr_odd, DL, MVT::i32, + SDValue(New, 0)); + OutChain = SDValue(New, 1); + } else { + New = CurDAG->getMachineNode(RISCV::PseudoLD_RV32_OPT, DL, + {MVT::i32, MVT::i32, MVT::Other}, Ops); + Lo = SDValue(New, 0); + Hi = SDValue(New, 1); + OutChain = SDValue(New, 2); + } + + CurDAG->setNodeMemRefs(New, {MemNode->getMemOperand()}); ReplaceUses(SDValue(Node, 0), Lo); ReplaceUses(SDValue(Node, 1), Hi); - ReplaceUses(SDValue(Node, 2), SDValue(New, 1)); + ReplaceUses(SDValue(Node, 2), OutChain); CurDAG->RemoveDeadNode(Node); return; } case RISCVISD::SD_RV32: { + auto *MemNode = cast(Node); + SDValue Base, Offset; - SDValue Chain = Node->getOperand(0); + SDValue Chain = MemNode->getChain(); SDValue Addr = Node->getOperand(3); SelectAddrRegImm(Addr, Base, Offset); SDValue Lo = Node->getOperand(1); SDValue Hi = Node->getOperand(2); - SDValue RegPair; - // Peephole to use X0_Pair for storing zero. - if (isNullConstant(Lo) && isNullConstant(Hi)) { - RegPair = CurDAG->getRegister(RISCV::X0_Pair, MVT::Untyped); - } else { - SDValue Ops[] = { - CurDAG->getTargetConstant(RISCV::GPRPairRegClassID, DL, MVT::i32), Lo, - CurDAG->getTargetConstant(RISCV::sub_gpr_even, DL, MVT::i32), Hi, - CurDAG->getTargetConstant(RISCV::sub_gpr_odd, DL, MVT::i32)}; + MachineSDNode *New; + if (MemNode->isVolatile()) { + SDValue RegPair; + // Peephole to use X0_Pair for storing zero. + if (isNullConstant(Lo) && isNullConstant(Hi)) { + RegPair = CurDAG->getRegister(RISCV::X0_Pair, MVT::Untyped); + } else { + SDValue Ops[] = { + CurDAG->getTargetConstant(RISCV::GPRPairRegClassID, DL, MVT::i32), + Lo, CurDAG->getTargetConstant(RISCV::sub_gpr_even, DL, MVT::i32), + Hi, CurDAG->getTargetConstant(RISCV::sub_gpr_odd, DL, MVT::i32)}; + + RegPair = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, + MVT::Untyped, Ops), + 0); + } - RegPair = SDValue(CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, - MVT::Untyped, Ops), - 0); + New = CurDAG->getMachineNode(RISCV::SD_RV32, DL, MVT::Other, + {RegPair, Base, Offset, Chain}); + } else { + New = CurDAG->getMachineNode(RISCV::PseudoSD_RV32_OPT, DL, MVT::Other, + {Lo, Hi, Base, Offset, Chain}); } - MachineSDNode *New = CurDAG->getMachineNode(RISCV::SD_RV32, DL, MVT::Other, - {RegPair, Base, Offset, Chain}); - CurDAG->setNodeMemRefs(New, {cast(Node)->getMemOperand()}); + CurDAG->setNodeMemRefs(New, {MemNode->getMemOperand()}); + ReplaceUses(SDValue(Node, 0), SDValue(New, 0)); CurDAG->RemoveDeadNode(Node); return; diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 3b250d7d9ad1f..6b5557383596f 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -23166,8 +23166,29 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, } } +static void addPairRegisterHints(MachineInstr &MI) { + assert((MI.getOpcode() == RISCV::PseudoLD_RV32_OPT || + MI.getOpcode() == RISCV::PseudoSD_RV32_OPT) && + "Needs LD/SD Pseudo"); + + Register FirstReg = MI.getOperand(0).getReg(); + Register SecondReg = MI.getOperand(1).getReg(); + + MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); + + if (FirstReg.isVirtual() && SecondReg.isVirtual()) { + MRI.setRegAllocationHint(FirstReg, RISCVRI::RegPairEven, SecondReg); + MRI.setRegAllocationHint(SecondReg, RISCVRI::RegPairOdd, FirstReg); + } +} + void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const { + + if (MI.getOpcode() == RISCV::PseudoLD_RV32_OPT || + MI.getOpcode() == RISCV::PseudoSD_RV32_OPT) + return addPairRegisterHints(MI); + // If instruction defines FRM operand, conservatively set it as non-dead to // express data dependency with FRM users and prevent incorrect instruction // reordering. diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZilsd.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZilsd.td index 4fc859f2547c1..e579ceebd18b7 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoZilsd.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZilsd.td @@ -51,17 +51,23 @@ def PseudoSD_RV32 : PseudoStore<"sd", GPRPairRV32>; def PseudoLD_RV32_OPT : Pseudo<(outs GPR:$rd1, GPR:$rd2), (ins GPR:$rs1, simm12_lo:$imm12), [], "", ""> { - let hasSideEffects = 0; - let mayLoad = 1; - let mayStore = 0; + let hasSideEffects = false; + let mayLoad = true; + let mayStore = false; + let isCodeGenOnly = true; + let Size = 8; // Might become two LWs + let hasPostISelHook = true; } def PseudoSD_RV32_OPT : Pseudo<(outs), (ins GPR:$rs1, GPR:$rs2, GPR:$rs3, simm12_lo:$imm12), [], "", ""> { - let hasSideEffects = 0; - let mayLoad = 0; - let mayStore = 1; + let hasSideEffects = false; + let mayLoad = false; + let mayStore = true; + let isCodeGenOnly = true; + let Size = 8; // Might become two SWs + let hasPostISelHook = true; } def : InstAlias<"ld $rd, (${rs1})", (LD_RV32 GPRPairRV32:$rd, GPR:$rs1, 0), 0>; diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td index 11b7a0a3c691a..d53dfbe8e4761 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td @@ -368,7 +368,7 @@ let RegAltNameIndices = [ABIRegAltName] in { } } -let RegInfos = XLenPairRI, CopyCost = 2 in { +let RegInfos = XLenPairRI, CopyCost = 2, AllocationPriority = 1 in { def GPRPair : RISCVRegisterClass<[XLenPairVT, XLenPairFVT], 64, (add X10_X11, X12_X13, X14_X15, X16_X17, X6_X7, diff --git a/llvm/test/CodeGen/RISCV/zilsd.ll b/llvm/test/CodeGen/RISCV/zilsd.ll index 27b1ff76f6f05..bd0c81b35e9b5 100644 --- a/llvm/test/CodeGen/RISCV/zilsd.ll +++ b/llvm/test/CodeGen/RISCV/zilsd.ll @@ -9,9 +9,10 @@ define i64 @load(ptr %a) nounwind { ; CHECK-LABEL: load: ; CHECK: # %bb.0: -; CHECK-NEXT: mv a2, a0 -; CHECK-NEXT: ld a0, 80(a0) -; CHECK-NEXT: ld zero, 0(a2) +; CHECK-NEXT: lw a2, 80(a0) +; CHECK-NEXT: lw a1, 84(a0) +; CHECK-NEXT: ld zero, 0(a0) +; CHECK-NEXT: mv a0, a2 ; CHECK-NEXT: ret %1 = getelementptr i64, ptr %a, i32 10 %2 = load i64, ptr %1 @@ -44,10 +45,10 @@ define i64 @load_align4(ptr %a) nounwind { define void @store(ptr %a, i64 %b) nounwind { ; CHECK-LABEL: store: ; CHECK: # %bb.0: -; CHECK-NEXT: mv a3, a2 -; CHECK-NEXT: mv a2, a1 -; CHECK-NEXT: sd a2, 0(a0) -; CHECK-NEXT: sd a2, 88(a0) +; CHECK-NEXT: sw a1, 0(a0) +; CHECK-NEXT: sw a2, 4(a0) +; CHECK-NEXT: sw a1, 88(a0) +; CHECK-NEXT: sw a2, 92(a0) ; CHECK-NEXT: ret store i64 %b, ptr %a %1 = getelementptr i64, ptr %a, i32 11 @@ -56,25 +57,11 @@ define void @store(ptr %a, i64 %b) nounwind { } define void @store_align4(ptr %a, i64 %b) nounwind { -; SLOW-LABEL: store_align4: -; SLOW: # %bb.0: -; SLOW-NEXT: sw a1, 88(a0) -; SLOW-NEXT: sw a2, 92(a0) -; SLOW-NEXT: ret -; -; FAST-LABEL: store_align4: -; FAST: # %bb.0: -; FAST-NEXT: mv a3, a2 -; FAST-NEXT: mv a2, a1 -; FAST-NEXT: sd a2, 88(a0) -; FAST-NEXT: ret -; -; 4BYTEALIGN-LABEL: store_align4: -; 4BYTEALIGN: # %bb.0: -; 4BYTEALIGN-NEXT: mv a3, a2 -; 4BYTEALIGN-NEXT: mv a2, a1 -; 4BYTEALIGN-NEXT: sd a2, 88(a0) -; 4BYTEALIGN-NEXT: ret +; CHECK-LABEL: store_align4: +; CHECK: # %bb.0: +; CHECK-NEXT: sw a1, 88(a0) +; CHECK-NEXT: sw a2, 92(a0) +; CHECK-NEXT: ret %1 = getelementptr i64, ptr %a, i32 11 store i64 %b, ptr %1, align 4 ret void @@ -158,9 +145,8 @@ define void @store_unaligned(ptr %p, i64 %v) { ; ; FAST-LABEL: store_unaligned: ; FAST: # %bb.0: -; FAST-NEXT: mv a3, a2 -; FAST-NEXT: mv a2, a1 -; FAST-NEXT: sd a2, 0(a0) +; FAST-NEXT: sw a1, 0(a0) +; FAST-NEXT: sw a2, 4(a0) ; FAST-NEXT: ret ; ; 4BYTEALIGN-LABEL: store_unaligned: @@ -226,3 +212,26 @@ entry: store i64 %b, ptr %add.ptr, align 8 ret void } + +define i64 @stack_access(ptr nocapture %p) nounwind { +; CHECK-LABEL: stack_access: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: addi sp, sp, -16 +; CHECK-NEXT: lw a2, 8(sp) +; CHECK-NEXT: lw a1, 12(sp) +; CHECK-NEXT: ld a4, 0(a0) +; CHECK-NEXT: sw a2, 0(a0) +; CHECK-NEXT: sw a1, 4(a0) +; CHECK-NEXT: sd a4, 8(sp) +; CHECK-NEXT: mv a0, a2 +; CHECK-NEXT: addi sp, sp, 16 +; CHECK-NEXT: ret +entry: + %stack = alloca i64, align 8 + %a = load i64, ptr %stack, align 8 + %b = load i64, ptr %p, align 8 + store i64 %a, ptr %p, align 8 + store i64 %b, ptr %stack, align 8 + %c = load i64, ptr %p, align 8 + ret i64 %c +}