Skip to content

[RISCV][LLVM] Enable atomics for 'Zalrsc' #163672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
+12,495 −44
Open

[RISCV][LLVM] Enable atomics for 'Zalrsc' #163672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
555a590
[RISCV][LLVM] Enable atomics for 'Zalrsc'
slachowsky Oct 14, 2025
ae9c04b
[LegalizeTypes][RISCV] Control sign-extend for atomicrmw input argument
slachowsky Oct 16, 2025
a676c2d
[RISCV][clang] Enable atomics for 'Zalrsc', not just 'A'
slachowsky Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion clang/lib/Basic/Targets/RISCV.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,8 +192,11 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
Builder.defineMacro("__riscv_muldiv");
}

if (ISAInfo->hasExtension("a")) {
// The "a" extension is composed of "zalrsc" and "zaamo"
if (ISAInfo->hasExtension("a"))
Builder.defineMacro("__riscv_atomic");

if (ISAInfo->hasExtension("zalrsc")) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

__riscv_atomic is tested under test/Preprocessor/riscv-target-features.c, but none of the __GCC* ones seem to be tested for RISCV. Please add a frontend test for those too, when enabling zalrsc

Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1");
Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2");
Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4");
Expand Down
6 changes: 4 additions & 2 deletions clang/lib/Basic/Targets/RISCV.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,8 @@ class LLVM_LIBRARY_VISIBILITY RISCV32TargetInfo : public RISCVTargetInfo {
void setMaxAtomicWidth() override {
MaxAtomicPromoteWidth = 128;

if (ISAInfo->hasExtension("a"))
// "a" implies "zalrsc" which is sufficient to inline atomics
if (ISAInfo->hasExtension("zalrsc"))
MaxAtomicInlineWidth = 32;
}
};
Expand Down Expand Up @@ -225,7 +226,8 @@ class LLVM_LIBRARY_VISIBILITY RISCV64TargetInfo : public RISCVTargetInfo {
void setMaxAtomicWidth() override {
MaxAtomicPromoteWidth = 128;

if (ISAInfo->hasExtension("a"))
// "a" implies "zalrsc" which is sufficient to inline atomics
if (ISAInfo->hasExtension("zalrsc"))
MaxAtomicInlineWidth = 64;
}
};
Expand Down
6 changes: 6 additions & 0 deletions llvm/include/llvm/CodeGen/TargetLowering.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2459,6 +2459,12 @@ class LLVM_ABI TargetLoweringBase {
return ISD::ANY_EXTEND;
}

/// Returns how the platform's atomic rmw operations expect their input
/// argument to be extended (ZERO_EXTEND, SIGN_EXTEND, or ANY_EXTEND).
virtual ISD::NodeType getExtendForAtomicRMWArg(unsigned Op) const {
return ISD::ANY_EXTEND;
}

/// @}

/// Returns true if we should normalize
Expand Down
15 changes: 14 additions & 1 deletion llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -429,7 +429,20 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
}

SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
SDValue Op2 = GetPromotedInteger(N->getOperand(2));
SDValue Op2 = N->getOperand(2);
switch (TLI.getExtendForAtomicRMWArg(N->getOpcode())) {
case ISD::SIGN_EXTEND:
Op2 = SExtPromotedInteger(Op2);
break;
case ISD::ZERO_EXTEND:
Op2 = ZExtPromotedInteger(Op2);
break;
case ISD::ANY_EXTEND:
Op2 = GetPromotedInteger(Op2);
break;
default:
llvm_unreachable("Invalid atomic op extension");
}
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
N->getMemoryVT(),
N->getChain(), N->getBasePtr(),
Expand Down
234 changes: 206 additions & 28 deletions llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,72 @@ bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB,
// expanded instructions for each pseudo is correct in the Size field of the
// tablegen definition for the pseudo.
switch (MBBI->getOpcode()) {
case RISCV::PseudoAtomicSwap32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 32,
NextMBBI);
case RISCV::PseudoAtomicSwap64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadAdd32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadAdd64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadSub32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadSub64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadAnd32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadAnd64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadOr32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadOr64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadXor32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadXor64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadNand32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadNand64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadMin32:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadMin64:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadMax32:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadMax64:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadUMin32:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadUMin64:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadUMax32:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadUMax64:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, false, 64,
NextMBBI);
case RISCV::PseudoMaskedAtomicSwap32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
NextMBBI);
Expand Down Expand Up @@ -277,6 +337,36 @@ static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
switch (BinOp) {
default:
llvm_unreachable("Unexpected AtomicRMW BinOp");
case AtomicRMWInst::Xchg:
BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
.addReg(IncrReg)
.addImm(0);
break;
case AtomicRMWInst::Add:
BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
.addReg(DestReg)
.addReg(IncrReg);
break;
case AtomicRMWInst::Sub:
BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
.addReg(DestReg)
.addReg(IncrReg);
break;
case AtomicRMWInst::And:
BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
.addReg(DestReg)
.addReg(IncrReg);
break;
case AtomicRMWInst::Or:
BuildMI(LoopMBB, DL, TII->get(RISCV::OR), ScratchReg)
.addReg(DestReg)
.addReg(IncrReg);
break;
case AtomicRMWInst::Xor:
BuildMI(LoopMBB, DL, TII->get(RISCV::XOR), ScratchReg)
.addReg(DestReg)
.addReg(IncrReg);
break;
case AtomicRMWInst::Nand:
BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
.addReg(DestReg)
Expand Down Expand Up @@ -433,38 +523,88 @@ static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
.addReg(ShamtReg);
}

bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
MachineBasicBlock::iterator &NextMBBI) {
assert(IsMasked == true &&
"Should only need to expand masked atomic max/min");
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
static void doAtomicMinMaxOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
DebugLoc DL, MachineBasicBlock *ThisMBB,
MachineBasicBlock *LoopHeadMBB,
MachineBasicBlock *LoopIfBodyMBB,
MachineBasicBlock *LoopTailMBB,
MachineBasicBlock *DoneMBB,
AtomicRMWInst::BinOp BinOp, int Width,
const RISCVSubtarget *STI) {
Register DestReg = MI.getOperand(0).getReg();
Register ScratchReg = MI.getOperand(1).getReg();
Register AddrReg = MI.getOperand(2).getReg();
Register IncrReg = MI.getOperand(3).getReg();
AtomicOrdering Ordering =
static_cast<AtomicOrdering>(MI.getOperand(4).getImm());

MachineInstr &MI = *MBBI;
DebugLoc DL = MI.getDebugLoc();
MachineFunction *MF = MBB.getParent();
auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
// .loophead:
// lr.[w|d] dest, (addr)
// mv scratch, dest
// ifnochangeneeded scratch, incr, .looptail
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width, STI)), DestReg)
.addReg(AddrReg);
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
.addReg(DestReg)
.addImm(0);
switch (BinOp) {
default:
llvm_unreachable("Unexpected AtomicRMW BinOp");
case AtomicRMWInst::Max: {
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
.addReg(ScratchReg)
.addReg(IncrReg)
.addMBB(LoopTailMBB);
break;
}
case AtomicRMWInst::Min: {
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
.addReg(IncrReg)
.addReg(ScratchReg)
.addMBB(LoopTailMBB);
break;
}
case AtomicRMWInst::UMax:
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
.addReg(ScratchReg)
.addReg(IncrReg)
.addMBB(LoopTailMBB);
break;
case AtomicRMWInst::UMin:
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
.addReg(IncrReg)
.addReg(ScratchReg)
.addMBB(LoopTailMBB);
break;
}

// Insert new MBBs.
MF->insert(++MBB.getIterator(), LoopHeadMBB);
MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
// .loopifbody:
// mv scratch, incr
BuildMI(LoopIfBodyMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
.addReg(IncrReg)
.addImm(0);

// Set up successors and transfer remaining instructions to DoneMBB.
LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
LoopHeadMBB->addSuccessor(LoopTailMBB);
LoopIfBodyMBB->addSuccessor(LoopTailMBB);
LoopTailMBB->addSuccessor(LoopHeadMBB);
LoopTailMBB->addSuccessor(DoneMBB);
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
DoneMBB->transferSuccessors(&MBB);
MBB.addSuccessor(LoopHeadMBB);
// .looptail:
// sc.[w|d] scratch, scratch, (addr)
// bnez scratch, loop
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width, STI)), ScratchReg)
.addReg(ScratchReg)
.addReg(AddrReg);
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
.addReg(ScratchReg)
.addReg(RISCV::X0)
.addMBB(LoopHeadMBB);
}

static void doMaskedAtomicMinMaxOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
DebugLoc DL, MachineBasicBlock *ThisMBB,
MachineBasicBlock *LoopHeadMBB,
MachineBasicBlock *LoopIfBodyMBB,
MachineBasicBlock *LoopTailMBB,
MachineBasicBlock *DoneMBB,
AtomicRMWInst::BinOp BinOp, int Width,
const RISCVSubtarget *STI) {
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
Register DestReg = MI.getOperand(0).getReg();
Register Scratch1Reg = MI.getOperand(1).getReg();
Register Scratch2Reg = MI.getOperand(2).getReg();
Expand Down Expand Up @@ -541,6 +681,44 @@ bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
.addReg(Scratch1Reg)
.addReg(RISCV::X0)
.addMBB(LoopHeadMBB);
}

bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
MachineBasicBlock::iterator &NextMBBI) {

MachineInstr &MI = *MBBI;
DebugLoc DL = MI.getDebugLoc();
MachineFunction *MF = MBB.getParent();
auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());

// Insert new MBBs.
MF->insert(++MBB.getIterator(), LoopHeadMBB);
MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
MF->insert(++LoopTailMBB->getIterator(), DoneMBB);

// Set up successors and transfer remaining instructions to DoneMBB.
LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
LoopHeadMBB->addSuccessor(LoopTailMBB);
LoopIfBodyMBB->addSuccessor(LoopTailMBB);
LoopTailMBB->addSuccessor(LoopHeadMBB);
LoopTailMBB->addSuccessor(DoneMBB);
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
DoneMBB->transferSuccessors(&MBB);
MBB.addSuccessor(LoopHeadMBB);

if (!IsMasked)
doAtomicMinMaxOpExpansion(TII, MI, DL, &MBB, LoopHeadMBB, LoopIfBodyMBB,
LoopTailMBB, DoneMBB, BinOp, Width, STI);
else
doMaskedAtomicMinMaxOpExpansion(TII, MI, DL, &MBB, LoopHeadMBB,
LoopIfBodyMBB, LoopTailMBB, DoneMBB, BinOp,
Width, STI);

NextMBBI = MBB.end();
MI.eraseFromParent();
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Target/RISCV/RISCVFeatures.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ def HasStdExtZaamo
: Predicate<"Subtarget->hasStdExtZaamo()">,
AssemblerPredicate<(any_of FeatureStdExtZaamo),
"'Zaamo' (Atomic Memory Operations)">;
def NoStdExtZaamo : Predicate<"!Subtarget->hasStdExtZaamo()">;

def FeatureStdExtZalrsc
: RISCVExtension<1, 0, "Load-Reserved/Store-Conditional">;
Expand Down Expand Up @@ -1861,7 +1862,7 @@ def FeatureForcedAtomics : SubtargetFeature<
"forced-atomics", "HasForcedAtomics", "true",
"Assume that lock-free native-width atomics are available">;
def HasAtomicLdSt
: Predicate<"Subtarget->hasStdExtA() || Subtarget->hasForcedAtomics()">;
: Predicate<"Subtarget->hasStdExtZalrsc() || Subtarget->hasForcedAtomics()">;

def FeatureTaggedGlobals : SubtargetFeature<"tagged-globals",
"AllowTaggedGlobals",
Expand Down
Loading
Loading