Skip to content

[RISCV][LLVM] Enable atomics for 'Zalrsc' #163672

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

[RISCV][LLVM] Enable atomics for 'Zalrsc' #163672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3f4346a
[RISCV][LLVM] Enable atomics for 'Zalrsc'
slachowsky Oct 14, 2025
645d868
[LegalizeTypes][RISCV] Control sign-extend for atomicrmw input argument
slachowsky Oct 16, 2025
2d87295
[RISCV][clang] Enable atomics for 'Zalrsc', not just 'A'
slachowsky Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion clang/lib/Basic/Targets/RISCV.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,8 +192,11 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
Builder.defineMacro("__riscv_muldiv");
}

if (ISAInfo->hasExtension("a")) {
// The "a" extension is composed of "zalrsc" and "zaamo"
if (ISAInfo->hasExtension("a"))
Builder.defineMacro("__riscv_atomic");

if (ISAInfo->hasExtension("zalrsc")) {
Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1");
Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2");
Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4");
Expand Down
6 changes: 4 additions & 2 deletions clang/lib/Basic/Targets/RISCV.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,8 @@ class LLVM_LIBRARY_VISIBILITY RISCV32TargetInfo : public RISCVTargetInfo {
void setMaxAtomicWidth() override {
MaxAtomicPromoteWidth = 128;

if (ISAInfo->hasExtension("a"))
// "a" implies "zalrsc" which is sufficient to inline atomics
if (ISAInfo->hasExtension("zalrsc"))
MaxAtomicInlineWidth = 32;
}
};
Expand Down Expand Up @@ -225,7 +226,8 @@ class LLVM_LIBRARY_VISIBILITY RISCV64TargetInfo : public RISCVTargetInfo {
void setMaxAtomicWidth() override {
MaxAtomicPromoteWidth = 128;

if (ISAInfo->hasExtension("a"))
// "a" implies "zalrsc" which is sufficient to inline atomics
if (ISAInfo->hasExtension("zalrsc"))
MaxAtomicInlineWidth = 64;
}
};
Expand Down
24 changes: 24 additions & 0 deletions clang/test/Preprocessor/riscv-atomics.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
// RUN: %clang --target=riscv32-unknown-linux-gnu -march=rv32ia -x c -E -dM %s \
// RUN: -o - | FileCheck %s
// RUN: %clang --target=riscv32-unknown-linux-gnu -march=rv32i_zalrsc -x c -E \
// RUN: -dM %s -o - | FileCheck %s
// RUN: %clang --target=riscv64-unknown-linux-gnu -march=rv64ia -x c -E -dM %s \
// RUN: -o - | FileCheck %s --check-prefixes=CHECK,CHECK-RV64
// RUN: %clang --target=riscv64-unknown-linux-gnu -march=rv64i_zalrsc -x c -E \
// RUN: -dM %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-RV64

// CHECK: #define __GCC_ATOMIC_BOOL_LOCK_FREE 2
// CHECK: #define __GCC_ATOMIC_CHAR16_T_LOCK_FREE 2
// CHECK: #define __GCC_ATOMIC_CHAR32_T_LOCK_FREE 2
// CHECK: #define __GCC_ATOMIC_CHAR_LOCK_FREE 2
// CHECK: #define __GCC_ATOMIC_INT_LOCK_FREE 2
// CHECK-RV64: #define __GCC_ATOMIC_LLONG_LOCK_FREE 2
// CHECK: #define __GCC_ATOMIC_LONG_LOCK_FREE 2
// CHECK: #define __GCC_ATOMIC_POINTER_LOCK_FREE 2
// CHECK: #define __GCC_ATOMIC_SHORT_LOCK_FREE 2
// CHECK: #define __GCC_ATOMIC_TEST_AND_SET_TRUEVAL 1
// CHECK: #define __GCC_ATOMIC_WCHAR_T_LOCK_FREE 2
// CHECK: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_1 1
// CHECK: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_2 1
// CHECK: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 1
// CHECK-RV64: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 1
6 changes: 6 additions & 0 deletions llvm/include/llvm/CodeGen/TargetLowering.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2459,6 +2459,12 @@ class LLVM_ABI TargetLoweringBase {
return ISD::ANY_EXTEND;
}

/// Returns how the platform's atomic rmw operations expect their input
/// argument to be extended (ZERO_EXTEND, SIGN_EXTEND, or ANY_EXTEND).
virtual ISD::NodeType getExtendForAtomicRMWArg(unsigned Op) const {
return ISD::ANY_EXTEND;
}

/// @}

/// Returns true if we should normalize
Expand Down
15 changes: 14 additions & 1 deletion llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -429,7 +429,20 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
}

SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
SDValue Op2 = GetPromotedInteger(N->getOperand(2));
SDValue Op2 = N->getOperand(2);
switch (TLI.getExtendForAtomicRMWArg(N->getOpcode())) {
case ISD::SIGN_EXTEND:
Op2 = SExtPromotedInteger(Op2);
break;
case ISD::ZERO_EXTEND:
Op2 = ZExtPromotedInteger(Op2);
break;
case ISD::ANY_EXTEND:
Op2 = GetPromotedInteger(Op2);
break;
default:
llvm_unreachable("Invalid atomic op extension");
}
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
N->getMemoryVT(),
N->getChain(), N->getBasePtr(),
Expand Down
229 changes: 201 additions & 28 deletions llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,70 @@ bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB,
// expanded instructions for each pseudo is correct in the Size field of the
// tablegen definition for the pseudo.
switch (MBBI->getOpcode()) {
case RISCV::PseudoAtomicSwap32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 32,
NextMBBI);
case RISCV::PseudoAtomicSwap64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadAdd32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadAdd64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadSub32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadSub64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadAnd32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadAnd64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadOr32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 32, NextMBBI);
case RISCV::PseudoAtomicLoadOr64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 64, NextMBBI);
case RISCV::PseudoAtomicLoadXor32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadXor64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadNand32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadNand64:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadMin32:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadMin64:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadMax32:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadMax64:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadUMin32:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadUMin64:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, false, 64,
NextMBBI);
case RISCV::PseudoAtomicLoadUMax32:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, false, 32,
NextMBBI);
case RISCV::PseudoAtomicLoadUMax64:
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, false, 64,
NextMBBI);
case RISCV::PseudoMaskedAtomicSwap32:
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
NextMBBI);
Expand Down Expand Up @@ -277,6 +335,36 @@ static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
switch (BinOp) {
default:
llvm_unreachable("Unexpected AtomicRMW BinOp");
case AtomicRMWInst::Xchg:
BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
.addReg(IncrReg)
.addImm(0);
break;
case AtomicRMWInst::Add:
BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
.addReg(DestReg)
.addReg(IncrReg);
break;
case AtomicRMWInst::Sub:
BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
.addReg(DestReg)
.addReg(IncrReg);
break;
case AtomicRMWInst::And:
BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
.addReg(DestReg)
.addReg(IncrReg);
break;
case AtomicRMWInst::Or:
BuildMI(LoopMBB, DL, TII->get(RISCV::OR), ScratchReg)
.addReg(DestReg)
.addReg(IncrReg);
break;
case AtomicRMWInst::Xor:
BuildMI(LoopMBB, DL, TII->get(RISCV::XOR), ScratchReg)
.addReg(DestReg)
.addReg(IncrReg);
break;
case AtomicRMWInst::Nand:
BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
.addReg(DestReg)
Expand Down Expand Up @@ -433,38 +521,85 @@ static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
.addReg(ShamtReg);
}

bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
MachineBasicBlock::iterator &NextMBBI) {
assert(IsMasked == true &&
"Should only need to expand masked atomic max/min");
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
static void doAtomicMinMaxOpExpansion(
const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopHeadMBB,
MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width,
const RISCVSubtarget *STI) {
Register DestReg = MI.getOperand(0).getReg();
Register ScratchReg = MI.getOperand(1).getReg();
Register AddrReg = MI.getOperand(2).getReg();
Register IncrReg = MI.getOperand(3).getReg();
AtomicOrdering Ordering =
static_cast<AtomicOrdering>(MI.getOperand(4).getImm());

MachineInstr &MI = *MBBI;
DebugLoc DL = MI.getDebugLoc();
MachineFunction *MF = MBB.getParent();
auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
// .loophead:
// lr.[w|d] dest, (addr)
// mv scratch, dest
// ifnochangeneeded scratch, incr, .looptail
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width, STI)), DestReg)
.addReg(AddrReg);
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
.addReg(DestReg)
.addImm(0);
switch (BinOp) {
default:
llvm_unreachable("Unexpected AtomicRMW BinOp");
case AtomicRMWInst::Max: {
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
.addReg(ScratchReg)
.addReg(IncrReg)
.addMBB(LoopTailMBB);
break;
}
case AtomicRMWInst::Min: {
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
.addReg(IncrReg)
.addReg(ScratchReg)
.addMBB(LoopTailMBB);
break;
}
case AtomicRMWInst::UMax:
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
.addReg(ScratchReg)
.addReg(IncrReg)
.addMBB(LoopTailMBB);
break;
case AtomicRMWInst::UMin:
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
.addReg(IncrReg)
.addReg(ScratchReg)
.addMBB(LoopTailMBB);
break;
}

// Insert new MBBs.
MF->insert(++MBB.getIterator(), LoopHeadMBB);
MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
// .loopifbody:
// mv scratch, incr
BuildMI(LoopIfBodyMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
.addReg(IncrReg)
.addImm(0);

// Set up successors and transfer remaining instructions to DoneMBB.
LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
LoopHeadMBB->addSuccessor(LoopTailMBB);
LoopIfBodyMBB->addSuccessor(LoopTailMBB);
LoopTailMBB->addSuccessor(LoopHeadMBB);
LoopTailMBB->addSuccessor(DoneMBB);
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
DoneMBB->transferSuccessors(&MBB);
MBB.addSuccessor(LoopHeadMBB);
// .looptail:
// sc.[w|d] scratch, scratch, (addr)
// bnez scratch, loop
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width, STI)),
ScratchReg)
.addReg(ScratchReg)
.addReg(AddrReg);
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
.addReg(ScratchReg)
.addReg(RISCV::X0)
.addMBB(LoopHeadMBB);
}

static void doMaskedAtomicMinMaxOpExpansion(
const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopHeadMBB,
MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width,
const RISCVSubtarget *STI) {
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
Register DestReg = MI.getOperand(0).getReg();
Register Scratch1Reg = MI.getOperand(1).getReg();
Register Scratch2Reg = MI.getOperand(2).getReg();
Expand Down Expand Up @@ -541,6 +676,44 @@ bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
.addReg(Scratch1Reg)
.addReg(RISCV::X0)
.addMBB(LoopHeadMBB);
}

bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
MachineBasicBlock::iterator &NextMBBI) {

MachineInstr &MI = *MBBI;
DebugLoc DL = MI.getDebugLoc();
MachineFunction *MF = MBB.getParent();
auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());

// Insert new MBBs.
MF->insert(++MBB.getIterator(), LoopHeadMBB);
MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
MF->insert(++LoopTailMBB->getIterator(), DoneMBB);

// Set up successors and transfer remaining instructions to DoneMBB.
LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
LoopHeadMBB->addSuccessor(LoopTailMBB);
LoopIfBodyMBB->addSuccessor(LoopTailMBB);
LoopTailMBB->addSuccessor(LoopHeadMBB);
LoopTailMBB->addSuccessor(DoneMBB);
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
DoneMBB->transferSuccessors(&MBB);
MBB.addSuccessor(LoopHeadMBB);

if (!IsMasked)
doAtomicMinMaxOpExpansion(TII, MI, DL, &MBB, LoopHeadMBB, LoopIfBodyMBB,
LoopTailMBB, DoneMBB, BinOp, Width, STI);
else
doMaskedAtomicMinMaxOpExpansion(TII, MI, DL, &MBB, LoopHeadMBB,
LoopIfBodyMBB, LoopTailMBB, DoneMBB, BinOp,
Width, STI);

NextMBBI = MBB.end();
MI.eraseFromParent();
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Target/RISCV/RISCVFeatures.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ def HasStdExtZaamo
: Predicate<"Subtarget->hasStdExtZaamo()">,
AssemblerPredicate<(any_of FeatureStdExtZaamo),
"'Zaamo' (Atomic Memory Operations)">;
def NoStdExtZaamo : Predicate<"!Subtarget->hasStdExtZaamo()">;

def FeatureStdExtZalrsc
: RISCVExtension<1, 0, "Load-Reserved/Store-Conditional">;
Expand Down Expand Up @@ -1864,7 +1865,7 @@ def FeatureForcedAtomics : SubtargetFeature<
"forced-atomics", "HasForcedAtomics", "true",
"Assume that lock-free native-width atomics are available">;
def HasAtomicLdSt
: Predicate<"Subtarget->hasStdExtA() || Subtarget->hasForcedAtomics()">;
: Predicate<"Subtarget->hasStdExtZalrsc() || Subtarget->hasForcedAtomics()">;

def FeatureTaggedGlobals : SubtargetFeature<"tagged-globals",
"AllowTaggedGlobals",
Expand Down
Loading