Skip to content

Commit 5fa7b22

Browse files
H-G-HristovH-G-Hristov
authored
Merge branch 'main' into hgh/libcxx/P0448R4-spanstream-A-strstream-replacement-using-span-charT-as-buffer
2 parents cf87545 + 70f7039 commit 5fa7b22

File tree

15 files changed

+12517
-47
lines changed

15 files changed

+12517
-47
lines changed

‎clang/lib/Basic/Targets/RISCV.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,8 +192,11 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
192192
Builder.defineMacro("__riscv_muldiv");
193193
}
194194

195-
if (ISAInfo->hasExtension("a")) {
195+
// The "a" extension is composed of "zalrsc" and "zaamo"
196+
if (ISAInfo->hasExtension("a"))
196197
Builder.defineMacro("__riscv_atomic");
198+
199+
if (ISAInfo->hasExtension("zalrsc")) {
197200
Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1");
198201
Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2");
199202
Builder.defineMacro("__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4");

‎clang/lib/Basic/Targets/RISCV.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,8 @@ class LLVM_LIBRARY_VISIBILITY RISCV32TargetInfo : public RISCVTargetInfo {
195195
void setMaxAtomicWidth() override {
196196
MaxAtomicPromoteWidth = 128;
197197

198-
if (ISAInfo->hasExtension("a"))
198+
// "a" implies "zalrsc" which is sufficient to inline atomics
199+
if (ISAInfo->hasExtension("zalrsc"))
199200
MaxAtomicInlineWidth = 32;
200201
}
201202
};
@@ -225,7 +226,8 @@ class LLVM_LIBRARY_VISIBILITY RISCV64TargetInfo : public RISCVTargetInfo {
225226
void setMaxAtomicWidth() override {
226227
MaxAtomicPromoteWidth = 128;
227228

228-
if (ISAInfo->hasExtension("a"))
229+
// "a" implies "zalrsc" which is sufficient to inline atomics
230+
if (ISAInfo->hasExtension("zalrsc"))
229231
MaxAtomicInlineWidth = 64;
230232
}
231233
};

‎clang/test/Preprocessor/riscv-atomics.c‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// RUN: %clang --target=riscv32-unknown-linux-gnu -march=rv32ia -x c -E -dM %s \
2+
// RUN: -o - | FileCheck %s
3+
// RUN: %clang --target=riscv32-unknown-linux-gnu -march=rv32i_zalrsc -x c -E \
4+
// RUN: -dM %s -o - | FileCheck %s
5+
// RUN: %clang --target=riscv64-unknown-linux-gnu -march=rv64ia -x c -E -dM %s \
6+
// RUN: -o - | FileCheck %s --check-prefixes=CHECK,CHECK-RV64
7+
// RUN: %clang --target=riscv64-unknown-linux-gnu -march=rv64i_zalrsc -x c -E \
8+
// RUN: -dM %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-RV64
9+
10+
// CHECK: #define __GCC_ATOMIC_BOOL_LOCK_FREE 2
11+
// CHECK: #define __GCC_ATOMIC_CHAR16_T_LOCK_FREE 2
12+
// CHECK: #define __GCC_ATOMIC_CHAR32_T_LOCK_FREE 2
13+
// CHECK: #define __GCC_ATOMIC_CHAR_LOCK_FREE 2
14+
// CHECK: #define __GCC_ATOMIC_INT_LOCK_FREE 2
15+
// CHECK-RV64: #define __GCC_ATOMIC_LLONG_LOCK_FREE 2
16+
// CHECK: #define __GCC_ATOMIC_LONG_LOCK_FREE 2
17+
// CHECK: #define __GCC_ATOMIC_POINTER_LOCK_FREE 2
18+
// CHECK: #define __GCC_ATOMIC_SHORT_LOCK_FREE 2
19+
// CHECK: #define __GCC_ATOMIC_TEST_AND_SET_TRUEVAL 1
20+
// CHECK: #define __GCC_ATOMIC_WCHAR_T_LOCK_FREE 2
21+
// CHECK: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_1 1
22+
// CHECK: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_2 1
23+
// CHECK: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 1
24+
// CHECK-RV64: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 1

‎llvm/include/llvm/CodeGen/TargetLowering.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2459,6 +2459,12 @@ class LLVM_ABI TargetLoweringBase {
24592459
return ISD::ANY_EXTEND;
24602460
}
24612461

2462+
/// Returns how the platform's atomic rmw operations expect their input
2463+
/// argument to be extended (ZERO_EXTEND, SIGN_EXTEND, or ANY_EXTEND).
2464+
virtual ISD::NodeType getExtendForAtomicRMWArg(unsigned Op) const {
2465+
return ISD::ANY_EXTEND;
2466+
}
2467+
24622468
/// @}
24632469

24642470
/// Returns true if we should normalize

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -429,7 +429,20 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
429429
}
430430

431431
SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
432-
SDValue Op2 = GetPromotedInteger(N->getOperand(2));
432+
SDValue Op2 = N->getOperand(2);
433+
switch (TLI.getExtendForAtomicRMWArg(N->getOpcode())) {
434+
case ISD::SIGN_EXTEND:
435+
Op2 = SExtPromotedInteger(Op2);
436+
break;
437+
case ISD::ZERO_EXTEND:
438+
Op2 = ZExtPromotedInteger(Op2);
439+
break;
440+
case ISD::ANY_EXTEND:
441+
Op2 = GetPromotedInteger(Op2);
442+
break;
443+
default:
444+
llvm_unreachable("Invalid atomic op extension");
445+
}
433446
SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
434447
N->getMemoryVT(),
435448
N->getChain(), N->getBasePtr(),

‎llvm/lib/Target/RISCV/RISCVExpandAtomicPseudoInsts.cpp‎

Lines changed: 201 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -109,12 +109,70 @@ bool RISCVExpandAtomicPseudo::expandMI(MachineBasicBlock &MBB,
109109
// expanded instructions for each pseudo is correct in the Size field of the
110110
// tablegen definition for the pseudo.
111111
switch (MBBI->getOpcode()) {
112+
case RISCV::PseudoAtomicSwap32:
113+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 32,
114+
NextMBBI);
115+
case RISCV::PseudoAtomicSwap64:
116+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, false, 64,
117+
NextMBBI);
118+
case RISCV::PseudoAtomicLoadAdd32:
119+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 32,
120+
NextMBBI);
121+
case RISCV::PseudoAtomicLoadAdd64:
122+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, false, 64,
123+
NextMBBI);
124+
case RISCV::PseudoAtomicLoadSub32:
125+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 32,
126+
NextMBBI);
127+
case RISCV::PseudoAtomicLoadSub64:
128+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, false, 64,
129+
NextMBBI);
130+
case RISCV::PseudoAtomicLoadAnd32:
131+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 32,
132+
NextMBBI);
133+
case RISCV::PseudoAtomicLoadAnd64:
134+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, false, 64,
135+
NextMBBI);
136+
case RISCV::PseudoAtomicLoadOr32:
137+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 32, NextMBBI);
138+
case RISCV::PseudoAtomicLoadOr64:
139+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, false, 64, NextMBBI);
140+
case RISCV::PseudoAtomicLoadXor32:
141+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 32,
142+
NextMBBI);
143+
case RISCV::PseudoAtomicLoadXor64:
144+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, false, 64,
145+
NextMBBI);
112146
case RISCV::PseudoAtomicLoadNand32:
113147
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
114148
NextMBBI);
115149
case RISCV::PseudoAtomicLoadNand64:
116150
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
117151
NextMBBI);
152+
case RISCV::PseudoAtomicLoadMin32:
153+
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, false, 32,
154+
NextMBBI);
155+
case RISCV::PseudoAtomicLoadMin64:
156+
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, false, 64,
157+
NextMBBI);
158+
case RISCV::PseudoAtomicLoadMax32:
159+
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, false, 32,
160+
NextMBBI);
161+
case RISCV::PseudoAtomicLoadMax64:
162+
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, false, 64,
163+
NextMBBI);
164+
case RISCV::PseudoAtomicLoadUMin32:
165+
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, false, 32,
166+
NextMBBI);
167+
case RISCV::PseudoAtomicLoadUMin64:
168+
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, false, 64,
169+
NextMBBI);
170+
case RISCV::PseudoAtomicLoadUMax32:
171+
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, false, 32,
172+
NextMBBI);
173+
case RISCV::PseudoAtomicLoadUMax64:
174+
return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, false, 64,
175+
NextMBBI);
118176
case RISCV::PseudoMaskedAtomicSwap32:
119177
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
120178
NextMBBI);
@@ -277,6 +335,36 @@ static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
277335
switch (BinOp) {
278336
default:
279337
llvm_unreachable("Unexpected AtomicRMW BinOp");
338+
case AtomicRMWInst::Xchg:
339+
BuildMI(LoopMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
340+
.addReg(IncrReg)
341+
.addImm(0);
342+
break;
343+
case AtomicRMWInst::Add:
344+
BuildMI(LoopMBB, DL, TII->get(RISCV::ADD), ScratchReg)
345+
.addReg(DestReg)
346+
.addReg(IncrReg);
347+
break;
348+
case AtomicRMWInst::Sub:
349+
BuildMI(LoopMBB, DL, TII->get(RISCV::SUB), ScratchReg)
350+
.addReg(DestReg)
351+
.addReg(IncrReg);
352+
break;
353+
case AtomicRMWInst::And:
354+
BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
355+
.addReg(DestReg)
356+
.addReg(IncrReg);
357+
break;
358+
case AtomicRMWInst::Or:
359+
BuildMI(LoopMBB, DL, TII->get(RISCV::OR), ScratchReg)
360+
.addReg(DestReg)
361+
.addReg(IncrReg);
362+
break;
363+
case AtomicRMWInst::Xor:
364+
BuildMI(LoopMBB, DL, TII->get(RISCV::XOR), ScratchReg)
365+
.addReg(DestReg)
366+
.addReg(IncrReg);
367+
break;
280368
case AtomicRMWInst::Nand:
281369
BuildMI(LoopMBB, DL, TII->get(RISCV::AND), ScratchReg)
282370
.addReg(DestReg)
@@ -433,38 +521,85 @@ static void insertSext(const RISCVInstrInfo *TII, DebugLoc DL,
433521
.addReg(ShamtReg);
434522
}
435523

436-
bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
437-
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
438-
AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
439-
MachineBasicBlock::iterator &NextMBBI) {
440-
assert(IsMasked == true &&
441-
"Should only need to expand masked atomic max/min");
442-
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
524+
static void doAtomicMinMaxOpExpansion(
525+
const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
526+
MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopHeadMBB,
527+
MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
528+
MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width,
529+
const RISCVSubtarget *STI) {
530+
Register DestReg = MI.getOperand(0).getReg();
531+
Register ScratchReg = MI.getOperand(1).getReg();
532+
Register AddrReg = MI.getOperand(2).getReg();
533+
Register IncrReg = MI.getOperand(3).getReg();
534+
AtomicOrdering Ordering =
535+
static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
443536

444-
MachineInstr &MI = *MBBI;
445-
DebugLoc DL = MI.getDebugLoc();
446-
MachineFunction *MF = MBB.getParent();
447-
auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
448-
auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
449-
auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
450-
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
537+
// .loophead:
538+
// lr.[w|d] dest, (addr)
539+
// mv scratch, dest
540+
// ifnochangeneeded scratch, incr, .looptail
541+
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width, STI)), DestReg)
542+
.addReg(AddrReg);
543+
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
544+
.addReg(DestReg)
545+
.addImm(0);
546+
switch (BinOp) {
547+
default:
548+
llvm_unreachable("Unexpected AtomicRMW BinOp");
549+
case AtomicRMWInst::Max: {
550+
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
551+
.addReg(ScratchReg)
552+
.addReg(IncrReg)
553+
.addMBB(LoopTailMBB);
554+
break;
555+
}
556+
case AtomicRMWInst::Min: {
557+
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGE))
558+
.addReg(IncrReg)
559+
.addReg(ScratchReg)
560+
.addMBB(LoopTailMBB);
561+
break;
562+
}
563+
case AtomicRMWInst::UMax:
564+
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
565+
.addReg(ScratchReg)
566+
.addReg(IncrReg)
567+
.addMBB(LoopTailMBB);
568+
break;
569+
case AtomicRMWInst::UMin:
570+
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BGEU))
571+
.addReg(IncrReg)
572+
.addReg(ScratchReg)
573+
.addMBB(LoopTailMBB);
574+
break;
575+
}
451576

452-
// Insert new MBBs.
453-
MF->insert(++MBB.getIterator(), LoopHeadMBB);
454-
MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
455-
MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
456-
MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
577+
// .loopifbody:
578+
// mv scratch, incr
579+
BuildMI(LoopIfBodyMBB, DL, TII->get(RISCV::ADDI), ScratchReg)
580+
.addReg(IncrReg)
581+
.addImm(0);
457582

458-
// Set up successors and transfer remaining instructions to DoneMBB.
459-
LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
460-
LoopHeadMBB->addSuccessor(LoopTailMBB);
461-
LoopIfBodyMBB->addSuccessor(LoopTailMBB);
462-
LoopTailMBB->addSuccessor(LoopHeadMBB);
463-
LoopTailMBB->addSuccessor(DoneMBB);
464-
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
465-
DoneMBB->transferSuccessors(&MBB);
466-
MBB.addSuccessor(LoopHeadMBB);
583+
// .looptail:
584+
// sc.[w|d] scratch, scratch, (addr)
585+
// bnez scratch, loop
586+
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width, STI)),
587+
ScratchReg)
588+
.addReg(ScratchReg)
589+
.addReg(AddrReg);
590+
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
591+
.addReg(ScratchReg)
592+
.addReg(RISCV::X0)
593+
.addMBB(LoopHeadMBB);
594+
}
467595

596+
static void doMaskedAtomicMinMaxOpExpansion(
597+
const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
598+
MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopHeadMBB,
599+
MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
600+
MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width,
601+
const RISCVSubtarget *STI) {
602+
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
468603
Register DestReg = MI.getOperand(0).getReg();
469604
Register Scratch1Reg = MI.getOperand(1).getReg();
470605
Register Scratch2Reg = MI.getOperand(2).getReg();
@@ -541,6 +676,44 @@ bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
541676
.addReg(Scratch1Reg)
542677
.addReg(RISCV::X0)
543678
.addMBB(LoopHeadMBB);
679+
}
680+
681+
bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
682+
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
683+
AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
684+
MachineBasicBlock::iterator &NextMBBI) {
685+
686+
MachineInstr &MI = *MBBI;
687+
DebugLoc DL = MI.getDebugLoc();
688+
MachineFunction *MF = MBB.getParent();
689+
auto LoopHeadMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
690+
auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
691+
auto LoopTailMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
692+
auto DoneMBB = MF->CreateMachineBasicBlock(MBB.getBasicBlock());
693+
694+
// Insert new MBBs.
695+
MF->insert(++MBB.getIterator(), LoopHeadMBB);
696+
MF->insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
697+
MF->insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
698+
MF->insert(++LoopTailMBB->getIterator(), DoneMBB);
699+
700+
// Set up successors and transfer remaining instructions to DoneMBB.
701+
LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
702+
LoopHeadMBB->addSuccessor(LoopTailMBB);
703+
LoopIfBodyMBB->addSuccessor(LoopTailMBB);
704+
LoopTailMBB->addSuccessor(LoopHeadMBB);
705+
LoopTailMBB->addSuccessor(DoneMBB);
706+
DoneMBB->splice(DoneMBB->end(), &MBB, MI, MBB.end());
707+
DoneMBB->transferSuccessors(&MBB);
708+
MBB.addSuccessor(LoopHeadMBB);
709+
710+
if (!IsMasked)
711+
doAtomicMinMaxOpExpansion(TII, MI, DL, &MBB, LoopHeadMBB, LoopIfBodyMBB,
712+
LoopTailMBB, DoneMBB, BinOp, Width, STI);
713+
else
714+
doMaskedAtomicMinMaxOpExpansion(TII, MI, DL, &MBB, LoopHeadMBB,
715+
LoopIfBodyMBB, LoopTailMBB, DoneMBB, BinOp,
716+
Width, STI);
544717

545718
NextMBBI = MBB.end();
546719
MI.eraseFromParent();

‎llvm/lib/Target/RISCV/RISCVFeatures.td‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,7 @@ def HasStdExtZaamo
218218
: Predicate<"Subtarget->hasStdExtZaamo()">,
219219
AssemblerPredicate<(any_of FeatureStdExtZaamo),
220220
"'Zaamo' (Atomic Memory Operations)">;
221+
def NoStdExtZaamo : Predicate<"!Subtarget->hasStdExtZaamo()">;
221222

222223
def FeatureStdExtZalrsc
223224
: RISCVExtension<1, 0, "Load-Reserved/Store-Conditional">;
@@ -1864,7 +1865,7 @@ def FeatureForcedAtomics : SubtargetFeature<
18641865
"forced-atomics", "HasForcedAtomics", "true",
18651866
"Assume that lock-free native-width atomics are available">;
18661867
def HasAtomicLdSt
1867-
: Predicate<"Subtarget->hasStdExtA() || Subtarget->hasForcedAtomics()">;
1868+
: Predicate<"Subtarget->hasStdExtZalrsc() || Subtarget->hasForcedAtomics()">;
18681869

18691870
def FeatureTaggedGlobals : SubtargetFeature<"tagged-globals",
18701871
"AllowTaggedGlobals",

0 commit comments

Comments
 (0)