Skip to content

Commit c1dc064

Browse files
LancernLancern
authored
[CIR] Add support for storing into _Atomic variables (#165872)
1 parent e856483 commit c1dc064

File tree

5 files changed

+162
-8
lines changed

5 files changed

+162
-8
lines changed

‎clang/include/clang/CIR/MissingFeatures.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,8 @@ struct MissingFeatures {
180180
static bool atomicSyncScopeID() { return false; }
181181
static bool atomicTypes() { return false; }
182182
static bool atomicUseLibCall() { return false; }
183+
static bool atomicMicrosoftVolatile() { return false; }
184+
static bool atomicOpenMP() { return false; }
183185

184186
// Global ctor handling
185187
static bool globalCtorLexOrder() { return false; }

‎clang/lib/CIR/CodeGen/CIRGenAtomic.cpp‎

Lines changed: 118 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ class AtomicInfo {
2727
CharUnits atomicAlign;
2828
CharUnits valueAlign;
2929
TypeEvaluationKind evaluationKind = cir::TEK_Scalar;
30+
bool useLibCall = true;
3031
LValue lvalue;
3132
mlir::Location loc;
3233

@@ -62,8 +63,8 @@ class AtomicInfo {
6263
assert(!cir::MissingFeatures::atomicInfo());
6364
cgf.cgm.errorNYI(loc, "AtomicInfo: non-simple lvalue");
6465
}
65-
66-
assert(!cir::MissingFeatures::atomicUseLibCall());
66+
useLibCall = !ctx.getTargetInfo().hasBuiltinAtomic(
67+
atomicSizeInBits, ctx.toBits(lvalue.getAlignment()));
6768
}
6869

6970
QualType getValueType() const { return valueTy; }
@@ -75,6 +76,8 @@ class AtomicInfo {
7576
assert(!cir::MissingFeatures::atomicInfoGetAtomicPointer());
7677
return nullptr;
7778
}
79+
bool shouldUseLibCall() const { return useLibCall; }
80+
const LValue &getAtomicLValue() const { return lvalue; }
7881
Address getAtomicAddress() const {
7982
mlir::Type elemTy;
8083
if (lvalue.isSimple()) {
@@ -96,6 +99,8 @@ class AtomicInfo {
9699

97100
bool emitMemSetZeroIfNecessary() const;
98101

102+
mlir::Value getScalarRValValueOrNull(RValue rvalue) const;
103+
99104
/// Cast the given pointer to an integer pointer suitable for atomic
100105
/// operations on the source.
101106
Address castToAtomicIntPointer(Address addr) const;
@@ -105,6 +110,9 @@ class AtomicInfo {
105110
/// copy the value across.
106111
Address convertToAtomicIntPointer(Address addr) const;
107112

113+
/// Converts a rvalue to integer value.
114+
mlir::Value convertRValueToInt(RValue rvalue, bool cmpxchg = false) const;
115+
108116
/// Copy an atomic r-value into atomic-layout memory.
109117
void emitCopyIntoMemory(RValue rvalue) const;
110118

@@ -195,6 +203,12 @@ Address AtomicInfo::createTempAlloca() const {
195203
return tempAlloca;
196204
}
197205

206+
mlir::Value AtomicInfo::getScalarRValValueOrNull(RValue rvalue) const {
207+
if (rvalue.isScalar() && (!hasPadding() || !lvalue.isSimple()))
208+
return rvalue.getValue();
209+
return nullptr;
210+
}
211+
198212
Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
199213
auto intTy = mlir::dyn_cast<cir::IntType>(addr.getElementType());
200214
// Don't bother with int casts if the integer size is the same.
@@ -211,10 +225,38 @@ bool AtomicInfo::emitMemSetZeroIfNecessary() const {
211225
return false;
212226

213227
cgf.cgm.errorNYI(loc,
214-
"AtomicInfo::emitMemSetZeroIfNecessary: emit memset zero");
228+
"AtomicInfo::emitMemSetZeroIfNecaessary: emit memset zero");
215229
return false;
216230
}
217231

232+
/// Return true if \param valueTy is a type that should be casted to integer
233+
/// around the atomic memory operation. If \param cmpxchg is true, then the
234+
/// cast of a floating point type is made as that instruction can not have
235+
/// floating point operands. TODO: Allow compare-and-exchange and FP - see
236+
/// comment in CIRGenAtomicExpandPass.cpp.
237+
static bool shouldCastToInt(mlir::Type valueTy, bool cmpxchg) {
238+
if (cir::isAnyFloatingPointType(valueTy))
239+
return isa<cir::FP80Type>(valueTy) || cmpxchg;
240+
return !isa<cir::IntType>(valueTy) && !isa<cir::PointerType>(valueTy);
241+
}
242+
243+
mlir::Value AtomicInfo::convertRValueToInt(RValue rvalue, bool cmpxchg) const {
244+
// If we've got a scalar value of the right size, try to avoid going
245+
// through memory. Floats get casted if needed by AtomicExpandPass.
246+
if (mlir::Value value = getScalarRValValueOrNull(rvalue)) {
247+
if (!shouldCastToInt(value.getType(), cmpxchg))
248+
return cgf.emitToMemory(value, valueTy);
249+
250+
cgf.cgm.errorNYI(
251+
loc, "AtomicInfo::convertRValueToInt: cast scalar rvalue to int");
252+
return nullptr;
253+
}
254+
255+
cgf.cgm.errorNYI(
256+
loc, "AtomicInfo::convertRValueToInt: cast non-scalar rvalue to int");
257+
return nullptr;
258+
}
259+
218260
/// Copy an r-value into memory as part of storing to an atomic type.
219261
/// This needs to create a bit-pattern suitable for atomic operations.
220262
void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
@@ -815,6 +857,79 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
815857
e->getExprLoc());
816858
}
817859

860+
void CIRGenFunction::emitAtomicStore(RValue rvalue, LValue dest, bool isInit) {
861+
bool isVolatile = dest.isVolatileQualified();
862+
auto order = cir::MemOrder::SequentiallyConsistent;
863+
if (!dest.getType()->isAtomicType()) {
864+
assert(!cir::MissingFeatures::atomicMicrosoftVolatile());
865+
}
866+
return emitAtomicStore(rvalue, dest, order, isVolatile, isInit);
867+
}
868+
869+
/// Emit a store to an l-value of atomic type.
870+
///
871+
/// Note that the r-value is expected to be an r-value of the atomic type; this
872+
/// means that for aggregate r-values, it should include storage for any padding
873+
/// that was necessary.
874+
void CIRGenFunction::emitAtomicStore(RValue rvalue, LValue dest,
875+
cir::MemOrder order, bool isVolatile,
876+
bool isInit) {
877+
// If this is an aggregate r-value, it should agree in type except
878+
// maybe for address-space qualification.
879+
mlir::Location loc = dest.getPointer().getLoc();
880+
assert(!rvalue.isAggregate() ||
881+
rvalue.getAggregateAddress().getElementType() ==
882+
dest.getAddress().getElementType());
883+
884+
AtomicInfo atomics(*this, dest, loc);
885+
LValue lvalue = atomics.getAtomicLValue();
886+
887+
if (lvalue.isSimple()) {
888+
// If this is an initialization, just put the value there normally.
889+
if (isInit) {
890+
atomics.emitCopyIntoMemory(rvalue);
891+
return;
892+
}
893+
894+
// Check whether we should use a library call.
895+
if (atomics.shouldUseLibCall()) {
896+
assert(!cir::MissingFeatures::atomicUseLibCall());
897+
cgm.errorNYI(loc, "emitAtomicStore: atomic store with library call");
898+
return;
899+
}
900+
901+
// Okay, we're doing this natively.
902+
mlir::Value valueToStore = atomics.convertRValueToInt(rvalue);
903+
904+
// Do the atomic store.
905+
Address addr = atomics.getAtomicAddress();
906+
if (mlir::Value value = atomics.getScalarRValValueOrNull(rvalue)) {
907+
if (shouldCastToInt(value.getType(), /*CmpXchg=*/false)) {
908+
addr = atomics.castToAtomicIntPointer(addr);
909+
valueToStore =
910+
builder.createIntCast(valueToStore, addr.getElementType());
911+
}
912+
}
913+
cir::StoreOp store = builder.createStore(loc, valueToStore, addr);
914+
915+
// Initializations don't need to be atomic.
916+
if (!isInit) {
917+
assert(!cir::MissingFeatures::atomicOpenMP());
918+
store.setMemOrder(order);
919+
}
920+
921+
// Other decoration.
922+
if (isVolatile)
923+
store.setIsVolatile(true);
924+
925+
assert(!cir::MissingFeatures::opLoadStoreTbaa());
926+
return;
927+
}
928+
929+
cgm.errorNYI(loc, "emitAtomicStore: non-simple atomic lvalue");
930+
assert(!cir::MissingFeatures::opLoadStoreAtomic());
931+
}
932+
818933
void CIRGenFunction::emitAtomicInit(Expr *init, LValue dest) {
819934
AtomicInfo atomics(*this, dest, getLoc(init->getSourceRange()));
820935

‎clang/lib/CIR/CodeGen/CIRGenExpr.cpp‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,8 @@ static LValue emitGlobalVarDeclLValue(CIRGenFunction &cgf, const Expr *e,
311311

312312
void CIRGenFunction::emitStoreOfScalar(mlir::Value value, Address addr,
313313
bool isVolatile, QualType ty,
314-
bool isInit, bool isNontemporal) {
314+
LValueBaseInfo baseInfo, bool isInit,
315+
bool isNontemporal) {
315316
assert(!cir::MissingFeatures::opLoadStoreThreadLocal());
316317

317318
if (const auto *clangVecTy = ty->getAs<clang::VectorType>()) {
@@ -333,7 +334,13 @@ void CIRGenFunction::emitStoreOfScalar(mlir::Value value, Address addr,
333334

334335
value = emitToMemory(value, ty);
335336

336-
assert(!cir::MissingFeatures::opLoadStoreAtomic());
337+
assert(!cir::MissingFeatures::opLoadStoreTbaa());
338+
LValue atomicLValue = LValue::makeAddr(addr, ty, baseInfo);
339+
if (ty->isAtomicType() ||
340+
(!isInit && isLValueSuitableForInlineAtomic(atomicLValue))) {
341+
emitAtomicStore(RValue::get(value), atomicLValue, isInit);
342+
return;
343+
}
337344

338345
// Update the alloca with more info on initialization.
339346
assert(addr.getPointer() && "expected pointer to exist");
@@ -550,7 +557,8 @@ void CIRGenFunction::emitStoreOfScalar(mlir::Value value, LValue lvalue,
550557
}
551558

552559
emitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(),
553-
lvalue.getType(), isInit, /*isNontemporal=*/false);
560+
lvalue.getType(), lvalue.getBaseInfo(), isInit,
561+
/*isNontemporal=*/false);
554562
}
555563

556564
mlir::Value CIRGenFunction::emitLoadOfScalar(Address addr, bool isVolatile,

‎clang/lib/CIR/CodeGen/CIRGenFunction.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1271,6 +1271,9 @@ class CIRGenFunction : public CIRGenTypeCache {
12711271

12721272
RValue emitAtomicExpr(AtomicExpr *e);
12731273
void emitAtomicInit(Expr *init, LValue dest);
1274+
void emitAtomicStore(RValue rvalue, LValue dest, bool isInit);
1275+
void emitAtomicStore(RValue rvalue, LValue dest, cir::MemOrder order,
1276+
bool isVolatile, bool isInit);
12741277

12751278
AutoVarEmission emitAutoVarAlloca(const clang::VarDecl &d,
12761279
mlir::OpBuilder::InsertPoint ip = {});
@@ -1680,8 +1683,8 @@ class CIRGenFunction : public CIRGenTypeCache {
16801683
bool isInit);
16811684

16821685
void emitStoreOfScalar(mlir::Value value, Address addr, bool isVolatile,
1683-
clang::QualType ty, bool isInit = false,
1684-
bool isNontemporal = false);
1686+
clang::QualType ty, LValueBaseInfo baseInfo,
1687+
bool isInit = false, bool isNontemporal = false);
16851688
void emitStoreOfScalar(mlir::Value value, LValue lvalue, bool isInit);
16861689

16871690
/// Store the specified rvalue into the specified

‎clang/test/CIR/CodeGen/atomic.c‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,32 @@ void f2(void) {
4646
// OGCG-NEXT: store i32 42, ptr %[[SLOT]], align 4
4747
// OGCG: }
4848

49+
void f3(_Atomic(int) *p) {
50+
*p = 42;
51+
}
52+
53+
// CIR-LABEL: @f3
54+
// CIR: cir.store align(4) atomic(seq_cst) %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
55+
56+
// LLVM-LABEL: @f3
57+
// LLVM: store atomic i32 42, ptr %{{.+}} seq_cst, align 4
58+
59+
// OGCG-LABEL: @f3
60+
// OGCG: store atomic i32 42, ptr %{{.+}} seq_cst, align 4
61+
62+
void f4(_Atomic(float) *p) {
63+
*p = 3.14;
64+
}
65+
66+
// CIR-LABEL: @f4
67+
// CIR: cir.store align(4) atomic(seq_cst) %{{.+}}, %{{.+}} : !cir.float, !cir.ptr<!cir.float>
68+
69+
// LLVM-LABEL: @f4
70+
// LLVM: store atomic float 0x40091EB860000000, ptr %{{.+}} seq_cst, align 4
71+
72+
// OGCG-LABEL: @f4
73+
// OGCG: store atomic float 0x40091EB860000000, ptr %{{.+}} seq_cst, align 4
74+
4975
void load(int *ptr) {
5076
int x;
5177
__atomic_load(ptr, &x, __ATOMIC_RELAXED);

0 commit comments

Comments
 (0)