[CIR] Add atomic exchange operation #158089
Merged
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Lancern
requested review from
lanza,
bcardosolopes,
xlauko and
andykaylor
as code owners
llvmbot
added
clang
|
@llvm/pr-subscribers-clangir @llvm/pr-subscribers-clang Author: Sirui Mu (Lancern) ChangesThis patch adds atomic exchange operation which covers the following C/C++ intrinsic functions:
Full diff: https://github.com/llvm/llvm-project/pull/158089.diff 6 Files Affected:
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index f3715bdb5ef42..0e01860a3e202 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -4033,6 +4033,47 @@ def CIR_ThrowOp : CIR_Op<"throw"> {
// Atomic operations
//===----------------------------------------------------------------------===//
+def CIR_AtomicXchg : CIR_Op<"atomic.xchg", [
+ AllTypesMatch<["result", "val"]>
+]> {
+ let summary = "Atomic exchange";
+ let description = [{
+ C/C++ atomic exchange operation. This operation implements the C/C++
+ builtin function `__atomic_exchange`, `__atomic_exchange_n`, and
+ `__c11_atomic_exchange`.
+
+ This operation takes two arguments: a pointer `ptr` and a value `val`. The
+ operation atomically replaces the value of the object pointed-to by `ptr`
+ with `val`, and returns the original value of the object.
+
+ Example:
+
+ ```mlir
+ %res = cir.atomic.xchg(%ptr : !cir.ptr<!u64i>,
+ %val : !u64i,
+ seq_cst) : !u64i
+ ```
+ }];
+
+ let results = (outs CIR_AnyType:$result);
+ let arguments = (ins Arg<CIR_PointerType, "", [MemRead, MemWrite]>:$ptr,
+ CIR_AnyType:$val,
+ Arg<CIR_MemOrder, "memory order">:$mem_order,
+ UnitAttr:$is_volatile);
+
+ let assemblyFormat = [{
+ `(`
+ $ptr `:` qualified(type($ptr)) `,`
+ $val `:` type($val) `,`
+ $mem_order
+ `)`
+ (`volatile` $is_volatile^)?
+ `:` type($result) attr-dict
+ }];
+
+ let hasVerifier = 1;
+}
+
def CIR_AtomicCmpXchg : CIR_Op<"atomic.cmpxchg", [
AllTypesMatch<["old", "expected", "desired"]>
]> {
diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
index 86ba0299af3cf..e943b0252bf4e 100644
--- a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
@@ -341,6 +341,7 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
}
assert(!cir::MissingFeatures::atomicSyncScopeID());
+ llvm::StringRef opName;
CIRGenBuilderTy &builder = cgf.getBuilder();
mlir::Location loc = cgf.getLoc(expr->getSourceRange());
@@ -400,6 +401,12 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
return;
}
+ case AtomicExpr::AO__c11_atomic_exchange:
+ case AtomicExpr::AO__atomic_exchange_n:
+ case AtomicExpr::AO__atomic_exchange:
+ opName = cir::AtomicXchg::getOperationName();
+ break;
+
case AtomicExpr::AO__opencl_atomic_init:
case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
@@ -421,11 +428,8 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
case AtomicExpr::AO__scoped_atomic_store:
case AtomicExpr::AO__scoped_atomic_store_n:
- case AtomicExpr::AO__c11_atomic_exchange:
case AtomicExpr::AO__hip_atomic_exchange:
case AtomicExpr::AO__opencl_atomic_exchange:
- case AtomicExpr::AO__atomic_exchange_n:
- case AtomicExpr::AO__atomic_exchange:
case AtomicExpr::AO__scoped_atomic_exchange_n:
case AtomicExpr::AO__scoped_atomic_exchange:
@@ -503,8 +507,23 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
case AtomicExpr::AO__atomic_clear:
cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: expr op NYI");
- break;
+ return;
}
+
+ assert(!opName.empty() && "expected operation name to build");
+ mlir::Value loadVal1 = builder.createLoad(loc, val1);
+
+ SmallVector<mlir::Value> atomicOperands = {ptr.getPointer(), loadVal1};
+ SmallVector<mlir::Type> atomicResTys = {loadVal1.getType()};
+ mlir::Operation *rmwOp = builder.create(loc, builder.getStringAttr(opName),
+ atomicOperands, atomicResTys);
+
+ rmwOp->setAttr("mem_order", orderAttr);
+ if (expr->isVolatile())
+ rmwOp->setAttr("is_volatile", builder.getUnitAttr());
+
+ mlir::Value result = rmwOp->getResult(0);
+ builder.createStore(loc, result, dest);
}
static bool isMemOrderValid(uint64_t order, bool isStore, bool isLoad) {
@@ -572,6 +591,11 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
val1 = emitPointerWithAlignment(e->getVal1());
break;
+ case AtomicExpr::AO__atomic_exchange:
+ val1 = emitPointerWithAlignment(e->getVal1());
+ dest = emitPointerWithAlignment(e->getVal2());
+ break;
+
case AtomicExpr::AO__atomic_compare_exchange:
case AtomicExpr::AO__atomic_compare_exchange_n:
case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
@@ -590,7 +614,9 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
isWeakExpr = e->getWeak();
break;
+ case AtomicExpr::AO__atomic_exchange_n:
case AtomicExpr::AO__atomic_store_n:
+ case AtomicExpr::AO__c11_atomic_exchange:
case AtomicExpr::AO__c11_atomic_store:
val1 = emitValToTemp(*this, e->getVal1());
break;
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 24aef693024f7..598f6a86796e0 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -2730,6 +2730,16 @@ mlir::LogicalResult cir::ThrowOp::verify() {
return failure();
}
+//===----------------------------------------------------------------------===//
+// AtomicXchg
+//===----------------------------------------------------------------------===//
+
+LogicalResult cir::AtomicXchg::verify() {
+ if (getPtr().getType().getPointee() != getVal().getType())
+ return emitOpError("ptr type and val type must match");
+ return success();
+}
+
//===----------------------------------------------------------------------===//
// AtomicCmpXchg
//===----------------------------------------------------------------------===//
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 816987ba48145..cf4d319f977a0 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -693,6 +693,17 @@ mlir::LogicalResult CIRToLLVMAtomicCmpXchgLowering::matchAndRewrite(
return mlir::success();
}
+mlir::LogicalResult CIRToLLVMAtomicXchgLowering::matchAndRewrite(
+ cir::AtomicXchg op, OpAdaptor adaptor,
+ mlir::ConversionPatternRewriter &rewriter) const {
+ assert(!cir::MissingFeatures::atomicSyncScopeID());
+ mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(adaptor.getMemOrder());
+ rewriter.replaceOpWithNewOp<mlir::LLVM::AtomicRMWOp>(
+ op, mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(), adaptor.getVal(),
+ llvmOrder);
+ return mlir::success();
+}
+
mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite(
cir::BitClrsbOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
@@ -2467,6 +2478,7 @@ void ConvertCIRToLLVMPass::runOnOperation() {
CIRToLLVMAssumeAlignedOpLowering,
CIRToLLVMAssumeSepStorageOpLowering,
CIRToLLVMAtomicCmpXchgLowering,
+ CIRToLLVMAtomicXchgLowering,
CIRToLLVMBaseClassAddrOpLowering,
CIRToLLVMBinOpLowering,
CIRToLLVMBitClrsbOpLowering,
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
index 34b121c88f677..d301c2b69f791 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -144,6 +144,16 @@ class CIRToLLVMAtomicCmpXchgLowering
mlir::ConversionPatternRewriter &) const override;
};
+class CIRToLLVMAtomicXchgLowering
+ : public mlir::OpConversionPattern<cir::AtomicXchg> {
+public:
+ using mlir::OpConversionPattern<cir::AtomicXchg>::OpConversionPattern;
+
+ mlir::LogicalResult
+ matchAndRewrite(cir::AtomicXchg op, OpAdaptor,
+ mlir::ConversionPatternRewriter &) const override;
+};
+
class CIRToLLVMBrCondOpLowering
: public mlir::OpConversionPattern<cir::BrCondOp> {
public:
diff --git a/clang/test/CIR/CodeGen/atomic.c b/clang/test/CIR/CodeGen/atomic.c
index 0eba2959c0ebc..9cdc639a8cf41 100644
--- a/clang/test/CIR/CodeGen/atomic.c
+++ b/clang/test/CIR/CodeGen/atomic.c
@@ -415,3 +415,102 @@ void atomic_cmpxchg_n(int *ptr, int *expected, int desired) {
// OGCG-NEXT: %[[SUCCESS_2:.+]] = zext i1 %[[SUCCESS]] to i8
// OGCG-NEXT: store i8 %[[SUCCESS_2]], ptr %{{.+}}, align 1
}
+
+void c11_atomic_exchange(_Atomic(int) *ptr, int value) {
+ // CIR-LABEL: @c11_atomic_exchange
+ // LLVM-LABEL: @c11_atomic_exchange
+ // OGCG-LABEL: @c11_atomic_exchange
+
+ __c11_atomic_exchange(ptr, value, __ATOMIC_RELAXED);
+ __c11_atomic_exchange(ptr, value, __ATOMIC_CONSUME);
+ __c11_atomic_exchange(ptr, value, __ATOMIC_ACQUIRE);
+ __c11_atomic_exchange(ptr, value, __ATOMIC_RELEASE);
+ __c11_atomic_exchange(ptr, value, __ATOMIC_ACQ_REL);
+ __c11_atomic_exchange(ptr, value, __ATOMIC_SEQ_CST);
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, relaxed) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, consume) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, acquire) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, release) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, acq_rel) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, seq_cst) : !s32i
+
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
+
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
+}
+
+void atomic_exchange(int *ptr, int *value, int *old) {
+ // CIR-LABEL: @atomic_exchange
+ // LLVM-LABEL: @atomic_exchange
+ // OGCG-LABEL: @atomic_exchange
+
+ __atomic_exchange(ptr, value, old, __ATOMIC_RELAXED);
+ __atomic_exchange(ptr, value, old, __ATOMIC_CONSUME);
+ __atomic_exchange(ptr, value, old, __ATOMIC_ACQUIRE);
+ __atomic_exchange(ptr, value, old, __ATOMIC_RELEASE);
+ __atomic_exchange(ptr, value, old, __ATOMIC_ACQ_REL);
+ __atomic_exchange(ptr, value, old, __ATOMIC_SEQ_CST);
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, relaxed) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, consume) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, acquire) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, release) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, acq_rel) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, seq_cst) : !s32i
+
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
+
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
+}
+
+void atomic_exchange_n(int *ptr, int value) {
+ // CIR-LABEL: @atomic_exchange_n
+ // LLVM-LABEL: @atomic_exchange_n
+ // OGCG-LABEL: @atomic_exchange_n
+
+ __atomic_exchange_n(ptr, value, __ATOMIC_RELAXED);
+ __atomic_exchange_n(ptr, value, __ATOMIC_CONSUME);
+ __atomic_exchange_n(ptr, value, __ATOMIC_ACQUIRE);
+ __atomic_exchange_n(ptr, value, __ATOMIC_RELEASE);
+ __atomic_exchange_n(ptr, value, __ATOMIC_ACQ_REL);
+ __atomic_exchange_n(ptr, value, __ATOMIC_SEQ_CST);
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, relaxed) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, consume) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, acquire) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, release) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, acq_rel) : !s32i
+ // CIR: %{{.+}} = cir.atomic.xchg(%{{.+}} : !cir.ptr<!s32i>, %{{.+}} : !s32i, seq_cst) : !s32i
+
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
+ // LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
+
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
+ // OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
+}
|
xlauko
reviewed
Sep 12, 2025
Lancern
force-pushed
the
cir/atomic-exchange
branch
2 times, most recently
from
308524a to
78170c9
Compare
This patch adds atomic exchange operation which covers the following C/C++ intrinsic functions: - `__c11_atomic_exchange` - `__atomic_exchange` - `__atomic_exchange_n`
Lancern
force-pushed
the
cir/atomic-exchange
branch
from
78170c9 to
163a61f
Compare
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
This patch adds atomic exchange operation which covers the following C/C++ intrinsic functions:
__c11_atomic_exchange__atomic_exchange__atomic_exchange_n