Skip to content

Commit 6744919

Browse files
LancernLancern
authored
[CIR] Add atomic exchange operation (#158089)
This patch adds atomic exchange operation which covers the following C/C++ intrinsic functions: - `__c11_atomic_exchange` - `__atomic_exchange` - `__atomic_exchange_n`
1 parent d510a69 commit 6744919

File tree

5 files changed

+200
-4
lines changed

5 files changed

+200
-4
lines changed

‎clang/include/clang/CIR/Dialect/IR/CIROps.td‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4131,6 +4131,45 @@ def CIR_ThrowOp : CIR_Op<"throw"> {
41314131
// Atomic operations
41324132
//===----------------------------------------------------------------------===//
41334133

4134+
def CIR_AtomicXchg : CIR_Op<"atomic.xchg", [
4135+
AllTypesMatch<["result", "val"]>,
4136+
TypesMatchWith<"type of 'val' must match the pointee type of 'ptr'",
4137+
"ptr", "val", "mlir::cast<cir::PointerType>($_self).getPointee()">
4138+
]> {
4139+
let summary = "Atomic exchange";
4140+
let description = [{
4141+
C/C++ atomic exchange operation. This operation implements the C/C++
4142+
builtin function `__atomic_exchange`, `__atomic_exchange_n`, and
4143+
`__c11_atomic_exchange`.
4144+
4145+
This operation takes two arguments: a pointer `ptr` and a value `val`. The
4146+
operation atomically replaces the value of the object pointed-to by `ptr`
4147+
with `val`, and returns the original value of the object.
4148+
4149+
Example:
4150+
4151+
```mlir
4152+
%res = cir.atomic.xchg(%ptr : !cir.ptr<!u64i>,
4153+
%val : !u64i,
4154+
seq_cst) : !u64i
4155+
```
4156+
}];
4157+
4158+
let results = (outs CIR_AnyType:$result);
4159+
let arguments = (ins
4160+
Arg<CIR_PointerType, "", [MemRead, MemWrite]>:$ptr,
4161+
CIR_AnyType:$val,
4162+
Arg<CIR_MemOrder, "memory order">:$mem_order,
4163+
UnitAttr:$is_volatile
4164+
);
4165+
4166+
let assemblyFormat = [{
4167+
$mem_order (`volatile` $is_volatile^)?
4168+
$ptr `,` $val
4169+
`:` qualified(type($ptr)) `->` type($result) attr-dict
4170+
}];
4171+
}
4172+
41344173
def CIR_AtomicCmpXchg : CIR_Op<"atomic.cmpxchg", [
41354174
AllTypesMatch<["old", "expected", "desired"]>
41364175
]> {

‎clang/lib/CIR/CodeGen/CIRGenAtomic.cpp‎

Lines changed: 30 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,7 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
341341
}
342342

343343
assert(!cir::MissingFeatures::atomicSyncScopeID());
344+
llvm::StringRef opName;
344345

345346
CIRGenBuilderTy &builder = cgf.getBuilder();
346347
mlir::Location loc = cgf.getLoc(expr->getSourceRange());
@@ -400,6 +401,12 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
400401
return;
401402
}
402403

404+
case AtomicExpr::AO__c11_atomic_exchange:
405+
case AtomicExpr::AO__atomic_exchange_n:
406+
case AtomicExpr::AO__atomic_exchange:
407+
opName = cir::AtomicXchg::getOperationName();
408+
break;
409+
403410
case AtomicExpr::AO__opencl_atomic_init:
404411

405412
case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
@@ -421,11 +428,8 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
421428
case AtomicExpr::AO__scoped_atomic_store:
422429
case AtomicExpr::AO__scoped_atomic_store_n:
423430

424-
case AtomicExpr::AO__c11_atomic_exchange:
425431
case AtomicExpr::AO__hip_atomic_exchange:
426432
case AtomicExpr::AO__opencl_atomic_exchange:
427-
case AtomicExpr::AO__atomic_exchange_n:
428-
case AtomicExpr::AO__atomic_exchange:
429433
case AtomicExpr::AO__scoped_atomic_exchange_n:
430434
case AtomicExpr::AO__scoped_atomic_exchange:
431435

@@ -503,8 +507,23 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
503507

504508
case AtomicExpr::AO__atomic_clear:
505509
cgf.cgm.errorNYI(expr->getSourceRange(), "emitAtomicOp: expr op NYI");
506-
break;
510+
return;
507511
}
512+
513+
assert(!opName.empty() && "expected operation name to build");
514+
mlir::Value loadVal1 = builder.createLoad(loc, val1);
515+
516+
SmallVector<mlir::Value> atomicOperands = {ptr.getPointer(), loadVal1};
517+
SmallVector<mlir::Type> atomicResTys = {loadVal1.getType()};
518+
mlir::Operation *rmwOp = builder.create(loc, builder.getStringAttr(opName),
519+
atomicOperands, atomicResTys);
520+
521+
rmwOp->setAttr("mem_order", orderAttr);
522+
if (expr->isVolatile())
523+
rmwOp->setAttr("is_volatile", builder.getUnitAttr());
524+
525+
mlir::Value result = rmwOp->getResult(0);
526+
builder.createStore(loc, result, dest);
508527
}
509528

510529
static bool isMemOrderValid(uint64_t order, bool isStore, bool isLoad) {
@@ -572,6 +591,11 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
572591
val1 = emitPointerWithAlignment(e->getVal1());
573592
break;
574593

594+
case AtomicExpr::AO__atomic_exchange:
595+
val1 = emitPointerWithAlignment(e->getVal1());
596+
dest = emitPointerWithAlignment(e->getVal2());
597+
break;
598+
575599
case AtomicExpr::AO__atomic_compare_exchange:
576600
case AtomicExpr::AO__atomic_compare_exchange_n:
577601
case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
@@ -590,7 +614,9 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
590614
isWeakExpr = e->getWeak();
591615
break;
592616

617+
case AtomicExpr::AO__atomic_exchange_n:
593618
case AtomicExpr::AO__atomic_store_n:
619+
case AtomicExpr::AO__c11_atomic_exchange:
594620
case AtomicExpr::AO__c11_atomic_store:
595621
val1 = emitValToTemp(*this, e->getVal1());
596622
break;

‎clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -717,6 +717,17 @@ mlir::LogicalResult CIRToLLVMAtomicCmpXchgLowering::matchAndRewrite(
717717
return mlir::success();
718718
}
719719

720+
mlir::LogicalResult CIRToLLVMAtomicXchgLowering::matchAndRewrite(
721+
cir::AtomicXchg op, OpAdaptor adaptor,
722+
mlir::ConversionPatternRewriter &rewriter) const {
723+
assert(!cir::MissingFeatures::atomicSyncScopeID());
724+
mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(adaptor.getMemOrder());
725+
rewriter.replaceOpWithNewOp<mlir::LLVM::AtomicRMWOp>(
726+
op, mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(), adaptor.getVal(),
727+
llvmOrder);
728+
return mlir::success();
729+
}
730+
720731
mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite(
721732
cir::BitClrsbOp op, OpAdaptor adaptor,
722733
mlir::ConversionPatternRewriter &rewriter) const {

‎clang/test/CIR/CodeGen/atomic.c‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -415,3 +415,102 @@ void atomic_cmpxchg_n(int *ptr, int *expected, int desired) {
415415
// OGCG-NEXT: %[[SUCCESS_2:.+]] = zext i1 %[[SUCCESS]] to i8
416416
// OGCG-NEXT: store i8 %[[SUCCESS_2]], ptr %{{.+}}, align 1
417417
}
418+
419+
void c11_atomic_exchange(_Atomic(int) *ptr, int value) {
420+
// CIR-LABEL: @c11_atomic_exchange
421+
// LLVM-LABEL: @c11_atomic_exchange
422+
// OGCG-LABEL: @c11_atomic_exchange
423+
424+
__c11_atomic_exchange(ptr, value, __ATOMIC_RELAXED);
425+
__c11_atomic_exchange(ptr, value, __ATOMIC_CONSUME);
426+
__c11_atomic_exchange(ptr, value, __ATOMIC_ACQUIRE);
427+
__c11_atomic_exchange(ptr, value, __ATOMIC_RELEASE);
428+
__c11_atomic_exchange(ptr, value, __ATOMIC_ACQ_REL);
429+
__c11_atomic_exchange(ptr, value, __ATOMIC_SEQ_CST);
430+
// CIR: %{{.+}} = cir.atomic.xchg relaxed %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
431+
// CIR: %{{.+}} = cir.atomic.xchg consume %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
432+
// CIR: %{{.+}} = cir.atomic.xchg acquire %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
433+
// CIR: %{{.+}} = cir.atomic.xchg release %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
434+
// CIR: %{{.+}} = cir.atomic.xchg acq_rel %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
435+
// CIR: %{{.+}} = cir.atomic.xchg seq_cst %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
436+
437+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
438+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
439+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
440+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
441+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
442+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
443+
444+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
445+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
446+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
447+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
448+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
449+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
450+
}
451+
452+
void atomic_exchange(int *ptr, int *value, int *old) {
453+
// CIR-LABEL: @atomic_exchange
454+
// LLVM-LABEL: @atomic_exchange
455+
// OGCG-LABEL: @atomic_exchange
456+
457+
__atomic_exchange(ptr, value, old, __ATOMIC_RELAXED);
458+
__atomic_exchange(ptr, value, old, __ATOMIC_CONSUME);
459+
__atomic_exchange(ptr, value, old, __ATOMIC_ACQUIRE);
460+
__atomic_exchange(ptr, value, old, __ATOMIC_RELEASE);
461+
__atomic_exchange(ptr, value, old, __ATOMIC_ACQ_REL);
462+
__atomic_exchange(ptr, value, old, __ATOMIC_SEQ_CST);
463+
// CIR: %{{.+}} = cir.atomic.xchg relaxed %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
464+
// CIR: %{{.+}} = cir.atomic.xchg consume %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
465+
// CIR: %{{.+}} = cir.atomic.xchg acquire %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
466+
// CIR: %{{.+}} = cir.atomic.xchg release %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
467+
// CIR: %{{.+}} = cir.atomic.xchg acq_rel %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
468+
// CIR: %{{.+}} = cir.atomic.xchg seq_cst %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
469+
470+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
471+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
472+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
473+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
474+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
475+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
476+
477+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
478+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
479+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
480+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
481+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
482+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
483+
}
484+
485+
void atomic_exchange_n(int *ptr, int value) {
486+
// CIR-LABEL: @atomic_exchange_n
487+
// LLVM-LABEL: @atomic_exchange_n
488+
// OGCG-LABEL: @atomic_exchange_n
489+
490+
__atomic_exchange_n(ptr, value, __ATOMIC_RELAXED);
491+
__atomic_exchange_n(ptr, value, __ATOMIC_CONSUME);
492+
__atomic_exchange_n(ptr, value, __ATOMIC_ACQUIRE);
493+
__atomic_exchange_n(ptr, value, __ATOMIC_RELEASE);
494+
__atomic_exchange_n(ptr, value, __ATOMIC_ACQ_REL);
495+
__atomic_exchange_n(ptr, value, __ATOMIC_SEQ_CST);
496+
// CIR: %{{.+}} = cir.atomic.xchg relaxed %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
497+
// CIR: %{{.+}} = cir.atomic.xchg consume %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
498+
// CIR: %{{.+}} = cir.atomic.xchg acquire %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
499+
// CIR: %{{.+}} = cir.atomic.xchg release %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
500+
// CIR: %{{.+}} = cir.atomic.xchg acq_rel %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
501+
// CIR: %{{.+}} = cir.atomic.xchg seq_cst %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
502+
503+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
504+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
505+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
506+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
507+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
508+
// LLVM: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
509+
510+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} monotonic, align 4
511+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
512+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acquire, align 4
513+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} release, align 4
514+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} acq_rel, align 4
515+
// OGCG: %{{.+}} = atomicrmw xchg ptr %{{.+}}, i32 %{{.+}} seq_cst, align 4
516+
}

‎clang/test/CIR/IR/atomic.cir‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// RUN: cir-opt %s | FileCheck %s
2+
3+
!s32i = !cir.int<s, 32>
4+
!u32i = !cir.int<u, 32>
5+
6+
cir.func @atomic_xchg(%ptr: !cir.ptr<!s32i>, %val: !s32i) {
7+
// CHECK-LABEL: @atomic_xchg
8+
%0 = cir.atomic.xchg relaxed %ptr, %val : !cir.ptr<!s32i> -> !s32i
9+
// CHECK: cir.atomic.xchg relaxed %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
10+
%1 = cir.atomic.xchg consume %ptr, %val : !cir.ptr<!s32i> -> !s32i
11+
// CHECK: cir.atomic.xchg consume %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
12+
%2 = cir.atomic.xchg acquire %ptr, %val : !cir.ptr<!s32i> -> !s32i
13+
// CHECK: cir.atomic.xchg acquire %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
14+
%3 = cir.atomic.xchg release %ptr, %val : !cir.ptr<!s32i> -> !s32i
15+
// CHECK: cir.atomic.xchg release %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
16+
%4 = cir.atomic.xchg acq_rel %ptr, %val : !cir.ptr<!s32i> -> !s32i
17+
// CHECK: cir.atomic.xchg acq_rel %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
18+
%5 = cir.atomic.xchg seq_cst %ptr, %val : !cir.ptr<!s32i> -> !s32i
19+
// CHECK: cir.atomic.xchg seq_cst %{{.+}}, %{{.+}} : !cir.ptr<!s32i> -> !s32i
20+
cir.return
21+
}

0 commit comments

Comments
 (0)