Skip to content

Commit 1962fbe

Browse files
LancernLancern
committed
[CIR] Atomic fetch operation
This patch adds CIR support for atomic fetch operations, including the intrinsic functions `__atomic_fetch_<binop>`, `__atomic_<binop>_fetch`, and `__c11_atomic_fetch_<binop>`, where `<binop>` could be `add`, `sub`, `max`, `min`, `and`, `or`, `xor`, and `nand`.
1 parent 3149a77 commit 1962fbe

File tree

6 files changed

+887
-24
lines changed

6 files changed

+887
-24
lines changed

‎clang/include/clang/CIR/Dialect/IR/CIROps.td‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4172,6 +4172,79 @@ def CIR_ThrowOp : CIR_Op<"throw"> {
41724172
// Atomic operations
41734173
//===----------------------------------------------------------------------===//
41744174

4175+
def CIR_AtomicFetchKind : CIR_I32EnumAttr<
4176+
"AtomicFetchKind", "Binary opcode for atomic fetch-and-update operations", [
4177+
I32EnumAttrCase<"Add", 0, "add">,
4178+
I32EnumAttrCase<"Sub", 1, "sub">,
4179+
I32EnumAttrCase<"And", 2, "and">,
4180+
I32EnumAttrCase<"Xor", 3, "xor">,
4181+
I32EnumAttrCase<"Or", 4, "or">,
4182+
I32EnumAttrCase<"Nand", 5, "nand">,
4183+
I32EnumAttrCase<"Max", 6, "max">,
4184+
I32EnumAttrCase<"Min", 7, "min">
4185+
]>;
4186+
4187+
def CIR_AtomicFetchOp : CIR_Op<"atomic.fetch", [
4188+
AllTypesMatch<["result", "val"]>,
4189+
TypesMatchWith<"type of 'val' must match the pointee type of 'ptr'",
4190+
"ptr", "val", "mlir::cast<cir::PointerType>($_self).getPointee()">
4191+
]> {
4192+
let summary = "Atomic fetch-and-update operation";
4193+
let description = [{
4194+
C/C++ atomic fetch-and-update operation. This operation implements the C/C++
4195+
builtin functions `__atomic_<binop>_fetch`, `__atomic_fetch_<binop>`, and
4196+
`__c11_atomic_fetch_<binop>`, where `<binop>` is one of the following binary
4197+
opcodes: `add`, `sub`, `and`, `xor`, `or`, `nand`, `max`, and `min`.
4198+
4199+
This operation takes 2 arguments: a pointer `ptr` and a value `val`. The
4200+
type of `val` must match the pointee type of `ptr`. And the type of `val`
4201+
must either be an integer or a floating-point type. If the binary operation
4202+
is neither `add` nor `sub`, then `val` must be an integer.
4203+
4204+
This operation atomically loads the value from `ptr`, performs the binary
4205+
operation as indicated by `binop` on the loaded value and `val`, and stores
4206+
the result back to `ptr`. If the `fetch_first` flag is present, the result
4207+
of this operation is the old value loaded from `ptr` before the binary
4208+
operation. Otherwise, the result of this operation is the result of the
4209+
binary operation.
4210+
4211+
Example:
4212+
%res = cir.atomic.fetch add seq_cst %ptr, %val
4213+
: (!cir.ptr<!s32i>, !s32i) -> !s32i
4214+
}];
4215+
let results = (outs CIR_AnyIntOrFloatType:$result);
4216+
let arguments = (ins
4217+
Arg<CIR_PtrToIntOrFloatType, "", [MemRead, MemWrite]>:$ptr,
4218+
CIR_AnyIntOrFloatType:$val,
4219+
CIR_AtomicFetchKind:$binop,
4220+
Arg<CIR_MemOrder, "memory order">:$mem_order,
4221+
UnitAttr:$is_volatile,
4222+
UnitAttr:$fetch_first
4223+
);
4224+
4225+
let assemblyFormat = [{
4226+
$binop $mem_order
4227+
(`fetch_first` $fetch_first^)?
4228+
$ptr `,` $val
4229+
(`volatile` $is_volatile^)?
4230+
`:` `(` qualified(type($ptr)) `,` qualified(type($val)) `)`
4231+
`->` type($result) attr-dict
4232+
}];
4233+
4234+
let hasVerifier = 1;
4235+
4236+
let extraLLVMLoweringPatternDecl = [{
4237+
mlir::Value buildPostOp(cir::AtomicFetchOp op, OpAdaptor adaptor,
4238+
mlir::ConversionPatternRewriter &rewriter,
4239+
mlir::Value rmwVal, bool isInt) const;
4240+
4241+
mlir::Value buildMinMaxPostOp(cir::AtomicFetchOp op, OpAdaptor adaptor,
4242+
mlir::ConversionPatternRewriter &rewriter,
4243+
mlir::Value rmwVal, bool isInt,
4244+
bool isSigned) const;
4245+
}];
4246+
}
4247+
41754248
def CIR_AtomicXchg : CIR_Op<"atomic.xchg", [
41764249
AllTypesMatch<["result", "val"]>,
41774250
TypesMatchWith<"type of 'val' must match the pointee type of 'ptr'",

‎clang/lib/CIR/CodeGen/CIRGenAtomic.cpp‎

Lines changed: 119 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,8 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
346346
CIRGenBuilderTy &builder = cgf.getBuilder();
347347
mlir::Location loc = cgf.getLoc(expr->getSourceRange());
348348
auto orderAttr = cir::MemOrderAttr::get(builder.getContext(), order);
349+
cir::AtomicFetchKindAttr fetchAttr;
350+
bool fetchFirst = true;
349351

350352
switch (expr->getOp()) {
351353
case AtomicExpr::AO__c11_atomic_init:
@@ -407,6 +409,86 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
407409
opName = cir::AtomicXchg::getOperationName();
408410
break;
409411

412+
case AtomicExpr::AO__atomic_add_fetch:
413+
fetchFirst = false;
414+
[[fallthrough]];
415+
case AtomicExpr::AO__c11_atomic_fetch_add:
416+
case AtomicExpr::AO__atomic_fetch_add:
417+
opName = cir::AtomicFetchOp::getOperationName();
418+
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
419+
cir::AtomicFetchKind::Add);
420+
break;
421+
422+
case AtomicExpr::AO__atomic_sub_fetch:
423+
fetchFirst = false;
424+
[[fallthrough]];
425+
case AtomicExpr::AO__c11_atomic_fetch_sub:
426+
case AtomicExpr::AO__atomic_fetch_sub:
427+
opName = cir::AtomicFetchOp::getOperationName();
428+
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
429+
cir::AtomicFetchKind::Sub);
430+
break;
431+
432+
case AtomicExpr::AO__atomic_min_fetch:
433+
fetchFirst = false;
434+
[[fallthrough]];
435+
case AtomicExpr::AO__c11_atomic_fetch_min:
436+
case AtomicExpr::AO__atomic_fetch_min:
437+
opName = cir::AtomicFetchOp::getOperationName();
438+
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
439+
cir::AtomicFetchKind::Min);
440+
break;
441+
442+
case AtomicExpr::AO__atomic_max_fetch:
443+
fetchFirst = false;
444+
[[fallthrough]];
445+
case AtomicExpr::AO__c11_atomic_fetch_max:
446+
case AtomicExpr::AO__atomic_fetch_max:
447+
opName = cir::AtomicFetchOp::getOperationName();
448+
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
449+
cir::AtomicFetchKind::Max);
450+
break;
451+
452+
case AtomicExpr::AO__atomic_and_fetch:
453+
fetchFirst = false;
454+
[[fallthrough]];
455+
case AtomicExpr::AO__c11_atomic_fetch_and:
456+
case AtomicExpr::AO__atomic_fetch_and:
457+
opName = cir::AtomicFetchOp::getOperationName();
458+
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
459+
cir::AtomicFetchKind::And);
460+
break;
461+
462+
case AtomicExpr::AO__atomic_or_fetch:
463+
fetchFirst = false;
464+
[[fallthrough]];
465+
case AtomicExpr::AO__c11_atomic_fetch_or:
466+
case AtomicExpr::AO__atomic_fetch_or:
467+
opName = cir::AtomicFetchOp::getOperationName();
468+
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
469+
cir::AtomicFetchKind::Or);
470+
break;
471+
472+
case AtomicExpr::AO__atomic_xor_fetch:
473+
fetchFirst = false;
474+
[[fallthrough]];
475+
case AtomicExpr::AO__c11_atomic_fetch_xor:
476+
case AtomicExpr::AO__atomic_fetch_xor:
477+
opName = cir::AtomicFetchOp::getOperationName();
478+
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
479+
cir::AtomicFetchKind::Xor);
480+
break;
481+
482+
case AtomicExpr::AO__atomic_nand_fetch:
483+
fetchFirst = false;
484+
[[fallthrough]];
485+
case AtomicExpr::AO__c11_atomic_fetch_nand:
486+
case AtomicExpr::AO__atomic_fetch_nand:
487+
opName = cir::AtomicFetchOp::getOperationName();
488+
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
489+
cir::AtomicFetchKind::Nand);
490+
break;
491+
410492
case AtomicExpr::AO__opencl_atomic_init:
411493

412494
case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
@@ -433,74 +515,50 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
433515
case AtomicExpr::AO__scoped_atomic_exchange_n:
434516
case AtomicExpr::AO__scoped_atomic_exchange:
435517

436-
case AtomicExpr::AO__atomic_add_fetch:
437518
case AtomicExpr::AO__scoped_atomic_add_fetch:
438519

439-
case AtomicExpr::AO__c11_atomic_fetch_add:
440520
case AtomicExpr::AO__hip_atomic_fetch_add:
441521
case AtomicExpr::AO__opencl_atomic_fetch_add:
442-
case AtomicExpr::AO__atomic_fetch_add:
443522
case AtomicExpr::AO__scoped_atomic_fetch_add:
444523

445-
case AtomicExpr::AO__atomic_sub_fetch:
446524
case AtomicExpr::AO__scoped_atomic_sub_fetch:
447525

448-
case AtomicExpr::AO__c11_atomic_fetch_sub:
449526
case AtomicExpr::AO__hip_atomic_fetch_sub:
450527
case AtomicExpr::AO__opencl_atomic_fetch_sub:
451-
case AtomicExpr::AO__atomic_fetch_sub:
452528
case AtomicExpr::AO__scoped_atomic_fetch_sub:
453529

454-
case AtomicExpr::AO__atomic_min_fetch:
455530
case AtomicExpr::AO__scoped_atomic_min_fetch:
456531

457-
case AtomicExpr::AO__c11_atomic_fetch_min:
458532
case AtomicExpr::AO__hip_atomic_fetch_min:
459533
case AtomicExpr::AO__opencl_atomic_fetch_min:
460-
case AtomicExpr::AO__atomic_fetch_min:
461534
case AtomicExpr::AO__scoped_atomic_fetch_min:
462535

463-
case AtomicExpr::AO__atomic_max_fetch:
464536
case AtomicExpr::AO__scoped_atomic_max_fetch:
465537

466-
case AtomicExpr::AO__c11_atomic_fetch_max:
467538
case AtomicExpr::AO__hip_atomic_fetch_max:
468539
case AtomicExpr::AO__opencl_atomic_fetch_max:
469-
case AtomicExpr::AO__atomic_fetch_max:
470540
case AtomicExpr::AO__scoped_atomic_fetch_max:
471541

472-
case AtomicExpr::AO__atomic_and_fetch:
473542
case AtomicExpr::AO__scoped_atomic_and_fetch:
474543

475-
case AtomicExpr::AO__c11_atomic_fetch_and:
476544
case AtomicExpr::AO__hip_atomic_fetch_and:
477545
case AtomicExpr::AO__opencl_atomic_fetch_and:
478-
case AtomicExpr::AO__atomic_fetch_and:
479546
case AtomicExpr::AO__scoped_atomic_fetch_and:
480547

481-
case AtomicExpr::AO__atomic_or_fetch:
482548
case AtomicExpr::AO__scoped_atomic_or_fetch:
483549

484-
case AtomicExpr::AO__c11_atomic_fetch_or:
485550
case AtomicExpr::AO__hip_atomic_fetch_or:
486551
case AtomicExpr::AO__opencl_atomic_fetch_or:
487-
case AtomicExpr::AO__atomic_fetch_or:
488552
case AtomicExpr::AO__scoped_atomic_fetch_or:
489553

490-
case AtomicExpr::AO__atomic_xor_fetch:
491554
case AtomicExpr::AO__scoped_atomic_xor_fetch:
492555

493-
case AtomicExpr::AO__c11_atomic_fetch_xor:
494556
case AtomicExpr::AO__hip_atomic_fetch_xor:
495557
case AtomicExpr::AO__opencl_atomic_fetch_xor:
496-
case AtomicExpr::AO__atomic_fetch_xor:
497558
case AtomicExpr::AO__scoped_atomic_fetch_xor:
498559

499-
case AtomicExpr::AO__atomic_nand_fetch:
500560
case AtomicExpr::AO__scoped_atomic_nand_fetch:
501561

502-
case AtomicExpr::AO__c11_atomic_fetch_nand:
503-
case AtomicExpr::AO__atomic_fetch_nand:
504562
case AtomicExpr::AO__scoped_atomic_fetch_nand:
505563

506564
case AtomicExpr::AO__atomic_test_and_set:
@@ -518,9 +576,13 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
518576
mlir::Operation *rmwOp = builder.create(loc, builder.getStringAttr(opName),
519577
atomicOperands, atomicResTys);
520578

579+
if (fetchAttr)
580+
rmwOp->setAttr("binop", fetchAttr);
521581
rmwOp->setAttr("mem_order", orderAttr);
522582
if (expr->isVolatile())
523583
rmwOp->setAttr("is_volatile", builder.getUnitAttr());
584+
if (fetchFirst && opName == cir::AtomicFetchOp::getOperationName())
585+
rmwOp->setAttr("fetch_first", builder.getUnitAttr());
524586

525587
mlir::Value result = rmwOp->getResult(0);
526588
builder.createStore(loc, result, dest);
@@ -614,8 +676,41 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
614676
isWeakExpr = e->getWeak();
615677
break;
616678

679+
case AtomicExpr::AO__c11_atomic_fetch_add:
680+
case AtomicExpr::AO__c11_atomic_fetch_sub:
681+
if (memTy->isPointerType()) {
682+
cgm.errorNYI(e->getSourceRange(),
683+
"atomic fetch-and-add and fetch-and-sub for pointers");
684+
return RValue::get(nullptr);
685+
}
686+
[[fallthrough]];
687+
case AtomicExpr::AO__atomic_fetch_add:
688+
case AtomicExpr::AO__atomic_fetch_max:
689+
case AtomicExpr::AO__atomic_fetch_min:
690+
case AtomicExpr::AO__atomic_fetch_sub:
691+
case AtomicExpr::AO__atomic_add_fetch:
692+
case AtomicExpr::AO__atomic_max_fetch:
693+
case AtomicExpr::AO__atomic_min_fetch:
694+
case AtomicExpr::AO__atomic_sub_fetch:
695+
case AtomicExpr::AO__c11_atomic_fetch_max:
696+
case AtomicExpr::AO__c11_atomic_fetch_min:
697+
shouldCastToIntPtrTy = !memTy->isFloatingType();
698+
[[fallthrough]];
699+
700+
case AtomicExpr::AO__atomic_fetch_and:
701+
case AtomicExpr::AO__atomic_fetch_nand:
702+
case AtomicExpr::AO__atomic_fetch_or:
703+
case AtomicExpr::AO__atomic_fetch_xor:
704+
case AtomicExpr::AO__atomic_and_fetch:
705+
case AtomicExpr::AO__atomic_nand_fetch:
706+
case AtomicExpr::AO__atomic_or_fetch:
707+
case AtomicExpr::AO__atomic_xor_fetch:
617708
case AtomicExpr::AO__atomic_exchange_n:
618709
case AtomicExpr::AO__atomic_store_n:
710+
case AtomicExpr::AO__c11_atomic_fetch_and:
711+
case AtomicExpr::AO__c11_atomic_fetch_nand:
712+
case AtomicExpr::AO__c11_atomic_fetch_or:
713+
case AtomicExpr::AO__c11_atomic_fetch_xor:
619714
case AtomicExpr::AO__c11_atomic_exchange:
620715
case AtomicExpr::AO__c11_atomic_store:
621716
val1 = emitValToTemp(*this, e->getVal1());

‎clang/lib/CIR/Dialect/IR/CIRDialect.cpp‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2860,6 +2860,21 @@ LogicalResult cir::AtomicCmpXchg::verify() {
28602860
return success();
28612861
}
28622862

2863+
//===----------------------------------------------------------------------===//
2864+
// AtomicFetchOp
2865+
//===----------------------------------------------------------------------===//
2866+
2867+
LogicalResult cir::AtomicFetchOp::verify() {
2868+
if (getBinop() != cir::AtomicFetchKind::Add &&
2869+
getBinop() != cir::AtomicFetchKind::Sub &&
2870+
getBinop() != cir::AtomicFetchKind::Max &&
2871+
getBinop() != cir::AtomicFetchKind::Min &&
2872+
!mlir::isa<cir::IntType>(getVal().getType()))
2873+
return emitError("only atomic add, sub, max, and min operation could "
2874+
"operate on floating-point values");
2875+
return success();
2876+
}
2877+
28632878
//===----------------------------------------------------------------------===//
28642879
// TypeInfoAttr
28652880
//===----------------------------------------------------------------------===//

0 commit comments

Comments
 (0)