Skip to content

[CIR] Atomic fetch operation #161631

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+887 −24
Open

[CIR] Atomic fetch operation #161631

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions clang/include/clang/CIR/Dialect/IR/CIROps.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4172,6 +4172,79 @@ def CIR_ThrowOp : CIR_Op<"throw"> {
// Atomic operations
//===----------------------------------------------------------------------===//

def CIR_AtomicFetchKind : CIR_I32EnumAttr<
"AtomicFetchKind", "Binary opcode for atomic fetch-and-update operations", [
I32EnumAttrCase<"Add", 0, "add">,
I32EnumAttrCase<"Sub", 1, "sub">,
I32EnumAttrCase<"And", 2, "and">,
I32EnumAttrCase<"Xor", 3, "xor">,
I32EnumAttrCase<"Or", 4, "or">,
I32EnumAttrCase<"Nand", 5, "nand">,
I32EnumAttrCase<"Max", 6, "max">,
I32EnumAttrCase<"Min", 7, "min">
]>;

def CIR_AtomicFetchOp : CIR_Op<"atomic.fetch", [
AllTypesMatch<["result", "val"]>,
TypesMatchWith<"type of 'val' must match the pointee type of 'ptr'",
"ptr", "val", "mlir::cast<cir::PointerType>($_self).getPointee()">
]> {
let summary = "Atomic fetch-and-update operation";
let description = [{
C/C++ atomic fetch-and-update operation. This operation implements the C/C++
builtin functions `__atomic_<binop>_fetch`, `__atomic_fetch_<binop>`, and
`__c11_atomic_fetch_<binop>`, where `<binop>` is one of the following binary
opcodes: `add`, `sub`, `and`, `xor`, `or`, `nand`, `max`, and `min`.

This operation takes 2 arguments: a pointer `ptr` and a value `val`. The
type of `val` must match the pointee type of `ptr`. And the type of `val`
must either be an integer or a floating-point type. If the binary operation
is neither `add` nor `sub`, then `val` must be an integer.

This operation atomically loads the value from `ptr`, performs the binary
operation as indicated by `binop` on the loaded value and `val`, and stores
the result back to `ptr`. If the `fetch_first` flag is present, the result
of this operation is the old value loaded from `ptr` before the binary
operation. Otherwise, the result of this operation is the result of the
binary operation.

Example:
%res = cir.atomic.fetch add seq_cst %ptr, %val
: (!cir.ptr<!s32i>, !s32i) -> !s32i
}];
let results = (outs CIR_AnyIntOrFloatType:$result);
let arguments = (ins
Arg<CIR_PtrToIntOrFloatType, "", [MemRead, MemWrite]>:$ptr,
CIR_AnyIntOrFloatType:$val,
CIR_AtomicFetchKind:$binop,
Arg<CIR_MemOrder, "memory order">:$mem_order,
UnitAttr:$is_volatile,
UnitAttr:$fetch_first
);

let assemblyFormat = [{
$binop $mem_order
(`fetch_first` $fetch_first^)?
$ptr `,` $val
(`volatile` $is_volatile^)?
`:` `(` qualified(type($ptr)) `,` qualified(type($val)) `)`
`->` type($result) attr-dict
}];

let hasVerifier = 1;

let extraLLVMLoweringPatternDecl = [{
mlir::Value buildPostOp(cir::AtomicFetchOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter,
mlir::Value rmwVal, bool isInt) const;

mlir::Value buildMinMaxPostOp(cir::AtomicFetchOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter,
mlir::Value rmwVal, bool isInt,
bool isSigned) const;
}];
}

def CIR_AtomicXchg : CIR_Op<"atomic.xchg", [
AllTypesMatch<["result", "val"]>,
TypesMatchWith<"type of 'val' must match the pointee type of 'ptr'",
Expand Down
143 changes: 119 additions & 24 deletions clang/lib/CIR/CodeGen/CIRGenAtomic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,8 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
CIRGenBuilderTy &builder = cgf.getBuilder();
mlir::Location loc = cgf.getLoc(expr->getSourceRange());
auto orderAttr = cir::MemOrderAttr::get(builder.getContext(), order);
cir::AtomicFetchKindAttr fetchAttr;
bool fetchFirst = true;

switch (expr->getOp()) {
case AtomicExpr::AO__c11_atomic_init:
Expand Down Expand Up @@ -407,6 +409,86 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
opName = cir::AtomicXchg::getOperationName();
break;

case AtomicExpr::AO__atomic_add_fetch:
fetchFirst = false;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_add:
opName = cir::AtomicFetchOp::getOperationName();
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
cir::AtomicFetchKind::Add);
break;

case AtomicExpr::AO__atomic_sub_fetch:
fetchFirst = false;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_sub:
case AtomicExpr::AO__atomic_fetch_sub:
opName = cir::AtomicFetchOp::getOperationName();
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
cir::AtomicFetchKind::Sub);
break;

case AtomicExpr::AO__atomic_min_fetch:
fetchFirst = false;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_min:
case AtomicExpr::AO__atomic_fetch_min:
opName = cir::AtomicFetchOp::getOperationName();
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
cir::AtomicFetchKind::Min);
break;

case AtomicExpr::AO__atomic_max_fetch:
fetchFirst = false;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_max:
case AtomicExpr::AO__atomic_fetch_max:
opName = cir::AtomicFetchOp::getOperationName();
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
cir::AtomicFetchKind::Max);
break;

case AtomicExpr::AO__atomic_and_fetch:
fetchFirst = false;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_and:
opName = cir::AtomicFetchOp::getOperationName();
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
cir::AtomicFetchKind::And);
break;

case AtomicExpr::AO__atomic_or_fetch:
fetchFirst = false;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_or:
opName = cir::AtomicFetchOp::getOperationName();
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
cir::AtomicFetchKind::Or);
break;

case AtomicExpr::AO__atomic_xor_fetch:
fetchFirst = false;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_xor:
opName = cir::AtomicFetchOp::getOperationName();
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
cir::AtomicFetchKind::Xor);
break;

case AtomicExpr::AO__atomic_nand_fetch:
fetchFirst = false;
[[fallthrough]];
case AtomicExpr::AO__c11_atomic_fetch_nand:
case AtomicExpr::AO__atomic_fetch_nand:
opName = cir::AtomicFetchOp::getOperationName();
fetchAttr = cir::AtomicFetchKindAttr::get(builder.getContext(),
cir::AtomicFetchKind::Nand);
break;

case AtomicExpr::AO__opencl_atomic_init:

case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
Expand All @@ -433,74 +515,50 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
case AtomicExpr::AO__scoped_atomic_exchange_n:
case AtomicExpr::AO__scoped_atomic_exchange:

case AtomicExpr::AO__atomic_add_fetch:
case AtomicExpr::AO__scoped_atomic_add_fetch:

case AtomicExpr::AO__c11_atomic_fetch_add:
case AtomicExpr::AO__hip_atomic_fetch_add:
case AtomicExpr::AO__opencl_atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_add:
case AtomicExpr::AO__scoped_atomic_fetch_add:

case AtomicExpr::AO__atomic_sub_fetch:
case AtomicExpr::AO__scoped_atomic_sub_fetch:

case AtomicExpr::AO__c11_atomic_fetch_sub:
case AtomicExpr::AO__hip_atomic_fetch_sub:
case AtomicExpr::AO__opencl_atomic_fetch_sub:
case AtomicExpr::AO__atomic_fetch_sub:
case AtomicExpr::AO__scoped_atomic_fetch_sub:

case AtomicExpr::AO__atomic_min_fetch:
case AtomicExpr::AO__scoped_atomic_min_fetch:

case AtomicExpr::AO__c11_atomic_fetch_min:
case AtomicExpr::AO__hip_atomic_fetch_min:
case AtomicExpr::AO__opencl_atomic_fetch_min:
case AtomicExpr::AO__atomic_fetch_min:
case AtomicExpr::AO__scoped_atomic_fetch_min:

case AtomicExpr::AO__atomic_max_fetch:
case AtomicExpr::AO__scoped_atomic_max_fetch:

case AtomicExpr::AO__c11_atomic_fetch_max:
case AtomicExpr::AO__hip_atomic_fetch_max:
case AtomicExpr::AO__opencl_atomic_fetch_max:
case AtomicExpr::AO__atomic_fetch_max:
case AtomicExpr::AO__scoped_atomic_fetch_max:

case AtomicExpr::AO__atomic_and_fetch:
case AtomicExpr::AO__scoped_atomic_and_fetch:

case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__hip_atomic_fetch_and:
case AtomicExpr::AO__opencl_atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_and:
case AtomicExpr::AO__scoped_atomic_fetch_and:

case AtomicExpr::AO__atomic_or_fetch:
case AtomicExpr::AO__scoped_atomic_or_fetch:

case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__hip_atomic_fetch_or:
case AtomicExpr::AO__opencl_atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_or:
case AtomicExpr::AO__scoped_atomic_fetch_or:

case AtomicExpr::AO__atomic_xor_fetch:
case AtomicExpr::AO__scoped_atomic_xor_fetch:

case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__hip_atomic_fetch_xor:
case AtomicExpr::AO__opencl_atomic_fetch_xor:
case AtomicExpr::AO__atomic_fetch_xor:
case AtomicExpr::AO__scoped_atomic_fetch_xor:

case AtomicExpr::AO__atomic_nand_fetch:
case AtomicExpr::AO__scoped_atomic_nand_fetch:

case AtomicExpr::AO__c11_atomic_fetch_nand:
case AtomicExpr::AO__atomic_fetch_nand:
case AtomicExpr::AO__scoped_atomic_fetch_nand:

case AtomicExpr::AO__atomic_test_and_set:
Expand All @@ -518,9 +576,13 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest,
mlir::Operation *rmwOp = builder.create(loc, builder.getStringAttr(opName),
atomicOperands, atomicResTys);

if (fetchAttr)
rmwOp->setAttr("binop", fetchAttr);
rmwOp->setAttr("mem_order", orderAttr);
if (expr->isVolatile())
rmwOp->setAttr("is_volatile", builder.getUnitAttr());
if (fetchFirst && opName == cir::AtomicFetchOp::getOperationName())
rmwOp->setAttr("fetch_first", builder.getUnitAttr());

mlir::Value result = rmwOp->getResult(0);
builder.createStore(loc, result, dest);
Expand Down Expand Up @@ -614,8 +676,41 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) {
isWeakExpr = e->getWeak();
break;

case AtomicExpr::AO__c11_atomic_fetch_add:
case AtomicExpr::AO__c11_atomic_fetch_sub:
if (memTy->isPointerType()) {
cgm.errorNYI(e->getSourceRange(),
"atomic fetch-and-add and fetch-and-sub for pointers");
return RValue::get(nullptr);
}
[[fallthrough]];
case AtomicExpr::AO__atomic_fetch_add:
case AtomicExpr::AO__atomic_fetch_max:
case AtomicExpr::AO__atomic_fetch_min:
case AtomicExpr::AO__atomic_fetch_sub:
case AtomicExpr::AO__atomic_add_fetch:
case AtomicExpr::AO__atomic_max_fetch:
case AtomicExpr::AO__atomic_min_fetch:
case AtomicExpr::AO__atomic_sub_fetch:
case AtomicExpr::AO__c11_atomic_fetch_max:
case AtomicExpr::AO__c11_atomic_fetch_min:
shouldCastToIntPtrTy = !memTy->isFloatingType();
[[fallthrough]];

case AtomicExpr::AO__atomic_fetch_and:
case AtomicExpr::AO__atomic_fetch_nand:
case AtomicExpr::AO__atomic_fetch_or:
case AtomicExpr::AO__atomic_fetch_xor:
case AtomicExpr::AO__atomic_and_fetch:
case AtomicExpr::AO__atomic_nand_fetch:
case AtomicExpr::AO__atomic_or_fetch:
case AtomicExpr::AO__atomic_xor_fetch:
case AtomicExpr::AO__atomic_exchange_n:
case AtomicExpr::AO__atomic_store_n:
case AtomicExpr::AO__c11_atomic_fetch_and:
case AtomicExpr::AO__c11_atomic_fetch_nand:
case AtomicExpr::AO__c11_atomic_fetch_or:
case AtomicExpr::AO__c11_atomic_fetch_xor:
case AtomicExpr::AO__c11_atomic_exchange:
case AtomicExpr::AO__c11_atomic_store:
val1 = emitValToTemp(*this, e->getVal1());
Expand Down
15 changes: 15 additions & 0 deletions clang/lib/CIR/Dialect/IR/CIRDialect.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2860,6 +2860,21 @@ LogicalResult cir::AtomicCmpXchg::verify() {
return success();
}

//===----------------------------------------------------------------------===//
// AtomicFetchOp
//===----------------------------------------------------------------------===//

LogicalResult cir::AtomicFetchOp::verify() {
if (getBinop() != cir::AtomicFetchKind::Add &&
getBinop() != cir::AtomicFetchKind::Sub &&
getBinop() != cir::AtomicFetchKind::Max &&
getBinop() != cir::AtomicFetchKind::Min &&
!mlir::isa<cir::IntType>(getVal().getType()))
return emitError("only atomic add, sub, max, and min operation could "
"operate on floating-point values");
return success();
}

//===----------------------------------------------------------------------===//
// TypeInfoAttr
//===----------------------------------------------------------------------===//
Expand Down
Loading
Loading