[llvm][mlir] Atomic Control Options Support #140932

anchuraj · 2025-05-21T17:16:09Z

Adding support for atomic control options for amd gpus. These are to support options -f[no-]atomic-remote-memory, -f[no-]atomic-fine-grained-memory, -f[no-]atomic-ignore-denormal-mode

llvmbot · 2025-05-21T17:38:56Z

@llvm/pr-subscribers-mlir

Author: Anchu Rajendran S (anchuraj)

Changes

Adding support for atomic control options for amd gpus. These are to support options -f[no-]atomic-remote-memory, -f[no-]atomic-fine-grained-memory, -f[no-]atomic-ignore-denormal-mode

Full diff: https://github.com/llvm/llvm-project/pull/140932.diff

3 Files Affected:

(modified) mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td (+28-5)
(modified) mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp (+6-1)
(modified) mlir/test/Target/LLVMIR/llvmir.mlir (+9)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index f19f9d5a3083c..8efb3c0ee4520 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -2164,12 +2164,18 @@ def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [
               LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering,
               OptionalAttr<StrAttr>:$syncscope,
               OptionalAttr<I64Attr>:$alignment,
-              UnitAttr:$volatile_);
+              UnitAttr:$volatile_,
+              UnitAttr:$amdgpu_no_fine_grained_memory,
+              UnitAttr:$amdgpu_ignore_denormal_mode,
+              UnitAttr:$amdgpu_no_remote_memory);
   // Append the aliasing related attributes defined in LLVM_MemAccessOpBase.
   let arguments = !con(args, aliasAttrs);
   let results = (outs LLVM_AtomicRMWType:$res);
   let assemblyFormat = [{
-    (`volatile` $volatile_^)? $bin_op $ptr `,` $val
+    (`volatile` $volatile_^)?
+    (`amdgpu_ignore_denormal_mode` $amdgpu_ignore_denormal_mode^)?
+    (`amdgpu_no_fine_grained_memory` $amdgpu_no_fine_grained_memory^)?
+    (`amdgpu_no_remote_memory` $amdgpu_no_remote_memory^)? $bin_op $ptr `,` $val
     (`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict `:`
     qualified(type($ptr)) `,` type($val)
   }];
@@ -2179,6 +2185,19 @@ def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [
         convertAtomicBinOpToLLVM($bin_op), $ptr, $val, llvm::MaybeAlign(),
         convertAtomicOrderingToLLVM($ordering));
     $res = inst;
+    auto &llvmContext = inst->getContext();
+    if($amdgpu_ignore_denormal_mode) {
+      llvm::MDNode *metadata = llvm::MDNode::get(llvmContext, std::nullopt);
+      inst->setMetadata((llvmContext).getMDKindID("amdgpu.ignore.denormal.mode"), metadata);
+    }
+    if($amdgpu_no_fine_grained_memory) {
+      llvm::MDNode *metadata = llvm::MDNode::get(llvmContext, std::nullopt);
+      inst->setMetadata(llvmContext.getMDKindID("amdgpu.no.fine.grained.memory"), metadata);
+    }
+    if($amdgpu_no_remote_memory) {
+      llvm::MDNode *metadata = llvm::MDNode::get(llvmContext, std::nullopt);
+      inst->setMetadata((llvmContext).getMDKindID("amdgpu.no.remote.memory"), metadata);
+    }
   }] # setVolatileCode
      # setSyncScopeCode
      # setAlignmentCode
@@ -2192,12 +2211,16 @@ def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [
         convertAtomicOrderingFromLLVM(atomicInst->getOrdering()),
         getLLVMSyncScope(atomicInst), alignment, atomicInst->isVolatile());
   }];
-  list<int> llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1, -1, -1, -1, -1];
+  list<int> llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+                              -1];
   let builders = [
-    OpBuilder<(ins "LLVM::AtomicBinOp":$binOp, "Value":$ptr, "Value":$val,
+      OpBuilder<(ins "LLVM::AtomicBinOp":$binOp, "Value":$ptr,"Value":$val,
       "LLVM::AtomicOrdering":$ordering,
       CArg<"StringRef", "StringRef()">:$syncscope,
-      CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile
+      CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile,
+      CArg<"bool", "false">:$isAmdgpuIgnoreDenormalMode,
+      CArg<"bool", "false">:$isAmdgpuNoFineGrainedMemory,
+      CArg<"bool", "false">:$isAmdgpuNoRemoteMemory
     )>
   ];
   let hasVerifier = 1;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index e17b9fd6eb8d3..90f7ba55b5b2f 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3274,10 +3274,15 @@ OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); }
 void AtomicRMWOp::build(OpBuilder &builder, OperationState &state,
                         AtomicBinOp binOp, Value ptr, Value val,
                         AtomicOrdering ordering, StringRef syncscope,
-                        unsigned alignment, bool isVolatile) {
+                        unsigned alignment, bool isVolatile,
+                        bool isAmdgpuIgnoreDenormalMode,
+                        bool isAmdgpuNoFineGrainedMemory,
+                        bool isAmdgpuNoRemoteMemory) {
   build(builder, state, val.getType(), binOp, ptr, val, ordering,
         !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
         alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
+        isAmdgpuIgnoreDenormalMode, isAmdgpuNoFineGrainedMemory,
+        isAmdgpuNoRemoteMemory,
         /*access_groups=*/nullptr,
         /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
 }
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index 4ef68fa83a70d..b4979fc11914c 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -1568,6 +1568,15 @@ llvm.func @atomicrmw(
   // CHECK-SAME:  syncscope("singlethread")
   // CHECK-SAME:  align 8
   %27 = llvm.atomicrmw volatile udec_wrap %i32_ptr, %i32 syncscope("singlethread") monotonic {alignment = 8 : i64} : !llvm.ptr, i32
+  // CHECK: atomicrmw
+  // CHECK-SAME: !amdgpu.ignore.denormal.mode
+  %28 = llvm.atomicrmw amdgpu_ignore_denormal_mode udec_wrap %i32_ptr, %i32 monotonic {alignment = 8 : i64} : !llvm.ptr, i32
+  // CHECK: atomicrmw
+  // CHECK-SAME: !amdgpu.no.fine.grained.memory
+  %29 = llvm.atomicrmw amdgpu_no_fine_grained_memory udec_wrap %i32_ptr, %i32 monotonic {alignment = 8 : i64} : !llvm.ptr, i32
+  // CHECK: atomicrmw
+  // CHECK-SAME: !amdgpu.no.remote.memory
+  %30 = llvm.atomicrmw amdgpu_no_remote_memory udec_wrap %i32_ptr, %i32 monotonic {alignment = 8 : i64} : !llvm.ptr, i32
   llvm.return
 }

llvmbot · 2025-05-21T17:38:57Z

@llvm/pr-subscribers-mlir-llvm

Author: Anchu Rajendran S (anchuraj)

Changes

Adding support for atomic control options for amd gpus. These are to support options -f[no-]atomic-remote-memory, -f[no-]atomic-fine-grained-memory, -f[no-]atomic-ignore-denormal-mode

Full diff: https://github.com/llvm/llvm-project/pull/140932.diff

3 Files Affected:

(modified) mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td (+28-5)
(modified) mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp (+6-1)
(modified) mlir/test/Target/LLVMIR/llvmir.mlir (+9)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index f19f9d5a3083c..8efb3c0ee4520 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -2164,12 +2164,18 @@ def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [
               LLVM_AtomicRMWType:$val, AtomicOrdering:$ordering,
               OptionalAttr<StrAttr>:$syncscope,
               OptionalAttr<I64Attr>:$alignment,
-              UnitAttr:$volatile_);
+              UnitAttr:$volatile_,
+              UnitAttr:$amdgpu_no_fine_grained_memory,
+              UnitAttr:$amdgpu_ignore_denormal_mode,
+              UnitAttr:$amdgpu_no_remote_memory);
   // Append the aliasing related attributes defined in LLVM_MemAccessOpBase.
   let arguments = !con(args, aliasAttrs);
   let results = (outs LLVM_AtomicRMWType:$res);
   let assemblyFormat = [{
-    (`volatile` $volatile_^)? $bin_op $ptr `,` $val
+    (`volatile` $volatile_^)?
+    (`amdgpu_ignore_denormal_mode` $amdgpu_ignore_denormal_mode^)?
+    (`amdgpu_no_fine_grained_memory` $amdgpu_no_fine_grained_memory^)?
+    (`amdgpu_no_remote_memory` $amdgpu_no_remote_memory^)? $bin_op $ptr `,` $val
     (`syncscope` `(` $syncscope^ `)`)? $ordering attr-dict `:`
     qualified(type($ptr)) `,` type($val)
   }];
@@ -2179,6 +2185,19 @@ def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [
         convertAtomicBinOpToLLVM($bin_op), $ptr, $val, llvm::MaybeAlign(),
         convertAtomicOrderingToLLVM($ordering));
     $res = inst;
+    auto &llvmContext = inst->getContext();
+    if($amdgpu_ignore_denormal_mode) {
+      llvm::MDNode *metadata = llvm::MDNode::get(llvmContext, std::nullopt);
+      inst->setMetadata((llvmContext).getMDKindID("amdgpu.ignore.denormal.mode"), metadata);
+    }
+    if($amdgpu_no_fine_grained_memory) {
+      llvm::MDNode *metadata = llvm::MDNode::get(llvmContext, std::nullopt);
+      inst->setMetadata(llvmContext.getMDKindID("amdgpu.no.fine.grained.memory"), metadata);
+    }
+    if($amdgpu_no_remote_memory) {
+      llvm::MDNode *metadata = llvm::MDNode::get(llvmContext, std::nullopt);
+      inst->setMetadata((llvmContext).getMDKindID("amdgpu.no.remote.memory"), metadata);
+    }
   }] # setVolatileCode
      # setSyncScopeCode
      # setAlignmentCode
@@ -2192,12 +2211,16 @@ def LLVM_AtomicRMWOp : LLVM_MemAccessOpBase<"atomicrmw", [
         convertAtomicOrderingFromLLVM(atomicInst->getOrdering()),
         getLLVMSyncScope(atomicInst), alignment, atomicInst->isVolatile());
   }];
-  list<int> llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1, -1, -1, -1, -1];
+  list<int> llvmArgIndices = [-1, 0, 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+                              -1];
   let builders = [
-    OpBuilder<(ins "LLVM::AtomicBinOp":$binOp, "Value":$ptr, "Value":$val,
+      OpBuilder<(ins "LLVM::AtomicBinOp":$binOp, "Value":$ptr,"Value":$val,
       "LLVM::AtomicOrdering":$ordering,
       CArg<"StringRef", "StringRef()">:$syncscope,
-      CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile
+      CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile,
+      CArg<"bool", "false">:$isAmdgpuIgnoreDenormalMode,
+      CArg<"bool", "false">:$isAmdgpuNoFineGrainedMemory,
+      CArg<"bool", "false">:$isAmdgpuNoRemoteMemory
     )>
   ];
   let hasVerifier = 1;
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index e17b9fd6eb8d3..90f7ba55b5b2f 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -3274,10 +3274,15 @@ OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); }
 void AtomicRMWOp::build(OpBuilder &builder, OperationState &state,
                         AtomicBinOp binOp, Value ptr, Value val,
                         AtomicOrdering ordering, StringRef syncscope,
-                        unsigned alignment, bool isVolatile) {
+                        unsigned alignment, bool isVolatile,
+                        bool isAmdgpuIgnoreDenormalMode,
+                        bool isAmdgpuNoFineGrainedMemory,
+                        bool isAmdgpuNoRemoteMemory) {
   build(builder, state, val.getType(), binOp, ptr, val, ordering,
         !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
         alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
+        isAmdgpuIgnoreDenormalMode, isAmdgpuNoFineGrainedMemory,
+        isAmdgpuNoRemoteMemory,
         /*access_groups=*/nullptr,
         /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
 }
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index 4ef68fa83a70d..b4979fc11914c 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -1568,6 +1568,15 @@ llvm.func @atomicrmw(
   // CHECK-SAME:  syncscope("singlethread")
   // CHECK-SAME:  align 8
   %27 = llvm.atomicrmw volatile udec_wrap %i32_ptr, %i32 syncscope("singlethread") monotonic {alignment = 8 : i64} : !llvm.ptr, i32
+  // CHECK: atomicrmw
+  // CHECK-SAME: !amdgpu.ignore.denormal.mode
+  %28 = llvm.atomicrmw amdgpu_ignore_denormal_mode udec_wrap %i32_ptr, %i32 monotonic {alignment = 8 : i64} : !llvm.ptr, i32
+  // CHECK: atomicrmw
+  // CHECK-SAME: !amdgpu.no.fine.grained.memory
+  %29 = llvm.atomicrmw amdgpu_no_fine_grained_memory udec_wrap %i32_ptr, %i32 monotonic {alignment = 8 : i64} : !llvm.ptr, i32
+  // CHECK: atomicrmw
+  // CHECK-SAME: !amdgpu.no.remote.memory
+  %30 = llvm.atomicrmw amdgpu_no_remote_memory udec_wrap %i32_ptr, %i32 monotonic {alignment = 8 : i64} : !llvm.ptr, i32
   llvm.return
 }

gysit

Thanks!

Can you also update the mlirImport builder to support importing the new attributes from LLVM IR?

gysit · 2025-05-21T18:54:03Z

mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td

+              UnitAttr:$amdgpu_no_fine_grained_memory,
+              UnitAttr:$amdgpu_ignore_denormal_mode,


Suggested change

UnitAttr:$amdgpu_no_fine_grained_memory,

UnitAttr:$amdgpu_ignore_denormal_mode,

UnitAttr:$amdgpu_ignore_denormal_mode,

UnitAttr:$amdgpu_no_fine_grained_memory,

nit: Let's use the same order as in the assembly format?

gysit · 2025-05-21T19:02:55Z

mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td

+      llvm::MDNode *metadata = llvm::MDNode::get(llvmContext, std::nullopt);
+      inst->setMetadata(llvmContext.getMDKindID("amdgpu.no.fine.grained.memory"), metadata);
+    }
+    if($amdgpu_no_remote_memory) {


Can you update the mlirBuilder below? It is used by the import of LLVM IR into MLIR LLVM dialect and we try to keep the two paths consistent.

A test for the import could be added in:
llvm-project/mlir/test/Target/LLVMIR/Import/instructions.ll
Or also in a separate file if you prefer.

Sure, Thank you! Working on this.

krzysz00

Main not is tests

krzysz00 · 2025-05-21T21:55:58Z

mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td

  let assemblyFormat = [{
-    (`volatile` $volatile_^)? $bin_op $ptr `,` $val
+    (`volatile` $volatile_^)?
+    (`amdgpu_ignore_denormal_mode` $amdgpu_ignore_denormal_mode^)?


Can we oilist these?

Or given that they're a bit obscure and platform-dependent, they could probably just live in the attr-dict

I am supportive of using discardable attributes for such platform-dependent flags. Otherwise, we may end-up with excessive amount of flags on the op itself.

krzysz00 · 2025-05-21T21:57:18Z

mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td

        convertAtomicOrderingToLLVM($ordering));
    $res = inst;
+    auto &llvmContext = inst->getContext();
+    if($amdgpu_ignore_denormal_mode) {


Nit: space after if

krzysz00 · 2025-05-21T22:02:21Z

... On further investigation, I'd like to point out that that rocdl.no_fine_grained_memory , rocdl.no_remote_memory, and so on can be used as discardable attributes and will lower to the metadata in question. Is there a reason that's insufficient?

anchuraj · 2025-05-27T18:54:04Z

... On further investigation, I'd like to point out that that rocdl.no_fine_grained_memory , rocdl.no_remote_memory, and so on can be used as discardable attributes and will lower to the metadata in question. Is there a reason that's insufficient?

I will be able to use the attributes defined. Thank you for pointing out

anchuraj force-pushed the atomic-options-llvm branch from 28b688e to f1e7b1e Compare May 21, 2025 17:35

anchuraj marked this pull request as ready for review May 21, 2025 17:38

anchuraj requested a review from gysit May 21, 2025 17:38

llvmbot added mlir:llvm mlir labels May 21, 2025

anchuraj requested review from Dinistro, jthackray and krzysz00 and removed request for jthackray May 21, 2025 17:39

anchuraj changed the title ~~Atomic Options Support~~ [llvm][mlir] Atomic Options Support May 21, 2025

anchuraj requested review from mjklemm and skatrak May 21, 2025 17:40

anchuraj changed the title ~~[llvm][mlir] Atomic Options Support~~ [llvm][mlir] Atomic Control Options Support May 21, 2025

Atomic Options Support

b0ceb7a

anchuraj force-pushed the atomic-options-llvm branch from f1e7b1e to b0ceb7a Compare May 21, 2025 17:52

gysit reviewed May 21, 2025

View reviewed changes

krzysz00 reviewed May 21, 2025

View reviewed changes

anchuraj closed this May 27, 2025

		UnitAttr:$amdgpu_no_fine_grained_memory,
		UnitAttr:$amdgpu_ignore_denormal_mode,

[llvm][mlir] Atomic Control Options Support #140932

[llvm][mlir] Atomic Control Options Support #140932

Uh oh!

Conversation

anchuraj commented May 21, 2025

Uh oh!

llvmbot commented May 21, 2025

Uh oh!

llvmbot commented May 21, 2025

Uh oh!

gysit left a comment

Choose a reason for hiding this comment

Uh oh!

gysit May 21, 2025

Choose a reason for hiding this comment

Uh oh!

gysit May 21, 2025

Choose a reason for hiding this comment

Uh oh!

anchuraj May 21, 2025

Choose a reason for hiding this comment

Uh oh!

krzysz00 left a comment

Choose a reason for hiding this comment

Uh oh!

krzysz00 May 21, 2025

Choose a reason for hiding this comment

Uh oh!

krzysz00 May 21, 2025

Choose a reason for hiding this comment

Uh oh!

gysit May 22, 2025

Choose a reason for hiding this comment

Uh oh!

krzysz00 May 21, 2025

Choose a reason for hiding this comment

Uh oh!

krzysz00 commented May 21, 2025

Uh oh!

anchuraj commented May 27, 2025

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

4 participants