diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td index a38dafa4d9cf3..cdb5ab5220e4f 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -229,6 +229,13 @@ class LLVM_MemOpPatterns { inst->setMetadata(llvm::LLVMContext::MD_nontemporal, metadata); } }]; + code setInvariantGroupCode = [{ + if ($invariantGroup) { + llvm::MDNode *metadata = llvm::MDNode::get(inst->getContext(), + std::nullopt); + inst->setMetadata(llvm::LLVMContext::MD_invariant_group, metadata); + } + }]; code setAccessGroupsMetadataCode = [{ moduleTranslation.setAccessGroupsMetadata(op, inst); }]; diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index dd0251eb6a285..9fc55c904ceaf 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -351,6 +351,7 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", UnitAttr:$volatile_, UnitAttr:$nontemporal, UnitAttr:$invariant, + UnitAttr:$invariantGroup, DefaultValuedAttr< AtomicOrdering, "AtomicOrdering::not_atomic">:$ordering, OptionalAttr:$syncscope); @@ -386,6 +387,7 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", (`volatile` $volatile_^)? $addr (`atomic` (`syncscope` `(` $syncscope^ `)`)? $ordering^)? (`invariant` $invariant^)? + (`invariant_group` $invariantGroup^)? attr-dict `:` qualified(type($addr)) `->` type($res) }]; string llvmBuilder = [{ @@ -399,6 +401,7 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", # setSyncScopeCode # setAlignmentCode # setNonTemporalMetadataCode + # setInvariantGroupCode # setAccessGroupsMetadataCode # setAliasAnalysisMetadataCode; string mlirBuilder = [{ @@ -408,6 +411,7 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", alignment, loadInst->isVolatile(), loadInst->hasMetadata(llvm::LLVMContext::MD_nontemporal), loadInst->hasMetadata(llvm::LLVMContext::MD_invariant_load), + loadInst->hasMetadata(llvm::LLVMContext::MD_invariant_group), convertAtomicOrderingFromLLVM(loadInst->getOrdering()), getLLVMSyncScope(loadInst)); }]; @@ -415,6 +419,7 @@ def LLVM_LoadOp : LLVM_MemAccessOpBase<"load", OpBuilder<(ins "Type":$type, "Value":$addr, CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile, CArg<"bool", "false">:$isNonTemporal, CArg<"bool", "false">:$isInvariant, + CArg<"bool", "false">:$isInvariantGroup, CArg<"AtomicOrdering", "AtomicOrdering::not_atomic">:$ordering, CArg<"StringRef", "StringRef()">:$syncscope)> ]; @@ -431,6 +436,7 @@ def LLVM_StoreOp : LLVM_MemAccessOpBase<"store", OptionalAttr:$alignment, UnitAttr:$volatile_, UnitAttr:$nontemporal, + UnitAttr:$invariantGroup, DefaultValuedAttr< AtomicOrdering, "AtomicOrdering::not_atomic">:$ordering, OptionalAttr:$syncscope); @@ -464,6 +470,7 @@ def LLVM_StoreOp : LLVM_MemAccessOpBase<"store", let assemblyFormat = [{ (`volatile` $volatile_^)? $value `,` $addr (`atomic` (`syncscope` `(` $syncscope^ `)`)? $ordering^)? + (`invariant_group` $invariantGroup^)? attr-dict `:` type($value) `,` qualified(type($addr)) }]; string llvmBuilder = [{ @@ -472,6 +479,7 @@ def LLVM_StoreOp : LLVM_MemAccessOpBase<"store", # setSyncScopeCode # setAlignmentCode # setNonTemporalMetadataCode + # setInvariantGroupCode # setAccessGroupsMetadataCode # setAliasAnalysisMetadataCode; string mlirBuilder = [{ @@ -480,6 +488,7 @@ def LLVM_StoreOp : LLVM_MemAccessOpBase<"store", $_op = $_builder.create($_location, $value, $addr, alignment, storeInst->isVolatile(), storeInst->hasMetadata(llvm::LLVMContext::MD_nontemporal), + storeInst->hasMetadata(llvm::LLVMContext::MD_invariant_group), convertAtomicOrderingFromLLVM(storeInst->getOrdering()), getLLVMSyncScope(storeInst)); }]; @@ -487,6 +496,7 @@ def LLVM_StoreOp : LLVM_MemAccessOpBase<"store", OpBuilder<(ins "Value":$value, "Value":$addr, CArg<"unsigned", "0">:$alignment, CArg<"bool", "false">:$isVolatile, CArg<"bool", "false">:$isNonTemporal, + CArg<"bool", "false">:$isInvariantGroup, CArg<"AtomicOrdering", "AtomicOrdering::not_atomic">:$ordering, CArg<"StringRef", "StringRef()">:$syncscope)> ]; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index 6b2d8943bf488..9bc9d2487833c 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -952,11 +952,11 @@ LogicalResult LoadOp::verify() { void LoadOp::build(OpBuilder &builder, OperationState &state, Type type, Value addr, unsigned alignment, bool isVolatile, - bool isNonTemporal, bool isInvariant, + bool isNonTemporal, bool isInvariant, bool isInvariantGroup, AtomicOrdering ordering, StringRef syncscope) { build(builder, state, type, addr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, - isNonTemporal, isInvariant, ordering, + isNonTemporal, isInvariant, isInvariantGroup, ordering, syncscope.empty() ? nullptr : builder.getStringAttr(syncscope), /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, @@ -991,11 +991,11 @@ LogicalResult StoreOp::verify() { void StoreOp::build(OpBuilder &builder, OperationState &state, Value value, Value addr, unsigned alignment, bool isVolatile, - bool isNonTemporal, AtomicOrdering ordering, - StringRef syncscope) { + bool isNonTemporal, bool isInvariantGroup, + AtomicOrdering ordering, StringRef syncscope) { build(builder, state, value, addr, alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile, - isNonTemporal, ordering, + isNonTemporal, isInvariantGroup, ordering, syncscope.empty() ? nullptr : builder.getStringAttr(syncscope), /*access_groups=*/nullptr, /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr); diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index 193ab53e6e820..d013df9609381 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -469,6 +469,20 @@ func.func @invariant_load(%ptr : !llvm.ptr) -> i32 { func.return %0 : i32 } +// CHECK-LABEL: @invariant_group_load +func.func @invariant_group_load(%ptr : !llvm.ptr) -> i32 { + // CHECK: llvm.load %{{.+}} invariant_group {alignment = 4 : i64} : !llvm.ptr -> i32 + %0 = llvm.load %ptr invariant_group {alignment = 4 : i64} : !llvm.ptr -> i32 + func.return %0 : i32 +} + +// CHECK-LABEL: @invariant_group_store +func.func @invariant_group_store(%val: i32, %ptr : !llvm.ptr) { + // CHECK: llvm.store %{{.+}}, %{{.+}} invariant_group : i32, !llvm.ptr + llvm.store %val, %ptr invariant_group : i32, !llvm.ptr + func.return +} + llvm.mlir.global external constant @_ZTIi() : !llvm.ptr llvm.func @bar(!llvm.ptr, !llvm.ptr, !llvm.ptr) llvm.func @__gxx_personality_v0(...) -> i32 diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll index f75c79ea63380..fff48bbc486bc 100644 --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -383,6 +383,33 @@ define float @invariant_load(ptr %ptr) { ; // ----- +; CHECK-LABEL: @invariant_group_load +; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] +define float @invariant_group_load(ptr %ptr) { + ; CHECK: %[[VAL:.+]] = llvm.load %[[PTR]] invariant_group {alignment = 4 : i64} : !llvm.ptr -> f32 + %1 = load float, ptr %ptr, align 4, !invariant.group !0 + ; CHECK: llvm.return %[[VAL]] + ret float %1 +} + +!0 = !{} + +; // ----- + +; CHECK-LABEL: @invariant_group_store +; CHECK-SAME: %[[VAL:[a-zA-Z0-9]+]] +; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] +define void @invariant_group_store(float %val, ptr %ptr) { + ; CHECK: llvm.store %[[VAL]], %[[PTR]] invariant_group {alignment = 4 : i64} : f32, !llvm.ptr + store float %val, ptr %ptr, align 4, !invariant.group !0 + ; CHECK: llvm.return + ret void +} + +!0 = !{} + +; // ----- + ; CHECK-LABEL: @atomic_load_store ; CHECK-SAME: %[[PTR:[a-zA-Z0-9]+]] define void @atomic_load_store(ptr %ptr) { diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir index 11d73ea7c84ad..ca40a8dfe49f0 100644 --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1967,6 +1967,20 @@ llvm.func @nontemporal_store_and_load(%ptr : !llvm.ptr) -> i32 { // ----- +// Check that invariant group attribute is exported as metadata node. +llvm.func @nontemporal_store_and_load(%ptr : !llvm.ptr) -> i32 { + %val = llvm.mlir.constant(42 : i32) : i32 + // CHECK: store i32 42, ptr %{{.*}} !invariant.group ![[NODE:[0-9]+]] + llvm.store %val, %ptr invariant_group : i32, !llvm.ptr + // CHECK: %{{.*}} = load i32, ptr %{{.*}} !invariant.group ![[NODE]] + %1 = llvm.load %ptr invariant_group : !llvm.ptr -> i32 + llvm.return %1 : i32 +} + +// CHECK: ![[NODE]] = !{} + +// ----- + llvm.func @atomic_store_and_load(%ptr : !llvm.ptr) { // CHECK: load atomic // CHECK-SAME: acquire, align 4