From 29ade41fb09d06538409ece5f84f09eb2dc36d7e Mon Sep 17 00:00:00 2001 From: pvanhout Date: Wed, 30 Jul 2025 14:18:46 +0200 Subject: [PATCH 1/5] [AMDGPU] Check noalias.addrspace in mayAccessScratchThroughFlat PR #149247 made the MD accessible by the backend so we can now leverage it in the memory model. The first use case here is detecting if a flat op can access scratch memory. Benefits both the MemoryLegalizer and InsertWaitCnt. --- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 22 ++++++++++++++++++- .../AMDGPU/gfx1250-scratch-scope-se.ll | 5 ++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 41885e45b4101..04f855d0bcf54 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4250,6 +4250,24 @@ bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode); } +static bool hasNoAliasAddrSpaceScratch(const MachineMemOperand *MemOp) { + const MDNode *MD = MemOp->getAAInfo().NoAliasAddrSpace; + if (!MD) + return false; + + // This MD is structured in ranges [A, B) + // Check if PRIVATE is included in any of them. + for (unsigned I = 0, E = MD->getNumOperands() / 2; I != E; ++I) { + auto *Low = mdconst::extract(MD->getOperand(2 * I + 0)); + auto *High = mdconst::extract(MD->getOperand(2 * I + 1)); + if (Low->getValue().ule(AMDGPUAS::PRIVATE_ADDRESS) && + High->getValue().ugt(AMDGPUAS::PRIVATE_ADDRESS)) + return true; + } + + return false; +} + bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { if (!isFLAT(MI) || isFLATGlobal(MI)) return false; @@ -4272,7 +4290,9 @@ bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { // See if any memory operand specifies an address space that involves scratch. return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) { unsigned AS = Memop->getAddrSpace(); - return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS; + if (AS == AMDGPUAS::FLAT_ADDRESS) + return !hasNoAliasAddrSpaceScratch(Memop); + return AS == AMDGPUAS::PRIVATE_ADDRESS; }); } diff --git a/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll b/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll index d1e82a06077f5..99025f0a983c0 100644 --- a/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll +++ b/llvm/test/CodeGen/AMDGPU/gfx1250-scratch-scope-se.ll @@ -39,20 +39,19 @@ define void @test_flat_store_no_scratch_alloc(ptr %ptr, i32 %val) #0 { ret void } -; TODO: handle define void @test_flat_store_noalias_addrspace(ptr %ptr, i32 %val) { ; GCN-LABEL: test_flat_store_noalias_addrspace: ; GCN: ; %bb.0: ; GCN-NEXT: s_wait_loadcnt_dscnt 0x0 ; GCN-NEXT: s_wait_kmcnt 0x0 -; GCN-NEXT: flat_store_b32 v[0:1], v2 scope:SCOPE_SE +; GCN-NEXT: flat_store_b32 v[0:1], v2 ; GCN-NEXT: s_wait_dscnt 0x0 ; GCN-NEXT: s_set_pc_i64 s[30:31] store i32 %val, ptr %ptr, !noalias.addrspace !{i32 5, i32 6} ret void } -; TODO: would be nice to handle too +; TODO: would be nice to handle define void @test_flat_store_select(ptr addrspace(1) %a, ptr addrspace(3) %b, i1 %cond, i32 %val) { ; GCN-SDAG-LABEL: test_flat_store_select: ; GCN-SDAG: ; %bb.0: From a58218c38c94623a2bff143899d31596352fc405 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Fri, 1 Aug 2025 12:03:07 +0200 Subject: [PATCH 2/5] Use shared helper --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 17 ++------------ llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 23 ++----------------- .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 17 ++++++++++++++ llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 5 ++++ 4 files changed, 26 insertions(+), 36 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index f58fde421f77d..29cf7c5dc9893 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -17795,21 +17795,8 @@ atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) { static bool flatInstrMayAccessPrivate(const Instruction *I) { const MDNode *NoaliasAddrSpaceMD = I->getMetadata(LLVMContext::MD_noalias_addrspace); - if (!NoaliasAddrSpaceMD) - return true; - - for (unsigned I = 0, E = NoaliasAddrSpaceMD->getNumOperands() / 2; I != E; - ++I) { - auto *Low = mdconst::extract( - NoaliasAddrSpaceMD->getOperand(2 * I + 0)); - if (Low->getValue().uge(AMDGPUAS::PRIVATE_ADDRESS)) { - auto *High = mdconst::extract( - NoaliasAddrSpaceMD->getOperand(2 * I + 1)); - return High->getValue().ule(AMDGPUAS::PRIVATE_ADDRESS); - } - } - - return true; + return !AMDGPU::hasValueInRange(NoaliasAddrSpaceMD, + AMDGPUAS::PRIVATE_ADDRESS); } TargetLowering::AtomicExpansionKind diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 04f855d0bcf54..4d3d3dc6cb4e1 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4250,24 +4250,6 @@ bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const { Opcode == AMDGPU::DS_SUB_GS_REG_RTN || isGWS(Opcode); } -static bool hasNoAliasAddrSpaceScratch(const MachineMemOperand *MemOp) { - const MDNode *MD = MemOp->getAAInfo().NoAliasAddrSpace; - if (!MD) - return false; - - // This MD is structured in ranges [A, B) - // Check if PRIVATE is included in any of them. - for (unsigned I = 0, E = MD->getNumOperands() / 2; I != E; ++I) { - auto *Low = mdconst::extract(MD->getOperand(2 * I + 0)); - auto *High = mdconst::extract(MD->getOperand(2 * I + 1)); - if (Low->getValue().ule(AMDGPUAS::PRIVATE_ADDRESS) && - High->getValue().ugt(AMDGPUAS::PRIVATE_ADDRESS)) - return true; - } - - return false; -} - bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { if (!isFLAT(MI) || isFLATGlobal(MI)) return false; @@ -4285,13 +4267,12 @@ bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { if (MI.memoperands_empty()) return true; - // TODO (?): Does this need to be taught how to read noalias.addrspace ? - // See if any memory operand specifies an address space that involves scratch. return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) { unsigned AS = Memop->getAddrSpace(); if (AS == AMDGPUAS::FLAT_ADDRESS) - return !hasNoAliasAddrSpaceScratch(Memop); + return !AMDGPU::hasValueInRange(Memop->getAAInfo().NoAliasAddrSpace, + AMDGPUAS::PRIVATE_ADDRESS); return AS == AMDGPUAS::PRIVATE_ADDRESS; }); } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index c41d62748c4be..0a6df9a06a6ff 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -21,6 +21,7 @@ #include "llvm/IR/IntrinsicsAMDGPU.h" #include "llvm/IR/IntrinsicsR600.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Metadata.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSubtargetInfo.h" @@ -1677,6 +1678,22 @@ getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) { return Vals; } +bool hasValueInRange(const MDNode *MD, unsigned Val) { + if (!MD) + return false; + + assert((MD->getNumOperands() % 2 == 0) && "invalid number of operands!"); + for (unsigned I = 0, E = MD->getNumOperands() / 2; I != E; ++I) { + auto *Low = mdconst::extract(MD->getOperand(2 * I + 0)); + auto *High = mdconst::extract(MD->getOperand(2 * I + 1)); + assert(Low->getValue().ult(High->getValue()) && "invalid range metadata!"); + if (Low->getValue().ule(Val) && High->getValue().ugt(Val)) + return true; + } + + return false; +} + unsigned getVmcntBitMask(const IsaVersion &Version) { return (1 << (getVmcntBitWidthLo(Version.Major) + getVmcntBitWidthHi(Version.Major))) - diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index befab68bb5698..67faaa7ed5ccf 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -35,6 +35,7 @@ class MCInstrInfo; class MCRegisterClass; class MCRegisterInfo; class MCSubtargetInfo; +class MDNode; class StringRef; class Triple; class raw_ostream; @@ -1064,6 +1065,10 @@ SmallVector getIntegerVecAttribute(const Function &F, StringRef Name, std::optional> getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size); +/// Checks if \p Val is inside \p MD, a !range-like metadata. +/// Returns false if \p MD is null. +bool hasValueInRange(const MDNode *MD, unsigned Val); + /// Represents the counter values to wait for in an s_waitcnt instruction. /// /// Large values (including the maximum possible integer) can be used to From a9c302b977e7ca51a94c5f3a56a3c2c890651359 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Mon, 4 Aug 2025 11:07:36 +0200 Subject: [PATCH 3/5] comments --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 7 +++---- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 8 +++++--- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 13 +++++-------- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 3 +-- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 29cf7c5dc9893..f6d5e8afb8a6b 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -17793,10 +17793,9 @@ atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) { /// Return if a flat address space atomicrmw can access private memory. static bool flatInstrMayAccessPrivate(const Instruction *I) { - const MDNode *NoaliasAddrSpaceMD = - I->getMetadata(LLVMContext::MD_noalias_addrspace); - return !AMDGPU::hasValueInRange(NoaliasAddrSpaceMD, - AMDGPUAS::PRIVATE_ADDRESS); + const MDNode *MD = I->getMetadata(LLVMContext::MD_noalias_addrspace); + return !(MD && + AMDGPU::hasValueInRangeLikeMetadata(*MD, AMDGPUAS::PRIVATE_ADDRESS)); } TargetLowering::AtomicExpansionKind diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 4d3d3dc6cb4e1..e5b29c64ca84d 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4270,9 +4270,11 @@ bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { // See if any memory operand specifies an address space that involves scratch. return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) { unsigned AS = Memop->getAddrSpace(); - if (AS == AMDGPUAS::FLAT_ADDRESS) - return !AMDGPU::hasValueInRange(Memop->getAAInfo().NoAliasAddrSpace, - AMDGPUAS::PRIVATE_ADDRESS); + if (AS == AMDGPUAS::FLAT_ADDRESS) { + const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace; + return !(MD && AMDGPU::hasValueInRangeLikeMetadata( + *MD, AMDGPUAS::PRIVATE_ADDRESS)); + } return AS == AMDGPUAS::PRIVATE_ADDRESS; }); } diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 0a6df9a06a6ff..6b35ec1783bc4 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -1678,14 +1678,11 @@ getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) { return Vals; } -bool hasValueInRange(const MDNode *MD, unsigned Val) { - if (!MD) - return false; - - assert((MD->getNumOperands() % 2 == 0) && "invalid number of operands!"); - for (unsigned I = 0, E = MD->getNumOperands() / 2; I != E; ++I) { - auto *Low = mdconst::extract(MD->getOperand(2 * I + 0)); - auto *High = mdconst::extract(MD->getOperand(2 * I + 1)); +bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) { + assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!"); + for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) { + auto *Low = mdconst::extract(MD.getOperand(2 * I + 0)); + auto *High = mdconst::extract(MD.getOperand(2 * I + 1)); assert(Low->getValue().ult(High->getValue()) && "invalid range metadata!"); if (Low->getValue().ule(Val) && High->getValue().ugt(Val)) return true; diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index 67faaa7ed5ccf..70dfb63cbe040 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -1066,8 +1066,7 @@ std::optional> getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size); /// Checks if \p Val is inside \p MD, a !range-like metadata. -/// Returns false if \p MD is null. -bool hasValueInRange(const MDNode *MD, unsigned Val); +bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val); /// Represents the counter values to wait for in an s_waitcnt instruction. /// From d5fa2c072b126a2a16bddc1b6144fb1289fea01f Mon Sep 17 00:00:00 2001 From: pvanhout Date: Thu, 7 Aug 2025 11:10:53 +0200 Subject: [PATCH 4/5] Comments --- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 ++-- llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index f6d5e8afb8a6b..6e32a03060fd2 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -17794,8 +17794,8 @@ atomicSupportedIfLegalIntType(const AtomicRMWInst *RMW) { /// Return if a flat address space atomicrmw can access private memory. static bool flatInstrMayAccessPrivate(const Instruction *I) { const MDNode *MD = I->getMetadata(LLVMContext::MD_noalias_addrspace); - return !(MD && - AMDGPU::hasValueInRangeLikeMetadata(*MD, AMDGPUAS::PRIVATE_ADDRESS)); + return !MD || + !AMDGPU::hasValueInRangeLikeMetadata(*MD, AMDGPUAS::PRIVATE_ADDRESS); } TargetLowering::AtomicExpansionKind diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index e5b29c64ca84d..a0170451266ce 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -4272,8 +4272,8 @@ bool SIInstrInfo::mayAccessScratchThroughFlat(const MachineInstr &MI) const { unsigned AS = Memop->getAddrSpace(); if (AS == AMDGPUAS::FLAT_ADDRESS) { const MDNode *MD = Memop->getAAInfo().NoAliasAddrSpace; - return !(MD && AMDGPU::hasValueInRangeLikeMetadata( - *MD, AMDGPUAS::PRIVATE_ADDRESS)); + return !MD || !AMDGPU::hasValueInRangeLikeMetadata( + *MD, AMDGPUAS::PRIVATE_ADDRESS); } return AS == AMDGPUAS::PRIVATE_ADDRESS; }); From 21bea22a1f0784a4cdfb356cd3e2a83c7920d344 Mon Sep 17 00:00:00 2001 From: pvanhout Date: Mon, 18 Aug 2025 10:15:58 +0200 Subject: [PATCH 5/5] fix test --- .../Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 20 ++++++++++++---- .../expand-cmpxchg-flat-maybe-private.ll | 24 ++++--------------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index 6b35ec1783bc4..a45cfc72d0f7d 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -1681,11 +1681,21 @@ getIntegerVecAttribute(const Function &F, StringRef Name, unsigned Size) { bool hasValueInRangeLikeMetadata(const MDNode &MD, int64_t Val) { assert((MD.getNumOperands() % 2 == 0) && "invalid number of operands!"); for (unsigned I = 0, E = MD.getNumOperands() / 2; I != E; ++I) { - auto *Low = mdconst::extract(MD.getOperand(2 * I + 0)); - auto *High = mdconst::extract(MD.getOperand(2 * I + 1)); - assert(Low->getValue().ult(High->getValue()) && "invalid range metadata!"); - if (Low->getValue().ule(Val) && High->getValue().ugt(Val)) - return true; + auto Low = + mdconst::extract(MD.getOperand(2 * I + 0))->getValue(); + auto High = + mdconst::extract(MD.getOperand(2 * I + 1))->getValue(); + // There are two types of [A; B) ranges: + // A < B, e.g. [4; 5) which is a range that only includes 4. + // A > B, e.g. [5; 4) which is a range that wraps around and includes + // everything except 4. + if (Low.ult(High)) { + if (Low.ule(Val) && High.ugt(Val)) + return true; + } else { + if (Low.uge(Val) && High.ult(Val)) + return true; + } } return false; diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-cmpxchg-flat-maybe-private.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-cmpxchg-flat-maybe-private.ll index 4b4994958e66a..a9e4f10dc9a53 100644 --- a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-cmpxchg-flat-maybe-private.ll +++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-cmpxchg-flat-maybe-private.ll @@ -308,24 +308,7 @@ define { i64, i1 } @cmpxchg_flat_agent_i64__noalias_addrspace_edge_case0(ptr %pt define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_6(ptr %ptr, i64 %val, i64 %swap) { ; CHECK-LABEL: define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_6( ; CHECK-SAME: ptr [[PTR:%.*]], i64 [[VAL:%.*]], i64 [[SWAP:%.*]]) { -; CHECK-NEXT: [[IS_PRIVATE:%.*]] = call i1 @llvm.amdgcn.is.private(ptr [[PTR]]) -; CHECK-NEXT: br i1 [[IS_PRIVATE]], label %[[ATOMICRMW_PRIVATE:.*]], label %[[ATOMICRMW_GLOBAL:.*]] -; CHECK: [[ATOMICRMW_PRIVATE]]: -; CHECK-NEXT: [[TMP1:%.*]] = addrspacecast ptr [[PTR]] to ptr addrspace(5) -; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr addrspace(5) [[TMP1]], align 8 -; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[TMP2]], [[VAL]] -; CHECK-NEXT: [[TMP4:%.*]] = select i1 [[TMP3]], i64 [[SWAP]], i64 [[TMP2]] -; CHECK-NEXT: store i64 [[TMP4]], ptr addrspace(5) [[TMP1]], align 8 -; CHECK-NEXT: [[TMP5:%.*]] = insertvalue { i64, i1 } poison, i64 [[TMP2]], 0 -; CHECK-NEXT: [[TMP6:%.*]] = insertvalue { i64, i1 } [[TMP5]], i1 [[TMP3]], 1 -; CHECK-NEXT: br label %[[ATOMICRMW_PHI:.*]] -; CHECK: [[ATOMICRMW_GLOBAL]]: -; CHECK-NEXT: [[TMP7:%.*]] = cmpxchg ptr [[PTR]], i64 [[VAL]], i64 [[SWAP]] syncscope("agent") monotonic seq_cst, align 8, !noalias.addrspace [[META0]] -; CHECK-NEXT: br label %[[ATOMICRMW_PHI]] -; CHECK: [[ATOMICRMW_PHI]]: -; CHECK-NEXT: [[RESULT:%.*]] = phi { i64, i1 } [ [[TMP6]], %[[ATOMICRMW_PRIVATE]] ], [ [[TMP7]], %[[ATOMICRMW_GLOBAL]] ] -; CHECK-NEXT: br label %[[ATOMICRMW_END:.*]] -; CHECK: [[ATOMICRMW_END]]: +; CHECK-NEXT: [[RESULT:%.*]] = cmpxchg ptr [[PTR]], i64 [[VAL]], i64 [[SWAP]] syncscope("agent") monotonic seq_cst, align 8, !noalias.addrspace [[META4:![0-9]+]] ; CHECK-NEXT: ret { i64, i1 } [[RESULT]] ; %result = cmpxchg ptr %ptr, i64 %val, i64 %swap syncscope("agent") monotonic seq_cst, !noalias.addrspace !7 @@ -335,7 +318,7 @@ define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_6(ptr %ptr, i64 %val, i64 %swap define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_3_5(ptr %ptr, i64 %val, i64 %swap) { ; CHECK-LABEL: define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_3_5( ; CHECK-SAME: ptr [[PTR:%.*]], i64 [[VAL:%.*]], i64 [[SWAP:%.*]]) { -; CHECK-NEXT: [[RESULT:%.*]] = cmpxchg ptr [[PTR]], i64 [[VAL]], i64 [[SWAP]] syncscope("agent") monotonic seq_cst, align 8, !noalias.addrspace [[META4:![0-9]+]] +; CHECK-NEXT: [[RESULT:%.*]] = cmpxchg ptr [[PTR]], i64 [[VAL]], i64 [[SWAP]] syncscope("agent") monotonic seq_cst, align 8, !noalias.addrspace [[META5:![0-9]+]] ; CHECK-NEXT: ret { i64, i1 } [[RESULT]] ; %result = cmpxchg ptr %ptr, i64 %val, i64 %swap syncscope("agent") monotonic seq_cst, !noalias.addrspace !8 @@ -357,5 +340,6 @@ define { i64, i1 } @cmpxchg_flat_agent_i64__no_2_3_5(ptr %ptr, i64 %val, i64 %sw ; CHECK: [[META1]] = !{[[META2:![0-9]+]], [[META3:![0-9]+]]} ; CHECK: [[META2]] = !{!"foo", !"bar"} ; CHECK: [[META3]] = !{!"bux", !"baz"} -; CHECK: [[META4]] = !{i32 2, i32 4, i32 5, i32 6} +; CHECK: [[META4]] = !{i32 2, i32 6} +; CHECK: [[META5]] = !{i32 2, i32 4, i32 5, i32 6} ;.