Skip to content

Commit 59c9c8a

Browse files
modikingmodiking
committed
change to shared_cluster, update tests
1 parent 4844c92 commit 59c9c8a

File tree

14 files changed

+111
-108
lines changed

14 files changed

+111
-108
lines changed

‎llvm/include/llvm/IR/IntrinsicsNVVM.td‎

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -127,11 +127,11 @@
127127
// * llvm.nvvm.atomic.load.inc.32 --> atomicrmw uinc_wrap
128128
// * llvm.nvvm.atomic.load.dec.32 --> atomicrmw udec_wrap
129129

130-
def llvm_global_ptr_ty : LLVMQualPointerType<1>; // (global)ptr
131-
def llvm_shared_ptr_ty : LLVMQualPointerType<3>; // (shared)ptr
132-
def llvm_local_ptr_ty : LLVMQualPointerType<5>; // (local)ptr
133-
def llvm_tmem_ptr_ty : LLVMQualPointerType<6>; // (tensor memory)ptr
134-
def llvm_dshared_ptr_ty : LLVMQualPointerType<7>; // (dshared)ptr
130+
def llvm_global_ptr_ty : LLVMQualPointerType<1>; // (global)ptr
131+
def llvm_shared_ptr_ty : LLVMQualPointerType<3>; // (shared)ptr
132+
def llvm_local_ptr_ty : LLVMQualPointerType<5>; // (local)ptr
133+
def llvm_tmem_ptr_ty : LLVMQualPointerType<6>; // (tensor memory)ptr
134+
def llvm_shared_cluster_ptr_ty : LLVMQualPointerType<7>; // (shared_cluster)ptr
135135

136136
//
137137
// MISC
@@ -692,15 +692,15 @@ class CP_ASYNC_BULK_TENSOR_G2S_INTR<int dim, string mode> {
692692
list<LLVMType> Im2ColOffsetsTy = !listsplat(llvm_i16_ty, NumIm2ColOffsets);
693693
list<LLVMType> TensorDimsTy = !listsplat(llvm_i32_ty, dim);
694694
list<LLVMType> ArgsTy = !listconcat(
695-
[llvm_dshared_ptr_ty, // dst_smem_ptr
696-
llvm_shared_ptr_ty, // mbarrier_smem_ptr
697-
llvm_ptr_ty], // tensormap_ptr
698-
TensorDimsTy, // actual tensor dims
699-
Im2ColOffsetsTy, // im2col offsets
700-
[llvm_i16_ty, // cta_mask
701-
llvm_i64_ty, // cache_hint
702-
llvm_i1_ty, // Flag for cta_mask
703-
llvm_i1_ty] // Flag for cache_hint
695+
[llvm_shared_cluster_ptr_ty, // dst_shared_cluster_ptr
696+
llvm_shared_ptr_ty, // mbarrier_smem_ptr
697+
llvm_ptr_ty], // tensormap_ptr
698+
TensorDimsTy, // actual tensor dims
699+
Im2ColOffsetsTy, // im2col offsets
700+
[llvm_i16_ty, // cta_mask
701+
llvm_i64_ty, // cache_hint
702+
llvm_i1_ty, // Flag for cta_mask
703+
llvm_i1_ty] // Flag for cache_hint
704704
);
705705

706706
int TempFlagsStartIdx = !add(dim, 5);
@@ -5119,7 +5119,7 @@ def int_nvvm_mapa
51195119
[IntrNoMem, IntrSpeculatable, NoCapture<ArgIndex<0>>],
51205120
"llvm.nvvm.mapa">;
51215121
def int_nvvm_mapa_shared_cluster
5122-
: DefaultAttrsIntrinsic<[llvm_dshared_ptr_ty], [llvm_shared_ptr_ty, llvm_i32_ty],
5122+
: DefaultAttrsIntrinsic<[llvm_shared_cluster_ptr_ty], [llvm_shared_ptr_ty, llvm_i32_ty],
51235123
[IntrNoMem, IntrSpeculatable, NoCapture<ArgIndex<0>>],
51245124
"llvm.nvvm.mapa.shared.cluster">;
51255125
def int_nvvm_getctarank
@@ -5219,14 +5219,14 @@ def int_nvvm_discard_L2 : DefaultAttrsIntrinsic<[],
52195219
// From Global to Shared Cluster
52205220
def int_nvvm_cp_async_bulk_global_to_shared_cluster
52215221
: DefaultAttrsIntrinsic<[],
5222-
[llvm_dshared_ptr_ty, // dst_dsmem_ptr
5223-
llvm_shared_ptr_ty, // mbarrier_ptr
5224-
llvm_global_ptr_ty, // src_gmem_ptr
5225-
llvm_i32_ty, // copy_size
5226-
llvm_i16_ty, // cta_mask
5227-
llvm_i64_ty, // cache_hint
5228-
llvm_i1_ty, // Flag for cta_mask
5229-
llvm_i1_ty], // Flag for cache_hint
5222+
[llvm_shared_cluster_ptr_ty, // dst_shared_cluster_ptr
5223+
llvm_shared_ptr_ty, // mbarrier_ptr
5224+
llvm_global_ptr_ty, // src_gmem_ptr
5225+
llvm_i32_ty, // copy_size
5226+
llvm_i16_ty, // cta_mask
5227+
llvm_i64_ty, // cache_hint
5228+
llvm_i1_ty, // Flag for cta_mask
5229+
llvm_i1_ty], // Flag for cache_hint
52305230
[IntrConvergent, IntrArgMemOnly,
52315231
WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>,
52325232
NoCapture<ArgIndex<0>>, NoCapture<ArgIndex<1>>,
@@ -5236,10 +5236,10 @@ def int_nvvm_cp_async_bulk_global_to_shared_cluster
52365236
// From Shared CTA to Shared Cluster
52375237
def int_nvvm_cp_async_bulk_shared_cta_to_cluster
52385238
: DefaultAttrsIntrinsic<[],
5239-
[llvm_dshared_ptr_ty, // dst_dsmem_ptr
5240-
llvm_shared_ptr_ty, // mbarrier_ptr
5241-
llvm_shared_ptr_ty, // src_smem_ptr
5242-
llvm_i32_ty], // copy_size
5239+
[llvm_shared_cluster_ptr_ty, // dst_shared_cluster_ptr
5240+
llvm_shared_ptr_ty, // mbarrier_ptr
5241+
llvm_shared_ptr_ty, // src_smem_ptr
5242+
llvm_i32_ty], // copy_size
52435243
[IntrConvergent, IntrArgMemOnly,
52445244
WriteOnly<ArgIndex<0>>, ReadOnly<ArgIndex<2>>,
52455245
NoCapture<ArgIndex<0>>, NoCapture<ArgIndex<1>>,

‎llvm/include/llvm/Support/NVPTXAddrSpace.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ enum AddressSpace : unsigned {
2525
ADDRESS_SPACE_CONST = 4,
2626
ADDRESS_SPACE_LOCAL = 5,
2727
ADDRESS_SPACE_TENSOR = 6,
28-
ADDRESS_SPACE_DSHARED = 7,
28+
ADDRESS_SPACE_SHARED_CLUSTER = 7,
2929

3030
ADDRESS_SPACE_PARAM = 101,
3131
};

‎llvm/lib/IR/AutoUpgrade.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -939,8 +939,8 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F,
939939
return false; // No other 'arm.*', 'aarch64.*'.
940940
}
941941

942-
static Intrinsic::ID shouldUpgradeNVPTXDSharedIntrinsic(Function *F,
943-
StringRef Name) {
942+
static Intrinsic::ID shouldUpgradeNVPTXSharedClusterIntrinsic(Function *F,
943+
StringRef Name) {
944944
if (Name.consume_front("mapa.shared.cluster"))
945945
if (F->getReturnType()->getPointerAddressSpace() ==
946946
NVPTXAS::ADDRESS_SPACE_SHARED)
@@ -1321,7 +1321,7 @@ static bool upgradeIntrinsicFunction1(Function *F, Function *&NewFn,
13211321
}
13221322

13231323
// Upgrade Distributed Shared Memory Intrinsics
1324-
Intrinsic::ID IID = shouldUpgradeNVPTXDSharedIntrinsic(F, Name);
1324+
Intrinsic::ID IID = shouldUpgradeNVPTXSharedClusterIntrinsic(F, Name);
13251325
if (IID != Intrinsic::not_intrinsic) {
13261326
rename(F);
13271327
NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), IID);
@@ -4797,7 +4797,7 @@ void llvm::UpgradeIntrinsicCall(CallBase *CI, Function *NewFn) {
47974797
Args[0] = Builder.CreateAddrSpaceCast(
47984798
Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_GENERIC));
47994799
Args[0] = Builder.CreateAddrSpaceCast(
4800-
Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_DSHARED));
4800+
Args[0], Builder.getPtrTy(NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER));
48014801

48024802
NewCall = Builder.CreateCall(NewFn, Args);
48034803
NewCall->takeName(CI);

‎llvm/lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@ void NVPTXInstPrinter::printLdStCode(const MCInst *MI, int OpNum,
285285
case NVPTX::AddressSpace::Global:
286286
case NVPTX::AddressSpace::Const:
287287
case NVPTX::AddressSpace::Shared:
288-
case NVPTX::AddressSpace::Dshared:
288+
case NVPTX::AddressSpace::SharedCluster:
289289
case NVPTX::AddressSpace::Param:
290290
case NVPTX::AddressSpace::Local:
291291
O << "." << A;

‎llvm/lib/Target/NVPTX/NVPTX.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ enum AddressSpace : AddressSpaceUnderlyingType {
176176
Shared = 3,
177177
Const = 4,
178178
Local = 5,
179-
Dshared = 7,
179+
SharedCluster = 7,
180180

181181
// NVPTX Backend Private:
182182
Param = 101

‎llvm/lib/Target/NVPTX/NVPTXAliasAnalysis.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,9 @@ static AliasResult::Kind getAliasResult(unsigned AS1, unsigned AS2) {
8787
// rules once it is added.
8888

8989
// Distributed shared memory aliases with shared memory.
90-
if (((AS1 == ADDRESS_SPACE_SHARED) && (AS2 == ADDRESS_SPACE_DSHARED)) ||
91-
((AS1 == ADDRESS_SPACE_DSHARED) && (AS2 == ADDRESS_SPACE_SHARED)))
90+
if (((AS1 == ADDRESS_SPACE_SHARED) &&
91+
(AS2 == ADDRESS_SPACE_SHARED_CLUSTER)) ||
92+
((AS1 == ADDRESS_SPACE_SHARED_CLUSTER) && (AS2 == ADDRESS_SPACE_SHARED)))
9293
return AliasResult::MayAlias;
9394

9495
return (AS1 == AS2 ? AliasResult::MayAlias : AliasResult::NoAlias);

‎llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -513,8 +513,8 @@ static std::optional<unsigned> convertAS(unsigned AS) {
513513
return NVPTX::AddressSpace::Global;
514514
case llvm::ADDRESS_SPACE_SHARED:
515515
return NVPTX::AddressSpace::Shared;
516-
case llvm::ADDRESS_SPACE_DSHARED:
517-
return NVPTX::AddressSpace::Dshared;
516+
case llvm::ADDRESS_SPACE_SHARED_CLUSTER:
517+
return NVPTX::AddressSpace::SharedCluster;
518518
case llvm::ADDRESS_SPACE_GENERIC:
519519
return NVPTX::AddressSpace::Generic;
520520
case llvm::ADDRESS_SPACE_PARAM:
@@ -661,7 +661,7 @@ getOperationOrderings(MemSDNode *N, const NVPTXSubtarget *Subtarget) {
661661
(CodeAddrSpace == NVPTX::AddressSpace::Generic ||
662662
CodeAddrSpace == NVPTX::AddressSpace::Global ||
663663
CodeAddrSpace == NVPTX::AddressSpace::Shared ||
664-
CodeAddrSpace == NVPTX::AddressSpace::Dshared);
664+
CodeAddrSpace == NVPTX::AddressSpace::SharedCluster);
665665
if (!AddrGenericOrGlobalOrShared)
666666
return NVPTX::Ordering::NotAtomic;
667667

@@ -982,8 +982,9 @@ void NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
982982
case ADDRESS_SPACE_SHARED:
983983
Opc = TM.is64Bit() ? NVPTX::cvta_shared_64 : NVPTX::cvta_shared;
984984
break;
985-
case ADDRESS_SPACE_DSHARED:
986-
Opc = TM.is64Bit() ? NVPTX::cvta_dshared_64 : NVPTX::cvta_dshared;
985+
case ADDRESS_SPACE_SHARED_CLUSTER:
986+
Opc = TM.is64Bit() ? NVPTX::cvta_shared_cluster_64
987+
: NVPTX::cvta_shared_cluster;
987988
break;
988989
case ADDRESS_SPACE_CONST:
989990
Opc = TM.is64Bit() ? NVPTX::cvta_const_64 : NVPTX::cvta_const;
@@ -1007,8 +1008,9 @@ void NVPTXDAGToDAGISel::SelectAddrSpaceCast(SDNode *N) {
10071008
case ADDRESS_SPACE_SHARED:
10081009
Opc = TM.is64Bit() ? NVPTX::cvta_to_shared_64 : NVPTX::cvta_to_shared;
10091010
break;
1010-
case ADDRESS_SPACE_DSHARED:
1011-
Opc = TM.is64Bit() ? NVPTX::cvta_to_dshared_64 : NVPTX::cvta_to_dshared;
1011+
case ADDRESS_SPACE_SHARED_CLUSTER:
1012+
Opc = TM.is64Bit() ? NVPTX::cvta_to_shared_cluster_64
1013+
: NVPTX::cvta_to_shared_cluster;
10121014
break;
10131015
case ADDRESS_SPACE_CONST:
10141016
Opc = TM.is64Bit() ? NVPTX::cvta_to_const_64 : NVPTX::cvta_to_const;

‎llvm/lib/Target/NVPTX/NVPTXIntrinsics.td‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ def AS_match {
3333
code shared = [{
3434
return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_SHARED);
3535
}];
36-
code dshared = [{
37-
return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_DSHARED);
36+
code shared_cluster = [{
37+
return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_SHARED_CLUSTER);
3838
}];
3939
code global = [{
4040
return ChkMemSDNodeAddressSpace(N, llvm::ADDRESS_SPACE_GLOBAL);
@@ -2042,8 +2042,8 @@ class ATOMIC_GLOBAL_CHK <dag frag>
20422042
: PatFrag<!setdagop(frag, ops), frag, AS_match.global>;
20432043
class ATOMIC_SHARED_CHK <dag frag>
20442044
: PatFrag<!setdagop(frag, ops), frag, AS_match.shared>;
2045-
class ATOMIC_DSHARED_CHK <dag frag>
2046-
: PatFrag<!setdagop(frag, ops), frag, AS_match.dshared>;
2045+
class ATOMIC_SHARED_CLUSTER_CHK <dag frag>
2046+
: PatFrag<!setdagop(frag, ops), frag, AS_match.shared_cluster>;
20472047
class ATOMIC_GENERIC_CHK <dag frag>
20482048
: PatFrag<!setdagop(frag, ops), frag, AS_match.generic>;
20492049

@@ -2098,15 +2098,15 @@ multiclass F_ATOMIC_2_AS<RegTyInfo t, SDPatternOperator frag, string op_str, lis
20982098
defvar frag_pat = (frag node:$a, node:$b);
20992099
defm _G : F_ATOMIC_2<t, "", ".global", op_str, ATOMIC_GLOBAL_CHK<frag_pat>, preds>;
21002100
defm _S : F_ATOMIC_2<t, "", ".shared", op_str, ATOMIC_SHARED_CHK<frag_pat>, preds>;
2101-
defm _DS : F_ATOMIC_2<t, "", ".shared::cluster", op_str, ATOMIC_DSHARED_CHK<frag_pat>, !listconcat([hasSM<80>], preds)>;
2101+
defm _S_C : F_ATOMIC_2<t, "", ".shared::cluster", op_str, ATOMIC_SHARED_CLUSTER_CHK<frag_pat>, !listconcat([hasSM<80>], preds)>;
21022102
defm _GEN : F_ATOMIC_2<t, "", "", op_str, ATOMIC_GENERIC_CHK<frag_pat>, preds>;
21032103
}
21042104

21052105
multiclass F_ATOMIC_3_AS<RegTyInfo t, SDPatternOperator frag, string sem_str, string op_str, list<Predicate> preds = []> {
21062106
defvar frag_pat = (frag node:$a, node:$b, node:$c);
21072107
defm _G : F_ATOMIC_3<t, sem_str, ".global", op_str, ATOMIC_GLOBAL_CHK<frag_pat>, preds>;
21082108
defm _S : F_ATOMIC_3<t, sem_str, ".shared", op_str, ATOMIC_SHARED_CHK<frag_pat>, preds>;
2109-
defm _DS : F_ATOMIC_3<t, sem_str, ".shared::cluster", op_str, ATOMIC_DSHARED_CHK<frag_pat>, !listconcat([hasSM<80>], preds)>;
2109+
defm _S_C : F_ATOMIC_3<t, sem_str, ".shared::cluster", op_str, ATOMIC_SHARED_CLUSTER_CHK<frag_pat>, !listconcat([hasSM<80>], preds)>;
21102110
defm _GEN : F_ATOMIC_3<t, sem_str, "", op_str, ATOMIC_GENERIC_CHK<frag_pat>, preds>;
21112111
}
21122112

@@ -2401,17 +2401,17 @@ multiclass G_TO_NG<string Str> {
24012401
"cvta.to." # Str # ".u64 \t$result, $src;", []>;
24022402
}
24032403

2404-
defm cvta_local : NG_TO_G<"local">;
2405-
defm cvta_shared : NG_TO_G<"shared">;
2406-
defm cvta_dshared : NG_TO_G<"shared::cluster">;
2407-
defm cvta_global : NG_TO_G<"global">;
2408-
defm cvta_const : NG_TO_G<"const">;
2404+
defm cvta_local : NG_TO_G<"local">;
2405+
defm cvta_shared : NG_TO_G<"shared">;
2406+
defm cvta_shared_cluster : NG_TO_G<"shared::cluster">;
2407+
defm cvta_global : NG_TO_G<"global">;
2408+
defm cvta_const : NG_TO_G<"const">;
24092409

2410-
defm cvta_to_local : G_TO_NG<"local">;
2411-
defm cvta_to_shared : G_TO_NG<"shared">;
2412-
defm cvta_to_dshared : G_TO_NG<"shared::cluster">;
2413-
defm cvta_to_global : G_TO_NG<"global">;
2414-
defm cvta_to_const : G_TO_NG<"const">;
2410+
defm cvta_to_local : G_TO_NG<"local">;
2411+
defm cvta_to_shared : G_TO_NG<"shared">;
2412+
defm cvta_to_shared_cluster : G_TO_NG<"shared::cluster">;
2413+
defm cvta_to_global : G_TO_NG<"global">;
2414+
defm cvta_to_const : G_TO_NG<"const">;
24152415

24162416
// nvvm.ptr.param.to.gen
24172417
defm cvta_param : NG_TO_G<"param">;

‎llvm/lib/Target/NVPTX/NVPTXUtilities.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ inline std::string AddressSpaceToString(AddressSpace A) {
168168
return "const";
169169
case AddressSpace::Shared:
170170
return "shared";
171-
case AddressSpace::Dshared:
171+
case AddressSpace::SharedCluster:
172172
return "shared::cluster";
173173
case AddressSpace::Param:
174174
return "param";

0 commit comments

Comments
 (0)