Skip to content

Commit 5c9352e

Browse files
arsenmarsenm
authored
DAG: Replace bitwidth with type in suffix in atomic tablegen ops (#94845)
1 parent b6bf402 commit 5c9352e

27 files changed

+649
-650
lines changed

‎llvm/include/llvm/Target/TargetSelectionDAG.td‎

Lines changed: 26 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1680,60 +1680,38 @@ multiclass ternary_atomic_op_ord {
16801680
}
16811681
}
16821682

1683-
multiclass binary_atomic_op<SDNode atomic_op, bit IsInt = 1> {
1684-
def _8 : PatFrag<(ops node:$ptr, node:$val),
1685-
(atomic_op node:$ptr, node:$val)> {
1686-
let IsAtomic = true;
1687-
let MemoryVT = !if(IsInt, i8, ?);
1688-
}
1689-
def _16 : PatFrag<(ops node:$ptr, node:$val),
1690-
(atomic_op node:$ptr, node:$val)> {
1691-
let IsAtomic = true;
1692-
let MemoryVT = !if(IsInt, i16, f16);
1693-
}
1694-
def _32 : PatFrag<(ops node:$ptr, node:$val),
1695-
(atomic_op node:$ptr, node:$val)> {
1696-
let IsAtomic = true;
1697-
let MemoryVT = !if(IsInt, i32, f32);
1698-
}
1699-
def _64 : PatFrag<(ops node:$ptr, node:$val),
1700-
(atomic_op node:$ptr, node:$val)> {
1701-
let IsAtomic = true;
1702-
let MemoryVT = !if(IsInt, i64, f64);
1683+
multiclass binary_atomic_op<SDNode atomic_op> {
1684+
foreach vt = [ i8, i16, i32, i64 ] in {
1685+
def _#vt : PatFrag<(ops node:$ptr, node:$val),
1686+
(atomic_op node:$ptr, node:$val)> {
1687+
let IsAtomic = true;
1688+
let MemoryVT = vt;
1689+
}
1690+
1691+
defm NAME#_#vt : binary_atomic_op_ord;
17031692
}
1693+
}
17041694

1705-
defm NAME#_8 : binary_atomic_op_ord;
1706-
defm NAME#_16 : binary_atomic_op_ord;
1707-
defm NAME#_32 : binary_atomic_op_ord;
1708-
defm NAME#_64 : binary_atomic_op_ord;
1695+
multiclass binary_atomic_op_fp<SDNode atomic_op> {
1696+
foreach vt = [ f16, bf16, v2f16, v2bf16, f32, f64 ] in {
1697+
def _#vt : PatFrag<(ops node:$ptr, node:$val),
1698+
(atomic_op node:$ptr, node:$val)> {
1699+
let IsAtomic = true;
1700+
let MemoryVT = vt;
1701+
}
1702+
}
17091703
}
17101704

17111705
multiclass ternary_atomic_op<SDNode atomic_op> {
1712-
def _8 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1713-
(atomic_op node:$ptr, node:$cmp, node:$val)> {
1714-
let IsAtomic = true;
1715-
let MemoryVT = i8;
1716-
}
1717-
def _16 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1718-
(atomic_op node:$ptr, node:$cmp, node:$val)> {
1719-
let IsAtomic = true;
1720-
let MemoryVT = i16;
1721-
}
1722-
def _32 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1723-
(atomic_op node:$ptr, node:$cmp, node:$val)> {
1724-
let IsAtomic = true;
1725-
let MemoryVT = i32;
1726-
}
1727-
def _64 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1728-
(atomic_op node:$ptr, node:$cmp, node:$val)> {
1729-
let IsAtomic = true;
1730-
let MemoryVT = i64;
1706+
foreach vt = [ i8, i16, i32, i64 ] in {
1707+
def _#vt : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
1708+
(atomic_op node:$ptr, node:$cmp, node:$val)> {
1709+
let IsAtomic = true;
1710+
let MemoryVT = vt;
1711+
}
1712+
1713+
defm NAME#_#vt : ternary_atomic_op_ord;
17311714
}
1732-
1733-
defm NAME#_8 : ternary_atomic_op_ord;
1734-
defm NAME#_16 : ternary_atomic_op_ord;
1735-
defm NAME#_32 : ternary_atomic_op_ord;
1736-
defm NAME#_64 : ternary_atomic_op_ord;
17371715
}
17381716

17391717
defm atomic_load_add : binary_atomic_op<atomic_load_add>;

‎llvm/lib/Target/AArch64/AArch64InstrFormats.td‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -11887,79 +11887,79 @@ multiclass LDOPregister<bits<3> opc, string op, bits<1> Acq, bits<1> Rel,
1188711887
// complex DAG for DstRHS.
1188811888
let Predicates = [HasLSE] in
1188911889
multiclass LDOPregister_patterns_ord_dag<string inst, string suffix, string op,
11890-
string size, dag SrcRHS, dag DstRHS> {
11891-
def : Pat<(!cast<PatFrag>(op#"_"#size#"_monotonic") GPR64sp:$Rn, SrcRHS),
11890+
ValueType vt, dag SrcRHS, dag DstRHS> {
11891+
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_monotonic") GPR64sp:$Rn, SrcRHS),
1189211892
(!cast<Instruction>(inst # suffix) DstRHS, GPR64sp:$Rn)>;
11893-
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acquire") GPR64sp:$Rn, SrcRHS),
11893+
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acquire") GPR64sp:$Rn, SrcRHS),
1189411894
(!cast<Instruction>(inst # "A" # suffix) DstRHS, GPR64sp:$Rn)>;
11895-
def : Pat<(!cast<PatFrag>(op#"_"#size#"_release") GPR64sp:$Rn, SrcRHS),
11895+
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_release") GPR64sp:$Rn, SrcRHS),
1189611896
(!cast<Instruction>(inst # "L" # suffix) DstRHS, GPR64sp:$Rn)>;
11897-
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acq_rel") GPR64sp:$Rn, SrcRHS),
11897+
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acq_rel") GPR64sp:$Rn, SrcRHS),
1189811898
(!cast<Instruction>(inst # "AL" # suffix) DstRHS, GPR64sp:$Rn)>;
11899-
def : Pat<(!cast<PatFrag>(op#"_"#size#"_seq_cst") GPR64sp:$Rn, SrcRHS),
11899+
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_seq_cst") GPR64sp:$Rn, SrcRHS),
1190011900
(!cast<Instruction>(inst # "AL" # suffix) DstRHS, GPR64sp:$Rn)>;
1190111901
}
1190211902

1190311903
multiclass LDOPregister_patterns_ord<string inst, string suffix, string op,
11904-
string size, dag RHS> {
11905-
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, size, RHS, RHS>;
11904+
ValueType vt, dag RHS> {
11905+
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, vt, RHS, RHS>;
1190611906
}
1190711907

1190811908
multiclass LDOPregister_patterns_ord_mod<string inst, string suffix, string op,
11909-
string size, dag LHS, dag RHS> {
11910-
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, size, LHS, RHS>;
11909+
ValueType vt, dag LHS, dag RHS> {
11910+
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, vt, LHS, RHS>;
1191111911
}
1191211912

1191311913
multiclass LDOPregister_patterns<string inst, string op> {
11914-
defm : LDOPregister_patterns_ord<inst, "X", op, "64", (i64 GPR64:$Rm)>;
11915-
defm : LDOPregister_patterns_ord<inst, "W", op, "32", (i32 GPR32:$Rm)>;
11916-
defm : LDOPregister_patterns_ord<inst, "H", op, "16", (i32 GPR32:$Rm)>;
11917-
defm : LDOPregister_patterns_ord<inst, "B", op, "8", (i32 GPR32:$Rm)>;
11914+
defm : LDOPregister_patterns_ord<inst, "X", op, i64, (i64 GPR64:$Rm)>;
11915+
defm : LDOPregister_patterns_ord<inst, "W", op, i32, (i32 GPR32:$Rm)>;
11916+
defm : LDOPregister_patterns_ord<inst, "H", op, i16, (i32 GPR32:$Rm)>;
11917+
defm : LDOPregister_patterns_ord<inst, "B", op, i8, (i32 GPR32:$Rm)>;
1191811918
}
1191911919

1192011920
multiclass LDOPregister_patterns_mod<string inst, string op, string mod> {
11921-
defm : LDOPregister_patterns_ord_mod<inst, "X", op, "64",
11921+
defm : LDOPregister_patterns_ord_mod<inst, "X", op, i64,
1192211922
(i64 GPR64:$Rm),
1192311923
(i64 (!cast<Instruction>(mod#Xrr) XZR, GPR64:$Rm))>;
11924-
defm : LDOPregister_patterns_ord_mod<inst, "W", op, "32",
11924+
defm : LDOPregister_patterns_ord_mod<inst, "W", op, i32,
1192511925
(i32 GPR32:$Rm),
1192611926
(i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
11927-
defm : LDOPregister_patterns_ord_mod<inst, "H", op, "16",
11927+
defm : LDOPregister_patterns_ord_mod<inst, "H", op, i16,
1192811928
(i32 GPR32:$Rm),
1192911929
(i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
11930-
defm : LDOPregister_patterns_ord_mod<inst, "B", op, "8",
11930+
defm : LDOPregister_patterns_ord_mod<inst, "B", op, i8,
1193111931
(i32 GPR32:$Rm),
1193211932
(i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
1193311933
}
1193411934

1193511935
let Predicates = [HasLSE] in
1193611936
multiclass CASregister_patterns_ord_dag<string inst, string suffix, string op,
11937-
string size, dag OLD, dag NEW> {
11938-
def : Pat<(!cast<PatFrag>(op#"_"#size#"_monotonic") GPR64sp:$Rn, OLD, NEW),
11937+
ValueType vt, dag OLD, dag NEW> {
11938+
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_monotonic") GPR64sp:$Rn, OLD, NEW),
1193911939
(!cast<Instruction>(inst # suffix) OLD, NEW, GPR64sp:$Rn)>;
11940-
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acquire") GPR64sp:$Rn, OLD, NEW),
11940+
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acquire") GPR64sp:$Rn, OLD, NEW),
1194111941
(!cast<Instruction>(inst # "A" # suffix) OLD, NEW, GPR64sp:$Rn)>;
11942-
def : Pat<(!cast<PatFrag>(op#"_"#size#"_release") GPR64sp:$Rn, OLD, NEW),
11942+
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_release") GPR64sp:$Rn, OLD, NEW),
1194311943
(!cast<Instruction>(inst # "L" # suffix) OLD, NEW, GPR64sp:$Rn)>;
11944-
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acq_rel") GPR64sp:$Rn, OLD, NEW),
11944+
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acq_rel") GPR64sp:$Rn, OLD, NEW),
1194511945
(!cast<Instruction>(inst # "AL" # suffix) OLD, NEW, GPR64sp:$Rn)>;
11946-
def : Pat<(!cast<PatFrag>(op#"_"#size#"_seq_cst") GPR64sp:$Rn, OLD, NEW),
11946+
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_seq_cst") GPR64sp:$Rn, OLD, NEW),
1194711947
(!cast<Instruction>(inst # "AL" # suffix) OLD, NEW, GPR64sp:$Rn)>;
1194811948
}
1194911949

1195011950
multiclass CASregister_patterns_ord<string inst, string suffix, string op,
11951-
string size, dag OLD, dag NEW> {
11952-
defm : CASregister_patterns_ord_dag<inst, suffix, op, size, OLD, NEW>;
11951+
ValueType vt, dag OLD, dag NEW> {
11952+
defm : CASregister_patterns_ord_dag<inst, suffix, op, vt, OLD, NEW>;
1195311953
}
1195411954

1195511955
multiclass CASregister_patterns<string inst, string op> {
11956-
defm : CASregister_patterns_ord<inst, "X", op, "64",
11956+
defm : CASregister_patterns_ord<inst, "X", op, i64,
1195711957
(i64 GPR64:$Rold), (i64 GPR64:$Rnew)>;
11958-
defm : CASregister_patterns_ord<inst, "W", op, "32",
11958+
defm : CASregister_patterns_ord<inst, "W", op, i32,
1195911959
(i32 GPR32:$Rold), (i32 GPR32:$Rnew)>;
11960-
defm : CASregister_patterns_ord<inst, "H", op, "16",
11960+
defm : CASregister_patterns_ord<inst, "H", op, i16,
1196111961
(i32 GPR32:$Rold), (i32 GPR32:$Rnew)>;
11962-
defm : CASregister_patterns_ord<inst, "B", op, "8",
11962+
defm : CASregister_patterns_ord<inst, "B", op, i8,
1196311963
(i32 GPR32:$Rold), (i32 GPR32:$Rnew)>;
1196411964
}
1196511965

‎llvm/lib/Target/AArch64/AArch64InstrGISel.td‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -346,16 +346,16 @@ let Predicates = [HasNEON] in {
346346
}
347347

348348
let Predicates = [HasNoLSE] in {
349-
def : Pat<(atomic_cmp_swap_8 GPR64:$addr, GPR32:$desired, GPR32:$new),
349+
def : Pat<(atomic_cmp_swap_i8 GPR64:$addr, GPR32:$desired, GPR32:$new),
350350
(CMP_SWAP_8 GPR64:$addr, GPR32:$desired, GPR32:$new)>;
351351

352-
def : Pat<(atomic_cmp_swap_16 GPR64:$addr, GPR32:$desired, GPR32:$new),
352+
def : Pat<(atomic_cmp_swap_i16 GPR64:$addr, GPR32:$desired, GPR32:$new),
353353
(CMP_SWAP_16 GPR64:$addr, GPR32:$desired, GPR32:$new)>;
354354

355-
def : Pat<(atomic_cmp_swap_32 GPR64:$addr, GPR32:$desired, GPR32:$new),
355+
def : Pat<(atomic_cmp_swap_i32 GPR64:$addr, GPR32:$desired, GPR32:$new),
356356
(CMP_SWAP_32 GPR64:$addr, GPR32:$desired, GPR32:$new)>;
357357

358-
def : Pat<(atomic_cmp_swap_64 GPR64:$addr, GPR64:$desired, GPR64:$new),
358+
def : Pat<(atomic_cmp_swap_i64 GPR64:$addr, GPR64:$desired, GPR64:$new),
359359
(CMP_SWAP_64 GPR64:$addr, GPR64:$desired, GPR64:$new)>;
360360
}
361361

‎llvm/lib/Target/AMDGPU/AMDGPUInstructions.td‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -637,21 +637,36 @@ defm int_amdgcn_atomic_cond_sub_u32 : local_addr_space_atomic_op;
637637
defm int_amdgcn_atomic_cond_sub_u32 : flat_addr_space_atomic_op;
638638
defm int_amdgcn_atomic_cond_sub_u32 : global_addr_space_atomic_op;
639639

640-
multiclass noret_binary_atomic_op<SDNode atomic_op, bit IsInt = 1> {
640+
multiclass noret_binary_atomic_op<SDNode atomic_op> {
641641
let HasNoUse = true in
642-
defm "_noret" : binary_atomic_op<atomic_op, IsInt>;
642+
defm "_noret" : binary_atomic_op<atomic_op>;
643+
}
644+
645+
multiclass noret_binary_atomic_op_fp<SDNode atomic_op> {
646+
let HasNoUse = true in
647+
defm "_noret" : binary_atomic_op_fp<atomic_op>;
643648
}
644649

645650
multiclass noret_ternary_atomic_op<SDNode atomic_op> {
646651
let HasNoUse = true in
647652
defm "_noret" : ternary_atomic_op<atomic_op>;
648653
}
649654

650-
multiclass binary_atomic_op_all_as<SDNode atomic_op, bit IsInt = 1> {
651-
foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in {
655+
defvar atomic_addrspace_names = [ "global", "flat", "constant", "local", "private", "region" ];
656+
657+
multiclass binary_atomic_op_all_as<SDNode atomic_op> {
658+
foreach as = atomic_addrspace_names in {
659+
let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
660+
defm "_"#as : binary_atomic_op<atomic_op>;
661+
defm "_"#as : noret_binary_atomic_op<atomic_op>;
662+
}
663+
}
664+
}
665+
multiclass binary_atomic_op_fp_all_as<SDNode atomic_op> {
666+
foreach as = atomic_addrspace_names in {
652667
let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
653-
defm "_"#as : binary_atomic_op<atomic_op, IsInt>;
654-
defm "_"#as : noret_binary_atomic_op<atomic_op, IsInt>;
668+
defm "_"#as : binary_atomic_op_fp<atomic_op>;
669+
defm "_"#as : noret_binary_atomic_op_fp<atomic_op>;
655670
}
656671
}
657672
}
@@ -666,11 +681,9 @@ defm atomic_load_sub : binary_atomic_op_all_as<atomic_load_sub>;
666681
defm atomic_load_umax : binary_atomic_op_all_as<atomic_load_umax>;
667682
defm atomic_load_umin : binary_atomic_op_all_as<atomic_load_umin>;
668683
defm atomic_load_xor : binary_atomic_op_all_as<atomic_load_xor>;
669-
defm atomic_load_fadd : binary_atomic_op_all_as<atomic_load_fadd, 0>;
684+
defm atomic_load_fadd : binary_atomic_op_fp_all_as<atomic_load_fadd>;
670685
defm atomic_load_uinc_wrap : binary_atomic_op_all_as<atomic_load_uinc_wrap>;
671686
defm atomic_load_udec_wrap : binary_atomic_op_all_as<atomic_load_udec_wrap>;
672-
let MemoryVT = v2f16 in
673-
defm atomic_load_fadd_v2f16 : binary_atomic_op_all_as<atomic_load_fadd, 0>;
674687
defm AMDGPUatomic_cmp_swap : binary_atomic_op_all_as<AMDGPUatomic_cmp_swap>;
675688

676689
def load_align8_local : PatFrag<(ops node:$ptr), (load_local node:$ptr)>,

‎llvm/lib/Target/AMDGPU/BUFInstructions.td‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1558,7 +1558,7 @@ multiclass BufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, bi
15581558

15591559
defvar Op = !cast<SDPatternOperator>(OpPrefix
15601560
# !if(!eq(RtnMode, "ret"), "", "_noret")
1561-
# !if(isIntr, "", "_" # vt.Size));
1561+
# !if(isIntr, "", "_" # vt));
15621562
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");
15631563

15641564
let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in {
@@ -1595,7 +1595,7 @@ multiclass BufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string
15951595

15961596
defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global"
15971597
# !if(!eq(RtnMode, "ret"), "", "_noret")
1598-
# "_" # vt.Size);
1598+
# "_" # vt);
15991599
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");
16001600
defvar data_vt_RC = getVregSrcForVT<data_vt>.ret.RegClass;
16011601

0 commit comments

Comments
 (0)