Skip to content

Commit 2ca071b

Browse files
topperctopperc
authored
[TableGen][RISCV][AArch64][GISel] Properly implement isAnyExtLoad/isSignExtLoad/isZeroExtLoad for IsAtomic in SelectionDAG. (#137096)
Support isAnyExtLoad() for IsAtomic in GISel. Modify atomic_load_az* to check for extload or zextload. And rename to atomic_load_azext* Add atomic_load_asext* and use in RISC-V. I used "asext" rather than "as" so it wouldn't be confused with the word "as".
1 parent a903c7b commit 2ca071b

File tree

7 files changed

+116
-73
lines changed

7 files changed

+116
-73
lines changed

‎llvm/include/llvm/Target/TargetSelectionDAG.td‎

Lines changed: 58 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1848,6 +1848,13 @@ def atomic_load_sext :
18481848
let IsSignExtLoad = true;
18491849
}
18501850

1851+
/// Atomic load which any extends the excess high bits.
1852+
def atomic_load_aext :
1853+
PatFrag<(ops node:$ptr), (atomic_load node:$ptr)> {
1854+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1855+
let IsAnyExtLoad = true;
1856+
}
1857+
18511858
def atomic_load_8 :
18521859
PatFrag<(ops node:$ptr),
18531860
(atomic_load node:$ptr)> {
@@ -1887,6 +1894,12 @@ def atomic_load_zext_16 :
18871894
let MemoryVT = i16;
18881895
}
18891896

1897+
def atomic_load_zext_32 :
1898+
PatFrag<(ops node:$ptr), (atomic_load_zext node:$ptr)> {
1899+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1900+
let MemoryVT = i32;
1901+
}
1902+
18901903
def atomic_load_sext_8 :
18911904
PatFrag<(ops node:$ptr), (atomic_load_sext node:$ptr)> {
18921905
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
@@ -1899,15 +1912,54 @@ def atomic_load_sext_16 :
18991912
let MemoryVT = i16;
19001913
}
19011914

1915+
def atomic_load_sext_32 :
1916+
PatFrag<(ops node:$ptr), (atomic_load_sext node:$ptr)> {
1917+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1918+
let MemoryVT = i32;
1919+
}
1920+
1921+
def atomic_load_aext_8 :
1922+
PatFrag<(ops node:$ptr), (atomic_load_aext node:$ptr)> {
1923+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1924+
let MemoryVT = i8;
1925+
}
1926+
1927+
def atomic_load_aext_16 :
1928+
PatFrag<(ops node:$ptr), (atomic_load_aext node:$ptr)> {
1929+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1930+
let MemoryVT = i16;
1931+
}
1932+
1933+
def atomic_load_aext_32 :
1934+
PatFrag<(ops node:$ptr), (atomic_load_aext node:$ptr)> {
1935+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1936+
let MemoryVT = i32;
1937+
}
1938+
19021939
// Atomic load which zeroes or anyextends the high bits.
1903-
def atomic_load_az_8 : PatFrags<(ops node:$op),
1904-
[(atomic_load_8 node:$op),
1905-
(atomic_load_zext_8 node:$op)]>;
1940+
def atomic_load_azext_8 : PatFrags<(ops node:$op),
1941+
[(atomic_load_aext_8 node:$op),
1942+
(atomic_load_zext_8 node:$op)]>;
19061943

19071944
// Atomic load which zeroes or anyextends the high bits.
1908-
def atomic_load_az_16 : PatFrags<(ops node:$op),
1909-
[(atomic_load_16 node:$op),
1910-
(atomic_load_zext_16 node:$op)]>;
1945+
def atomic_load_azext_16 : PatFrags<(ops node:$op),
1946+
[(atomic_load_aext_16 node:$op),
1947+
(atomic_load_zext_16 node:$op)]>;
1948+
1949+
// Atomic load which sign extends or anyextends the high bits.
1950+
def atomic_load_asext_8 : PatFrags<(ops node:$op),
1951+
[(atomic_load_aext_8 node:$op),
1952+
(atomic_load_sext_8 node:$op)]>;
1953+
1954+
// Atomic load which sign extends or anyextends the high bits.
1955+
def atomic_load_asext_16 : PatFrags<(ops node:$op),
1956+
[(atomic_load_aext_16 node:$op),
1957+
(atomic_load_sext_16 node:$op)]>;
1958+
1959+
// Atomic load which sign extends or anyextends the high bits.
1960+
def atomic_load_asext_32 : PatFrags<(ops node:$op),
1961+
[(atomic_load_aext_32 node:$op),
1962+
(atomic_load_sext_32 node:$op)]>;
19111963

19121964
def nonext_masked_gather :
19131965
PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),

‎llvm/lib/Target/AArch64/AArch64InstrAtomics.td‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -61,34 +61,34 @@ let Predicates = [HasRCPC] in {
6161
}
6262

6363
// 8-bit loads
64-
def : Pat<(seq_cst_load<atomic_load_az_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
65-
def : Pat<(acquiring_load<atomic_load_az_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
66-
def : Pat<(relaxed_load<atomic_load_az_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm,
67-
ro_Wextend8:$offset)),
64+
def : Pat<(seq_cst_load<atomic_load_azext_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
65+
def : Pat<(acquiring_load<atomic_load_azext_8> GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
66+
def : Pat<(relaxed_load<atomic_load_azext_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm,
67+
ro_Wextend8:$offset)),
6868
(LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>;
69-
def : Pat<(relaxed_load<atomic_load_az_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm,
70-
ro_Xextend8:$offset)),
69+
def : Pat<(relaxed_load<atomic_load_azext_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm,
70+
ro_Xextend8:$offset)),
7171
(LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>;
72-
def : Pat<(relaxed_load<atomic_load_az_8> (am_indexed8 GPR64sp:$Rn,
73-
uimm12s1:$offset)),
72+
def : Pat<(relaxed_load<atomic_load_azext_8> (am_indexed8 GPR64sp:$Rn,
73+
uimm12s1:$offset)),
7474
(LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
75-
def : Pat<(relaxed_load<atomic_load_az_8>
75+
def : Pat<(relaxed_load<atomic_load_azext_8>
7676
(am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
7777
(LDURBBi GPR64sp:$Rn, simm9:$offset)>;
7878

7979
// 16-bit loads
80-
def : Pat<(seq_cst_load<atomic_load_az_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
81-
def : Pat<(acquiring_load<atomic_load_az_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
82-
def : Pat<(relaxed_load<atomic_load_az_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
83-
ro_Wextend16:$extend)),
80+
def : Pat<(seq_cst_load<atomic_load_azext_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
81+
def : Pat<(acquiring_load<atomic_load_azext_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
82+
def : Pat<(relaxed_load<atomic_load_azext_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
83+
ro_Wextend16:$extend)),
8484
(LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
85-
def : Pat<(relaxed_load<atomic_load_az_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
86-
ro_Xextend16:$extend)),
85+
def : Pat<(relaxed_load<atomic_load_azext_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
86+
ro_Xextend16:$extend)),
8787
(LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
88-
def : Pat<(relaxed_load<atomic_load_az_16> (am_indexed16 GPR64sp:$Rn,
89-
uimm12s2:$offset)),
88+
def : Pat<(relaxed_load<atomic_load_azext_16> (am_indexed16 GPR64sp:$Rn,
89+
uimm12s2:$offset)),
9090
(LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>;
91-
def : Pat<(relaxed_load<atomic_load_az_16>
91+
def : Pat<(relaxed_load<atomic_load_azext_16>
9292
(am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
9393
(LDURHHi GPR64sp:$Rn, simm9:$offset)>;
9494

@@ -591,10 +591,10 @@ let Predicates = [HasRCPC3, HasNEON] in {
591591
// v8.4a FEAT_LRCPC2 patterns
592592
let Predicates = [HasRCPC_IMMO, UseLDAPUR] in {
593593
// Load-Acquire RCpc Register unscaled loads
594-
def : Pat<(acquiring_load<atomic_load_az_8>
594+
def : Pat<(acquiring_load<atomic_load_azext_8>
595595
(am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
596596
(LDAPURBi GPR64sp:$Rn, simm9:$offset)>;
597-
def : Pat<(acquiring_load<atomic_load_az_16>
597+
def : Pat<(acquiring_load<atomic_load_azext_16>
598598
(am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
599599
(LDAPURHi GPR64sp:$Rn, simm9:$offset)>;
600600
def : Pat<(acquiring_load<atomic_load_32>

‎llvm/lib/Target/BPF/BPFInstrInfo.td‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1342,11 +1342,11 @@ let Predicates = [BPFHasALU32] in {
13421342

13431343
let Predicates = [BPFHasLoadAcqStoreRel] in {
13441344
foreach P = [[relaxed_load<atomic_load_32>, LDW32],
1345-
[relaxed_load<atomic_load_az_16>, LDH32],
1346-
[relaxed_load<atomic_load_az_8>, LDB32],
1345+
[relaxed_load<atomic_load_azext_16>, LDH32],
1346+
[relaxed_load<atomic_load_azext_8>, LDB32],
13471347
[acquiring_load<atomic_load_32>, LDWACQ32],
1348-
[acquiring_load<atomic_load_az_16>, LDHACQ32],
1349-
[acquiring_load<atomic_load_az_8>, LDBACQ32],
1348+
[acquiring_load<atomic_load_azext_16>, LDHACQ32],
1349+
[acquiring_load<atomic_load_azext_8>, LDBACQ32],
13501350
] in {
13511351
def : Pat<(P[0] ADDRri:$addr), (P[1] ADDRri:$addr)>;
13521352
}

‎llvm/lib/Target/RISCV/RISCVInstrInfoA.td‎

Lines changed: 3 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -118,29 +118,6 @@ defm AMOMAXU_D : AMO_rr_aq_rl<0b11100, 0b011, "amomaxu.d">,
118118
// Pseudo-instructions and codegen patterns
119119
//===----------------------------------------------------------------------===//
120120

121-
def riscv_atomic_asextload : PatFrag<(ops node:$ptr), (atomic_load node:$ptr), [{
122-
ISD::LoadExtType ETy = cast<AtomicSDNode>(N)->getExtensionType();
123-
return ETy == ISD::EXTLOAD || ETy == ISD::SEXTLOAD;
124-
}]>;
125-
126-
def riscv_atomic_asextload_8 : PatFrag<(ops node:$ptr),
127-
(riscv_atomic_asextload node:$ptr)> {
128-
let IsAtomic = true;
129-
let MemoryVT = i8;
130-
}
131-
132-
def riscv_atomic_asextload_16 : PatFrag<(ops node:$ptr),
133-
(riscv_atomic_asextload node:$ptr)> {
134-
let IsAtomic = true;
135-
let MemoryVT = i16;
136-
}
137-
138-
def riscv_atomic_asextload_32 : PatFrag<(ops node:$ptr),
139-
(riscv_atomic_asextload node:$ptr)> {
140-
let IsAtomic = true;
141-
let MemoryVT = i32;
142-
}
143-
144121
let IsAtomic = 1 in {
145122
// An atomic load operation that does not need either acquire or release
146123
// semantics.
@@ -188,8 +165,8 @@ class seq_cst_store<PatFrag base>
188165
// any ordering. This is necessary because AtomicExpandPass has added fences to
189166
// atomic load/stores and changed them to unordered ones.
190167
let Predicates = [HasAtomicLdSt] in {
191-
def : LdPat<relaxed_load<riscv_atomic_asextload_8>, LB>;
192-
def : LdPat<relaxed_load<riscv_atomic_asextload_16>, LH>;
168+
def : LdPat<relaxed_load<atomic_load_asext_8>, LB>;
169+
def : LdPat<relaxed_load<atomic_load_asext_16>, LH>;
193170

194171
def : StPat<relaxed_store<atomic_store_8>, SB, GPR, XLenVT>;
195172
def : StPat<relaxed_store<atomic_store_16>, SH, GPR, XLenVT>;
@@ -201,7 +178,7 @@ let Predicates = [HasAtomicLdSt, IsRV32] in {
201178
}
202179

203180
let Predicates = [HasAtomicLdSt, IsRV64] in {
204-
def : LdPat<relaxed_load<riscv_atomic_asextload_32>, LW>;
181+
def : LdPat<relaxed_load<atomic_load_asext_32>, LW>;
205182
def : LdPat<relaxed_load<atomic_load_64>, LD, i64>;
206183
def : StPat<relaxed_store<atomic_store_64>, SD, GPR, i64>;
207184
}

‎llvm/lib/Target/RISCV/RISCVInstrInfoZalasr.td‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,11 +76,11 @@ class PatSRL<SDPatternOperator OpNode, RVInst Inst, ValueType vt = XLenVT>
7676
let Predicates = [HasStdExtZalasr] in {
7777
// the sequentially consistent loads use
7878
// .aq instead of .aqrl to match the psABI/A.7
79-
def : PatLAQ<acquiring_load<riscv_atomic_asextload_8>, LB_AQ>;
80-
def : PatLAQ<seq_cst_load<riscv_atomic_asextload_8>, LB_AQ>;
79+
def : PatLAQ<acquiring_load<atomic_load_asext_8>, LB_AQ>;
80+
def : PatLAQ<seq_cst_load<atomic_load_asext_8>, LB_AQ>;
8181

82-
def : PatLAQ<acquiring_load<riscv_atomic_asextload_16>, LH_AQ>;
83-
def : PatLAQ<seq_cst_load<riscv_atomic_asextload_16>, LH_AQ>;
82+
def : PatLAQ<acquiring_load<atomic_load_asext_16>, LH_AQ>;
83+
def : PatLAQ<seq_cst_load<atomic_load_asext_16>, LH_AQ>;
8484

8585
// the sequentially consistent stores use
8686
// .rl instead of .aqrl to match the psABI/A.7
@@ -101,8 +101,8 @@ let Predicates = [HasStdExtZalasr, IsRV32] in {
101101
} // Predicates = [HasStdExtZalasr, IsRV64]
102102

103103
let Predicates = [HasStdExtZalasr, IsRV64] in {
104-
def : PatLAQ<acquiring_load<riscv_atomic_asextload_32>, LW_AQ>;
105-
def : PatLAQ<seq_cst_load<riscv_atomic_asextload_32>, LW_AQ>;
104+
def : PatLAQ<acquiring_load<atomic_load_asext_32>, LW_AQ>;
105+
def : PatLAQ<seq_cst_load<atomic_load_asext_32>, LW_AQ>;
106106

107107
def : PatLAQ<acquiring_load<atomic_load_64>, LD_AQ>;
108108
def : PatLAQ<seq_cst_load<atomic_load_64>, LD_AQ>;

‎llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -910,7 +910,7 @@ std::string TreePredicateFn::getPredCode() const {
910910

911911
if (!isLoad() && !isStore() && !isAtomic() && getMemoryVT())
912912
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
913-
"MemoryVT requires IsLoad or IsStore");
913+
"MemoryVT requires IsLoad or IsStore or IsAtomic");
914914

915915
if (!isLoad() && !isStore()) {
916916
if (isUnindexed())
@@ -937,11 +937,10 @@ std::string TreePredicateFn::getPredCode() const {
937937
if (isNonExtLoad())
938938
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
939939
"IsNonExtLoad requires IsLoad");
940-
if (isAnyExtLoad())
941-
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
942-
"IsAnyExtLoad requires IsLoad");
943-
944940
if (!isAtomic()) {
941+
if (isAnyExtLoad())
942+
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
943+
"IsAnyExtLoad requires IsLoad or IsAtomic");
945944
if (isSignExtLoad())
946945
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
947946
"IsSignExtLoad requires IsLoad or IsAtomic");
@@ -970,8 +969,9 @@ std::string TreePredicateFn::getPredCode() const {
970969
if (getMemoryVT() == nullptr && !isAtomicOrderingMonotonic() &&
971970
getAddressSpaces() == nullptr &&
972971
// FIXME: Should atomic loads be IsLoad, IsAtomic, or both?
973-
!isZeroExtLoad() && !isSignExtLoad() && !isAtomicOrderingAcquire() &&
974-
!isAtomicOrderingRelease() && !isAtomicOrderingAcquireRelease() &&
972+
!isAnyExtLoad() && !isZeroExtLoad() && !isSignExtLoad() &&
973+
!isAtomicOrderingAcquire() && !isAtomicOrderingRelease() &&
974+
!isAtomicOrderingAcquireRelease() &&
975975
!isAtomicOrderingSequentiallyConsistent() &&
976976
!isAtomicOrderingAcquireOrStronger() &&
977977
!isAtomicOrderingReleaseOrStronger() &&
@@ -1075,9 +1075,22 @@ std::string TreePredicateFn::getPredCode() const {
10751075
"if (isReleaseOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) "
10761076
"return false;\n";
10771077

1078-
// TODO: Handle atomic sextload/zextload normally when ATOMIC_LOAD is removed.
1079-
if (isAtomic() && (isZeroExtLoad() || isSignExtLoad()))
1080-
Code += "return false;\n";
1078+
if (isAtomic()) {
1079+
if ((isAnyExtLoad() + isSignExtLoad() + isZeroExtLoad()) > 1)
1080+
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
1081+
"IsAnyExtLoad, IsSignExtLoad, and IsZeroExtLoad are "
1082+
"mutually exclusive");
1083+
1084+
if (isAnyExtLoad())
1085+
Code += "if (cast<AtomicSDNode>(N)->getExtensionType() != ISD::EXTLOAD) "
1086+
"return false;\n";
1087+
if (isSignExtLoad())
1088+
Code += "if (cast<AtomicSDNode>(N)->getExtensionType() != ISD::SEXTLOAD) "
1089+
"return false;\n";
1090+
if (isZeroExtLoad())
1091+
Code += "if (cast<AtomicSDNode>(N)->getExtensionType() != ISD::ZEXTLOAD) "
1092+
"return false;\n";
1093+
}
10811094

10821095
if (isLoad() || isStore()) {
10831096
StringRef SDNodeName = isLoad() ? "LoadSDNode" : "StoreSDNode";

‎llvm/utils/TableGen/GlobalISelEmitter.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -627,7 +627,8 @@ Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
627627
0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0);
628628
return InsnMatcher;
629629
}
630-
if (Predicate.isLoad() && Predicate.isAnyExtLoad()) {
630+
if ((Predicate.isLoad() || Predicate.isAtomic()) &&
631+
Predicate.isAnyExtLoad()) {
631632
InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
632633
0, MemoryVsLLTSizePredicateMatcher::LessThan, 0);
633634
return InsnMatcher;

0 commit comments

Comments
 (0)