Skip to content

Commit 3032131

Browse files
topperctopperc
committed
[TableGen][RISCV][AArch64] Properly implement isAnyExtLoad/isSignExtLoad/isZeroExtLoad for IsAtomic in SelectionDAG.
Support isAnyExtLoad() for IsAtomic in GISel. Modify atomic_load_az* to check for extload or zextload. Add atomic_load_asext* and use in RISC-V. I used "asext" rather than "as" so it wouldn't be confused with the word "as".
1 parent 6ba704a commit 3032131

File tree

5 files changed

+88
-45
lines changed

5 files changed

+88
-45
lines changed

‎llvm/include/llvm/Target/TargetSelectionDAG.td‎

Lines changed: 54 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1852,6 +1852,13 @@ def atomic_load_sext :
18521852
let IsSignExtLoad = true;
18531853
}
18541854

1855+
/// Atomic load which any extends the excess high bits.
1856+
def atomic_load_aext :
1857+
PatFrag<(ops node:$ptr), (atomic_load node:$ptr)> {
1858+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1859+
let IsAnyExtLoad = true;
1860+
}
1861+
18551862
def atomic_load_8 :
18561863
PatFrag<(ops node:$ptr),
18571864
(atomic_load node:$ptr)> {
@@ -1891,6 +1898,12 @@ def atomic_load_zext_16 :
18911898
let MemoryVT = i16;
18921899
}
18931900

1901+
def atomic_load_zext_32 :
1902+
PatFrag<(ops node:$ptr), (atomic_load_zext node:$ptr)> {
1903+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1904+
let MemoryVT = i32;
1905+
}
1906+
18941907
def atomic_load_sext_8 :
18951908
PatFrag<(ops node:$ptr), (atomic_load_sext node:$ptr)> {
18961909
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
@@ -1903,16 +1916,55 @@ def atomic_load_sext_16 :
19031916
let MemoryVT = i16;
19041917
}
19051918

1919+
def atomic_load_sext_32 :
1920+
PatFrag<(ops node:$ptr), (atomic_load_sext node:$ptr)> {
1921+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1922+
let MemoryVT = i32;
1923+
}
1924+
1925+
def atomic_load_aext_8 :
1926+
PatFrag<(ops node:$ptr), (atomic_load_aext node:$ptr)> {
1927+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1928+
let MemoryVT = i8;
1929+
}
1930+
1931+
def atomic_load_aext_16 :
1932+
PatFrag<(ops node:$ptr), (atomic_load_aext node:$ptr)> {
1933+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1934+
let MemoryVT = i16;
1935+
}
1936+
1937+
def atomic_load_aext_32 :
1938+
PatFrag<(ops node:$ptr), (atomic_load_aext node:$ptr)> {
1939+
let IsAtomic = true; // FIXME: Should be IsLoad and/or IsAtomic?
1940+
let MemoryVT = i32;
1941+
}
1942+
19061943
// Atomic load which zeroes or anyextends the high bits.
19071944
def atomic_load_az_8 : PatFrags<(ops node:$op),
1908-
[(atomic_load_8 node:$op),
1945+
[(atomic_load_aext_8 node:$op),
19091946
(atomic_load_zext_8 node:$op)]>;
19101947

19111948
// Atomic load which zeroes or anyextends the high bits.
19121949
def atomic_load_az_16 : PatFrags<(ops node:$op),
1913-
[(atomic_load_16 node:$op),
1950+
[(atomic_load_aext_16 node:$op),
19141951
(atomic_load_zext_16 node:$op)]>;
19151952

1953+
// Atomic load which sign extends or anyextends the high bits.
1954+
def atomic_load_asext_8 : PatFrags<(ops node:$op),
1955+
[(atomic_load_aext_8 node:$op),
1956+
(atomic_load_sext_8 node:$op)]>;
1957+
1958+
// Atomic load which sign extends or anyextends the high bits.
1959+
def atomic_load_asext_16 : PatFrags<(ops node:$op),
1960+
[(atomic_load_aext_16 node:$op),
1961+
(atomic_load_sext_16 node:$op)]>;
1962+
1963+
// Atomic load which sign extends or anyextends the high bits.
1964+
def atomic_load_asext_32 : PatFrags<(ops node:$op),
1965+
[(atomic_load_aext_32 node:$op),
1966+
(atomic_load_sext_32 node:$op)]>;
1967+
19161968
def nonext_masked_gather :
19171969
PatFrag<(ops node:$def, node:$pred, node:$ptr, node:$idx),
19181970
(masked_gather node:$def, node:$pred, node:$ptr, node:$idx), [{

‎llvm/lib/Target/RISCV/RISCVInstrInfoA.td‎

Lines changed: 3 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -118,29 +118,6 @@ defm AMOMAXU_D : AMO_rr_aq_rl<0b11100, 0b011, "amomaxu.d">,
118118
// Pseudo-instructions and codegen patterns
119119
//===----------------------------------------------------------------------===//
120120

121-
def riscv_atomic_asextload : PatFrag<(ops node:$ptr), (atomic_load node:$ptr), [{
122-
ISD::LoadExtType ETy = cast<AtomicSDNode>(N)->getExtensionType();
123-
return ETy == ISD::EXTLOAD || ETy == ISD::SEXTLOAD;
124-
}]>;
125-
126-
def riscv_atomic_asextload_8 : PatFrag<(ops node:$ptr),
127-
(riscv_atomic_asextload node:$ptr)> {
128-
let IsAtomic = true;
129-
let MemoryVT = i8;
130-
}
131-
132-
def riscv_atomic_asextload_16 : PatFrag<(ops node:$ptr),
133-
(riscv_atomic_asextload node:$ptr)> {
134-
let IsAtomic = true;
135-
let MemoryVT = i16;
136-
}
137-
138-
def riscv_atomic_asextload_32 : PatFrag<(ops node:$ptr),
139-
(riscv_atomic_asextload node:$ptr)> {
140-
let IsAtomic = true;
141-
let MemoryVT = i32;
142-
}
143-
144121
let IsAtomic = 1 in {
145122
// An atomic load operation that does not need either acquire or release
146123
// semantics.
@@ -188,8 +165,8 @@ class seq_cst_store<PatFrag base>
188165
// any ordering. This is necessary because AtomicExpandPass has added fences to
189166
// atomic load/stores and changed them to unordered ones.
190167
let Predicates = [HasAtomicLdSt] in {
191-
def : LdPat<relaxed_load<riscv_atomic_asextload_8>, LB>;
192-
def : LdPat<relaxed_load<riscv_atomic_asextload_16>, LH>;
168+
def : LdPat<relaxed_load<atomic_load_asext_8>, LB>;
169+
def : LdPat<relaxed_load<atomic_load_asext_16>, LH>;
193170

194171
def : StPat<relaxed_store<atomic_store_8>, SB, GPR, XLenVT>;
195172
def : StPat<relaxed_store<atomic_store_16>, SH, GPR, XLenVT>;
@@ -201,7 +178,7 @@ let Predicates = [HasAtomicLdSt, IsRV32] in {
201178
}
202179

203180
let Predicates = [HasAtomicLdSt, IsRV64] in {
204-
def : LdPat<relaxed_load<riscv_atomic_asextload_32>, LW>;
181+
def : LdPat<relaxed_load<atomic_load_asext_32>, LW>;
205182
def : LdPat<relaxed_load<atomic_load_64>, LD, i64>;
206183
def : StPat<relaxed_store<atomic_store_64>, SD, GPR, i64>;
207184
}

‎llvm/lib/Target/RISCV/RISCVInstrInfoZalasr.td‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,11 +76,11 @@ class PatSRL<SDPatternOperator OpNode, RVInst Inst, ValueType vt = XLenVT>
7676
let Predicates = [HasStdExtZalasr] in {
7777
// the sequentially consistent loads use
7878
// .aq instead of .aqrl to match the psABI/A.7
79-
def : PatLAQ<acquiring_load<riscv_atomic_asextload_8>, LB_AQ>;
80-
def : PatLAQ<seq_cst_load<riscv_atomic_asextload_8>, LB_AQ>;
79+
def : PatLAQ<acquiring_load<atomic_load_asext_8>, LB_AQ>;
80+
def : PatLAQ<seq_cst_load<atomic_load_asext_8>, LB_AQ>;
8181

82-
def : PatLAQ<acquiring_load<riscv_atomic_asextload_16>, LH_AQ>;
83-
def : PatLAQ<seq_cst_load<riscv_atomic_asextload_16>, LH_AQ>;
82+
def : PatLAQ<acquiring_load<atomic_load_asext_16>, LH_AQ>;
83+
def : PatLAQ<seq_cst_load<atomic_load_asext_16>, LH_AQ>;
8484

8585
// the sequentially consistent stores use
8686
// .rl instead of .aqrl to match the psABI/A.7
@@ -101,8 +101,8 @@ let Predicates = [HasStdExtZalasr, IsRV32] in {
101101
} // Predicates = [HasStdExtZalasr, IsRV64]
102102

103103
let Predicates = [HasStdExtZalasr, IsRV64] in {
104-
def : PatLAQ<acquiring_load<riscv_atomic_asextload_32>, LW_AQ>;
105-
def : PatLAQ<seq_cst_load<riscv_atomic_asextload_32>, LW_AQ>;
104+
def : PatLAQ<acquiring_load<atomic_load_asext_32>, LW_AQ>;
105+
def : PatLAQ<seq_cst_load<atomic_load_asext_32>, LW_AQ>;
106106

107107
def : PatLAQ<acquiring_load<atomic_load_64>, LD_AQ>;
108108
def : PatLAQ<seq_cst_load<atomic_load_64>, LD_AQ>;

‎llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -910,7 +910,7 @@ std::string TreePredicateFn::getPredCode() const {
910910

911911
if (!isLoad() && !isStore() && !isAtomic() && getMemoryVT())
912912
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
913-
"MemoryVT requires IsLoad or IsStore");
913+
"MemoryVT requires IsLoad or IsStore or IsAtomic");
914914

915915
if (!isLoad() && !isStore()) {
916916
if (isUnindexed())
@@ -937,11 +937,10 @@ std::string TreePredicateFn::getPredCode() const {
937937
if (isNonExtLoad())
938938
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
939939
"IsNonExtLoad requires IsLoad");
940-
if (isAnyExtLoad())
941-
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
942-
"IsAnyExtLoad requires IsLoad");
943-
944940
if (!isAtomic()) {
941+
if (isAnyExtLoad())
942+
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
943+
"IsAnyExtLoad requires IsLoad or IsAtomic");
945944
if (isSignExtLoad())
946945
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
947946
"IsSignExtLoad requires IsLoad or IsAtomic");
@@ -970,8 +969,9 @@ std::string TreePredicateFn::getPredCode() const {
970969
if (getMemoryVT() == nullptr && !isAtomicOrderingMonotonic() &&
971970
getAddressSpaces() == nullptr &&
972971
// FIXME: Should atomic loads be IsLoad, IsAtomic, or both?
973-
!isZeroExtLoad() && !isSignExtLoad() && !isAtomicOrderingAcquire() &&
974-
!isAtomicOrderingRelease() && !isAtomicOrderingAcquireRelease() &&
972+
!isAnyExtLoad() && !isZeroExtLoad() && !isSignExtLoad() &&
973+
!isAtomicOrderingAcquire() && !isAtomicOrderingRelease() &&
974+
!isAtomicOrderingAcquireRelease() &&
975975
!isAtomicOrderingSequentiallyConsistent() &&
976976
!isAtomicOrderingAcquireOrStronger() &&
977977
!isAtomicOrderingReleaseOrStronger() &&
@@ -1075,9 +1075,22 @@ std::string TreePredicateFn::getPredCode() const {
10751075
"if (isReleaseOrStronger(cast<AtomicSDNode>(N)->getMergedOrdering())) "
10761076
"return false;\n";
10771077

1078-
// TODO: Handle atomic sextload/zextload normally when ATOMIC_LOAD is removed.
1079-
if (isAtomic() && (isZeroExtLoad() || isSignExtLoad()))
1080-
Code += "return false;\n";
1078+
if (isAtomic()) {
1079+
if ((isAnyExtLoad() + isSignExtLoad() + isZeroExtLoad()) > 1)
1080+
PrintFatalError(getOrigPatFragRecord()->getRecord()->getLoc(),
1081+
"IsNonExtLoad, IsAnyExtLoad, IsSignExtLoad, and "
1082+
"IsZeroExtLoad are mutually exclusive");
1083+
1084+
if (isAnyExtLoad())
1085+
Code += "if (cast<AtomicSDNode>(N)->getExtensionType() != ISD::EXTLOAD) "
1086+
"return false;\n";
1087+
if (isSignExtLoad())
1088+
Code += "if (cast<AtomicSDNode>(N)->getExtensionType() != ISD::SEXTLOAD) "
1089+
"return false;\n";
1090+
if (isZeroExtLoad())
1091+
Code += "if (cast<AtomicSDNode>(N)->getExtensionType() != ISD::ZEXTLOAD) "
1092+
"return false;\n";
1093+
}
10811094

10821095
if (isLoad() || isStore()) {
10831096
StringRef SDNodeName = isLoad() ? "LoadSDNode" : "StoreSDNode";

‎llvm/utils/TableGen/GlobalISelEmitter.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -624,7 +624,8 @@ Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
624624
0, MemoryVsLLTSizePredicateMatcher::EqualTo, 0);
625625
return InsnMatcher;
626626
}
627-
if (Predicate.isLoad() && Predicate.isAnyExtLoad()) {
627+
if ((Predicate.isLoad() || Predicate.isAtomic()) &&
628+
Predicate.isAnyExtLoad()) {
628629
InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
629630
0, MemoryVsLLTSizePredicateMatcher::LessThan, 0);
630631
return InsnMatcher;

0 commit comments

Comments
 (0)