Skip to content

Commit be23cdc

Browse files
lukel97lukel97
authored
[RegAlloc] Account for use availability when applying rematerializable weight discount (#159180)
This aims to fix the issue that caused https://reviews.llvm.org/D106408 to be reverted. CalcSpillWeights will reduce the weight of an interval by half if it's considered rematerializable, so it will be evicted before others. It does this by checking TII.isTriviallyReMaterializable. However rematerialization may still fail if any of the defining MI's uses aren't available at the locations it needs to be rematerialized. LiveRangeEdit::canRematerializeAt calls allUsesAvailableAt to check this but CalcSpillWeights doesn't, so the two diverge. This fixes it by also checking allUsesAvailableAt in CalcSpillWeights. In practice this has zero change AArch64/X86-64/RISC-V as measured on llvm-test-suite, but prevents weights from being perturbed in an upcoming patch which enables more rematerialization by re-attempting https://reviews.llvm.org/D106408
1 parent 6634148 commit be23cdc

32 files changed

+34938
-35320
lines changed

‎llvm/include/llvm/CodeGen/CalcSpillWeights.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,14 @@ class VirtRegMap;
8181
static bool isRematerializable(const LiveInterval &LI,
8282
const LiveIntervals &LIS,
8383
const VirtRegMap &VRM,
84+
const MachineRegisterInfo &MRI,
85+
const TargetInstrInfo &TII);
86+
87+
/// \returns true if all registers used by \p MI are also available with the
88+
/// same value at \p UseIdx.
89+
static bool allUsesAvailableAt(const MachineInstr *MI, SlotIndex UseIdx,
90+
const LiveIntervals &LIS,
91+
const MachineRegisterInfo &MRI,
8492
const TargetInstrInfo &TII);
8593

8694
protected:

‎llvm/include/llvm/CodeGen/LiveRangeEdit.h‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -189,11 +189,6 @@ class LiveRangeEdit : private MachineRegisterInfo::Delegate {
189189
explicit Remat(const VNInfo *ParentVNI) : ParentVNI(ParentVNI) {}
190190
};
191191

192-
/// allUsesAvailableAt - Return true if all registers used by OrigMI at
193-
/// OrigIdx are also available with the same value at UseIdx.
194-
bool allUsesAvailableAt(const MachineInstr *OrigMI, SlotIndex OrigIdx,
195-
SlotIndex UseIdx) const;
196-
197192
/// canRematerializeAt - Determine if ParentVNI can be rematerialized at
198193
/// UseIdx. It is assumed that parent_.getVNINfoAt(UseIdx) == ParentVNI.
199194
bool canRematerializeAt(Remat &RM, VNInfo *OrigVNI, SlotIndex UseIdx);

‎llvm/lib/CodeGen/CalcSpillWeights.cpp‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,12 +81,15 @@ Register VirtRegAuxInfo::copyHint(const MachineInstr *MI, Register Reg,
8181
bool VirtRegAuxInfo::isRematerializable(const LiveInterval &LI,
8282
const LiveIntervals &LIS,
8383
const VirtRegMap &VRM,
84+
const MachineRegisterInfo &MRI,
8485
const TargetInstrInfo &TII) {
8586
Register Reg = LI.reg();
8687
Register Original = VRM.getOriginal(Reg);
88+
SmallDenseMap<unsigned, MachineInstr *> VNIDefs;
8789
for (LiveInterval::const_vni_iterator I = LI.vni_begin(), E = LI.vni_end();
8890
I != E; ++I) {
8991
const VNInfo *VNI = *I;
92+
const VNInfo *OrigVNI = VNI;
9093
if (VNI->isUnused())
9194
continue;
9295
if (VNI->isPHIDef())
@@ -124,6 +127,75 @@ bool VirtRegAuxInfo::isRematerializable(const LiveInterval &LI,
124127

125128
if (!TII.isReMaterializable(*MI))
126129
return false;
130+
131+
VNIDefs[OrigVNI->id] = MI;
132+
}
133+
134+
// If MI has register uses, it will only be rematerializable if its uses are
135+
// also live at the indices it will be rematerialized at.
136+
for (MachineOperand &MO : MRI.reg_nodbg_operands(LI.reg())) {
137+
if (!MO.readsReg())
138+
continue;
139+
SlotIndex UseIdx = LIS.getInstructionIndex(*MO.getParent());
140+
MachineInstr *Def = VNIDefs[LI.getVNInfoAt(UseIdx)->id];
141+
assert(Def && "Use with no def");
142+
if (!allUsesAvailableAt(Def, UseIdx, LIS, MRI, TII))
143+
return false;
144+
}
145+
146+
return true;
147+
}
148+
149+
bool VirtRegAuxInfo::allUsesAvailableAt(const MachineInstr *MI,
150+
SlotIndex UseIdx,
151+
const LiveIntervals &LIS,
152+
const MachineRegisterInfo &MRI,
153+
const TargetInstrInfo &TII) {
154+
SlotIndex OrigIdx = LIS.getInstructionIndex(*MI).getRegSlot(true);
155+
UseIdx = std::max(UseIdx, UseIdx.getRegSlot(true));
156+
for (const MachineOperand &MO : MI->operands()) {
157+
if (!MO.isReg() || !MO.getReg() || !MO.readsReg())
158+
continue;
159+
160+
// We can't remat physreg uses, unless it is a constant or target wants
161+
// to ignore this use.
162+
if (MO.getReg().isPhysical()) {
163+
if (MRI.isConstantPhysReg(MO.getReg()) || TII.isIgnorableUse(MO))
164+
continue;
165+
return false;
166+
}
167+
168+
const LiveInterval &li = LIS.getInterval(MO.getReg());
169+
const VNInfo *OVNI = li.getVNInfoAt(OrigIdx);
170+
if (!OVNI)
171+
continue;
172+
173+
// Don't allow rematerialization immediately after the original def.
174+
// It would be incorrect if OrigMI redefines the register.
175+
// See PR14098.
176+
if (SlotIndex::isSameInstr(OrigIdx, UseIdx))
177+
return false;
178+
179+
if (OVNI != li.getVNInfoAt(UseIdx))
180+
return false;
181+
182+
// Check that subrange is live at UseIdx.
183+
if (li.hasSubRanges()) {
184+
const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
185+
unsigned SubReg = MO.getSubReg();
186+
LaneBitmask LM = SubReg ? TRI->getSubRegIndexLaneMask(SubReg)
187+
: MRI.getMaxLaneMaskForVReg(MO.getReg());
188+
for (const LiveInterval::SubRange &SR : li.subranges()) {
189+
if ((SR.LaneMask & LM).none())
190+
continue;
191+
if (!SR.liveAt(UseIdx))
192+
return false;
193+
// Early exit if all used lanes are checked. No need to continue.
194+
LM &= ~SR.LaneMask;
195+
if (LM.none())
196+
break;
197+
}
198+
}
127199
}
128200
return true;
129201
}
@@ -339,7 +411,7 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI, SlotIndex *Start,
339411
// it is a preferred candidate for spilling.
340412
// FIXME: this gets much more complicated once we support non-trivial
341413
// re-materialization.
342-
if (isRematerializable(LI, LIS, VRM, *MF.getSubtarget().getInstrInfo()))
414+
if (isRematerializable(LI, LIS, VRM, MRI, *MF.getSubtarget().getInstrInfo()))
343415
TotalWeight *= 0.5F;
344416

345417
// Finally, we scale the weight by the scale factor of register class.

‎llvm/lib/CodeGen/LiveRangeEdit.cpp‎

Lines changed: 3 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -92,60 +92,6 @@ bool LiveRangeEdit::anyRematerializable() {
9292
return !Remattable.empty();
9393
}
9494

95-
/// allUsesAvailableAt - Return true if all registers used by OrigMI at
96-
/// OrigIdx are also available with the same value at UseIdx.
97-
bool LiveRangeEdit::allUsesAvailableAt(const MachineInstr *OrigMI,
98-
SlotIndex OrigIdx,
99-
SlotIndex UseIdx) const {
100-
OrigIdx = OrigIdx.getRegSlot(true);
101-
UseIdx = std::max(UseIdx, UseIdx.getRegSlot(true));
102-
for (const MachineOperand &MO : OrigMI->operands()) {
103-
if (!MO.isReg() || !MO.getReg() || !MO.readsReg())
104-
continue;
105-
106-
// We can't remat physreg uses, unless it is a constant or target wants
107-
// to ignore this use.
108-
if (MO.getReg().isPhysical()) {
109-
if (MRI.isConstantPhysReg(MO.getReg()) || TII.isIgnorableUse(MO))
110-
continue;
111-
return false;
112-
}
113-
114-
LiveInterval &li = LIS.getInterval(MO.getReg());
115-
const VNInfo *OVNI = li.getVNInfoAt(OrigIdx);
116-
if (!OVNI)
117-
continue;
118-
119-
// Don't allow rematerialization immediately after the original def.
120-
// It would be incorrect if OrigMI redefines the register.
121-
// See PR14098.
122-
if (SlotIndex::isSameInstr(OrigIdx, UseIdx))
123-
return false;
124-
125-
if (OVNI != li.getVNInfoAt(UseIdx))
126-
return false;
127-
128-
// Check that subrange is live at UseIdx.
129-
if (li.hasSubRanges()) {
130-
const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
131-
unsigned SubReg = MO.getSubReg();
132-
LaneBitmask LM = SubReg ? TRI->getSubRegIndexLaneMask(SubReg)
133-
: MRI.getMaxLaneMaskForVReg(MO.getReg());
134-
for (LiveInterval::SubRange &SR : li.subranges()) {
135-
if ((SR.LaneMask & LM).none())
136-
continue;
137-
if (!SR.liveAt(UseIdx))
138-
return false;
139-
// Early exit if all used lanes are checked. No need to continue.
140-
LM &= ~SR.LaneMask;
141-
if (LM.none())
142-
break;
143-
}
144-
}
145-
}
146-
return true;
147-
}
148-
14995
bool LiveRangeEdit::canRematerializeAt(Remat &RM, VNInfo *OrigVNI,
15096
SlotIndex UseIdx) {
15197
assert(ScannedRemattable && "Call anyRematerializable first");
@@ -155,12 +101,10 @@ bool LiveRangeEdit::canRematerializeAt(Remat &RM, VNInfo *OrigVNI,
155101
return false;
156102

157103
// No defining instruction provided.
158-
SlotIndex DefIdx;
159104
assert(RM.OrigMI && "No defining instruction for remattable value");
160-
DefIdx = LIS.getInstructionIndex(*RM.OrigMI);
161105

162106
// Verify that all used registers are available with the same values.
163-
if (!allUsesAvailableAt(RM.OrigMI, DefIdx, UseIdx))
107+
if (!VirtRegAuxInfo::allUsesAvailableAt(RM.OrigMI, UseIdx, LIS, MRI, TII))
164108
return false;
165109

166110
return true;
@@ -221,8 +165,8 @@ bool LiveRangeEdit::foldAsLoad(LiveInterval *LI,
221165

222166
// Since we're moving the DefMI load, make sure we're not extending any live
223167
// ranges.
224-
if (!allUsesAvailableAt(DefMI, LIS.getInstructionIndex(*DefMI),
225-
LIS.getInstructionIndex(*UseMI)))
168+
if (!VirtRegAuxInfo::allUsesAvailableAt(
169+
DefMI, LIS.getInstructionIndex(*UseMI), LIS, MRI, TII))
226170
return false;
227171

228172
// We also need to make sure it is safe to move the load.

‎llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -927,7 +927,7 @@ MLEvictAdvisor::getLIFeatureComponents(const LiveInterval &LI) const {
927927
Ret.HintWeights += Freq;
928928
}
929929
Ret.IsRemat = VirtRegAuxInfo::isRematerializable(
930-
LI, *LIS, *VRM, *MF.getSubtarget().getInstrInfo());
930+
LI, *LIS, *VRM, *MRI, *MF.getSubtarget().getInstrInfo());
931931
return Ret;
932932
}
933933

‎llvm/lib/CodeGen/RegisterCoalescer.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "llvm/ADT/SmallPtrSet.h"
2121
#include "llvm/ADT/SmallVector.h"
2222
#include "llvm/ADT/Statistic.h"
23+
#include "llvm/CodeGen/CalcSpillWeights.h"
2324
#include "llvm/CodeGen/LiveInterval.h"
2425
#include "llvm/CodeGen/LiveIntervals.h"
2526
#include "llvm/CodeGen/LiveRangeEdit.h"
@@ -1393,10 +1394,7 @@ bool RegisterCoalescer::reMaterializeDef(const CoalescerPair &CP,
13931394
}
13941395
}
13951396

1396-
SmallVector<Register, 8> NewRegs;
1397-
LiveRangeEdit Edit(&SrcInt, NewRegs, *MF, *LIS, nullptr, this);
1398-
SlotIndex DefIdx = LIS->getInstructionIndex(*DefMI);
1399-
if (!Edit.allUsesAvailableAt(DefMI, DefIdx, CopyIdx))
1397+
if (!VirtRegAuxInfo::allUsesAvailableAt(DefMI, CopyIdx, *LIS, *MRI, *TII))
14001398
return false;
14011399

14021400
DebugLoc DL = CopyMI->getDebugLoc();
@@ -1405,6 +1403,8 @@ bool RegisterCoalescer::reMaterializeDef(const CoalescerPair &CP,
14051403
std::next(MachineBasicBlock::iterator(CopyMI));
14061404
LiveRangeEdit::Remat RM(ValNo);
14071405
RM.OrigMI = DefMI;
1406+
SmallVector<Register, 8> NewRegs;
1407+
LiveRangeEdit Edit(&SrcInt, NewRegs, *MF, *LIS, nullptr, this);
14081408
Edit.rematerializeAt(*MBB, MII, DstReg, RM, *TRI, false, SrcIdx, CopyMI);
14091409
MachineInstr &NewMI = *std::prev(MII);
14101410
NewMI.setDebugLoc(DL);

0 commit comments

Comments
 (0)