Skip to content

release/20.x: [llvm][CodeGen] avoid repeated interval calculation in window scheduler (#132352) #134775

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 14, 2025
Merged

release/20.x: [llvm][CodeGen] avoid repeated interval calculation in window scheduler (#132352) #134775

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
73d1e85
[CodeGen] Avoid repeated hash lookups (NFC) (#130237)
kazutakahirata Mar 7, 2025
d88cd35
[llvm][CodeGen] Fix the empty interval issue in Window Scheduler (#12…
huaatian Mar 17, 2025
a141e58
[llvm][CodeGen] avoid repeated interval calculation in window schedul…
huaatian Apr 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 36 additions & 20 deletions llvm/lib/CodeGen/ModuloSchedule.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ void ModuloScheduleExpander::generatePipelinedLoop() {
MachineInstr *NewMI = cloneInstr(CI, MaxStageCount, StageNum);
updateInstruction(NewMI, false, MaxStageCount, StageNum, VRMap);
KernelBB->push_back(NewMI);
LIS.InsertMachineInstrInMaps(*NewMI);
InstrMap[NewMI] = CI;
}

Expand All @@ -150,6 +151,7 @@ void ModuloScheduleExpander::generatePipelinedLoop() {
MachineInstr *NewMI = MF.CloneMachineInstr(&MI);
updateInstruction(NewMI, false, MaxStageCount, 0, VRMap);
KernelBB->push_back(NewMI);
LIS.InsertMachineInstrInMaps(*NewMI);
InstrMap[NewMI] = &MI;
}

Expand Down Expand Up @@ -226,6 +228,7 @@ void ModuloScheduleExpander::generateProlog(unsigned LastStage,
cloneAndChangeInstr(&*BBI, i, (unsigned)StageNum);
updateInstruction(NewMI, false, i, (unsigned)StageNum, VRMap);
NewBB->push_back(NewMI);
LIS.InsertMachineInstrInMaps(*NewMI);
InstrMap[NewMI] = &*BBI;
}
}
Expand Down Expand Up @@ -303,6 +306,7 @@ void ModuloScheduleExpander::generateEpilog(
MachineInstr *NewMI = cloneInstr(In, UINT_MAX, 0);
updateInstruction(NewMI, i == 1, EpilogStage, 0, VRMap);
NewBB->push_back(NewMI);
LIS.InsertMachineInstrInMaps(*NewMI);
InstrMap[NewMI] = In;
}
}
Expand Down Expand Up @@ -343,14 +347,11 @@ void ModuloScheduleExpander::generateEpilog(
/// basic block with ToReg.
static void replaceRegUsesAfterLoop(unsigned FromReg, unsigned ToReg,
MachineBasicBlock *MBB,
MachineRegisterInfo &MRI,
LiveIntervals &LIS) {
MachineRegisterInfo &MRI) {
for (MachineOperand &O :
llvm::make_early_inc_range(MRI.use_operands(FromReg)))
if (O.getParent()->getParent() != MBB)
O.setReg(ToReg);
if (!LIS.hasInterval(ToReg))
LIS.createEmptyInterval(ToReg);
}

/// Return true if the register has a use that occurs outside the
Expand Down Expand Up @@ -464,10 +465,12 @@ void ModuloScheduleExpander::generateExistingPhis(
InstOp1 = MRI.getVRegDef(PhiOp1);
int PhiOpStage = Schedule.getStage(InstOp1);
int StageAdj = (PhiOpStage != -1 ? PhiStage - PhiOpStage : 0);
if (PhiOpStage != -1 && PrologStage - StageAdj >= Indirects + np &&
VRMap[PrologStage - StageAdj - Indirects - np].count(PhiOp1)) {
PhiOp1 = VRMap[PrologStage - StageAdj - Indirects - np][PhiOp1];
break;
if (PhiOpStage != -1 && PrologStage - StageAdj >= Indirects + np) {
auto &M = VRMap[PrologStage - StageAdj - Indirects - np];
if (auto It = M.find(PhiOp1); It != M.end()) {
PhiOp1 = It->second;
break;
}
}
++Indirects;
}
Expand Down Expand Up @@ -540,7 +543,7 @@ void ModuloScheduleExpander::generateExistingPhis(
PhiOp2 = VRMap[LastStageNum - np - 1][LoopVal];

if (IsLast && np == NumPhis - 1)
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI, LIS);
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
continue;
}
}
Expand All @@ -558,6 +561,7 @@ void ModuloScheduleExpander::generateExistingPhis(
TII->get(TargetOpcode::PHI), NewReg);
NewPhi.addReg(PhiOp1).addMBB(BB1);
NewPhi.addReg(PhiOp2).addMBB(BB2);
LIS.InsertMachineInstrInMaps(*NewPhi);
if (np == 0)
InstrMap[NewPhi] = &*BBI;

Expand All @@ -580,7 +584,7 @@ void ModuloScheduleExpander::generateExistingPhis(
// register to replace depends on whether the Phi is scheduled in the
// epilog.
if (IsLast && np == NumPhis - 1)
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI, LIS);
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);

// In the kernel, a dependent Phi uses the value from this Phi.
if (InKernel)
Expand All @@ -597,8 +601,12 @@ void ModuloScheduleExpander::generateExistingPhis(

// Check if we need to rename a Phi that has been eliminated due to
// scheduling.
if (NumStages == 0 && IsLast && VRMap[CurStageNum].count(LoopVal))
replaceRegUsesAfterLoop(Def, VRMap[CurStageNum][LoopVal], BB, MRI, LIS);
if (NumStages == 0 && IsLast) {
auto &CurStageMap = VRMap[CurStageNum];
auto It = CurStageMap.find(LoopVal);
if (It != CurStageMap.end())
replaceRegUsesAfterLoop(Def, It->second, BB, MRI);
}
}
}

Expand Down Expand Up @@ -697,6 +705,7 @@ void ModuloScheduleExpander::generatePhis(
TII->get(TargetOpcode::PHI), NewReg);
NewPhi.addReg(PhiOp1).addMBB(BB1);
NewPhi.addReg(PhiOp2).addMBB(BB2);
LIS.InsertMachineInstrInMaps(*NewPhi);
if (np == 0)
InstrMap[NewPhi] = &*BBI;

Expand All @@ -717,7 +726,7 @@ void ModuloScheduleExpander::generatePhis(
NewReg);
}
if (IsLast && np == NumPhis - 1)
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI, LIS);
replaceRegUsesAfterLoop(Def, NewReg, BB, MRI);
}
}
}
Expand Down Expand Up @@ -826,9 +835,11 @@ void ModuloScheduleExpander::splitLifetimes(MachineBasicBlock *KernelBB,
// We split the lifetime when we find the first use.
if (SplitReg == 0) {
SplitReg = MRI.createVirtualRegister(MRI.getRegClass(Def));
BuildMI(*KernelBB, MI, MI->getDebugLoc(),
TII->get(TargetOpcode::COPY), SplitReg)
.addReg(Def);
MachineInstr *newCopy =
BuildMI(*KernelBB, MI, MI->getDebugLoc(),
TII->get(TargetOpcode::COPY), SplitReg)
.addReg(Def);
LIS.InsertMachineInstrInMaps(*newCopy);
}
BBJ.substituteRegister(Def, SplitReg, 0, *TRI);
}
Expand Down Expand Up @@ -896,13 +907,17 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock &PreheaderBB,
removePhis(Epilog, LastEpi);
// Remove the blocks that are no longer referenced.
if (LastPro != LastEpi) {
for (auto &MI : *LastEpi)
LIS.RemoveMachineInstrFromMaps(MI);
LastEpi->clear();
LastEpi->eraseFromParent();
}
if (LastPro == KernelBB) {
LoopInfo->disposed(&LIS);
NewKernel = nullptr;
}
for (auto &MI : *LastPro)
LIS.RemoveMachineInstrFromMaps(MI);
LastPro->clear();
LastPro->eraseFromParent();
} else {
Expand Down Expand Up @@ -1044,7 +1059,7 @@ void ModuloScheduleExpander::updateInstruction(MachineInstr *NewMI,
MO.setReg(NewReg);
VRMap[CurStageNum][reg] = NewReg;
if (LastDef)
replaceRegUsesAfterLoop(reg, NewReg, BB, MRI, LIS);
replaceRegUsesAfterLoop(reg, NewReg, BB, MRI);
} else if (MO.isUse()) {
MachineInstr *Def = MRI.getVRegDef(reg);
// Compute the stage that contains the last definition for instruction.
Expand Down Expand Up @@ -1193,10 +1208,11 @@ void ModuloScheduleExpander::rewriteScheduledInstr(
UseOp.setReg(ReplaceReg);
else {
Register SplitReg = MRI.createVirtualRegister(MRI.getRegClass(OldReg));
BuildMI(*BB, UseMI, UseMI->getDebugLoc(), TII->get(TargetOpcode::COPY),
SplitReg)
.addReg(ReplaceReg);
MachineInstr *newCopy = BuildMI(*BB, UseMI, UseMI->getDebugLoc(),
TII->get(TargetOpcode::COPY), SplitReg)
.addReg(ReplaceReg);
UseOp.setReg(SplitReg);
LIS.InsertMachineInstrInMaps(*newCopy);
}
}
}
Expand Down
103 changes: 103 additions & 0 deletions llvm/test/CodeGen/AArch64/aarch64-swp-ws-live-intervals.mir
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
# RUN: llc --mtriple=aarch64 %s -run-pass=pipeliner -o - | FileCheck %s

...
---
name: foo
tracksRegLiveness: true
body: |
; CHECK-LABEL: name: foo
; CHECK: bb.0:
; CHECK-NEXT: successors: %bb.3(0x80000000)
; CHECK-NEXT: liveins: $x0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
; CHECK-NEXT: [[FMOVD0_:%[0-9]+]]:fpr64 = FMOVD0
; CHECK-NEXT: [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 1
; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:gpr64sp = SUBREG_TO_REG 0, [[MOVi32imm]], %subreg.sub_32
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3:
; CHECK-NEXT: successors: %bb.4(0x40000000), %bb.7(0x40000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[FADDDrr:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[FMOVD0_]], [[FMOVD0_]], implicit $fpcr
; CHECK-NEXT: [[SUBSXri:%[0-9]+]]:gpr64 = nsw SUBSXri [[SUBREG_TO_REG]], 1, 0, implicit-def $nzcv
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY [[SUBSXri]]
; CHECK-NEXT: [[FMOVDi:%[0-9]+]]:fpr64 = FMOVDi 112
; CHECK-NEXT: Bcc 0, %bb.7, implicit $nzcv
; CHECK-NEXT: B %bb.4
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.4:
; CHECK-NEXT: successors: %bb.5(0x80000000), %bb.6(0x00000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[FADDDrr1:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[FADDDrr]], [[FMOVD0_]], implicit $fpcr
; CHECK-NEXT: [[FADDDrr2:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[FMOVD0_]], [[FMOVD0_]], implicit $fpcr
; CHECK-NEXT: [[SUBSXri1:%[0-9]+]]:gpr64 = nsw SUBSXri [[COPY1]], 1, 0, implicit-def $nzcv
; CHECK-NEXT: [[COPY2:%[0-9]+]]:gpr64all = COPY [[SUBSXri1]]
; CHECK-NEXT: [[FMOVDi1:%[0-9]+]]:fpr64 = FMOVDi 112
; CHECK-NEXT: Bcc 0, %bb.6, implicit $nzcv
; CHECK-NEXT: B %bb.5
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: successors: %bb.6(0x04000000), %bb.5(0x7c000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[PHI:%[0-9]+]]:gpr64sp = PHI [[COPY2]], %bb.4, %24, %bb.5
; CHECK-NEXT: [[PHI1:%[0-9]+]]:fpr64 = PHI [[FMOVDi1]], %bb.4, %25, %bb.5
; CHECK-NEXT: [[PHI2:%[0-9]+]]:fpr64 = PHI [[FMOVDi]], %bb.4, [[PHI1]], %bb.5
; CHECK-NEXT: [[PHI3:%[0-9]+]]:fpr64 = PHI [[FADDDrr2]], %bb.4, %22, %bb.5
; CHECK-NEXT: [[PHI4:%[0-9]+]]:fpr64 = PHI [[FADDDrr1]], %bb.4, %23, %bb.5
; CHECK-NEXT: [[SUBSXri2:%[0-9]+]]:gpr64 = nsw SUBSXri [[PHI]], 1, 0, implicit-def $nzcv
; CHECK-NEXT: [[FADDDrr3:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[PHI2]], [[FMOVD0_]], implicit $fpcr
; CHECK-NEXT: [[FADDDrr4:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[PHI3]], [[PHI2]], implicit $fpcr
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gpr64all = COPY [[SUBSXri2]]
; CHECK-NEXT: STRDui [[PHI4]], [[COPY]], 0
; CHECK-NEXT: [[FMOVDi2:%[0-9]+]]:fpr64 = FMOVDi 112
; CHECK-NEXT: Bcc 1, %bb.5, implicit $nzcv
; CHECK-NEXT: B %bb.6
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6:
; CHECK-NEXT: successors: %bb.7(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[PHI5:%[0-9]+]]:fpr64 = PHI [[FMOVDi]], %bb.4, [[PHI1]], %bb.5
; CHECK-NEXT: [[PHI6:%[0-9]+]]:fpr64 = PHI [[FADDDrr2]], %bb.4, [[FADDDrr3]], %bb.5
; CHECK-NEXT: [[PHI7:%[0-9]+]]:fpr64 = PHI [[FADDDrr1]], %bb.4, [[FADDDrr4]], %bb.5
; CHECK-NEXT: STRDui [[PHI7]], [[COPY]], 0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.7:
; CHECK-NEXT: successors: %bb.2(0x80000000)
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[PHI8:%[0-9]+]]:fpr64 = PHI [[FMOVD0_]], %bb.3, [[PHI5]], %bb.6
; CHECK-NEXT: [[PHI9:%[0-9]+]]:fpr64 = PHI [[FADDDrr]], %bb.3, [[PHI6]], %bb.6
; CHECK-NEXT: [[FADDDrr5:%[0-9]+]]:fpr64 = nofpexcept FADDDrr [[PHI9]], [[PHI8]], implicit $fpcr
; CHECK-NEXT: STRDui [[FADDDrr5]], [[COPY]], 0
; CHECK-NEXT: B %bb.2
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.2:
; CHECK-NEXT: RET_ReallyLR
bb.0:
successors: %bb.1(0x80000000)
liveins: $x0

%0:gpr64common = COPY $x0
%1:fpr64 = FMOVD0
%2:gpr32 = MOVi32imm 1
%3:gpr64all = SUBREG_TO_REG 0, killed %2, %subreg.sub_32

bb.1:
successors: %bb.2(0x04000000), %bb.1(0x7c000000)

%4:gpr64sp = PHI %3, %bb.0, %5, %bb.1
%6:fpr64 = PHI %1, %bb.0, %7, %bb.1
%8:fpr64 = PHI %1, %bb.0, %6, %bb.1
%9:fpr64 = nofpexcept FADDDrr %8, %1, implicit $fpcr
%10:fpr64 = nofpexcept FADDDrr killed %9, %6, implicit $fpcr
STRDui killed %10, %0, 0
%11:gpr64 = nsw SUBSXri %4, 1, 0, implicit-def $nzcv
%5:gpr64all = COPY %11
%7:fpr64 = FMOVDi 112
Bcc 1, %bb.1, implicit $nzcv
B %bb.2

bb.2:
RET_ReallyLR

...
Loading
Loading