Skip to content

[NVPTX] Fixup NVPTXPrologEpilogPass for opt-bisect-limit #144136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 27, 2025
Merged

[NVPTX] Fixup NVPTXPrologEpilogPass for opt-bisect-limit #144136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7f09ca2
[NVPTX] Fixup NVPTXPrologEpilogPass for opt-bisect-limit
AlexMaclean Jun 13, 2025
1840841
address comments
AlexMaclean Jun 17, 2025
1336642
Merge branch 'main' into dev/amaclean/upstream/fixup-pro-epi
AlexMaclean Jun 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions llvm/lib/Target/NVPTX/NVPTX.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ void initializeNVPTXAAWrapperPassPass(PassRegistry &);
void initializeNVPTXExternalAAWrapperPass(PassRegistry &);
void initializeNVPTXPeepholePass(PassRegistry &);
void initializeNVPTXTagInvariantLoadLegacyPassPass(PassRegistry &);
void initializeNVPTXPrologEpilogPassPass(PassRegistry &);

struct NVVMIntrRangePass : PassInfoMixin<NVVMIntrRangePass> {
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
Expand Down
84 changes: 54 additions & 30 deletions llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,52 @@ class NVPTXPrologEpilogPass : public MachineFunctionPass {
private:
void calculateFrameObjectOffsets(MachineFunction &Fn);
};
}
} // end anonymous namespace

MachineFunctionPass *llvm::createNVPTXPrologEpilogPass() {
return new NVPTXPrologEpilogPass();
}

char NVPTXPrologEpilogPass::ID = 0;

INITIALIZE_PASS(NVPTXPrologEpilogPass, DEBUG_TYPE,
"NVPTX Prologue/Epilogue Insertion", false, false)

static bool replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI,
unsigned OpIdx) {
const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
if (MI.isDebugValue()) {

MachineOperand &Op = MI.getOperand(OpIdx);
assert(MI.isDebugOperand(&Op) &&
"Frame indices can only appear as a debug operand in a DBG_VALUE*"
" machine instruction");
Register Reg;
unsigned FrameIdx = Op.getIndex();

StackOffset Offset = TFI->getFrameIndexReference(MF, FrameIdx, Reg);
Op.ChangeToRegister(Reg, false /*isDef*/);

const DIExpression *DIExpr = MI.getDebugExpression();
if (MI.isNonListDebugValue()) {
DIExpr = TRI.prependOffsetExpression(MI.getDebugExpression(),
DIExpression::ApplyOffset, Offset);
} else {
// The debug operand at DebugOpIndex was a frame index at offset
// `Offset`; now the operand has been replaced with the frame
// register, we must add Offset with `register x, plus Offset`.
unsigned DebugOpIndex = MI.getDebugOperandIndex(&Op);
SmallVector<uint64_t, 3> Ops;
TRI.getOffsetOpcodes(Offset, Ops);
DIExpr = DIExpression::appendOpsToArg(DIExpr, Ops, DebugOpIndex);
}
MI.getDebugExpressionOp().setMetadata(DIExpr);
return true;
}
return false;
}

bool NVPTXPrologEpilogPass::runOnMachineFunction(MachineFunction &MF) {
const TargetSubtargetInfo &STI = MF.getSubtarget();
const TargetFrameLowering &TFI = *STI.getFrameLowering();
Expand All @@ -57,41 +95,27 @@ bool NVPTXPrologEpilogPass::runOnMachineFunction(MachineFunction &MF) {

calculateFrameObjectOffsets(MF);

for (MachineBasicBlock &MBB : MF) {
for (MachineInstr &MI : MBB) {
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
if (!MI.getOperand(i).isFI())
for (MachineBasicBlock &BB : MF) {
for (MachineBasicBlock::iterator I = BB.end(); I != BB.begin();) {
MachineInstr &MI = *std::prev(I);

bool RemovedMI = false;
for (const auto &[Idx, Op] : enumerate(MI.operands())) {
if (!Op.isFI())
continue;

// Frame indices in debug values are encoded in a target independent
// way with simply the frame index and offset rather than any
// target-specific addressing mode.
if (MI.isDebugValue()) {
MachineOperand &Op = MI.getOperand(i);
assert(
MI.isDebugOperand(&Op) &&
"Frame indices can only appear as a debug operand in a DBG_VALUE*"
" machine instruction");
Register Reg;
auto Offset =
TFI.getFrameIndexReference(MF, Op.getIndex(), Reg);
Op.ChangeToRegister(Reg, /*isDef=*/false);
const DIExpression *DIExpr = MI.getDebugExpression();
if (MI.isNonListDebugValue()) {
DIExpr = TRI.prependOffsetExpression(MI.getDebugExpression(), DIExpression::ApplyOffset, Offset);
} else {
SmallVector<uint64_t, 3> Ops;
TRI.getOffsetOpcodes(Offset, Ops);
unsigned OpIdx = MI.getDebugOperandIndex(&Op);
DIExpr = DIExpression::appendOpsToArg(DIExpr, Ops, OpIdx);
}
MI.getDebugExpressionOp().setMetadata(DIExpr);
if (replaceFrameIndexDebugInstr(MF, MI, Idx))
continue;
}

TRI.eliminateFrameIndex(MI, 0, i, nullptr);
// Eliminate this FrameIndex operand.
RemovedMI = TRI.eliminateFrameIndex(MI, 0, Idx, nullptr);
Modified = true;
if (RemovedMI)
break;
}

if (!RemovedMI)
--I;
}
}

Expand Down
15 changes: 10 additions & 5 deletions llvm/lib/Target/NVPTX/NVPTXRegisterInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,15 +103,20 @@ BitVector NVPTXRegisterInfo::getReservedRegs(const MachineFunction &MF) const {

bool NVPTXRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
int SPAdj, unsigned FIOperandNum,
RegScavenger *RS) const {
RegScavenger *) const {
assert(SPAdj == 0 && "Unexpected");

MachineInstr &MI = *II;
int FrameIndex = MI.getOperand(FIOperandNum).getIndex();
if (MI.isLifetimeMarker()) {
MI.eraseFromParent();
return true;
}

const int FrameIndex = MI.getOperand(FIOperandNum).getIndex();

MachineFunction &MF = *MI.getParent()->getParent();
int Offset = MF.getFrameInfo().getObjectOffset(FrameIndex) +
MI.getOperand(FIOperandNum + 1).getImm();
const MachineFunction &MF = *MI.getParent()->getParent();
const int Offset = MF.getFrameInfo().getObjectOffset(FrameIndex) +
MI.getOperand(FIOperandNum + 1).getImm();

// Using I0 as the frame pointer
MI.getOperand(FIOperandNum).ChangeToRegister(getFrameRegister(MF), false);
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Target/NVPTX/NVPTXTargetMachine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeNVPTXTarget() {
initializeNVPTXExternalAAWrapperPass(PR);
initializeNVPTXPeepholePass(PR);
initializeNVPTXTagInvariantLoadLegacyPassPass(PR);
initializeNVPTXPrologEpilogPassPass(PR);
}

static std::string computeDataLayout(bool is64Bit, bool UseShortPointers) {
Expand Down
51 changes: 51 additions & 0 deletions llvm/test/CodeGen/NVPTX/frameindex-lifetime.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t
; RUN: llc -mtriple=nvptx64-nvidia-cuda -opt-bisect-limit=%(line-1) -O3 %s -o %t

;; This test is intended to verify that we don't crash when -opt-bisect-limit
;; is used in conjunction with lifetime markers. Previously, later passes
;; would not handle these intructions correctly and relied on earlier passes
;; to remove them.

declare void @bar(ptr)

define void @foo() {
%p = alloca i32
call void @llvm.lifetime.start(i64 4, ptr %p)
call void @bar(ptr %p)
call void @llvm.lifetime.end(i64 4, ptr %p)
ret void
}
Loading