Skip to content

X86: Fix win64 tail call regression for tail call to loaded pointer #158055

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 11, 2025
Merged

X86: Fix win64 tail call regression for tail call to loaded pointer #158055

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions llvm/lib/Target/X86/X86AsmPrinter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -478,9 +478,9 @@ static bool isIndirectBranchOrTailCall(const MachineInstr &MI) {
Opc == X86::TAILJMPr64 || Opc == X86::TAILJMPm64 ||
Opc == X86::TCRETURNri || Opc == X86::TCRETURN_WIN64ri ||
Opc == X86::TCRETURN_HIPE32ri || Opc == X86::TCRETURNmi ||
Opc == X86::TCRETURNri64 || Opc == X86::TCRETURNmi64 ||
Opc == X86::TCRETURNri64_ImpCall || Opc == X86::TAILJMPr64_REX ||
Opc == X86::TAILJMPm64_REX;
Opc == X86::TCRETURN_WINmi64 || Opc == X86::TCRETURNri64 ||
Opc == X86::TCRETURNmi64 || Opc == X86::TCRETURNri64_ImpCall ||
Opc == X86::TAILJMPr64_REX || Opc == X86::TAILJMPm64_REX;
}

void X86AsmPrinter::emitBasicBlockEnd(const MachineBasicBlock &MBB) {
Expand Down
9 changes: 6 additions & 3 deletions llvm/lib/Target/X86/X86ExpandPseudo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,8 +276,10 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,
case X86::TCRETURNdi64cc:
case X86::TCRETURNri64:
case X86::TCRETURNri64_ImpCall:
case X86::TCRETURNmi64: {
bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64;
case X86::TCRETURNmi64:
case X86::TCRETURN_WINmi64: {
bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64 ||
Opcode == X86::TCRETURN_WINmi64;
MachineOperand &JumpTarget = MBBI->getOperand(0);
MachineOperand &StackAdjust = MBBI->getOperand(isMem ? X86::AddrNumOperands
: 1);
Expand Down Expand Up @@ -341,7 +343,8 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB,
MIB.addImm(MBBI->getOperand(2).getImm());
}

} else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64) {
} else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64 ||
Opcode == X86::TCRETURN_WINmi64) {
unsigned Op = (Opcode == X86::TCRETURNmi)
? X86::TAILJMPm
: (IsX64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Target/X86/X86FrameLowering.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2402,7 +2402,7 @@ static bool isTailCallOpcode(unsigned Opc) {
Opc == X86::TCRETURN_HIPE32ri || Opc == X86::TCRETURNdi ||
Opc == X86::TCRETURNmi || Opc == X86::TCRETURNri64 ||
Opc == X86::TCRETURNri64_ImpCall || Opc == X86::TCRETURNdi64 ||
Opc == X86::TCRETURNmi64;
Opc == X86::TCRETURNmi64 || Opc == X86::TCRETURN_WINmi64;
}

void X86FrameLowering::emitEpilogue(MachineFunction &MF,
Expand Down
12 changes: 8 additions & 4 deletions llvm/lib/Target/X86/X86InstrCompiler.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1364,15 +1364,19 @@ def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off),
// There wouldn't be enough scratch registers for base+index.
def : Pat<(X86tcret_6regs (load addr:$dst), timm:$off),
(TCRETURNmi64 addr:$dst, timm:$off)>,
Requires<[In64BitMode, NotUseIndirectThunkCalls]>;
Requires<[In64BitMode, IsNotWin64CCFunc, NotUseIndirectThunkCalls]>;

def : Pat<(X86tcret_6regs (load addr:$dst), timm:$off),
(TCRETURN_WINmi64 addr:$dst, timm:$off)>,
Requires<[IsWin64CCFunc, NotUseIndirectThunkCalls]>;

def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off),
(INDIRECT_THUNK_TCRETURN64 ptr_rc_tailcall:$dst, timm:$off)>,
Requires<[In64BitMode, UseIndirectThunkCalls]>;
Requires<[In64BitMode, IsNotWin64CCFunc, UseIndirectThunkCalls]>;

def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off),
(INDIRECT_THUNK_TCRETURN32 ptr_rc_tailcall:$dst, timm:$off)>,
Requires<[Not64BitMode, UseIndirectThunkCalls]>;
Requires<[Not64BitMode, IsNotWin64CCFunc, UseIndirectThunkCalls]>;

def : Pat<(X86tcret (i64 tglobaladdr:$dst), timm:$off),
(TCRETURNdi64 tglobaladdr:$dst, timm:$off)>,
Expand Down Expand Up @@ -2215,7 +2219,7 @@ let Predicates = [HasZU] in {
def : Pat<(i64 (zext (i16 (mul (loadi16 addr:$src1), imm:$src2)))),
(SUBREG_TO_REG (i64 0), (IMULZU16rmi addr:$src1, imm:$src2), sub_16bit)>;
}

// mul reg, imm
def : Pat<(mul GR16:$src1, imm:$src2),
(IMUL16rri GR16:$src1, imm:$src2)>;
Expand Down
3 changes: 3 additions & 0 deletions llvm/lib/Target/X86/X86InstrControl.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,9 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
def TCRETURNmi64 : PseudoI<(outs),
(ins i64mem_TC:$dst, i32imm:$offset),
[]>, Sched<[WriteJumpLd]>;
def TCRETURN_WINmi64 : PseudoI<(outs),
(ins i64mem_w64TC:$dst, i32imm:$offset),
[]>, Sched<[WriteJumpLd]>;

def TAILJMPd64 : PseudoI<(outs), (ins i64i32imm_brtarget:$dst),
[]>, Sched<[WriteJump]>;
Expand Down
5 changes: 5 additions & 0 deletions llvm/lib/Target/X86/X86InstrOperands.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,11 @@ def i64mem_TC : X86MemOperand<"printqwordmem", X86Mem64AsmOperand, 64> {
ptr_rc_tailcall, i32imm, SEGMENT_REG);
}

def i64mem_w64TC : X86MemOperand<"printqwordmem", X86Mem64AsmOperand, 64> {
let MIOperandInfo = (ops GR64_TCW64, i8imm,
GR64_TCW64, i32imm, SEGMENT_REG);
}

// Special parser to detect 16-bit mode to select 16-bit displacement.
def X86AbsMemMode16AsmOperand : AsmOperandClass {
let Name = "AbsMemMode16";
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Target/X86/X86RegisterInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1010,6 +1010,7 @@ unsigned X86RegisterInfo::findDeadCallerSavedReg(
case X86::TCRETURNri64:
case X86::TCRETURNri64_ImpCall:
case X86::TCRETURNmi64:
case X86::TCRETURN_WINmi64:
case X86::EH_RETURN:
case X86::EH_RETURN64: {
LiveRegUnits LRU(*this);
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -893,6 +893,7 @@ void X86SpeculativeLoadHardeningPass::unfoldCallAndJumpLoads(
case X86::TAILJMPm64_REX:
case X86::TAILJMPm:
case X86::TCRETURNmi64:
case X86::TCRETURN_WINmi64:
case X86::TCRETURNmi: {
// Use the generic unfold logic now that we know we're dealing with
// expected instructions.
Expand Down
48 changes: 48 additions & 0 deletions llvm/test/CodeGen/X86/win64-tailcall-memory.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=x86_64-unknown-windows-gnu < %s | FileCheck %s

; Check calling convention is correct for win64 when doing a tailcall
; for a pointer loaded from memory.

declare void @foo(i64, ptr)

define void @do_tailcall(ptr %objp) nounwind {
; CHECK-LABEL: do_tailcall:
; CHECK: # %bb.0:
; CHECK-NEXT: pushq %rsi
; CHECK-NEXT: subq $32, %rsp
; CHECK-NEXT: movq %rcx, %rsi
; CHECK-NEXT: xorl %ecx, %ecx
; CHECK-NEXT: xorl %edx, %edx
; CHECK-NEXT: callq foo
; CHECK-NEXT: xorl %ecx, %ecx
; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: addq $32, %rsp
; CHECK-NEXT: popq %rsi
; CHECK-NEXT: rex64 jmpq *(%rax) # TAILCALL
tail call void @foo(i64 0, ptr null)
%fptr = load ptr, ptr %objp, align 8
tail call void %fptr(ptr null)
ret void
}

; Make sure aliases of ccc are also treated as win64 functions
define fastcc void @do_tailcall_fastcc(ptr %objp) nounwind {
; CHECK-LABEL: do_tailcall_fastcc:
; CHECK: # %bb.0:
; CHECK-NEXT: pushq %rsi
; CHECK-NEXT: subq $32, %rsp
; CHECK-NEXT: movq %rcx, %rsi
; CHECK-NEXT: xorl %ecx, %ecx
; CHECK-NEXT: xorl %edx, %edx
; CHECK-NEXT: callq foo
; CHECK-NEXT: xorl %ecx, %ecx
; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: addq $32, %rsp
; CHECK-NEXT: popq %rsi
; CHECK-NEXT: rex64 jmpq *(%rax) # TAILCALL
tail call void @foo(i64 0, ptr null)
%fptr = load ptr, ptr %objp, align 8
tail call fastcc void %fptr(ptr null)
ret void
}