Skip to content

Commit 39a148d

Browse files
guy-davidguy-david
committed
[AArch64] Skip storing of stack arguments when lowering tail calls
When possible, do not emit trivial load and stores to the same offset on the stack.
1 parent fd452da commit 39a148d

File tree

9 files changed

+266
-57
lines changed

9 files changed

+266
-57
lines changed

‎llvm/include/llvm/CodeGen/GlobalISel/Utils.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,10 @@ LLVM_ABI bool isGuaranteedNotToBeUndef(Register Reg,
617617
/// estimate of the type.
618618
LLVM_ABI Type *getTypeForLLT(LLT Ty, LLVMContext &C);
619619

620+
/// Returns true if the instruction \p MI is one of the assert
621+
/// instructions.
622+
LLVM_ABI bool isAssertMI(const MachineInstr &MI);
623+
620624
/// An integer-like constant.
621625
///
622626
/// It abstracts over scalar, fixed-length vectors, and scalable vectors.

‎llvm/include/llvm/CodeGen/SelectionDAGNodes.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -724,6 +724,19 @@ END_TWO_BYTE_PACK()
724724
}
725725
}
726726

727+
/// Test if this node is an assert operation.
728+
bool isAssert() const {
729+
switch (NodeType) {
730+
default:
731+
return false;
732+
case ISD::AssertAlign:
733+
case ISD::AssertNoFPClass:
734+
case ISD::AssertSext:
735+
case ISD::AssertZext:
736+
return true;
737+
}
738+
}
739+
727740
/// Test if this node is a vector predication operation.
728741
bool isVPOpcode() const { return ISD::isVPOpcode(getOpcode()); }
729742

‎llvm/lib/CodeGen/GlobalISel/Utils.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2003,6 +2003,17 @@ Type *llvm::getTypeForLLT(LLT Ty, LLVMContext &C) {
20032003
return IntegerType::get(C, Ty.getSizeInBits());
20042004
}
20052005

2006+
bool llvm::isAssertMI(const MachineInstr &MI) {
2007+
switch (MI.getOpcode()) {
2008+
default:
2009+
return false;
2010+
case TargetOpcode::G_ASSERT_ALIGN:
2011+
case TargetOpcode::G_ASSERT_SEXT:
2012+
case TargetOpcode::G_ASSERT_ZEXT:
2013+
return true;
2014+
}
2015+
}
2016+
20062017
APInt llvm::GIConstant::getScalarValue() const {
20072018
assert(Kind == GIConstantKind::Scalar && "Expected scalar constant");
20082019

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8761,6 +8761,45 @@ getSMToggleCondition(const SMECallAttrs &CallAttrs) {
87618761
llvm_unreachable("Unsupported attributes");
87628762
}
87638763

8764+
/// Check whether a stack argument requires lowering in a tail call.
8765+
static bool shouldLowerTailCallStackArg(const MachineFunction &MF,
8766+
const CCValAssign &VA, SDValue Arg,
8767+
ISD::ArgFlagsTy Flags, int CallOffset) {
8768+
// FIXME: We should be able to handle this case, but it's not clear how to.
8769+
if (Flags.isZExt() || Flags.isSExt())
8770+
return true;
8771+
8772+
for (;;) {
8773+
// Look through nodes that don't alter the bits of the incoming value.
8774+
unsigned Op = Arg.getOpcode();
8775+
if (Op == ISD::ZERO_EXTEND || Op == ISD::ANY_EXTEND || Op == ISD::BITCAST ||
8776+
Arg->isAssert() || Op == AArch64ISD::ASSERT_ZEXT_BOOL) {
8777+
Arg = Arg.getOperand(0);
8778+
continue;
8779+
}
8780+
break;
8781+
}
8782+
8783+
// If the argument is a load from the same immutable stack slot, we can reuse
8784+
// it.
8785+
if (auto *LoadNode = dyn_cast<LoadSDNode>(Arg)) {
8786+
if (auto *FINode = dyn_cast<FrameIndexSDNode>(LoadNode->getBasePtr())) {
8787+
const MachineFrameInfo &MFI = MF.getFrameInfo();
8788+
int FI = FINode->getIndex();
8789+
if (!MFI.isImmutableObjectIndex(FI))
8790+
return true;
8791+
if (CallOffset != MFI.getObjectOffset(FI))
8792+
return true;
8793+
uint64_t SizeInBits = LoadNode->getMemoryVT().getFixedSizeInBits();
8794+
if (SizeInBits / 8 != MFI.getObjectSize(FI))
8795+
return true;
8796+
return false;
8797+
}
8798+
}
8799+
8800+
return true;
8801+
}
8802+
87648803
/// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain,
87658804
/// and add input and output parameter nodes.
87668805
SDValue
@@ -9183,10 +9222,13 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
91839222
}
91849223
unsigned LocMemOffset = VA.getLocMemOffset();
91859224
int32_t Offset = LocMemOffset + BEAlign;
9186-
SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
9187-
PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
91889225

91899226
if (IsTailCall) {
9227+
// When the frame pointer is perfectly aligned for the tail call and the
9228+
// same stack argument is passed down intact, we can reuse it.
9229+
if (!FPDiff && !shouldLowerTailCallStackArg(MF, VA, Arg, Flags, Offset))
9230+
continue;
9231+
91909232
Offset = Offset + FPDiff;
91919233
int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
91929234

‎llvm/lib/Target/AArch64/GISel/AArch64CallLowering.cpp‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "llvm/CodeGen/Analysis.h"
2626
#include "llvm/CodeGen/CallingConvLower.h"
2727
#include "llvm/CodeGen/FunctionLoweringInfo.h"
28+
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
2829
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
2930
#include "llvm/CodeGen/GlobalISel/Utils.h"
3031
#include "llvm/CodeGen/LowLevelTypeUtils.h"
@@ -35,6 +36,7 @@
3536
#include "llvm/CodeGen/MachineMemOperand.h"
3637
#include "llvm/CodeGen/MachineOperand.h"
3738
#include "llvm/CodeGen/MachineRegisterInfo.h"
39+
#include "llvm/CodeGen/TargetOpcodes.h"
3840
#include "llvm/CodeGen/TargetRegisterInfo.h"
3941
#include "llvm/CodeGen/TargetSubtargetInfo.h"
4042
#include "llvm/CodeGen/ValueTypes.h"
@@ -296,10 +298,60 @@ struct OutgoingArgHandler : public CallLowering::OutgoingValueHandler {
296298
MIRBuilder.buildCopy(PhysReg, ExtReg);
297299
}
298300

301+
/// Check whether a stack argument requires lowering in a tail call.
302+
static bool shouldLowerTailCallStackArg(const MachineFunction &MF,
303+
const CCValAssign &VA,
304+
Register ValVReg,
305+
Register StoreAddr) {
306+
const MachineRegisterInfo &MRI = MF.getRegInfo();
307+
// Print the defining instruction for the value.
308+
auto *DefMI = MRI.getVRegDef(ValVReg);
309+
assert(DefMI && "No defining instruction");
310+
for (;;) {
311+
// Look through nodes that don't alter the bits of the incoming value.
312+
unsigned Op = DefMI->getOpcode();
313+
if (Op == TargetOpcode::G_ZEXT || Op == TargetOpcode::G_ANYEXT ||
314+
Op == TargetOpcode::G_BITCAST || isAssertMI(*DefMI)) {
315+
DefMI = MRI.getVRegDef(DefMI->getOperand(1).getReg());
316+
continue;
317+
}
318+
break;
319+
}
320+
321+
auto *Load = dyn_cast<GLoad>(DefMI);
322+
if (!Load)
323+
return true;
324+
Register LoadReg = Load->getPointerReg();
325+
auto *LoadAddrDef = MRI.getVRegDef(LoadReg);
326+
assert(LoadAddrDef && "No defining instruction");
327+
if (LoadAddrDef->getOpcode() != TargetOpcode::G_FRAME_INDEX)
328+
return true;
329+
assert(LoadAddrDef && "No defining instruction");
330+
const MachineFrameInfo &MFI = MF.getFrameInfo();
331+
int LoadFI = LoadAddrDef->getOperand(1).getIndex();
332+
333+
auto *StoreAddrDef = MRI.getVRegDef(StoreAddr);
334+
assert(StoreAddrDef && "No defining instruction");
335+
if (StoreAddrDef->getOpcode() != TargetOpcode::G_FRAME_INDEX)
336+
return true;
337+
int StoreFI = StoreAddrDef->getOperand(1).getIndex();
338+
339+
if (!MFI.isImmutableObjectIndex(LoadFI))
340+
return true;
341+
if (MFI.getObjectOffset(LoadFI) != MFI.getObjectOffset(StoreFI))
342+
return true;
343+
if (Load->getMemSize() != MFI.getObjectSize(StoreFI))
344+
return true;
345+
346+
return false;
347+
}
348+
299349
void assignValueToAddress(Register ValVReg, Register Addr, LLT MemTy,
300350
const MachinePointerInfo &MPO,
301351
const CCValAssign &VA) override {
302352
MachineFunction &MF = MIRBuilder.getMF();
353+
if (!FPDiff && !shouldLowerTailCallStackArg(MF, VA, ValVReg, Addr))
354+
return;
303355
auto MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, MemTy,
304356
inferAlignFromPtrInfo(MF, MPO));
305357
MIRBuilder.buildStore(ValVReg, Addr, *MMO);

‎llvm/test/CodeGen/AArch64/darwinpcs-tail.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@
55

66
; CHECK-LABEL: __ZThn16_N1C3addEPKcz:
77
; CHECK: b __ZN1C3addEPKcz
8+
89
; CHECK-LABEL: _tailTest:
910
; CHECK: b __ZN1C3addEPKcz
11+
1012
; CHECK-LABEL: __ZThn8_N1C1fEiiiiiiiiiz:
11-
; CHECK: ldr w9, [sp, #4]
12-
; CHECK: str w9, [sp, #4]
1313
; CHECK: b __ZN1C1fEiiiiiiiiiz
1414

1515
%class.C = type { %class.A.base, [4 x i8], %class.B.base, [4 x i8] }

‎llvm/test/CodeGen/AArch64/scavenge-large-call.ll‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
; CHECK: add {{x[0-9]+}}, sp,
55

66
define void @caller(ptr %0, i16 %1, i16 %2, i8 %3, double %4, i16 %5, i8 %6, ptr %7, double %8, i32 %9, ptr %10, double %11, double %12, [2 x i64] %13, [2 x i64] %14, [2 x i64] %15, double %16, double %17, [2 x i64] %18, [2 x i64] %19, i16 %20, i32 %21, double %22, i8 %23, [2 x i64] %24, [2 x i64] %25, [2 x i64] %26, i8 %27, i16 %28, i16 %29, i16 %30, i32 %31, [2 x i64] %32, [2 x i64] %33, [2 x i64] %34, [2 x i64] %35, [2 x i64] %36, i32 %37, i32 %38) {
7-
tail call void @callee(ptr %0, i16 %1, i16 %2, i8 %3, double 0.000000e+00, i16 %5, i8 %6, ptr %7, double 0.000000e+00, i32 %9, ptr %10, double 0.000000e+00, double 0.000000e+00, [2 x i64] %13, [2 x i64] %14, [2 x i64] %15, double 0.000000e+00, double 0.000000e+00, [2 x i64] %18, [2 x i64] %19, i16 %20, i32 %21, double 0.000000e+00, i8 %23, [2 x i64] %24, [2 x i64] %25, [2 x i64] zeroinitializer, i8 %27, i16 0, i16 0, i16 %28, i32 0, [2 x i64] zeroinitializer, [2 x i64] zeroinitializer, [2 x i64] zeroinitializer, [2 x i64] %35, [2 x i64] %36, i32 0, i32 0)
7+
call void @callee(ptr %0, i16 %1, i16 %2, i8 %3, double 0.000000e+00, i16 %5, i8 %6, ptr %7, double 0.000000e+00, i32 %9, ptr %10, double 0.000000e+00, double 0.000000e+00, [2 x i64] %13, [2 x i64] %14, [2 x i64] %15, double 0.000000e+00, double 0.000000e+00, [2 x i64] %18, [2 x i64] %19, i16 %20, i32 %21, double 0.000000e+00, i8 %23, [2 x i64] %24, [2 x i64] %25, [2 x i64] zeroinitializer, i8 %27, i16 0, i16 0, i16 %28, i32 0, [2 x i64] zeroinitializer, [2 x i64] zeroinitializer, [2 x i64] zeroinitializer, [2 x i64] %35, [2 x i64] %36, i32 0, i32 0)
88
ret void
99
}
1010

‎llvm/test/CodeGen/AArch64/sve-fixed-length-frame-offests-crash.ll‎

Lines changed: 50 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -11,66 +11,64 @@ target triple = "aarch64-unknown-linux-gnu"
1111
define dso_local void @func1(ptr %v1, ptr %v2, ptr %v3, ptr %v4, ptr %v5, ptr %v6, ptr %v7, ptr %v8,
1212
; CHECK-LABEL: func1:
1313
; CHECK: // %bb.0:
14-
; CHECK-NEXT: str x29, [sp, #-48]! // 8-byte Folded Spill
15-
; CHECK-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill
16-
; CHECK-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill
17-
; CHECK-NEXT: .cfi_def_cfa_offset 48
18-
; CHECK-NEXT: .cfi_offset w19, -8
19-
; CHECK-NEXT: .cfi_offset w20, -16
20-
; CHECK-NEXT: .cfi_offset w21, -24
21-
; CHECK-NEXT: .cfi_offset w22, -32
22-
; CHECK-NEXT: .cfi_offset w29, -48
23-
; CHECK-NEXT: add x10, sp, #176
24-
; CHECK-NEXT: add x8, sp, #48
25-
; CHECK-NEXT: add x9, sp, #144
26-
; CHECK-NEXT: ldr z3, [x10]
14+
; CHECK-NEXT: sub sp, sp, #368
15+
; CHECK-NEXT: stp x29, x30, [sp, #336] // 16-byte Folded Spill
16+
; CHECK-NEXT: str x28, [sp, #352] // 8-byte Folded Spill
17+
; CHECK-NEXT: add x29, sp, #336
18+
; CHECK-NEXT: .cfi_def_cfa w29, 32
19+
; CHECK-NEXT: .cfi_offset w28, -16
20+
; CHECK-NEXT: .cfi_offset w30, -24
21+
; CHECK-NEXT: .cfi_offset w29, -32
22+
; CHECK-NEXT: add x8, x29, #32
23+
; CHECK-NEXT: add x9, x29, #72
24+
; CHECK-NEXT: ptrue p0.d
2725
; CHECK-NEXT: ldr z0, [x8]
28-
; CHECK-NEXT: add x8, sp, #112
29-
; CHECK-NEXT: ldr z2, [x9]
26+
; CHECK-NEXT: add x8, x29, #256
27+
; CHECK-NEXT: ldr z3, [x9]
3028
; CHECK-NEXT: ldr z1, [x8]
31-
; CHECK-NEXT: add x20, sp, #176
32-
; CHECK-NEXT: ldp x9, x8, [sp, #328]
33-
; CHECK-NEXT: ldr x15, [sp, #104]
34-
; CHECK-NEXT: ldp x11, x10, [sp, #312]
35-
; CHECK-NEXT: ldur q4, [sp, #88]
36-
; CHECK-NEXT: ldp x13, x12, [sp, #296]
37-
; CHECK-NEXT: ldr x19, [sp, #272]
38-
; CHECK-NEXT: ldp x18, x14, [sp, #280]
39-
; CHECK-NEXT: ldp x16, x17, [sp, #208]
40-
; CHECK-NEXT: ldp x21, x22, [sp, #352]
41-
; CHECK-NEXT: str z3, [x20]
42-
; CHECK-NEXT: add x20, sp, #144
43-
; CHECK-NEXT: str z2, [x20]
44-
; CHECK-NEXT: add x20, sp, #112
45-
; CHECK-NEXT: str z1, [x20]
46-
; CHECK-NEXT: add x20, sp, #48
47-
; CHECK-NEXT: str z0, [x20]
48-
; CHECK-NEXT: stp x21, x22, [sp, #352]
49-
; CHECK-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
50-
; CHECK-NEXT: stp x19, x18, [sp, #272]
51-
; CHECK-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
52-
; CHECK-NEXT: stp x16, x17, [sp, #208]
53-
; CHECK-NEXT: stur q4, [sp, #88]
54-
; CHECK-NEXT: str x15, [sp, #104]
55-
; CHECK-NEXT: stp x14, x13, [sp, #288]
56-
; CHECK-NEXT: stp x12, x11, [sp, #304]
57-
; CHECK-NEXT: stp x10, x9, [sp, #320]
58-
; CHECK-NEXT: str x8, [sp, #336]
59-
; CHECK-NEXT: ldr x29, [sp], #48 // 8-byte Folded Reload
60-
; CHECK-NEXT: b func2
29+
; CHECK-NEXT: add x8, x29, #288
30+
; CHECK-NEXT: add x9, x29, #168
31+
; CHECK-NEXT: ldr z2, [x8]
32+
; CHECK-NEXT: add x8, x29, #104
33+
; CHECK-NEXT: ldr z6, [x9]
34+
; CHECK-NEXT: ldr z4, [x8]
35+
; CHECK-NEXT: add x8, x29, #136
36+
; CHECK-NEXT: mov x12, #17 // =0x11
37+
; CHECK-NEXT: ldr z5, [x8]
38+
; CHECK-NEXT: ldp x10, x11, [x29, #336]
39+
; CHECK-NEXT: st1d { z6.d }, p0, [sp, x12, lsl #3]
40+
; CHECK-NEXT: mov x12, #13 // =0xd
41+
; CHECK-NEXT: ldr x8, [x29, #200]
42+
; CHECK-NEXT: ldr x9, [x29, #320]
43+
; CHECK-NEXT: st1d { z5.d }, p0, [sp, x12, lsl #3]
44+
; CHECK-NEXT: mov x12, #9 // =0x9
45+
; CHECK-NEXT: st1d { z4.d }, p0, [sp, x12, lsl #3]
46+
; CHECK-NEXT: mov x12, #5 // =0x5
47+
; CHECK-NEXT: st1d { z3.d }, p0, [sp, x12, lsl #3]
48+
; CHECK-NEXT: stp x10, x11, [sp, #304]
49+
; CHECK-NEXT: str x9, [sp, #288]
50+
; CHECK-NEXT: str z2, [sp, #8, mul vl]
51+
; CHECK-NEXT: str z1, [sp, #7, mul vl]
52+
; CHECK-NEXT: str x8, [sp, #168]
53+
; CHECK-NEXT: str z0, [sp]
54+
; CHECK-NEXT: bl func2
55+
; CHECK-NEXT: ldp x29, x30, [sp, #336] // 16-byte Folded Reload
56+
; CHECK-NEXT: ldr x28, [sp, #352] // 8-byte Folded Reload
57+
; CHECK-NEXT: add sp, sp, #368
58+
; CHECK-NEXT: ret
6159
ptr %v9, ptr %v10, ptr %v11, ptr %v12, ptr %v13, ptr %v14, ptr %v15, ptr %v16,
6260
ptr %v17, ptr %v18, ptr %v19, ptr %v20, ptr %v21, ptr %v22, ptr %v23, ptr %v24,
6361
ptr %v25, ptr %v26, ptr %v27, ptr %v28, ptr %v29, ptr %v30, ptr %v31, ptr %v32,
6462
ptr %v33, ptr %v34, ptr %v35, ptr %v36, ptr %v37, ptr %v38, ptr %v39, ptr %v40,
6563
ptr %v41, ptr %v42, ptr %v43, ptr %v44, ptr %v45, ptr %v46, ptr %v47, ptr %v48,
6664
i64 %v49) #0 {
67-
tail call void @func2(ptr %v1, ptr %v2, ptr %v3, ptr %v4, ptr %v5, ptr %v6, ptr %v7, ptr %v8,
68-
ptr %v9, ptr %v10, ptr %v11, ptr %v12, ptr undef, ptr %v14, ptr %v15, ptr %v16,
69-
ptr %v17, ptr %v18, ptr %v19, ptr %v20, ptr %v21, ptr %v22, ptr %v23, ptr %v24,
70-
ptr %v25, ptr %v26, ptr %v27, ptr %v28, ptr %v29, ptr %v30, ptr undef, ptr undef,
71-
ptr undef, ptr undef, ptr undef, ptr undef, ptr %v37, ptr %v38, ptr %v39, ptr %v40,
72-
ptr %v41, ptr %v42, ptr %v43, ptr %v44, ptr %v45, ptr undef, ptr %v47, ptr %v48,
73-
i64 undef)
65+
call void @func2(ptr %v1, ptr %v2, ptr %v3, ptr %v4, ptr %v5, ptr %v6, ptr %v7, ptr %v8,
66+
ptr %v9, ptr %v10, ptr %v11, ptr %v12, ptr undef, ptr %v14, ptr %v15, ptr %v16,
67+
ptr %v17, ptr %v18, ptr %v19, ptr %v20, ptr %v21, ptr %v22, ptr %v23, ptr %v24,
68+
ptr %v25, ptr %v26, ptr %v27, ptr %v28, ptr %v29, ptr %v30, ptr undef, ptr undef,
69+
ptr undef, ptr undef, ptr undef, ptr undef, ptr %v37, ptr %v38, ptr %v39, ptr %v40,
70+
ptr %v41, ptr %v42, ptr %v43, ptr %v44, ptr %v45, ptr undef, ptr %v47, ptr %v48,
71+
i64 undef)
7472
ret void
7573
}
7674

0 commit comments

Comments
 (0)