Skip to content

Commit efe2fc3

Browse files
hassnaaHamdihassnaaHamdi
committed
[Analysis]: Allow inlining recursive call IF recursion depth is 1.
1 parent 65c7ea7 commit efe2fc3

File tree

4 files changed

+261
-2
lines changed

4 files changed

+261
-2
lines changed

‎llvm/include/llvm/Analysis/InlineCost.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ struct InlineParams {
236236
std::optional<bool> EnableDeferral;
237237

238238
/// Indicate whether we allow inlining for recursive call.
239-
std::optional<bool> AllowRecursiveCall = false;
239+
std::optional<bool> AllowRecursiveCall = true;
240240
};
241241

242242
std::optional<int> getStringFnAttrAsInt(CallBase &CB, StringRef AttrKind);

‎llvm/lib/Analysis/InlineCost.cpp‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "llvm/Analysis/CodeMetrics.h"
2222
#include "llvm/Analysis/ConstantFolding.h"
2323
#include "llvm/Analysis/EphemeralValuesCache.h"
24+
#include "llvm/Analysis/DomConditionCache.h"
2425
#include "llvm/Analysis/InstructionSimplify.h"
2526
#include "llvm/Analysis/LoopInfo.h"
2627
#include "llvm/Analysis/MemoryBuiltins.h"
@@ -1170,6 +1171,10 @@ class InlineCostCallAnalyzer final : public CallAnalyzer {
11701171
std::optional<CostBenefitPair> getCostBenefitPair() { return CostBenefit; }
11711172
bool wasDecidedByCostBenefit() const { return DecidedByCostBenefit; }
11721173
bool wasDecidedByCostThreshold() const { return DecidedByCostThreshold; }
1174+
bool shouldCheckRecursiveCall() {
1175+
return IsRecursiveCall && AllowRecursiveCall;
1176+
}
1177+
bool shouldInlineRecursiveCall(CallBase &Call);
11731178
};
11741179

11751180
// Return true if CB is the sole call to local function Callee.
@@ -2895,6 +2900,68 @@ InlineResult CallAnalyzer::analyze() {
28952900
return finalizeAnalysis();
28962901
}
28972902

2903+
bool InlineCostCallAnalyzer::shouldInlineRecursiveCall(CallBase &Call) {
2904+
CallInst *CI = cast<CallInst>(&Call);
2905+
auto CIB = CI->getParent();
2906+
// Only handle case when we have sinlge predecessor
2907+
if (auto Predecessor = CIB->getSinglePredecessor()) {
2908+
BranchInst *Br = dyn_cast<BranchInst>(Predecessor->getTerminator());
2909+
if (!Br || Br->isUnconditional()) {
2910+
return false;
2911+
}
2912+
Value *Var = Br->getCondition();
2913+
CmpInst *CmpInstr = dyn_cast<CmpInst>(Var);
2914+
if (CmpInstr && !isa<Constant>(CmpInstr->getOperand(1))) {
2915+
// Current logic of ValueTracking/DomConditionCache works only if RHS is
2916+
// constant.
2917+
return false;
2918+
}
2919+
unsigned ArgNum = 0;
2920+
Value *FuncArg = nullptr, *CallArg = nullptr;
2921+
// Check which func argument the cmp instr is using:
2922+
for (; ArgNum < CI->getFunction()->arg_size(); ArgNum++) {
2923+
FuncArg = CI->getFunction()->getArg(ArgNum);
2924+
CallArg = CI->getArgOperand(ArgNum);
2925+
if (CmpInstr) {
2926+
if ((FuncArg == CmpInstr->getOperand(0)) &&
2927+
(CallArg != CmpInstr->getOperand(0)))
2928+
break;
2929+
} else if (FuncArg == Var && (CallArg != Var))
2930+
break;
2931+
}
2932+
// Only handle the case when a func argument controls the cmp instruction:
2933+
if (ArgNum < CI->getFunction()->arg_size()) {
2934+
bool isTrueSuccessor = CIB == Br->getSuccessor(0);
2935+
if (CmpInstr) {
2936+
SimplifyQuery SQ(CI->getFunction()->getDataLayout(),
2937+
dyn_cast<Instruction>(CallArg));
2938+
DomConditionCache DC;
2939+
DC.registerBranch(Br);
2940+
SQ.DC = &DC;
2941+
DominatorTree DT(*CI->getFunction());
2942+
SQ.DT = &DT;
2943+
Value *simplifiedInstruction = llvm::simplifyInstructionWithOperands(
2944+
CmpInstr, {CallArg, CmpInstr->getOperand(1)}, SQ);
2945+
if (!simplifiedInstruction)
2946+
return false;
2947+
if (auto *ConstVal =
2948+
dyn_cast<llvm::ConstantInt>(simplifiedInstruction)) {
2949+
if (ConstVal->isOne())
2950+
return !isTrueSuccessor;
2951+
return isTrueSuccessor;
2952+
}
2953+
} else {
2954+
if (auto *ConstVal = dyn_cast<llvm::ConstantInt>(CallArg)) {
2955+
if (ConstVal->isOne())
2956+
return !isTrueSuccessor;
2957+
return isTrueSuccessor;
2958+
}
2959+
}
2960+
}
2961+
}
2962+
return false;
2963+
}
2964+
28982965
void InlineCostCallAnalyzer::print(raw_ostream &OS) {
28992966
#define DEBUG_PRINT_STAT(x) OS << " " #x ": " << x << "\n"
29002967
if (PrintInstructionComments)
@@ -3126,6 +3193,12 @@ InlineCost llvm::getInlineCost(
31263193

31273194
LLVM_DEBUG(CA.dump());
31283195

3196+
// Check if callee function is recursive:
3197+
if (ShouldInline.isSuccess()) {
3198+
if (CA.shouldCheckRecursiveCall() && !CA.shouldInlineRecursiveCall(Call))
3199+
return InlineCost::getNever("deep recursion");
3200+
}
3201+
31293202
// Always make cost benefit based decision explicit.
31303203
// We use always/never here since threshold is not meaningful,
31313204
// as it's not what drives cost-benefit analysis.
@@ -3168,6 +3241,13 @@ InlineResult llvm::isInlineViable(Function &F) {
31683241

31693242
// Disallow recursive calls.
31703243
Function *Callee = Call->getCalledFunction();
3244+
// This function is called when we have "alwaysinline" attribute.
3245+
// If we allowed the inlining here given that the recursive inlining is
3246+
// allowed, then there will be problem in the second trial of inlining,
3247+
// because the Inliner pass allow only one time inlining and then it
3248+
// inserts "noinline" attribute which will be in conflict with the
3249+
// attribute of "alwaysinline" so, "alwaysinline" for recursive function
3250+
// will be disallowed to avoid conflict of attributes.
31713251
if (&F == Callee)
31723252
return InlineResult::failure("recursive call");
31733253

‎llvm/test/Transforms/Inline/inline-recursive-fn.ll‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
2+
; RUN: opt -S -passes='inline,instcombine' < %s | FileCheck %s
3+
4+
define float @inline_rec_true_successor(float %x, float %scale) {
5+
; CHECK-LABEL: define float @inline_rec_true_successor(
6+
; CHECK-SAME: float [[X:%.*]], float [[SCALE:%.*]]) {
7+
; CHECK-NEXT: [[ENTRY:.*:]]
8+
; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[X]], 0.000000e+00
9+
; CHECK-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
10+
; CHECK: [[COMMON_RET18:.*]]:
11+
; CHECK-NEXT: [[COMMON_RET18_OP:%.*]] = phi float [ [[COMMON_RET18_OP_I:%.*]], %[[INLINE_REC_TRUE_SUCCESSOR_EXIT:.*]] ], [ [[MUL:%.*]], %[[IF_END]] ]
12+
; CHECK-NEXT: ret float [[COMMON_RET18_OP]]
13+
; CHECK: [[IF_THEN]]:
14+
; CHECK-NEXT: br i1 false, label %[[IF_THEN_I:.*]], label %[[IF_END_I:.*]]
15+
; CHECK: [[IF_THEN_I]]:
16+
; CHECK-NEXT: br label %[[INLINE_REC_TRUE_SUCCESSOR_EXIT]]
17+
; CHECK: [[IF_END_I]]:
18+
; CHECK-NEXT: [[FNEG:%.*]] = fneg float [[X]]
19+
; CHECK-NEXT: [[MUL_I:%.*]] = fmul float [[SCALE]], [[FNEG]]
20+
; CHECK-NEXT: br label %[[INLINE_REC_TRUE_SUCCESSOR_EXIT]]
21+
; CHECK: [[INLINE_REC_TRUE_SUCCESSOR_EXIT]]:
22+
; CHECK-NEXT: [[COMMON_RET18_OP_I]] = phi float [ poison, %[[IF_THEN_I]] ], [ [[MUL_I]], %[[IF_END_I]] ]
23+
; CHECK-NEXT: br label %[[COMMON_RET18]]
24+
; CHECK: [[IF_END]]:
25+
; CHECK-NEXT: [[MUL]] = fmul float [[X]], [[SCALE]]
26+
; CHECK-NEXT: br label %[[COMMON_RET18]]
27+
;
28+
entry:
29+
%cmp = fcmp olt float %x, 0.000000e+00
30+
br i1 %cmp, label %if.then, label %if.end
31+
32+
common.ret18: ; preds = %if.then, %if.end
33+
%common.ret18.op = phi float [ %call, %if.then ], [ %mul, %if.end ]
34+
ret float %common.ret18.op
35+
36+
if.then: ; preds = %entry
37+
%fneg = fneg float %x
38+
%call = tail call float @inline_rec_true_successor(float %fneg, float %scale)
39+
br label %common.ret18
40+
41+
if.end: ; preds = %entry
42+
%mul = fmul float %x, %scale
43+
br label %common.ret18
44+
}
45+
46+
define float @test_inline_rec_true_successor(float %x, float %scale) {
47+
entry:
48+
%res = tail call float @inline_rec_true_successor(float %x, float %scale)
49+
ret float %res
50+
}
51+
52+
; Same as previous test except that the recursive callsite is in the false successor
53+
define float @inline_rec_false_successor(float %x, float %scale) {
54+
; CHECK-LABEL: define float @inline_rec_false_successor(
55+
; CHECK-SAME: float [[Y:%.*]], float [[SCALE:%.*]]) {
56+
; CHECK-NEXT: [[ENTRY:.*:]]
57+
; CHECK-NEXT: [[CMP:%.*]] = fcmp uge float [[Y]], 0.000000e+00
58+
; CHECK-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
59+
; CHECK: [[COMMON_RET18:.*]]:
60+
; CHECK-NEXT: [[COMMON_RET18_OP:%.*]] = phi float [ [[MUL:%.*]], %[[IF_THEN]] ], [ [[COMMON_RET18_OP_I:%.*]], %[[INLINE_REC_FALSE_SUCCESSOR_EXIT:.*]] ]
61+
; CHECK-NEXT: ret float [[COMMON_RET18_OP]]
62+
; CHECK: [[IF_THEN]]:
63+
; CHECK-NEXT: [[MUL]] = fmul float [[Y]], [[SCALE]]
64+
; CHECK-NEXT: br label %[[COMMON_RET18]]
65+
; CHECK: [[IF_END]]:
66+
; CHECK-NEXT: br i1 true, label %[[IF_THEN_I:.*]], label %[[IF_END_I:.*]]
67+
; CHECK: [[IF_THEN_I]]:
68+
; CHECK-NEXT: [[FNEG:%.*]] = fneg float [[Y]]
69+
; CHECK-NEXT: [[MUL_I:%.*]] = fmul float [[SCALE]], [[FNEG]]
70+
; CHECK-NEXT: br label %[[INLINE_REC_FALSE_SUCCESSOR_EXIT]]
71+
; CHECK: [[IF_END_I]]:
72+
; CHECK-NEXT: br label %[[INLINE_REC_FALSE_SUCCESSOR_EXIT]]
73+
; CHECK: [[INLINE_REC_FALSE_SUCCESSOR_EXIT]]:
74+
; CHECK-NEXT: [[COMMON_RET18_OP_I]] = phi float [ [[MUL_I]], %[[IF_THEN_I]] ], [ poison, %[[IF_END_I]] ]
75+
; CHECK-NEXT: br label %[[COMMON_RET18]]
76+
;
77+
entry:
78+
%cmp = fcmp uge float %x, 0.000000e+00
79+
br i1 %cmp, label %if.then, label %if.end
80+
81+
common.ret18: ; preds = %if.then, %if.end
82+
%common.ret18.op = phi float [ %mul, %if.then ], [ %call, %if.end ]
83+
ret float %common.ret18.op
84+
85+
if.then: ; preds = %entry
86+
%mul = fmul float %x, %scale
87+
br label %common.ret18
88+
89+
if.end: ; preds = %entry
90+
%fneg = fneg float %x
91+
%call = tail call float @inline_rec_false_successor(float %fneg, float %scale)
92+
br label %common.ret18
93+
}
94+
95+
define float @test_inline_rec_false_successor(float %x, float %scale) {
96+
entry:
97+
%res = tail call float @inline_rec_false_successor(float %x, float %scale)
98+
ret float %res
99+
}
100+
101+
; Test when the BR has Value not cmp instruction
102+
define float @inline_rec_no_cmp(i1 %flag, float %scale) {
103+
; CHECK-LABEL: define float @inline_rec_no_cmp(
104+
; CHECK-SAME: i1 [[FLAG:%.*]], float [[SCALE:%.*]]) {
105+
; CHECK-NEXT: [[ENTRY:.*:]]
106+
; CHECK-NEXT: br i1 [[FLAG]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
107+
; CHECK: [[IF_THEN]]:
108+
; CHECK-NEXT: [[SUM:%.*]] = fadd float [[SCALE]], 5.000000e+00
109+
; CHECK-NEXT: [[SUM1:%.*]] = fadd float [[SUM]], [[SCALE]]
110+
; CHECK-NEXT: br label %[[COMMON_RET:.*]]
111+
; CHECK: [[IF_END]]:
112+
; CHECK-NEXT: [[SUM2:%.*]] = fadd float [[SCALE]], 5.000000e+00
113+
; CHECK-NEXT: br label %[[COMMON_RET]]
114+
; CHECK: [[COMMON_RET]]:
115+
; CHECK-NEXT: [[COMMON_RET_RES:%.*]] = phi float [ [[SUM1]], %[[IF_THEN]] ], [ [[SUM2]], %[[IF_END]] ]
116+
; CHECK-NEXT: ret float [[COMMON_RET_RES]]
117+
;
118+
entry:
119+
br i1 %flag, label %if.then, label %if.end
120+
if.then:
121+
%res = tail call float @inline_rec_no_cmp(i1 false, float %scale)
122+
%sum1 = fadd float %res, %scale
123+
br label %common.ret
124+
if.end:
125+
%sum2 = fadd float %scale, 5.000000e+00
126+
br label %common.ret
127+
common.ret:
128+
%common.ret.res = phi float [ %sum1, %if.then ], [ %sum2, %if.end ]
129+
ret float %common.ret.res
130+
}
131+
132+
define float @test_inline_rec_no_cmp(i1 %flag, float %scale) {
133+
entry:
134+
%res = tail call float @inline_rec_no_cmp(i1 %flag, float %scale)
135+
ret float %res
136+
}
137+
138+
define float @no_inline_rec(float %x, float %scale) {
139+
; CHECK-LABEL: define float @no_inline_rec(
140+
; CHECK-SAME: float [[Z:%.*]], float [[SCALE:%.*]]) {
141+
; CHECK-NEXT: [[ENTRY:.*:]]
142+
; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[Z]], 5.000000e+00
143+
; CHECK-NEXT: br i1 [[CMP]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
144+
; CHECK: [[COMMON_RET18:.*]]:
145+
; CHECK-NEXT: [[COMMON_RET18_OP:%.*]] = phi float [ [[FNEG1:%.*]], %[[IF_THEN]] ], [ [[MUL:%.*]], %[[IF_END]] ]
146+
; CHECK-NEXT: ret float [[COMMON_RET18_OP]]
147+
; CHECK: [[IF_THEN]]:
148+
; CHECK-NEXT: [[FADD:%.*]] = fadd float [[Z]], 5.000000e+00
149+
; CHECK-NEXT: [[CALL:%.*]] = tail call float @no_inline_rec(float [[FADD]], float [[SCALE]])
150+
; CHECK-NEXT: [[FNEG1]] = fneg float [[CALL]]
151+
; CHECK-NEXT: br label %[[COMMON_RET18]]
152+
; CHECK: [[IF_END]]:
153+
; CHECK-NEXT: [[MUL]] = fmul float [[Z]], [[SCALE]]
154+
; CHECK-NEXT: br label %[[COMMON_RET18]]
155+
;
156+
entry:
157+
%cmp = fcmp olt float %x, 5.000000e+00
158+
br i1 %cmp, label %if.then, label %if.end
159+
160+
common.ret18: ; preds = %if.then, %if.end
161+
%common.ret18.op = phi float [ %fneg1, %if.then ], [ %mul, %if.end ]
162+
ret float %common.ret18.op
163+
164+
if.then: ; preds = %entry
165+
%fadd = fadd float %x, 5.000000e+00
166+
%call = tail call float @no_inline_rec(float %fadd, float %scale)
167+
%fneg1 = fneg float %call
168+
br label %common.ret18
169+
170+
if.end: ; preds = %entry
171+
%mul = fmul float %x, %scale
172+
br label %common.ret18
173+
}
174+
175+
define float @test_no_inline(float %x, float %scale) {
176+
entry:
177+
%res = tail call float @no_inline_rec(float %x, float %scale)
178+
ret float %res
179+
}

‎llvm/test/Transforms/Inline/inline-remark.ll‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,6 @@ define void @test3() {
5656
}
5757

5858
; CHECK: attributes [[ATTR1]] = { "inline-remark"="(cost=25, threshold=0)" }
59-
; CHECK: attributes [[ATTR2]] = { "inline-remark"="(cost=never): recursive" }
59+
; CHECK: attributes [[ATTR2]] = { "inline-remark"="(cost=0, threshold=0)" }
6060
; CHECK: attributes [[ATTR3]] = { "inline-remark"="unsupported operand bundle; (cost={{.*}}, threshold={{.*}})" }
6161
; CHECK: attributes [[ATTR4]] = { alwaysinline "inline-remark"="(cost=never): recursive call" }

0 commit comments

Comments
 (0)