diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h index 93b0a8d3cef04..ae86f353f16d9 100644 --- a/llvm/include/llvm/Analysis/InlineCost.h +++ b/llvm/include/llvm/Analysis/InlineCost.h @@ -336,7 +336,11 @@ LLVM_ABI std::optional getInliningCostFeatures( ProfileSummaryInfo *PSI = nullptr, OptimizationRemarkEmitter *ORE = nullptr); -/// Minimal filter to detect invalid constructs for inlining. +/// Check if it is mechanically possible to inline the function \p Callee, based +/// on the contents of the function. +/// +/// See also \p CanInlineCallSite as an additional precondition necessary to +/// perform a valid inline in a particular use context. LLVM_ABI InlineResult isInlineViable(Function &Callee); // This pass is used to annotate instructions during the inline process for diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 6b56230a6e1d4..cfa06a5be79fd 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -279,6 +279,9 @@ class InlineFunctionInfo { /// `InlinedCalls` above is used. SmallVector InlinedCallSites; + Value *ConvergenceControlToken = nullptr; + Instruction *CallSiteEHPad = nullptr; + /// Update profile for callee as well as cloned version. We need to do this /// for regular inlining, but not for inlining from sample profile loader. bool UpdateProfile; @@ -287,9 +290,36 @@ class InlineFunctionInfo { StaticAllocas.clear(); InlinedCalls.clear(); InlinedCallSites.clear(); + ConvergenceControlToken = nullptr; + CallSiteEHPad = nullptr; } }; +/// Check if it is legal to perform inlining of the function called by \p CB +/// into the caller at this particular use, and sets fields in \p IFI. +/// +/// This does not consider whether it is possible for the function callee itself +/// to be inlined; for that see isInlineViable. +LLVM_ABI InlineResult CanInlineCallSite(const CallBase &CB, + InlineFunctionInfo &IFI); + +/// This should generally not be used, use InlineFunction instead. +/// +/// Perform mechanical inlining of \p CB into the caller. +/// +/// This does not perform any legality or profitability checks for the +/// inlining. This assumes that CanInlineCallSite was already called, populated +/// \p IFI, and returned InlineResult::success. +/// +/// Also assumes that isInlineViable returned InlineResult::success for the +/// called function. +LLVM_ABI void InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI, + bool MergeAttributes = false, + AAResults *CalleeAAR = nullptr, + bool InsertLifetime = true, + Function *ForwardVarArgsTo = nullptr, + OptimizationRemarkEmitter *ORE = nullptr); + /// This function inlines the called function into the basic /// block of the caller. This returns false if it is not possible to inline /// this call. The program is still in a well defined state if this occurs @@ -328,7 +358,8 @@ LLVM_ABI InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes = false, AAResults *CalleeAAR = nullptr, bool InsertLifetime = true, - Function *ForwardVarArgsTo = nullptr); + Function *ForwardVarArgsTo = nullptr, + OptimizationRemarkEmitter *ORE = nullptr); /// Clones a loop \p OrigLoop. Returns the loop and the blocks in \p /// Blocks. diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index ed3dca2f7c307..59a47a9bb0d3e 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -2361,15 +2361,13 @@ remapIndices(Function &Caller, BasicBlock *StartBB, // Updating the contextual profile after an inlining means, at a high level, // copying over the data of the callee, **intentionally without any value // scaling**, and copying over the callees of the inlined callee. -llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, - PGOContextualProfile &CtxProf, - bool MergeAttributes, - AAResults *CalleeAAR, - bool InsertLifetime, - Function *ForwardVarArgsTo) { +llvm::InlineResult llvm::InlineFunction( + CallBase &CB, InlineFunctionInfo &IFI, PGOContextualProfile &CtxProf, + bool MergeAttributes, AAResults *CalleeAAR, bool InsertLifetime, + Function *ForwardVarArgsTo, OptimizationRemarkEmitter *ORE) { if (!CtxProf.isInSpecializedModule()) return InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime, - ForwardVarArgsTo); + ForwardVarArgsTo, ORE); auto &Caller = *CB.getCaller(); auto &Callee = *CB.getCalledFunction(); @@ -2387,7 +2385,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, const auto NumCalleeCallsites = CtxProf.getNumCallsites(Callee); auto Ret = InlineFunction(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime, - ForwardVarArgsTo); + ForwardVarArgsTo, ORE); if (!Ret.isSuccess()) return Ret; @@ -2457,20 +2455,8 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, return Ret; } -/// This function inlines the called function into the basic block of the -/// caller. This returns false if it is not possible to inline this call. -/// The program is still in a well defined state if this occurs though. -/// -/// Note that this only does one level of inlining. For example, if the -/// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now -/// exists in the instruction stream. Similarly this will inline a recursive -/// function by one level. -llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, - bool MergeAttributes, - AAResults *CalleeAAR, - bool InsertLifetime, - Function *ForwardVarArgsTo, - OptimizationRemarkEmitter *ORE) { +llvm::InlineResult llvm::CanInlineCallSite(const CallBase &CB, + InlineFunctionInfo &IFI) { assert(CB.getParent() && CB.getFunction() && "Instruction not in function!"); // FIXME: we don't inline callbr yet. @@ -2487,7 +2473,6 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, // The inliner does not know how to inline through calls with operand bundles // in general ... - Value *ConvergenceControlToken = nullptr; if (CB.hasOperandBundles()) { for (int i = 0, e = CB.getNumOperandBundles(); i != e; ++i) { auto OBUse = CB.getOperandBundleAt(i); @@ -2503,7 +2488,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, if (Tag == LLVMContext::OB_kcfi) continue; if (Tag == LLVMContext::OB_convergencectrl) { - ConvergenceControlToken = OBUse.Inputs[0].get(); + IFI.ConvergenceControlToken = OBUse.Inputs[0].get(); continue; } @@ -2521,28 +2506,22 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, // fully implements convergence control tokens, there is no mixing of // controlled and uncontrolled convergent operations in the whole program. if (CB.isConvergent()) { - if (!ConvergenceControlToken && + if (!IFI.ConvergenceControlToken && getConvergenceEntry(CalledFunc->getEntryBlock())) { return InlineResult::failure( "convergent call needs convergencectrl operand"); } } - // If the call to the callee cannot throw, set the 'nounwind' flag on any - // calls that we inline. - bool MarkNoUnwind = CB.doesNotThrow(); - - BasicBlock *OrigBB = CB.getParent(); - Function *Caller = OrigBB->getParent(); + const BasicBlock *OrigBB = CB.getParent(); + const Function *Caller = OrigBB->getParent(); // GC poses two hazards to inlining, which only occur when the callee has GC: // 1. If the caller has no GC, then the callee's GC must be propagated to the // caller. // 2. If the caller has a differing GC, it is invalid to inline. if (CalledFunc->hasGC()) { - if (!Caller->hasGC()) - Caller->setGC(CalledFunc->getGC()); - else if (CalledFunc->getGC() != Caller->getGC()) + if (Caller->hasGC() && CalledFunc->getGC() != Caller->getGC()) return InlineResult::failure("incompatible GC"); } @@ -2560,34 +2539,31 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, ? Caller->getPersonalityFn()->stripPointerCasts() : nullptr; if (CalledPersonality) { - if (!CallerPersonality) - Caller->setPersonalityFn(CalledPersonality); // If the personality functions match, then we can perform the // inlining. Otherwise, we can't inline. // TODO: This isn't 100% true. Some personality functions are proper // supersets of others and can be used in place of the other. - else if (CalledPersonality != CallerPersonality) + if (CallerPersonality && CalledPersonality != CallerPersonality) return InlineResult::failure("incompatible personality"); } // We need to figure out which funclet the callsite was in so that we may // properly nest the callee. - Instruction *CallSiteEHPad = nullptr; if (CallerPersonality) { EHPersonality Personality = classifyEHPersonality(CallerPersonality); if (isScopedEHPersonality(Personality)) { std::optional ParentFunclet = CB.getOperandBundle(LLVMContext::OB_funclet); if (ParentFunclet) - CallSiteEHPad = cast(ParentFunclet->Inputs.front()); + IFI.CallSiteEHPad = cast(ParentFunclet->Inputs.front()); // OK, the inlining site is legal. What about the target function? - if (CallSiteEHPad) { + if (IFI.CallSiteEHPad) { if (Personality == EHPersonality::MSVC_CXX) { // The MSVC personality cannot tolerate catches getting inlined into // cleanup funclets. - if (isa(CallSiteEHPad)) { + if (isa(IFI.CallSiteEHPad)) { // Ok, the call site is within a cleanuppad. Let's check the callee // for catchpads. for (const BasicBlock &CalledBB : *CalledFunc) { @@ -2607,13 +2583,34 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, } } + return InlineResult::success(); +} + +/// This function inlines the called function into the basic block of the +/// caller. This returns false if it is not possible to inline this call. +/// The program is still in a well defined state if this occurs though. +/// +/// Note that this only does one level of inlining. For example, if the +/// instruction 'call B' is inlined, and 'B' calls 'C', then the call to 'C' now +/// exists in the instruction stream. Similarly this will inline a recursive +/// function by one level. +void llvm::InlineFunctionImpl(CallBase &CB, InlineFunctionInfo &IFI, + bool MergeAttributes, AAResults *CalleeAAR, + bool InsertLifetime, Function *ForwardVarArgsTo, + OptimizationRemarkEmitter *ORE) { + BasicBlock *OrigBB = CB.getParent(); + Function *Caller = OrigBB->getParent(); + Function *CalledFunc = CB.getCalledFunction(); + assert(CalledFunc && !CalledFunc->isDeclaration() && + "CanInlineCallSite should have verified direct call to definition"); + // Determine if we are dealing with a call in an EHPad which does not unwind // to caller. bool EHPadForCallUnwindsLocally = false; - if (CallSiteEHPad && isa(CB)) { + if (IFI.CallSiteEHPad && isa(CB)) { UnwindDestMemoTy FuncletUnwindMap; Value *CallSiteUnwindDestToken = - getUnwindDestToken(CallSiteEHPad, FuncletUnwindMap); + getUnwindDestToken(IFI.CallSiteEHPad, FuncletUnwindMap); EHPadForCallUnwindsLocally = CallSiteUnwindDestToken && @@ -2630,6 +2627,30 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, ClonedCodeInfo InlinedFunctionInfo; Function::iterator FirstNewBlock; + // GC poses two hazards to inlining, which only occur when the callee has GC: + // 1. If the caller has no GC, then the callee's GC must be propagated to the + // caller. + // 2. If the caller has a differing GC, it is invalid to inline. + if (CalledFunc->hasGC()) { + if (!Caller->hasGC()) + Caller->setGC(CalledFunc->getGC()); + else { + assert(CalledFunc->getGC() == Caller->getGC() && + "CanInlineCallSite should have verified compatible GCs"); + } + } + + if (CalledFunc->hasPersonalityFn()) { + Constant *CalledPersonality = + CalledFunc->getPersonalityFn()->stripPointerCasts(); + if (!Caller->hasPersonalityFn()) { + Caller->setPersonalityFn(CalledPersonality); + } else + assert(Caller->getPersonalityFn()->stripPointerCasts() == + CalledPersonality && + "CanInlineCallSite should have verified compatible personality"); + } + { // Scope to destroy VMap after cloning. ValueToValueMapTy VMap; struct ByValInit { @@ -2819,10 +2840,10 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, IFI.GetAssumptionCache(*Caller).registerAssumption(II); } - if (ConvergenceControlToken) { + if (IFI.ConvergenceControlToken) { IntrinsicInst *IntrinsicCall = getConvergenceEntry(*FirstNewBlock); if (IntrinsicCall) { - IntrinsicCall->replaceAllUsesWith(ConvergenceControlToken); + IntrinsicCall->replaceAllUsesWith(IFI.ConvergenceControlToken); IntrinsicCall->eraseFromParent(); } } @@ -2869,6 +2890,10 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, } } + // If the call to the callee cannot throw, set the 'nounwind' flag on any + // calls that we inline. + bool MarkNoUnwind = CB.doesNotThrow(); + SmallVector VarArgsToForward; SmallVector VarArgsAttrs; for (unsigned i = CalledFunc->getFunctionType()->getNumParams(); @@ -3055,12 +3080,12 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, // Update the lexical scopes of the new funclets and callsites. // Anything that had 'none' as its parent is now nested inside the callsite's // EHPad. - if (CallSiteEHPad) { + if (IFI.CallSiteEHPad) { for (Function::iterator BB = FirstNewBlock->getIterator(), E = Caller->end(); BB != E; ++BB) { // Add bundle operands to inlined call sites. - PropagateOperandBundles(BB, CallSiteEHPad); + PropagateOperandBundles(BB, IFI.CallSiteEHPad); // It is problematic if the inlinee has a cleanupret which unwinds to // caller and we inline it into a call site which doesn't unwind but into @@ -3076,11 +3101,11 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, if (auto *CatchSwitch = dyn_cast(I)) { if (isa(CatchSwitch->getParentPad())) - CatchSwitch->setParentPad(CallSiteEHPad); + CatchSwitch->setParentPad(IFI.CallSiteEHPad); } else { auto *FPI = cast(I); if (isa(FPI->getParentPad())) - FPI->setParentPad(CallSiteEHPad); + FPI->setParentPad(IFI.CallSiteEHPad); } } } @@ -3236,7 +3261,7 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, AttributeFuncs::mergeAttributesForInlining(*Caller, *CalledFunc); // We are now done with the inlining. - return InlineResult::success(); + return; } // Otherwise, we have the normal case, of more than one block to inline or @@ -3404,6 +3429,19 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, if (MergeAttributes) AttributeFuncs::mergeAttributesForInlining(*Caller, *CalledFunc); +} - return InlineResult::success(); +llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, + bool MergeAttributes, + AAResults *CalleeAAR, + bool InsertLifetime, + Function *ForwardVarArgsTo, + OptimizationRemarkEmitter *ORE) { + llvm::InlineResult Result = CanInlineCallSite(CB, IFI); + if (Result.isSuccess()) { + InlineFunctionImpl(CB, IFI, MergeAttributes, CalleeAAR, InsertLifetime, + ForwardVarArgsTo, ORE); + } + + return Result; }