diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 02aa21e33bf4f..f0f85c706866b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -494,12 +494,7 @@ class InnerLoopVectorizer { MinProfitableTripCount(MinProfitableTripCount), UF(UnrollFactor), Builder(PSE.getSE()->getContext()), Legal(LVL), Cost(CM), BFI(BFI), PSI(PSI), RTChecks(RTChecks), Plan(Plan), - VectorPHVPB(Plan.getEntry()->getSingleSuccessor()) { - // Query this against the original loop and save it here because the profile - // of the original loop header may change as the transformation happens. - OptForSizeBasedOnProfile = llvm::shouldOptimizeForSize( - OrigLoop->getHeader(), PSI, BFI, PGSOQueryType::IRPass); - } + VectorPHVPB(Plan.getEntry()->getSingleSuccessor()) {} virtual ~InnerLoopVectorizer() = default; @@ -672,10 +667,6 @@ class InnerLoopVectorizer { BlockFrequencyInfo *BFI; ProfileSummaryInfo *PSI; - // Whether this loop should be optimized for size based on profile guided size - // optimizatios. - bool OptForSizeBasedOnProfile; - /// Structure to hold information about generated runtime checks, responsible /// for cleaning the checks, if vectorization turns out unprofitable. GeneratedRTChecks &RTChecks; @@ -987,13 +978,18 @@ class LoopVectorizationCostModel { AssumptionCache *AC, OptimizationRemarkEmitter *ORE, const Function *F, const LoopVectorizeHints *Hints, - InterleavedAccessInfo &IAI) + InterleavedAccessInfo &IAI, + ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) : ScalarEpilogueStatus(SEL), TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), DB(DB), AC(AC), ORE(ORE), TheFunction(F), Hints(Hints), InterleaveInfo(IAI) { if (TTI.supportsScalableVectors() || ForceTargetSupportsScalableVectors) initializeVScaleForTuning(); CostKind = F->hasMinSize() ? TTI::TCK_CodeSize : TTI::TCK_RecipThroughput; + // Query this against the original loop and save it here because the profile + // of the original loop header may change as the transformation happens. + OptForSize = llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI, + PGSOQueryType::IRPass); } /// \return An upper bound for the vectorization factors (both fixed and @@ -1830,6 +1826,10 @@ class LoopVectorizationCostModel { /// The kind of cost that we are calculating TTI::TargetCostKind CostKind; + + /// Whether this loop should be optimized for size based on function attribute + /// or profile information. + bool OptForSize; }; } // end namespace llvm @@ -2602,9 +2602,8 @@ BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) { if (!SCEVCheckBlock) return nullptr; - assert(!(SCEVCheckBlock->getParent()->hasOptSize() || - (OptForSizeBasedOnProfile && - Cost->Hints->getForce() != LoopVectorizeHints::FK_Enabled)) && + assert((!Cost->OptForSize || + Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled) && "Cannot SCEV check stride or overflow when optimizing for size"); assert(!LoopBypassBlocks.empty() && "Should already be a bypass block due to iteration count check"); @@ -2629,7 +2628,7 @@ BasicBlock *InnerLoopVectorizer::emitMemRuntimeChecks(BasicBlock *Bypass) { assert((!EnableVPlanNativePath || OrigLoop->begin() == OrigLoop->end()) && "Runtime checks are not supported for outer loops yet"); - if (MemCheckBlock->getParent()->hasOptSize() || OptForSizeBasedOnProfile) { + if (Cost->OptForSize) { assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled && "Cannot emit memory checks when optimizing for size, unless forced " "to vectorize."); @@ -10083,7 +10082,7 @@ static bool processLoopInVPlanNativePath( getScalarEpilogueLowering(F, L, Hints, PSI, BFI, TTI, TLI, *LVL, &IAI); LoopVectorizationCostModel CM(SEL, L, PSE, LI, LVL, *TTI, TLI, DB, AC, ORE, F, - &Hints, IAI); + &Hints, IAI, PSI, BFI); // Use the planner for outer loop vectorization. // TODO: CM is not used at this point inside the planner. Turn CM into an // optional argument if we don't need it in the future. @@ -10620,7 +10619,7 @@ bool LoopVectorizePass::processLoop(Loop *L) { // Use the cost model. LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, DB, AC, ORE, - F, &Hints, IAI); + F, &Hints, IAI, PSI, BFI); // Use the planner for vectorization. LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, IAI, PSE, Hints, ORE);