diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 57b7358049bce..5375d2be9c875 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -355,6 +355,16 @@ cl::opt EnableVPlanNativePath( "enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization.")); + +cl::opt + VerifyEachVPlan("vplan-verify-each", +#ifdef EXPENSIVE_CHECKS + cl::init(true), +#else + cl::init(false), +#endif + cl::Hidden, + cl::desc("Verfiy VPlans after VPlan transforms.")); } // namespace llvm // This flag enables the stress testing of the VPlan H-CFG construction in the @@ -7651,8 +7661,8 @@ DenseMap LoopVectorizationPlanner::executePlan( // TODO: Move to VPlan transform stage once the transition to the VPlan-based // cost model is complete for better cost estimates. - VPlanTransforms::unrollByUF(BestVPlan, BestUF, - OrigLoop->getHeader()->getContext()); + VPlanTransforms::runPass(VPlanTransforms::unrollByUF, BestVPlan, BestUF, + OrigLoop->getHeader()->getContext()); VPlanTransforms::optimizeForVFAndUF(BestVPlan, BestVF, BestUF, PSE); VPlanTransforms::convertToConcreteRecipes(BestVPlan); @@ -8908,13 +8918,14 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, if (auto Plan = tryToBuildVPlanWithVPRecipes(SubRange)) { // Now optimize the initial VPlan. if (!Plan->hasVF(ElementCount::getFixed(1))) - VPlanTransforms::truncateToMinimalBitwidths(*Plan, - CM.getMinimalBitwidths()); + VPlanTransforms::runPass(VPlanTransforms::truncateToMinimalBitwidths, + *Plan, CM.getMinimalBitwidths()); VPlanTransforms::optimize(*Plan); // TODO: try to put it close to addActiveLaneMask(). // Discard the plan if it is not EVL-compatible - if (CM.foldTailWithEVL() && !VPlanTransforms::tryAddExplicitVectorLength( - *Plan, CM.getMaxSafeElements())) + if (CM.foldTailWithEVL() && + !VPlanTransforms::runPass(VPlanTransforms::tryAddExplicitVectorLength, + *Plan, CM.getMaxSafeElements())) break; assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid"); VPlans.push_back(std::move(Plan)); @@ -9423,8 +9434,9 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { // Interleave memory: for each Interleave Group we marked earlier as relevant // for this VPlan, replace the Recipes widening its memory instructions with a // single VPInterleaveRecipe at its insertion point. - VPlanTransforms::createInterleaveGroups( - *Plan, InterleaveGroups, RecipeBuilder, CM.isScalarEpilogueAllowed()); + VPlanTransforms::runPass(VPlanTransforms::createInterleaveGroups, *Plan, + InterleaveGroups, RecipeBuilder, + CM.isScalarEpilogueAllowed()); for (ElementCount VF : Range) Plan->addVF(VF); @@ -9466,13 +9478,16 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { } } - VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) { + auto BlockNeedsPredication = [this](BasicBlock *BB) { return Legal->blockNeedsPredication(BB); - }); + }; + VPlanTransforms::runPass(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan, + BlockNeedsPredication); // Sink users of fixed-order recurrence past the recipe defining the previous // value and introduce FirstOrderRecurrenceSplice VPInstructions. - if (!VPlanTransforms::adjustFixedOrderRecurrences(*Plan, Builder)) + if (!VPlanTransforms::runPass(VPlanTransforms::adjustFixedOrderRecurrences, + *Plan, Builder)) return nullptr; if (useActiveLaneMask(Style)) { @@ -9815,10 +9830,10 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue())); } } - - VPlanTransforms::clearReductionWrapFlags(*Plan); for (VPRecipeBase *R : ToDelete) R->eraseFromParent(); + + VPlanTransforms::runPass(VPlanTransforms::clearReductionWrapFlags, *Plan); } void VPDerivedIVRecipe::execute(VPTransformState &State) { @@ -10182,7 +10197,7 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) { VPIRInst->eraseFromParent(); ResumePhi->eraseFromParent(); } - VPlanTransforms::removeDeadRecipes(MainPlan); + VPlanTransforms::runPass(VPlanTransforms::removeDeadRecipes, MainPlan); using namespace VPlanPatternMatch; VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader(); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 35da93ee3b407..52d15c535276a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -52,6 +52,7 @@ using namespace llvm::VPlanPatternMatch; namespace llvm { extern cl::opt EnableVPlanNativePath; } + extern cl::opt ForceTargetInstructionCost; static cl::opt PrintVPlansInDotFormat( diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 714250a56ff57..242dc8a636a6f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -19,6 +19,7 @@ #include "VPlanDominatorTree.h" #include "VPlanPatternMatch.h" #include "VPlanUtils.h" +#include "VPlanVerifier.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" @@ -964,10 +965,10 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) { /// Try to simplify the recipes in \p Plan. Use \p CanonicalIVTy as type for all /// un-typed live-ins in VPTypeAnalysis. -static void simplifyRecipes(VPlan &Plan, Type *CanonicalIVTy) { +static void simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) { ReversePostOrderTraversal> RPOT( Plan.getEntry()); - VPTypeAnalysis TypeInfo(CanonicalIVTy); + VPTypeAnalysis TypeInfo(&CanonicalIVTy); for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly(RPOT)) { for (VPRecipeBase &R : make_early_inc_range(*VPBB)) { simplifyRecipe(R, TypeInfo); @@ -1029,7 +1030,7 @@ void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, VPBlockUtils::connectBlocks(Preheader, Header); VPBlockUtils::connectBlocks(ExitingVPBB, Exit); - simplifyRecipes(Plan, CanIVTy); + simplifyRecipes(Plan, *CanIVTy); } else { // The vector region contains header phis for which we cannot remove the // loop region yet. @@ -1439,19 +1440,19 @@ void VPlanTransforms::truncateToMinimalBitwidths( } void VPlanTransforms::optimize(VPlan &Plan) { - removeRedundantCanonicalIVs(Plan); - removeRedundantInductionCasts(Plan); - - simplifyRecipes(Plan, Plan.getCanonicalIV()->getScalarType()); - removeDeadRecipes(Plan); - legalizeAndOptimizeInductions(Plan); - removeRedundantExpandSCEVRecipes(Plan); - simplifyRecipes(Plan, Plan.getCanonicalIV()->getScalarType()); - removeDeadRecipes(Plan); - - createAndOptimizeReplicateRegions(Plan); - mergeBlocksIntoPredecessors(Plan); - licm(Plan); + runPass(removeRedundantCanonicalIVs, Plan); + runPass(removeRedundantInductionCasts, Plan); + + runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType()); + runPass(removeDeadRecipes, Plan); + runPass(legalizeAndOptimizeInductions, Plan); + runPass(removeRedundantExpandSCEVRecipes, Plan); + runPass(simplifyRecipes, Plan, *Plan.getCanonicalIV()->getScalarType()); + runPass(removeDeadRecipes, Plan); + + runPass(createAndOptimizeReplicateRegions, Plan); + runPass(mergeBlocksIntoPredecessors, Plan); + runPass(licm, Plan); } // Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace @@ -1871,7 +1872,8 @@ bool VPlanTransforms::tryAddExplicitVectorLength( } void VPlanTransforms::dropPoisonGeneratingRecipes( - VPlan &Plan, function_ref BlockNeedsPredication) { + VPlan &Plan, + const std::function &BlockNeedsPredication) { // Collect recipes in the backward slice of `Root` that may generate a poison // value that is used after vectorization. SmallPtrSet Visited; @@ -1971,7 +1973,7 @@ void VPlanTransforms::createInterleaveGroups( VPlan &Plan, const SmallPtrSetImpl *> &InterleaveGroups, - VPRecipeBuilder &RecipeBuilder, bool ScalarEpilogueAllowed) { + VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed) { if (InterleaveGroups.empty()) return; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index b31fef5d62456..ad24d9f146822 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -14,7 +14,9 @@ #define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H #include "VPlan.h" +#include "VPlanVerifier.h" #include "llvm/ADT/STLFunctionalExtras.h" +#include "llvm/Support/CommandLine.h" namespace llvm { @@ -27,7 +29,29 @@ class TargetLibraryInfo; class VPBuilder; class VPRecipeBuilder; +extern cl::opt VerifyEachVPlan; + struct VPlanTransforms { + /// Helper to run a VPlan transform \p Transform on \p VPlan, forwarding extra + /// arguments to the transform. Returns the boolean returned by the transform. + template + static bool runPass(bool (*Transform)(VPlan &, ArgsTy...), VPlan &Plan, + typename std::remove_reference::type &...Args) { + bool Res = Transform(Plan, Args...); + if (VerifyEachVPlan) + verifyVPlanIsValid(Plan); + return Res; + } + /// Helper to run a VPlan transform \p Transform on \p VPlan, forwarding extra + /// arguments to the transform. + template + static void runPass(void (*Fn)(VPlan &, ArgsTy...), VPlan &Plan, + typename std::remove_reference::type &...Args) { + Fn(Plan, Args...); + if (VerifyEachVPlan) + verifyVPlanIsValid(Plan); + } + /// Replaces the VPInstructions in \p Plan with corresponding /// widen recipes. static void @@ -100,7 +124,8 @@ struct VPlanTransforms { /// TODO: Replace BlockNeedsPredication callback with retrieving info from /// VPlan directly. static void dropPoisonGeneratingRecipes( - VPlan &Plan, function_ref BlockNeedsPredication); + VPlan &Plan, + const std::function &BlockNeedsPredication); /// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and /// replaces all uses except the canonical IV increment of @@ -119,7 +144,7 @@ struct VPlanTransforms { VPlan &Plan, const SmallPtrSetImpl *> &InterleaveGroups, - VPRecipeBuilder &RecipeBuilder, bool ScalarEpilogueAllowed); + VPRecipeBuilder &RecipeBuilder, const bool &ScalarEpilogueAllowed); /// Remove dead recipes from \p Plan. static void removeDeadRecipes(VPlan &Plan);