Skip to content
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9744,7 +9744,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
Legal->getUncountableEarlyExitingBlock()) {
VPlanTransforms::runPass(VPlanTransforms::handleUncountableEarlyExit, *Plan,
*PSE.getSE(), OrigLoop, UncountableExitingBlock,
RecipeBuilder);
RecipeBuilder, Range);
}
DenseMap<VPValue *, VPValue *> IVEndValues;
addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
Expand Down
6 changes: 4 additions & 2 deletions llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanUtils.h"
#include "VPlanVerifier.h"
Expand Down Expand Up @@ -2380,7 +2381,8 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {

void VPlanTransforms::handleUncountableEarlyExit(
VPlan &Plan, ScalarEvolution &SE, Loop *OrigLoop,
BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder) {
BasicBlock *UncountableExitingBlock, VPRecipeBuilder &RecipeBuilder,
const VFRange &Range) {
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
auto *LatchVPBB = cast<VPBasicBlock>(LoopRegion->getExiting());
VPBuilder Builder(LatchVPBB->getTerminator());
Expand Down Expand Up @@ -2436,7 +2438,7 @@ void VPlanTransforms::handleUncountableEarlyExit(
ExitIRI->extractLastLaneOfOperand(MiddleBuilder);
}
// Add the incoming value from the early exit.
if (!IncomingFromEarlyExit->isLiveIn() && !Plan.hasScalarVFOnly()) {
if (!IncomingFromEarlyExit->isLiveIn() && !Range.Start.isScalar()) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One more thing; should it additionally check that the Range only has a single VF (i.e. distance(start, end) == 1)?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will need to clamp the range to only contain the scalar VF, otherwise we may end up with a plan that is missing the extract and VF > 1

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done, thanks!

VPValue *FirstActiveLane = EarlyExitB.createNaryOp(
VPInstruction::FirstActiveLane, {EarlyExitTakenCond}, nullptr,
"first.active.lane");
Expand Down
4 changes: 3 additions & 1 deletion llvm/lib/Transforms/Vectorize/VPlanTransforms.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class PredicatedScalarEvolution;
class TargetLibraryInfo;
class VPBuilder;
class VPRecipeBuilder;
class VFRange;

extern cl::opt<bool> VerifyEachVPlan;

Expand Down Expand Up @@ -174,7 +175,8 @@ struct VPlanTransforms {
static void handleUncountableEarlyExit(VPlan &Plan, ScalarEvolution &SE,
Loop *OrigLoop,
BasicBlock *UncountableExitingBlock,
VPRecipeBuilder &RecipeBuilder);
VPRecipeBuilder &RecipeBuilder,
const VFRange &Range);

/// Lower abstract recipes to concrete ones, that can be codegen'd.
static void convertToConcreteRecipes(VPlan &Plan);
Expand Down
Loading