Skip to content

Commit de30701

Browse files
fhahnfhahn
committed
[LV] Vectorize maxnum/minnum w/o fast-math flags.
Update LV to vectorize maxnum/minnum reductions without fast-math flags, by adding an extra check in the loop if any inputs to maxnum/minnum are NaN. If any input is NaN, *exit the vector loop, *compute the reduction result up to the vector iteration that contained NaN inputs and * resume in the scalar loop New recurrence kinds are added for reductions using maxnum/minnum without fast-math flags. The new recurrence kinds are not supported in the code to generate IR to perform the reductions to prevent accidential mis-use. Users need to add the required checks ensuring no NaN inputs, and convert to regular FMin/FMax recurrence kinds.
1 parent cfdd5ca commit de30701

16 files changed

+497
-65
lines changed

‎llvm/include/llvm/Analysis/IVDescriptors.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ enum class RecurKind {
4747
FMul, ///< Product of floats.
4848
FMin, ///< FP min implemented in terms of select(cmp()).
4949
FMax, ///< FP max implemented in terms of select(cmp()).
50+
FMinNumNoFMFs, ///< FP min with llvm.minnum semantics and no fast-math flags.
51+
FMaxNumNoFMFs, ///< FP max with llvm.maxnumsemantics and no fast-math flags.
5052
FMinimum, ///< FP min with llvm.minimum semantics
5153
FMaximum, ///< FP max with llvm.maximum semantics
5254
FMinimumNum, ///< FP min with llvm.minimumnum semantics

‎llvm/lib/Analysis/IVDescriptors.cpp

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -941,10 +941,27 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
941941
m_Intrinsic<Intrinsic::minimumnum>(m_Value(), m_Value())) ||
942942
match(I, m_Intrinsic<Intrinsic::maximumnum>(m_Value(), m_Value()));
943943
};
944-
if (isIntMinMaxRecurrenceKind(Kind) ||
945-
(HasRequiredFMF() && isFPMinMaxRecurrenceKind(Kind)))
944+
if (isIntMinMaxRecurrenceKind(Kind))
946945
return isMinMaxPattern(I, Kind, Prev);
947-
else if (isFMulAddIntrinsic(I))
946+
if (isFPMinMaxRecurrenceKind(Kind)) {
947+
if (HasRequiredFMF())
948+
return isMinMaxPattern(I, Kind, Prev);
949+
// We may be able to vectorize FMax/FMin reductions using maxnum/minnum
950+
// intrinsics with extra checks ensuring the inputs are not NaN.
951+
auto *StartV = dyn_cast<ConstantFP>(
952+
OrigPhi->getIncomingValueForBlock(L->getLoopPredecessor()));
953+
if (StartV && !StartV->getValue().isNaN() &&
954+
isMinMaxPattern(I, Kind, Prev).isRecurrence()) {
955+
if (((Kind == RecurKind::FMax &&
956+
match(I, m_Intrinsic<Intrinsic::maxnum>(m_Value(), m_Value()))) ||
957+
Kind == RecurKind::FMaxNumNoFMFs))
958+
return InstDesc(I, RecurKind::FMaxNumNoFMFs);
959+
if (((Kind == RecurKind::FMin &&
960+
match(I, m_Intrinsic<Intrinsic::minnum>(m_Value(), m_Value()))) ||
961+
Kind == RecurKind::FMinNumNoFMFs))
962+
return InstDesc(I, RecurKind::FMinNumNoFMFs);
963+
}
964+
} else if (isFMulAddIntrinsic(I))
948965
return InstDesc(Kind == RecurKind::FMulAdd, I,
949966
I->hasAllowReassoc() ? nullptr : I);
950967
return InstDesc(false, I);

‎llvm/lib/Transforms/Utils/LoopUtils.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -938,8 +938,10 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
938938
case RecurKind::UMin:
939939
return Intrinsic::vector_reduce_umin;
940940
case RecurKind::FMax:
941+
case RecurKind::FMaxNumNoFMFs:
941942
return Intrinsic::vector_reduce_fmax;
942943
case RecurKind::FMin:
944+
case RecurKind::FMinNumNoFMFs:
943945
return Intrinsic::vector_reduce_fmin;
944946
case RecurKind::FMaximum:
945947
return Intrinsic::vector_reduce_fmaximum;

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4347,8 +4347,15 @@ bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
43474347
ElementCount VF) const {
43484348
// Cross iteration phis such as reductions need special handling and are
43494349
// currently unsupported.
4350-
if (any_of(OrigLoop->getHeader()->phis(),
4351-
[&](PHINode &Phi) { return Legal->isFixedOrderRecurrence(&Phi); }))
4350+
if (any_of(OrigLoop->getHeader()->phis(), [&](PHINode &Phi) {
4351+
if (Legal->isReductionVariable(&Phi)) {
4352+
RecurKind RK =
4353+
Legal->getRecurrenceDescriptor(&Phi).getRecurrenceKind();
4354+
return RK == RecurKind::FMinNumNoFMFs ||
4355+
RK == RecurKind::FMaxNumNoFMFs;
4356+
}
4357+
return Legal->isFixedOrderRecurrence(&Phi);
4358+
}))
43524359
return false;
43534360

43544361
// Phis with uses outside of the loop require special handling and are
@@ -8816,6 +8823,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
88168823

88178824
// Adjust the recipes for any inloop reductions.
88188825
adjustRecipesForReductions(Plan, RecipeBuilder, Range.Start);
8826+
if (!VPlanTransforms::runPass(
8827+
VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath, *Plan))
8828+
return nullptr;
88198829

88208830
// Transform recipes to abstract recipes if it is legal and beneficial and
88218831
// clamp the range for better cost estimation.

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23196,6 +23196,8 @@ class HorizontalReduction {
2319623196
case RecurKind::FindFirstIVUMin:
2319723197
case RecurKind::FindLastIVSMax:
2319823198
case RecurKind::FindLastIVUMax:
23199+
case RecurKind::FMaxNumNoFMFs:
23200+
case RecurKind::FMinNumNoFMFs:
2319923201
case RecurKind::FMaximumNum:
2320023202
case RecurKind::FMinimumNum:
2320123203
case RecurKind::None:
@@ -23333,6 +23335,8 @@ class HorizontalReduction {
2333323335
case RecurKind::FindFirstIVUMin:
2333423336
case RecurKind::FindLastIVSMax:
2333523337
case RecurKind::FindLastIVUMax:
23338+
case RecurKind::FMaxNumNoFMFs:
23339+
case RecurKind::FMinNumNoFMFs:
2333623340
case RecurKind::FMaximumNum:
2333723341
case RecurKind::FMinimumNum:
2333823342
case RecurKind::None:
@@ -23435,6 +23439,8 @@ class HorizontalReduction {
2343523439
case RecurKind::FindFirstIVUMin:
2343623440
case RecurKind::FindLastIVSMax:
2343723441
case RecurKind::FindLastIVUMax:
23442+
case RecurKind::FMaxNumNoFMFs:
23443+
case RecurKind::FMinNumNoFMFs:
2343823444
case RecurKind::FMaximumNum:
2343923445
case RecurKind::FMinimumNum:
2344023446
case RecurKind::None:

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1356,20 +1356,22 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
13561356
unsigned Opcode;
13571357

13581358
public:
1359-
VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
1360-
const VPIRFlags &Flags, DebugLoc DL)
1361-
: VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
1362-
Opcode(Opcode) {}
1363-
13641359
VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
13651360
: VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
13661361
Opcode(I.getOpcode()) {}
13671362

1363+
VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
1364+
const VPIRFlags &Flags, const VPIRMetadata &Metadata,
1365+
DebugLoc DL)
1366+
: VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
1367+
VPIRMetadata(Metadata), Opcode(Opcode) {}
1368+
13681369
~VPWidenRecipe() override = default;
13691370

13701371
VPWidenRecipe *clone() override {
1371-
auto *R = new VPWidenRecipe(*getUnderlyingInstr(), operands());
1372-
R->transferFlags(*this);
1372+
auto *R =
1373+
new VPWidenRecipe(getOpcode(), operands(), *this, *this, getDebugLoc());
1374+
R->setUnderlyingValue(getUnderlyingValue());
13731375
return R;
13741376
}
13751377

‎llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
8484
return ResTy;
8585
}
8686
case Instruction::ICmp:
87+
case Instruction::FCmp:
8788
case VPInstruction::ActiveLaneMask:
8889
assert(inferScalarType(R->getOperand(0)) ==
8990
inferScalarType(R->getOperand(1)) &&

‎llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -628,3 +628,140 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
628628
Term->addMetadata(LLVMContext::MD_prof, BranchWeights);
629629
}
630630
}
631+
632+
static VPValue *getMinMaxCompareValue(VPSingleDefRecipe *MinMaxOp,
633+
VPReductionPHIRecipe *RedPhi) {
634+
auto *RepR = dyn_cast<VPReplicateRecipe>(MinMaxOp);
635+
if (!isa<VPWidenIntrinsicRecipe>(MinMaxOp) &&
636+
!(RepR && (isa<IntrinsicInst>(RepR->getUnderlyingInstr()))))
637+
return nullptr;
638+
639+
if (MinMaxOp->getOperand(0) == RedPhi)
640+
return MinMaxOp->getOperand(1);
641+
return MinMaxOp->getOperand(0);
642+
}
643+
644+
/// Returns true if there VPlan is read-only and execution can be resumed at the
645+
/// beginning of the last vector iteration in the scalar loop
646+
static bool canResumeInScalarLoopFromVectorLoop(VPlan &Plan) {
647+
for (VPBlockBase *VPB : vp_depth_first_shallow(
648+
Plan.getVectorLoopRegion()->getEntryBasicBlock())) {
649+
auto *VPBB = dyn_cast<VPBasicBlock>(VPB);
650+
if (!VPBB)
651+
return false;
652+
for (auto &R : *VPBB) {
653+
if (match(&R, m_BranchOnCount(m_VPValue(), m_VPValue())))
654+
continue;
655+
if (R.mayWriteToMemory())
656+
return false;
657+
}
658+
}
659+
return true;
660+
}
661+
662+
bool VPlanTransforms::handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan) {
663+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
664+
VPValue *AnyNaN = nullptr;
665+
VPReductionPHIRecipe *RedPhiR = nullptr;
666+
VPRecipeWithIRFlags *MinMaxOp = nullptr;
667+
bool HasUnsupportedPhi = false;
668+
for (auto &R : LoopRegion->getEntryBasicBlock()->phis()) {
669+
HasUnsupportedPhi |=
670+
!isa<VPCanonicalIVPHIRecipe, VPWidenIntOrFpInductionRecipe,
671+
VPReductionPHIRecipe>(&R);
672+
auto *Cur = dyn_cast<VPReductionPHIRecipe>(&R);
673+
if (!Cur)
674+
continue;
675+
if (RedPhiR)
676+
return false;
677+
if (Cur->getRecurrenceKind() != RecurKind::FMaxNumNoFMFs &&
678+
Cur->getRecurrenceKind() != RecurKind::FMinNumNoFMFs)
679+
continue;
680+
681+
RedPhiR = Cur;
682+
MinMaxOp = dyn_cast<VPRecipeWithIRFlags>(
683+
RedPhiR->getBackedgeValue()->getDefiningRecipe());
684+
if (!MinMaxOp)
685+
return false;
686+
VPValue *In = getMinMaxCompareValue(MinMaxOp, RedPhiR);
687+
if (!In)
688+
return false;
689+
690+
auto *IsNaN =
691+
new VPInstruction(Instruction::FCmp, {In, In}, {CmpInst::FCMP_UNO}, {});
692+
IsNaN->insertBefore(MinMaxOp);
693+
AnyNaN = new VPInstruction(VPInstruction::AnyOf, {IsNaN});
694+
AnyNaN->getDefiningRecipe()->insertAfter(IsNaN);
695+
}
696+
697+
if (!AnyNaN)
698+
return true;
699+
700+
if (HasUnsupportedPhi || !canResumeInScalarLoopFromVectorLoop(Plan))
701+
return false;
702+
703+
auto *MiddleVPBB = Plan.getMiddleBlock();
704+
auto *RdxResult = dyn_cast<VPInstruction>(&MiddleVPBB->front());
705+
if (!RdxResult ||
706+
RdxResult->getOpcode() != VPInstruction::ComputeReductionResult ||
707+
RdxResult->getOperand(0) != RedPhiR)
708+
return false;
709+
710+
auto *ScalarPH = Plan.getScalarPreheader();
711+
// Update the resume phis in the scalar preheader. They all must either resume
712+
// from the reduction result or the canonical induction. Bail out if there are
713+
// other resume phis.
714+
for (auto &R : ScalarPH->phis()) {
715+
auto *ResumeR = cast<VPPhi>(&R);
716+
VPValue *VecV = ResumeR->getOperand(0);
717+
VPValue *BypassV = ResumeR->getOperand(ResumeR->getNumOperands() - 1);
718+
if (VecV != RdxResult && VecV != &Plan.getVectorTripCount())
719+
return false;
720+
ResumeR->setOperand(
721+
1, VecV == &Plan.getVectorTripCount() ? Plan.getCanonicalIV() : VecV);
722+
ResumeR->addOperand(BypassV);
723+
}
724+
725+
// Create a new reduction phi recipe with either FMin/FMax, replacing
726+
// FMinNumNoFMFs/FMaxNumNoFMFs.
727+
RecurKind NewRK = RedPhiR->getRecurrenceKind() != RecurKind::FMinNumNoFMFs
728+
? RecurKind::FMin
729+
: RecurKind::FMax;
730+
auto *NewRedPhiR = new VPReductionPHIRecipe(
731+
cast<PHINode>(RedPhiR->getUnderlyingValue()), NewRK,
732+
*RedPhiR->getStartValue(), RedPhiR->isInLoop(), RedPhiR->isOrdered());
733+
NewRedPhiR->addOperand(RedPhiR->getOperand(1));
734+
NewRedPhiR->insertBefore(RedPhiR);
735+
RedPhiR->replaceAllUsesWith(NewRedPhiR);
736+
RedPhiR->eraseFromParent();
737+
738+
// Update the loop exit condition to exit if either any of the inputs is NaN
739+
// or the vector trip count is reached.
740+
VPBasicBlock *LatchVPBB = LoopRegion->getExitingBasicBlock();
741+
VPBuilder Builder(LatchVPBB->getTerminator());
742+
auto *LatchExitingBranch = cast<VPInstruction>(LatchVPBB->getTerminator());
743+
assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCount &&
744+
"Unexpected terminator");
745+
auto *IsLatchExitTaken =
746+
Builder.createICmp(CmpInst::ICMP_EQ, LatchExitingBranch->getOperand(0),
747+
LatchExitingBranch->getOperand(1));
748+
auto *AnyExitTaken =
749+
Builder.createNaryOp(Instruction::Or, {AnyNaN, IsLatchExitTaken});
750+
Builder.createNaryOp(VPInstruction::BranchOnCond, AnyExitTaken);
751+
LatchExitingBranch->eraseFromParent();
752+
753+
// Split the middle block and introduce a new block, branching to the scalar
754+
// preheader to resume iteration in the scalar loop if any NaNs have been
755+
// encountered.
756+
MiddleVPBB->splitAt(std::prev(MiddleVPBB->end()));
757+
Builder.setInsertPoint(MiddleVPBB, MiddleVPBB->begin());
758+
auto *NewSel =
759+
Builder.createSelect(AnyNaN, NewRedPhiR, RdxResult->getOperand(1));
760+
RdxResult->setOperand(1, NewSel);
761+
Builder.setInsertPoint(MiddleVPBB);
762+
Builder.createNaryOp(VPInstruction::BranchOnCond, AnyNaN);
763+
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
764+
MiddleVPBB->swapSuccessors();
765+
std::swap(ScalarPH->getPredecessors()[1], ScalarPH->getPredecessors().back());
766+
return true;
767+
}

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
587587
Value *Op = State.get(getOperand(0), vputils::onlyFirstLaneUsed(this));
588588
return Builder.CreateFreeze(Op, Name);
589589
}
590+
case Instruction::FCmp:
590591
case Instruction::ICmp: {
591592
bool OnlyFirstLaneUsed = vputils::onlyFirstLaneUsed(this);
592593
Value *A = State.get(getOperand(0), OnlyFirstLaneUsed);
@@ -860,7 +861,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
860861
Value *Res = State.get(getOperand(0));
861862
for (VPValue *Op : drop_begin(operands()))
862863
Res = Builder.CreateOr(Res, State.get(Op));
863-
return Builder.CreateOrReduce(Res);
864+
return State.VF.isScalar() ? Res : Builder.CreateOrReduce(Res);
864865
}
865866
case VPInstruction::FirstActiveLane: {
866867
if (getNumOperands() == 1) {
@@ -1033,6 +1034,7 @@ bool VPInstruction::opcodeMayReadOrWriteFromMemory() const {
10331034
switch (getOpcode()) {
10341035
case Instruction::ExtractElement:
10351036
case Instruction::Freeze:
1037+
case Instruction::FCmp:
10361038
case Instruction::ICmp:
10371039
case Instruction::Select:
10381040
case VPInstruction::AnyOf:
@@ -1068,6 +1070,7 @@ bool VPInstruction::onlyFirstLaneUsed(const VPValue *Op) const {
10681070
return Op == getOperand(1);
10691071
case Instruction::PHI:
10701072
return true;
1073+
case Instruction::FCmp:
10711074
case Instruction::ICmp:
10721075
case Instruction::Select:
10731076
case Instruction::Or:
@@ -1100,6 +1103,7 @@ bool VPInstruction::onlyFirstPartUsed(const VPValue *Op) const {
11001103
switch (getOpcode()) {
11011104
default:
11021105
return false;
1106+
case Instruction::FCmp:
11031107
case Instruction::ICmp:
11041108
case Instruction::Select:
11051109
return vputils::onlyFirstPartUsed(this);
@@ -1786,7 +1790,7 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
17861790
return Opcode == Instruction::ZExt;
17871791
break;
17881792
case OperationType::Cmp:
1789-
return Opcode == Instruction::ICmp;
1793+
return Opcode == Instruction::FCmp || Opcode == Instruction::ICmp;
17901794
case OperationType::Other:
17911795
return true;
17921796
}

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,11 @@ struct VPlanTransforms {
9999
/// not valid.
100100
static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
101101

102+
/// Check if \p Plan contains any FMaxNumNoFMFs or FMinNumNoFMFs reductions.
103+
/// If they do, try to update the vector loop to exit early if any input is
104+
/// NaN and resume executing in the scalar loop to handle the NaNs there.
105+
static bool handleMaxMinNumReductionsWithoutFastMath(VPlan &Plan);
106+
102107
/// Clear NSW/NUW flags from reduction instructions if necessary.
103108
static void clearReductionWrapFlags(VPlan &Plan);
104109

0 commit comments

Comments
 (0)