@@ -39,38 +39,43 @@ const SCEV *getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE);
3939
4040// / Returns true if \p VPV is uniform after vectorization.
4141inline bool isUniformAfterVectorization (const VPValue *VPV) {
42- auto IsKnownUniformOpcode = [](auto *R) -> bool {
43- return Instruction::isBinaryOp (R->getOpcode ()) ||
44- Instruction::isCast (R->getOpcode ()) ||
45- R->getOpcode () == Instruction::GetElementPtr ||
46- R->getOpcode () == Instruction::ICmp ||
47- R->getOpcode () == Instruction::FCmp;
42+ auto PreservesUniformity = [](unsigned Opcode) -> bool {
43+ if (Instruction::isBinaryOp (Opcode) || Instruction::isCast (Opcode))
44+ return true ;
45+ switch (Opcode) {
46+ case Instruction::GetElementPtr:
47+ case Instruction::ICmp:
48+ case Instruction::FCmp:
49+ case VPInstruction::Broadcast:
50+ case VPInstruction::PtrAdd:
51+ return true ;
52+ default :
53+ return false ;
54+ }
4855 };
4956
50- // A value defined outside the vector region must be uniform after
51- // vectorization inside a vector region.
57+ // A live-in must be uniform across the scope of VPlan.
5258 if (VPV->isLiveIn ())
5359 return true ;
60+
5461 if (auto *Rep = dyn_cast<VPReplicateRecipe>(VPV)) {
55- const VPRegionBlock *ParentR = Rep->getParent ()->getParent ();
62+ const VPRegionBlock *RegionOfR = Rep->getParent ()->getParent ();
63+ if (RegionOfR && RegionOfR->isReplicator ())
64+ return false ;
5665 return Rep->isUniform () ||
57- (IsKnownUniformOpcode (Rep) &&
58- (!ParentR || !ParentR->isReplicator ()) &&
66+ (PreservesUniformity (Rep->getOpcode ()) &&
5967 all_of (Rep->operands (), isUniformAfterVectorization));
6068 }
61-
6269 if (isa<VPWidenGEPRecipe, VPDerivedIVRecipe, VPBlendRecipe>(VPV))
6370 return all_of (VPV->getDefiningRecipe ()->operands (),
6471 isUniformAfterVectorization);
6572 if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
66- return IsKnownUniformOpcode (WidenR) &&
73+ return PreservesUniformity (WidenR-> getOpcode () ) &&
6774 all_of (WidenR->operands (), isUniformAfterVectorization);
6875 }
6976 if (auto *VPI = dyn_cast<VPInstruction>(VPV))
7077 return VPI->isSingleScalar () || VPI->isVectorToScalar () ||
71- ((IsKnownUniformOpcode (VPI) ||
72- VPI->getOpcode () == VPInstruction::Broadcast ||
73- VPI->getOpcode () == VPInstruction::PtrAdd) &&
78+ (PreservesUniformity (VPI->getOpcode ()) &&
7479 all_of (VPI->operands (), isUniformAfterVectorization));
7580
7681 // VPExpandSCEVRecipes must be placed in the entry and are alway uniform.
0 commit comments