@@ -9539,10 +9539,9 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
95399539 const TargetLibraryInfo &TLI,
95409540 const InstructionsState &S,
95419541 const BoUpSLP::EdgeInfo &UserTreeIdx,
9542- bool DoNotFail ) {
9542+ bool TryPad ) {
95439543 // Check that every instruction appears once in this bundle.
95449544 SmallVector<Value *> UniqueValues;
9545- SmallVector<Value *> NonUniqueValueVL;
95469545 SmallDenseMap<Value *, unsigned, 16> UniquePositions(VL.size());
95479546 for (Value *V : VL) {
95489547 if (isConstant(V)) {
@@ -9578,27 +9577,32 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
95789577 (UniquePositions.size() == 1 && all_of(UniqueValues, [](Value *V) {
95799578 return isa<UndefValue>(V) || !isConstant(V);
95809579 }))) {
9581- if (DoNotFail && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 &&
9580+ if (TryPad && UniquePositions.size() > 1 && NumUniqueScalarValues > 1 &&
95829581 S.getMainOp()->isSafeToRemove() &&
95839582 all_of(UniqueValues, IsaPred<Instruction, PoisonValue>)) {
95849583 // Find the number of elements, which forms full vectors.
95859584 unsigned PWSz = getFullVectorNumberOfElements(
95869585 TTI, UniqueValues.front()->getType(), UniqueValues.size());
95879586 PWSz = std::min<unsigned>(PWSz, VL.size());
95889587 if (PWSz == VL.size()) {
9588+ // We ended up with the same size after removing duplicates and
9589+ // upgrading the resulting vector size to a "nice size". Just keep
9590+ // the initial VL then.
95899591 ReuseShuffleIndices.clear();
95909592 } else {
9591- NonUniqueValueVL.assign(UniqueValues.begin(), UniqueValues.end());
9592- NonUniqueValueVL.append(
9593+ // Pad unique values with poison to grow the vector to a "nice" size
9594+ SmallVector<Value *> PaddedUniqueValues(UniqueValues.begin(),
9595+ UniqueValues.end());
9596+ PaddedUniqueValues.append(
95939597 PWSz - UniqueValues.size(),
95949598 PoisonValue::get(UniqueValues.front()->getType()));
95959599 // Check that extended with poisons operations are still valid for
95969600 // vectorization (div/rem are not allowed).
9597- if (!getSameOpcode(NonUniqueValueVL , TLI).valid()) {
9601+ if (!getSameOpcode(PaddedUniqueValues , TLI).valid()) {
95989602 LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
95999603 return false;
96009604 }
9601- VL = NonUniqueValueVL ;
9605+ VL = std::move(PaddedUniqueValues) ;
96029606 }
96039607 return true;
96049608 }
@@ -10014,9 +10018,9 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VL, unsigned Depth,
1001410018 SmallVector<int> ReuseShuffleIndices;
1001510019 SmallVector<Value *> NonUniqueValueVL(VL.begin(), VL.end());
1001610020 auto TryToFindDuplicates = [&](const InstructionsState &S,
10017- bool DoNotFail = false) {
10021+ bool TryPad = false) {
1001810022 if (tryToFindDuplicates(NonUniqueValueVL, ReuseShuffleIndices, *TTI, *TLI,
10019- S, UserTreeIdx, DoNotFail )) {
10023+ S, UserTreeIdx, TryPad )) {
1002010024 VL = NonUniqueValueVL;
1002110025 return true;
1002210026 }
@@ -10082,7 +10086,7 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VL, unsigned Depth,
1008210086 return;
1008310087
1008410088 // Check that every instruction appears once in this bundle.
10085- if (!TryToFindDuplicates(S, /*DoNotFail =*/true))
10089+ if (!TryToFindDuplicates(S, /*TryPad =*/true))
1008610090 return;
1008710091
1008810092 // Perform specific checks for each particular instruction kind.
0 commit comments