Skip to content

Commit 5225c47

Browse files
ronliebbcahoon
authored andcommitted
Reland:
[SLP][NFC]Add missing finalize params in the CostEstimator, NFC. Prepare functions for generalization of codegen/cost estimation. Update revert_patches.txt Change-Id: Iedf59193b2b9c567c2da3c75e030ff798d9548a7
1 parent 005d151 commit 5225c47

File tree

2 files changed

+82
-19
lines changed

2 files changed

+82
-19
lines changed

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 68 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7135,7 +7135,6 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
71357135
// into a vector and can be represented as a permutation elements in a
71367136
// single input vector or of 2 input vectors.
71377137
Cost += computeExtractCost(VL, Mask, ShuffleKind);
7138-
InVectors.assign(1, E);
71397138
return VecBase;
71407139
}
71417140
void add(const TreeEntry *E1, const TreeEntry *E2, ArrayRef<int> Mask) {
@@ -7146,18 +7145,57 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
71467145
CommonMask.assign(Mask.begin(), Mask.end());
71477146
InVectors.assign(1, E1);
71487147
}
7149-
void gather(ArrayRef<Value *> VL, Value *Root = nullptr) {
7148+
/// Adds another one input vector and the mask for the shuffling.
7149+
void add(Value *V1, ArrayRef<int> Mask) {
7150+
assert(CommonMask.empty() && InVectors.empty() &&
7151+
"Expected empty input mask/vectors.");
7152+
CommonMask.assign(Mask.begin(), Mask.end());
7153+
InVectors.assign(1, V1);
7154+
}
7155+
Value *gather(ArrayRef<Value *> VL, Value *Root = nullptr) {
71507156
Cost += getBuildVectorCost(VL, Root);
71517157
if (!Root) {
71527158
assert(InVectors.empty() && "Unexpected input vectors for buildvector.");
71537159
// FIXME: Need to find a way to avoid use of getNullValue here.
7154-
InVectors.assign(1, Constant::getNullValue(FixedVectorType::get(
7155-
VL.front()->getType(), VL.size())));
7160+
SmallVector<Constant *> Vals;
7161+
for (Value *V : VL) {
7162+
if (isa<UndefValue>(V)) {
7163+
Vals.push_back(cast<Constant>(V));
7164+
continue;
7165+
}
7166+
Vals.push_back(Constant::getNullValue(V->getType()));
7167+
}
7168+
return ConstantVector::get(Vals);
71567169
}
7170+
return ConstantVector::getSplat(
7171+
ElementCount::getFixed(VL.size()),
7172+
Constant::getNullValue(VL.front()->getType()));
71577173
}
71587174
/// Finalize emission of the shuffles.
7159-
InstructionCost finalize(ArrayRef<int> ExtMask) {
7175+
InstructionCost
7176+
finalize(ArrayRef<int> ExtMask, unsigned VF = 0,
7177+
function_ref<void(Value *&, SmallVectorImpl<int> &)> Action = {}) {
71607178
IsFinalized = true;
7179+
if (Action) {
7180+
const PointerUnion<Value *, const TreeEntry *> &Vec = InVectors.front();
7181+
if (InVectors.size() == 2) {
7182+
Cost += createShuffle(Vec, InVectors.back(), CommonMask);
7183+
InVectors.pop_back();
7184+
} else {
7185+
Cost += createShuffle(Vec, nullptr, CommonMask);
7186+
}
7187+
for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx)
7188+
if (CommonMask[Idx] != PoisonMaskElem)
7189+
CommonMask[Idx] = Idx;
7190+
assert(VF > 0 &&
7191+
"Expected vector length for the final value before action.");
7192+
Value *V = Vec.dyn_cast<Value *>();
7193+
if (!Vec.isNull() && !V)
7194+
V = Constant::getNullValue(FixedVectorType::get(
7195+
Vec.get<const TreeEntry *>()->Scalars.front()->getType(),
7196+
CommonMask.size()));
7197+
Action(V, CommonMask);
7198+
}
71617199
::addMask(CommonMask, ExtMask, /*ExtendingManyInputs=*/true);
71627200
if (CommonMask.empty())
71637201
return Cost;
@@ -7291,18 +7329,31 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
72917329
Estimator.add(Entries.front(), Mask);
72927330
else
72937331
Estimator.add(Entries.front(), Entries.back(), Mask);
7294-
Estimator.gather(
7295-
GatheredScalars,
7296-
Constant::getNullValue(FixedVectorType::get(
7297-
GatheredScalars.front()->getType(), GatheredScalars.size())));
7298-
return Estimator.finalize(E->ReuseShuffleIndices);
7299-
}
7300-
Estimator.gather(
7301-
GatheredScalars,
7302-
VL.equals(GatheredScalars)
7303-
? nullptr
7304-
: Constant::getNullValue(FixedVectorType::get(
7305-
GatheredScalars.front()->getType(), GatheredScalars.size())));
7332+
if (all_of(GatheredScalars, PoisonValue ::classof))
7333+
return Estimator.finalize(E->ReuseShuffleIndices);
7334+
return Estimator.finalize(
7335+
E->ReuseShuffleIndices, E->Scalars.size(),
7336+
[&](Value *&Vec, SmallVectorImpl<int> &Mask) {
7337+
Vec = Estimator.gather(GatheredScalars,
7338+
Constant::getNullValue(FixedVectorType::get(
7339+
GatheredScalars.front()->getType(),
7340+
GatheredScalars.size())));
7341+
});
7342+
}
7343+
if (!all_of(GatheredScalars, PoisonValue::classof)) {
7344+
auto Gathers = ArrayRef(GatheredScalars).take_front(VL.size());
7345+
bool SameGathers = VL.equals(Gathers);
7346+
Value *BV = Estimator.gather(
7347+
Gathers, SameGathers ? nullptr
7348+
: Constant::getNullValue(FixedVectorType::get(
7349+
GatheredScalars.front()->getType(),
7350+
GatheredScalars.size())));
7351+
SmallVector<int> ReuseMask(Gathers.size(), PoisonMaskElem);
7352+
std::iota(ReuseMask.begin(), ReuseMask.end(), 0);
7353+
Estimator.add(BV, ReuseMask);
7354+
}
7355+
if (ExtractShuffle)
7356+
Estimator.add(E, std::nullopt);
73067357
return Estimator.finalize(E->ReuseShuffleIndices);
73077358
}
73087359
InstructionCost CommonCost = 0;

revert_patches.txt

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,15 @@
1+
ee2a35ac4ab [X86] Add AVX-NE-CONVERT instruc
2+
Revert "[X86] Support -march=sierraforest, grandridge,
3+
Revert "[AMDGPU] Enable OMod on more VOP3 instructions" # rocrand assert fails
4+
5+
#break nekbone, cxx20 offsetof
6+
e327b52766ed [C2x] reject type definitions in offsetof
7+
8+
7f12efa88 [X86][FP16] Lower half->i16 into vcvttph2[u]w directly
9+
10+
FP16
11+
Revert "[X86][MemFold] Update some records for X86MemFoldTables.inc"
12+
113
SROA breaks MIOpen
214
Revert "[SROA] Create additional vector type candidates based on store and load slices"
315
Revert "[SROA] Fix bug where CandidateTys is appended while being iterated"
@@ -12,8 +24,8 @@ Johannes: breakage
1224
Revert "[OpenMP] Disable early vectorization of loads/stores in the runtime "
1325
asserts in snap and MI-Teams
1426

15-
Nicole and Saiyed:
16-
Revert " [OpenMP][DeviceRTL][AMDGPU] Support code object version 5"
27+
28+
Revert Complete the implementation of P2361 Unevaluated string literals
1729

1830
Revert "[Parser] Parse string literal arguments of 'avail"
1931

0 commit comments

Comments
 (0)