Skip to content

Commit 1dac302

Browse files
authored
[LV] Explicitly disallow interleaved access requiring gap mask for scalable VFs. nfc (#154122)
Currently, VPInterleaveRecipe::execute does not support generating LLVM IR for interleaved accesses that require a gap mask for scalable VFs. It would be better to detect and prevent such groups from being vectorized as interleaved accesses in LoopVectorizationCostModel::interleavedAccessCanBeWidened, rather than relying on the TTI function getInterleavedMemoryOpCost to return an invalid cost.
1 parent fb4450c commit 1dac302

File tree

2 files changed

+8
-2
lines changed

2 files changed

+8
-2
lines changed

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3095,6 +3095,12 @@ bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
30953095
if (Group->isReverse())
30963096
return false;
30973097

3098+
// TODO: Support interleaved access that requires a gap mask for scalable VFs.
3099+
bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
3100+
StoreAccessWithGapsRequiresMasking;
3101+
if (VF.isScalable() && NeedsMaskForGaps)
3102+
return false;
3103+
30983104
auto *Ty = getLoadStoreType(I);
30993105
const Align Alignment = getLoadStoreAlignment(I);
31003106
unsigned AS = getLoadStoreAddressSpace(I);

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3496,6 +3496,8 @@ static Value *interleaveVectors(IRBuilderBase &Builder, ArrayRef<Value *> Vals,
34963496
// store <12 x i32> %interleaved.vec ; Write 4 tuples of R,G,B
34973497
void VPInterleaveRecipe::execute(VPTransformState &State) {
34983498
assert(!State.Lane && "Interleave group being replicated.");
3499+
assert((!NeedsMaskForGaps || !State.VF.isScalable()) &&
3500+
"Masking gaps for scalable vectors is not yet supported.");
34993501
const InterleaveGroup<Instruction> *Group = IG;
35003502
Instruction *Instr = Group->getInsertPos();
35013503

@@ -3613,8 +3615,6 @@ void VPInterleaveRecipe::execute(VPTransformState &State) {
36133615
createBitMaskForGaps(State.Builder, State.VF.getKnownMinValue(), *Group);
36143616
assert(((MaskForGaps != nullptr) == NeedsMaskForGaps) &&
36153617
"Mismatch between NeedsMaskForGaps and MaskForGaps");
3616-
assert((!MaskForGaps || !State.VF.isScalable()) &&
3617-
"masking gaps for scalable vectors is not yet supported.");
36183618
ArrayRef<VPValue *> StoredValues = getStoredValues();
36193619
// Collect the stored vector from each member.
36203620
SmallVector<Value *, 4> StoredVecs;

0 commit comments

Comments
 (0)