Skip to content

Commit a834320

Browse files
sjoerdmeijersjoerdmeijer
committed
[AArch64] Set MaxInterleaving to 4 for Neoverse V2
1 parent acfa8a0 commit a834320

File tree

13 files changed

+172
-5
lines changed

13 files changed

+172
-5
lines changed

‎llvm/include/llvm/Analysis/TargetTransformInfo.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -626,6 +626,10 @@ class TargetTransformInfo {
626626
AssumptionCache &AC, TargetLibraryInfo *LibInfo,
627627
HardwareLoopInfo &HWLoopInfo) const;
628628

629+
// Query the target for which minimum vectorization factor epilogue
630+
// vectorization should be considered.
631+
unsigned getEpilogueVectorizationMinVF() const;
632+
629633
/// Query the target whether it would be prefered to create a predicated
630634
/// vector loop, which can avoid the need to emit a scalar epilogue loop.
631635
bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const;
@@ -1865,6 +1869,7 @@ class TargetTransformInfo::Concept {
18651869
AssumptionCache &AC,
18661870
TargetLibraryInfo *LibInfo,
18671871
HardwareLoopInfo &HWLoopInfo) = 0;
1872+
virtual unsigned getEpilogueVectorizationMinVF() = 0;
18681873
virtual bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) = 0;
18691874
virtual TailFoldingStyle
18701875
getPreferredTailFoldingStyle(bool IVUpdateMayOverflow = true) = 0;
@@ -2319,6 +2324,9 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
23192324
HardwareLoopInfo &HWLoopInfo) override {
23202325
return Impl.isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
23212326
}
2327+
unsigned getEpilogueVectorizationMinVF() override {
2328+
return Impl.getEpilogueVectorizationMinVF();
2329+
}
23222330
bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) override {
23232331
return Impl.preferPredicateOverEpilogue(TFI);
23242332
}

‎llvm/include/llvm/Analysis/TargetTransformInfoImpl.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,8 @@ class TargetTransformInfoImplBase {
192192
return false;
193193
}
194194

195+
unsigned getEpilogueVectorizationMinVF() const { return 16; }
196+
195197
bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) const { return false; }
196198

197199
TailFoldingStyle

‎llvm/include/llvm/CodeGen/BasicTTIImpl.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -667,6 +667,10 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
667667
return BaseT::isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
668668
}
669669

670+
unsigned getEpilogueVectorizationMinVF() {
671+
return BaseT::getEpilogueVectorizationMinVF();
672+
}
673+
670674
bool preferPredicateOverEpilogue(TailFoldingInfo *TFI) {
671675
return BaseT::preferPredicateOverEpilogue(TFI);
672676
}

‎llvm/lib/Analysis/TargetTransformInfo.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -352,6 +352,10 @@ bool TargetTransformInfo::isHardwareLoopProfitable(
352352
return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo);
353353
}
354354

355+
unsigned TargetTransformInfo::getEpilogueVectorizationMinVF() const {
356+
return TTIImpl->getEpilogueVectorizationMinVF();
357+
}
358+
355359
bool TargetTransformInfo::preferPredicateOverEpilogue(
356360
TailFoldingInfo *TFI) const {
357361
return TTIImpl->preferPredicateOverEpilogue(TFI);

‎llvm/lib/Target/AArch64/AArch64Subtarget.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,12 +234,13 @@ void AArch64Subtarget::initializeProperties(bool HasMinSize) {
234234
MaxBytesForLoopAlignment = 16;
235235
break;
236236
case NeoverseV2:
237-
// Specialize cost for Neoverse-V2.
237+
case NeoverseV3:
238+
EpilogueVectorizationMinVF = 8;
239+
MaxInterleaveFactor = 4;
238240
ScatterOverhead = 13;
239241
LLVM_FALLTHROUGH;
240242
case NeoverseN2:
241243
case NeoverseN3:
242-
case NeoverseV3:
243244
PrefFunctionAlignment = Align(16);
244245
PrefLoopAlignment = Align(32);
245246
MaxBytesForLoopAlignment = 16;

‎llvm/lib/Target/AArch64/AArch64Subtarget.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
5656
bool ATTRIBUTE = DEFAULT;
5757
#include "AArch64GenSubtargetInfo.inc"
5858

59+
unsigned EpilogueVectorizationMinVF = 16;
5960
uint8_t MaxInterleaveFactor = 2;
6061
uint8_t VectorInsertExtractBaseCost = 2;
6162
uint16_t CacheLineSize = 0;
@@ -225,6 +226,9 @@ class AArch64Subtarget final : public AArch64GenSubtargetInfo {
225226
hasFuseAdrpAdd() || hasFuseLiterals();
226227
}
227228

229+
unsigned getEpilogueVectorizationMinVF() const {
230+
return EpilogueVectorizationMinVF;
231+
}
228232
unsigned getMaxInterleaveFactor() const { return MaxInterleaveFactor; }
229233
unsigned getVectorInsertExtractBaseCost() const;
230234
unsigned getCacheLineSize() const override { return CacheLineSize; }

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4581,6 +4581,10 @@ static bool containsDecreasingPointers(Loop *TheLoop,
45814581
return false;
45824582
}
45834583

4584+
unsigned AArch64TTIImpl::getEpilogueVectorizationMinVF() const {
4585+
return ST->getEpilogueVectorizationMinVF();
4586+
}
4587+
45844588
bool AArch64TTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) {
45854589
if (!ST->hasSVE())
45864590
return false;

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,8 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {
376376
return ST->useFixedOverScalableIfEqualCost();
377377
}
378378

379+
unsigned getEpilogueVectorizationMinVF() const;
380+
379381
bool preferPredicateOverEpilogue(TailFoldingInfo *TFI);
380382

381383
bool supportsScalableVectors() const {

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ static cl::opt<unsigned> EpilogueVectorizationForceVF(
185185
"loops."));
186186

187187
static cl::opt<unsigned> EpilogueVectorizationMinVF(
188-
"epilogue-vectorization-minimum-VF", cl::init(16), cl::Hidden,
188+
"epilogue-vectorization-minimum-VF", cl::Hidden,
189189
cl::desc("Only loops with vectorization factor equal to or larger than "
190190
"the specified value are considered for epilogue vectorization."));
191191

@@ -4644,7 +4644,11 @@ bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
46444644
if (TTI.getMaxInterleaveFactor(VF) <= 1)
46454645
return false;
46464646

4647-
if ((Multiplier * VF.getKnownMinValue()) >= EpilogueVectorizationMinVF)
4647+
unsigned MinVFThreshold = EpilogueVectorizationMinVF.getNumOccurrences() > 0
4648+
? EpilogueVectorizationMinVF
4649+
: TTI.getEpilogueVectorizationMinVF();
4650+
4651+
if ((Multiplier * VF.getKnownMinValue()) >= MinVFThreshold)
46484652
return true;
46494653
return false;
46504654
}

‎llvm/test/Transforms/LoopVectorize/AArch64/interleaving-load-store.ll‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
; RUN: opt -passes=loop-vectorize -mtriple=arm64-apple-macos -mcpu=apple-a14 -S %s | FileCheck --check-prefix=INTERLEAVE-4 %s
66
; RUN: opt -passes=loop-vectorize -mtriple=arm64-apple-macos -mcpu=apple-a15 -S %s | FileCheck --check-prefix=INTERLEAVE-4 %s
77
; RUN: opt -passes=loop-vectorize -mtriple=arm64-apple-macos -mcpu=apple-a16 -S %s | FileCheck --check-prefix=INTERLEAVE-4 %s
8+
; RUN: opt -passes=loop-vectorize -mtriple=arm64 -mcpu=neoverse-v2 -S %s | FileCheck --check-prefix=INTERLEAVE-4 %s
9+
; RUN: opt -passes=loop-vectorize -mtriple=arm64 -mcpu=neoverse-v3 -S %s | FileCheck --check-prefix=INTERLEAVE-4-VLA %s
810

911
; Tests for selecting interleave counts for loops with loads and stores.
1012

@@ -213,6 +215,12 @@ define void @interleave_single_load_store(ptr %src, ptr %dst, i64 %N, i8 %a, i8
213215
; INTERLEAVE-2: exit:
214216
; INTERLEAVE-2-NEXT: ret void
215217
;
218+
; INTERLEAVE-4-VLA-LABEL: @interleave_single_load_store(
219+
; INTERLEAVE-4-VLA: call <vscale x 16 x i8> @llvm.smax.nxv16i8(
220+
; INTERLEAVE-4-VLA-NEXT: call <vscale x 16 x i8> @llvm.smax.nxv16i8(
221+
; INTERLEAVE-4-VLA-NEXT: call <vscale x 16 x i8> @llvm.smax.nxv16i8(
222+
; INTERLEAVE-4-VLA-NEXT: call <vscale x 16 x i8> @llvm.smax.nxv16i8(
223+
;
216224
entry:
217225
br label %loop
218226

0 commit comments

Comments
 (0)