From 6b78004592a9fc358a739ba28be5a93579a15923 Mon Sep 17 00:00:00 2001 From: Sam Tebbs Date: Wed, 28 May 2025 10:23:09 +0100 Subject: [PATCH 1/8] [LV] Fix MVE regression from #132190 Register pressure was only considered if the vector bandwidth was being maximised (chosen either by the target or user options), but #132190 inadvertently caused high pressure VFs to be pruned even when max bandwidth wasn't enabled. This PR returns to the previous behaviour. --- .../Transforms/Vectorize/LoopVectorize.cpp | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 427c1460fcfc9..83bc5cde95abf 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -956,6 +956,10 @@ class LoopVectorizationCostModel { return expectedCost(UserVF).isValid(); } + /// \return True if maximizing vector bandwidth is enabled by the target or + /// user options. + bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind); + /// \return The size (in bits) of the smallest and widest types in the code /// that needs to be vectorized. We ignore values that remain scalar such as /// 64 bit loop indices. @@ -3918,6 +3922,14 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) { return FixedScalableVFPair::getNone(); } +bool LoopVectorizationCostModel::useMaxBandwidth( + TargetTransformInfo::RegisterKind RegKind) { + return MaximizeBandwidth || (MaximizeBandwidth.getNumOccurrences() == 0 && + (TTI.shouldMaximizeVectorBandwidth(RegKind) || + (UseWiderVFIfCallVariantsPresent && + Legal->hasVectorCallVariants()))); +} + ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget( unsigned MaxTripCount, unsigned SmallestType, unsigned WidestType, ElementCount MaxSafeVF, bool FoldTailByMasking) { @@ -3983,10 +3995,7 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget( ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector : TargetTransformInfo::RGK_FixedWidthVector; ElementCount MaxVF = MaxVectorElementCount; - if (MaximizeBandwidth || - (MaximizeBandwidth.getNumOccurrences() == 0 && - (TTI.shouldMaximizeVectorBandwidth(RegKind) || - (UseWiderVFIfCallVariantsPresent && Legal->hasVectorCallVariants())))) { + if (useMaxBandwidth(RegKind)) { auto MaxVectorElementCountMaxBW = ElementCount::get( llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType), ComputeScalableMaxVF); @@ -4349,7 +4358,10 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() { /// Don't consider the VF if it exceeds the number of registers for the /// target. - if (RU.exceedsMaxNumRegs(TTI)) + if (CM.useMaxBandwidth(VF.isScalable() + ? TargetTransformInfo::RGK_ScalableVector + : TargetTransformInfo::RGK_FixedWidthVector) && + RU.exceedsMaxNumRegs(TTI)) continue; InstructionCost C = CM.expectedCost(VF); @@ -7119,7 +7131,10 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() { InstructionCost Cost = cost(*P, VF); VectorizationFactor CurrentFactor(VF, Cost, ScalarCost); - if (RU.exceedsMaxNumRegs(TTI)) { + if (CM.useMaxBandwidth(VF.isScalable() + ? TargetTransformInfo::RGK_ScalableVector + : TargetTransformInfo::RGK_FixedWidthVector) && + RU.exceedsMaxNumRegs(TTI)) { LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width " << VF << " because it uses too many registers\n"); continue; From 95543d1a90fe1f3aecdb486678c131b3b809b0cb Mon Sep 17 00:00:00 2001 From: Samuel Tebbs Date: Thu, 29 May 2025 13:46:02 +0100 Subject: [PATCH 2/8] Add test --- .../ARM/mve-reg-pressure-vmla.ll | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll new file mode 100644 index 0000000000000..667022e0470e4 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll @@ -0,0 +1,147 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:" --version 5 +; RUN: opt -passes=loop-vectorize < %s -S -o - | FileCheck %s + +source_filename = "" +target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64" +target triple = "thumbv8.1m.main-unknown-none-eabihf" + +; Even though it has high register pressure, this example should still vectorise since the mul+add chains become VMLAs. + +define void @fn(i32 noundef %n, ptr %in, ptr %out) #0 { +; CHECK-LABEL: define void @fn( +; CHECK-SAME: i32 noundef [[N:%.*]], ptr [[IN:%.*]], ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[CMP46_NOT:%.*]] = icmp eq i32 [[N]], 0 +; CHECK-NEXT: br i1 [[CMP46_NOT]], [[FOR_COND_CLEANUP:label %.*]], label %[[FOR_BODY_PREHEADER:.*]] +; CHECK: [[FOR_BODY_PREHEADER]]: +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; CHECK: [[VECTOR_MEMCHECK]]: +; CHECK-NEXT: [[TMP0:%.*]] = mul i32 [[N]], 3 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT]], i32 [[TMP0]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[IN]], i32 [[TMP0]] +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[OUT]], [[SCEVGEP1]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[IN]], [[SCEVGEP]] +; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 [[N]], 3 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[IN]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[POINTER_PHI2:%.*]] = phi ptr [ [[OUT]], %[[VECTOR_PH]] ], [ [[PTR_IND3:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <4 x i32> +; CHECK-NEXT: [[VECTOR_GEP4:%.*]] = getelementptr i8, ptr [[POINTER_PHI2]], <4 x i32> +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 [[INDEX]], i32 [[N]]) +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 1 +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[VECTOR_GEP]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0:![0-9]+]] +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP]], i32 2 +; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[TMP1]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER6:%.*]] = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> [[TMP2]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]], <4 x i8> poison), !alias.scope [[META0]] +; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[WIDE_MASKED_GATHER]] to <4 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], splat (i32 19595) +; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[WIDE_MASKED_GATHER5]] to <4 x i32> +; CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw <4 x i32> [[TMP5]], splat (i32 38470) +; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i8> [[WIDE_MASKED_GATHER6]] to <4 x i32> +; CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw <4 x i32> [[TMP7]], splat (i32 7471) +; CHECK-NEXT: [[TMP9:%.*]] = add nuw nsw <4 x i32> [[TMP4]], splat (i32 32768) +; CHECK-NEXT: [[TMP10:%.*]] = add nuw nsw <4 x i32> [[TMP9]], [[TMP6]] +; CHECK-NEXT: [[TMP11:%.*]] = add nuw nsw <4 x i32> [[TMP10]], [[TMP8]] +; CHECK-NEXT: [[TMP12:%.*]] = lshr <4 x i32> [[TMP11]], splat (i32 16) +; CHECK-NEXT: [[TMP13:%.*]] = trunc <4 x i32> [[TMP12]] to <4 x i8> +; CHECK-NEXT: [[TMP14:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], splat (i32 32767) +; CHECK-NEXT: [[TMP15:%.*]] = mul nuw <4 x i32> [[TMP5]], splat (i32 16762097) +; CHECK-NEXT: [[TMP16:%.*]] = mul nuw <4 x i32> [[TMP7]], splat (i32 16759568) +; CHECK-NEXT: [[TMP17:%.*]] = add nuw nsw <4 x i32> [[TMP14]], splat (i32 32768) +; CHECK-NEXT: [[TMP18:%.*]] = add nuw <4 x i32> [[TMP17]], [[TMP15]] +; CHECK-NEXT: [[TMP19:%.*]] = add <4 x i32> [[TMP18]], [[TMP16]] +; CHECK-NEXT: [[TMP20:%.*]] = lshr <4 x i32> [[TMP19]], splat (i32 16) +; CHECK-NEXT: [[TMP21:%.*]] = trunc <4 x i32> [[TMP20]] to <4 x i8> +; CHECK-NEXT: [[TMP22:%.*]] = mul nuw nsw <4 x i32> [[TMP3]], splat (i32 13282) +; CHECK-NEXT: [[TMP23:%.*]] = mul nuw <4 x i32> [[TMP5]], splat (i32 16744449) +; CHECK-NEXT: [[TMP24:%.*]] = mul nuw nsw <4 x i32> [[TMP7]], splat (i32 19485) +; CHECK-NEXT: [[TMP25:%.*]] = add nuw nsw <4 x i32> [[TMP22]], splat (i32 32768) +; CHECK-NEXT: [[TMP26:%.*]] = add nuw <4 x i32> [[TMP25]], [[TMP23]] +; CHECK-NEXT: [[TMP27:%.*]] = add nuw <4 x i32> [[TMP26]], [[TMP24]] +; CHECK-NEXT: [[TMP28:%.*]] = lshr <4 x i32> [[TMP27]], splat (i32 16) +; CHECK-NEXT: [[TMP29:%.*]] = trunc <4 x i32> [[TMP28]] to <4 x i8> +; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP4]], i32 1 +; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP13]], <4 x ptr> [[VECTOR_GEP4]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]] +; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds nuw i8, <4 x ptr> [[VECTOR_GEP4]], i32 2 +; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP21]], <4 x ptr> [[TMP30]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3]], !noalias [[META0]] +; CHECK-NEXT: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> [[TMP29]], <4 x ptr> [[TMP31]], i32 1, <4 x i1> [[ACTIVE_LANE_MASK]]), !alias.scope [[META3]], !noalias [[META0]] +; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4 +; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i32 12 +; CHECK-NEXT: [[PTR_IND3]] = getelementptr i8, ptr [[POINTER_PHI2]], i32 12 +; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br [[FOR_COND_CLEANUP_LOOPEXIT:label %.*]] +; CHECK: [[SCALAR_PH]]: +; +entry: + %cmp46.not = icmp eq i32 %n, 0 + br i1 %cmp46.not, label %for.cond.cleanup, label %for.body.preheader + +for.body.preheader: ; preds = %entry + br label %for.body + +for.cond.cleanup.loopexit: ; preds = %for.body + br label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry + ret void + +for.body: ; preds = %for.body.preheader, %for.body + %in.addr.049 = phi ptr [ %incdec.ptr2, %for.body ], [ %in, %for.body.preheader ] + %out.addr.048 = phi ptr [ %incdec.ptr34, %for.body ], [ %out, %for.body.preheader ] + %i.047 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ] + %incdec.ptr = getelementptr inbounds nuw i8, ptr %in.addr.049, i32 1 + %0 = load i8, ptr %in.addr.049, align 1 + %incdec.ptr1 = getelementptr inbounds nuw i8, ptr %in.addr.049, i32 2 + %1 = load i8, ptr %incdec.ptr, align 1 + %incdec.ptr2 = getelementptr inbounds nuw i8, ptr %in.addr.049, i32 3 + %2 = load i8, ptr %incdec.ptr1, align 1 + %conv = zext i8 %0 to i32 + %mul = mul nuw nsw i32 %conv, 19595 + %conv3 = zext i8 %1 to i32 + %mul4 = mul nuw nsw i32 %conv3, 38470 + %conv5 = zext i8 %2 to i32 + %mul6 = mul nuw nsw i32 %conv5, 7471 + %add = add nuw nsw i32 %mul, 32768 + %add7 = add nuw nsw i32 %add, %mul4 + %add8 = add nuw nsw i32 %add7, %mul6 + %shr = lshr i32 %add8, 16 + %conv9 = trunc nuw i32 %shr to i8 + %mul11 = mul nuw nsw i32 %conv, 32767 + %mul13 = mul nuw i32 %conv3, 16762097 + %mul16 = mul nuw i32 %conv5, 16759568 + %add14 = add nuw nsw i32 %mul11, 32768 + %add17 = add nuw i32 %add14, %mul13 + %add18 = add i32 %add17, %mul16 + %shr19 = lshr i32 %add18, 16 + %conv20 = trunc i32 %shr19 to i8 + %mul22 = mul nuw nsw i32 %conv, 13282 + %mul24 = mul nuw i32 %conv3, 16744449 + %mul27 = mul nuw nsw i32 %conv5, 19485 + %add25 = add nuw nsw i32 %mul22, 32768 + %add28 = add nuw i32 %add25, %mul24 + %add29 = add nuw i32 %add28, %mul27 + %shr30 = lshr i32 %add29, 16 + %conv31 = trunc i32 %shr30 to i8 + %incdec.ptr32 = getelementptr inbounds nuw i8, ptr %out.addr.048, i32 1 + store i8 %conv9, ptr %out.addr.048, align 1 + %incdec.ptr33 = getelementptr inbounds nuw i8, ptr %out.addr.048, i32 2 + store i8 %conv20, ptr %incdec.ptr32, align 1 + %incdec.ptr34 = getelementptr inbounds nuw i8, ptr %out.addr.048, i32 3 + store i8 %conv31, ptr %incdec.ptr33, align 1 + %inc = add nuw i32 %i.047, 1 + %exitcond.not = icmp eq i32 %inc, %n + br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body, !llvm.loop !7 +} + +attributes #0 = { nofree norecurse nosync nounwind memory(argmem: readwrite) "target-features"="+mve" } + +!7 = !{!"llvm.loop.mustprogress"} From 8663143b359108b980be96f75796cca02b6b4d59 Mon Sep 17 00:00:00 2001 From: Samuel Tebbs Date: Thu, 29 May 2025 14:23:23 +0100 Subject: [PATCH 3/8] Only compute reg usage if needed --- .../Transforms/Vectorize/LoopVectorize.cpp | 23 ++++++++--- .../LoopVectorize/RISCV/reg-usage-bf16.ll | 3 +- .../LoopVectorize/RISCV/reg-usage-f16.ll | 6 +-- .../LoopVectorize/RISCV/reg-usage.ll | 40 ++++++++----------- 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 83bc5cde95abf..b0c2f9ee46a9b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -4350,8 +4350,13 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() { for (auto &P : VPlans) { ArrayRef VFs(P->vectorFactors().begin(), P->vectorFactors().end()); - auto RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); - for (auto [VF, RU] : zip_equal(VFs, RUs)) { + SmallVector RUs; + if (CM.useMaxBandwidth(TargetTransformInfo::RGK_ScalableVector) || + CM.useMaxBandwidth(TargetTransformInfo::RGK_FixedWidthVector)) + RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); + + for (unsigned I = 0; I < VFs.size(); I++) { + ElementCount VF = VFs[I]; // The cost for scalar VF=1 is already calculated, so ignore it. if (VF.isScalar()) continue; @@ -4361,7 +4366,7 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() { if (CM.useMaxBandwidth(VF.isScalable() ? TargetTransformInfo::RGK_ScalableVector : TargetTransformInfo::RGK_FixedWidthVector) && - RU.exceedsMaxNumRegs(TTI)) + RUs[I].exceedsMaxNumRegs(TTI)) continue; InstructionCost C = CM.expectedCost(VF); @@ -7108,8 +7113,14 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() { for (auto &P : VPlans) { ArrayRef VFs(P->vectorFactors().begin(), P->vectorFactors().end()); - auto RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); - for (auto [VF, RU] : zip_equal(VFs, RUs)) { + + SmallVector RUs; + if (CM.useMaxBandwidth(TargetTransformInfo::RGK_ScalableVector) || + CM.useMaxBandwidth(TargetTransformInfo::RGK_FixedWidthVector)) + RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); + + for (unsigned I = 0; I < VFs.size(); I++) { + ElementCount VF = VFs[I]; if (VF.isScalar()) continue; if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) { @@ -7134,7 +7145,7 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() { if (CM.useMaxBandwidth(VF.isScalable() ? TargetTransformInfo::RGK_ScalableVector : TargetTransformInfo::RGK_FixedWidthVector) && - RU.exceedsMaxNumRegs(TTI)) { + RUs[I].exceedsMaxNumRegs(TTI)) { LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width " << VF << " because it uses too many registers\n"); continue; diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll index cb071f989dafa..5a67b54c7a3d5 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll @@ -3,8 +3,7 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) { ; CHECK-LABEL: add -; CHECK: LV(REG): VF = 8 -; CHECK-NEXT: LV(REG): Found max usage: 2 item +; CHECK: LV(REG): Found max usage: 2 item ; CHECK-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers ; CHECK-NEXT: LV(REG): Found invariant usage: 1 item diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll index 15facfc48137b..d4909fa61b4f5 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll @@ -4,14 +4,12 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) { ; CHECK-LABEL: add -; ZVFH: LV(REG): VF = 8 -; ZVFH-NEXT: LV(REG): Found max usage: 2 item +; ZVFH: LV(REG): Found max usage: 2 item ; ZVFH-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; ZVFH-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers ; ZVFH-NEXT: LV(REG): Found invariant usage: 1 item ; ZVFH-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; ZVFHMIN: LV(REG): VF = 8 -; ZVFHMIN-NEXT: LV(REG): Found max usage: 2 item +; ZVFHMIN: LV(REG): Found max usage: 2 item ; ZVFHMIN-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; ZVFHMIN-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers ; ZVFHMIN-NEXT: LV(REG): Found invariant usage: 1 item diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll index 870f52876c5a9..318ecd5934e13 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll @@ -28,28 +28,24 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture rea ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::FPRRC, 2 registers ; CHECK-SCALAR-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL1: LV(REG): VF = 2 -; CHECK-LMUL1-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL1: LV(REG): Found max usage: 2 item ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers ; CHECK-LMUL1-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL2: LV(REG): VF = 4 -; CHECK-LMUL2-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL2: LV(REG): Found max usage: 2 item ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers +; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers ; CHECK-LMUL2-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL4: LV(REG): VF = 8 -; CHECK-LMUL4-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL4: LV(REG): Found max usage: 2 item ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers +; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers ; CHECK-LMUL4-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL8: LV(REG): VF = 16 -; CHECK-LMUL8-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL8: LV(REG): Found max usage: 2 item ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers +; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 16 registers ; CHECK-LMUL8-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers @@ -80,21 +76,17 @@ define void @goo(ptr nocapture noundef %a, i32 noundef signext %n) { ; CHECK-SCALAR: LV(REG): VF = 1 ; CHECK-SCALAR-NEXT: LV(REG): Found max usage: 1 item ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL1: LV(REG): VF = 2 -; CHECK-LMUL1-NEXT: LV(REG): Found max usage: 2 item -; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers -; CHECK-LMUL2: LV(REG): VF = 4 -; CHECK-LMUL2-NEXT: LV(REG): Found max usage: 2 item -; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers +; CHECK-LMUL1: LV(REG): Found max usage: 2 item +; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers +; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 1 registers +; CHECK-LMUL2: LV(REG): Found max usage: 2 item +; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers -; CHECK-LMUL4: LV(REG): VF = 8 -; CHECK-LMUL4-NEXT: LV(REG): Found max usage: 2 item -; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers +; CHECK-LMUL4: LV(REG): Found max usage: 2 item +; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers -; CHECK-LMUL8: LV(REG): VF = 16 -; CHECK-LMUL8-NEXT: LV(REG): Found max usage: 2 item -; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers +; CHECK-LMUL8: LV(REG): Found max usage: 2 item +; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers entry: %cmp3 = icmp sgt i32 %n, 0 From 07f6b77fa5b05d32d7e7b18d09eeac7786b2296a Mon Sep 17 00:00:00 2001 From: Sam Tebbs Date: Fri, 30 May 2025 17:38:51 +0100 Subject: [PATCH 4/8] Interpret VPWidenPointerInductionRecipe as a scalar instead --- .../Transforms/Vectorize/LoopVectorize.cpp | 46 ++++--------------- .../Transforms/Vectorize/VPlanAnalysis.cpp | 2 +- .../LoopVectorize/RISCV/reg-usage-bf16.ll | 3 +- .../LoopVectorize/RISCV/reg-usage-f16.ll | 6 ++- .../LoopVectorize/RISCV/reg-usage.ll | 40 +++++++++------- 5 files changed, 41 insertions(+), 56 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index b0c2f9ee46a9b..427c1460fcfc9 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -956,10 +956,6 @@ class LoopVectorizationCostModel { return expectedCost(UserVF).isValid(); } - /// \return True if maximizing vector bandwidth is enabled by the target or - /// user options. - bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind); - /// \return The size (in bits) of the smallest and widest types in the code /// that needs to be vectorized. We ignore values that remain scalar such as /// 64 bit loop indices. @@ -3922,14 +3918,6 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) { return FixedScalableVFPair::getNone(); } -bool LoopVectorizationCostModel::useMaxBandwidth( - TargetTransformInfo::RegisterKind RegKind) { - return MaximizeBandwidth || (MaximizeBandwidth.getNumOccurrences() == 0 && - (TTI.shouldMaximizeVectorBandwidth(RegKind) || - (UseWiderVFIfCallVariantsPresent && - Legal->hasVectorCallVariants()))); -} - ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget( unsigned MaxTripCount, unsigned SmallestType, unsigned WidestType, ElementCount MaxSafeVF, bool FoldTailByMasking) { @@ -3995,7 +3983,10 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget( ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector : TargetTransformInfo::RGK_FixedWidthVector; ElementCount MaxVF = MaxVectorElementCount; - if (useMaxBandwidth(RegKind)) { + if (MaximizeBandwidth || + (MaximizeBandwidth.getNumOccurrences() == 0 && + (TTI.shouldMaximizeVectorBandwidth(RegKind) || + (UseWiderVFIfCallVariantsPresent && Legal->hasVectorCallVariants())))) { auto MaxVectorElementCountMaxBW = ElementCount::get( llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType), ComputeScalableMaxVF); @@ -4350,23 +4341,15 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() { for (auto &P : VPlans) { ArrayRef VFs(P->vectorFactors().begin(), P->vectorFactors().end()); - SmallVector RUs; - if (CM.useMaxBandwidth(TargetTransformInfo::RGK_ScalableVector) || - CM.useMaxBandwidth(TargetTransformInfo::RGK_FixedWidthVector)) - RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); - - for (unsigned I = 0; I < VFs.size(); I++) { - ElementCount VF = VFs[I]; + auto RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); + for (auto [VF, RU] : zip_equal(VFs, RUs)) { // The cost for scalar VF=1 is already calculated, so ignore it. if (VF.isScalar()) continue; /// Don't consider the VF if it exceeds the number of registers for the /// target. - if (CM.useMaxBandwidth(VF.isScalable() - ? TargetTransformInfo::RGK_ScalableVector - : TargetTransformInfo::RGK_FixedWidthVector) && - RUs[I].exceedsMaxNumRegs(TTI)) + if (RU.exceedsMaxNumRegs(TTI)) continue; InstructionCost C = CM.expectedCost(VF); @@ -7113,14 +7096,8 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() { for (auto &P : VPlans) { ArrayRef VFs(P->vectorFactors().begin(), P->vectorFactors().end()); - - SmallVector RUs; - if (CM.useMaxBandwidth(TargetTransformInfo::RGK_ScalableVector) || - CM.useMaxBandwidth(TargetTransformInfo::RGK_FixedWidthVector)) - RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); - - for (unsigned I = 0; I < VFs.size(); I++) { - ElementCount VF = VFs[I]; + auto RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); + for (auto [VF, RU] : zip_equal(VFs, RUs)) { if (VF.isScalar()) continue; if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) { @@ -7142,10 +7119,7 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() { InstructionCost Cost = cost(*P, VF); VectorizationFactor CurrentFactor(VF, Cost, ScalarCost); - if (CM.useMaxBandwidth(VF.isScalable() - ? TargetTransformInfo::RGK_ScalableVector - : TargetTransformInfo::RGK_FixedWidthVector) && - RUs[I].exceedsMaxNumRegs(TTI)) { + if (RU.exceedsMaxNumRegs(TTI)) { LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width " << VF << " because it uses too many registers\n"); continue; diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index 76da5b0314a8e..23ddd74f5d2be 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -547,7 +547,7 @@ SmallVector llvm::calculateRegisterUsageForPlan( if (VFs[J].isScalar() || isa(R) || + VPScalarIVStepsRecipe, VPWidenPointerInductionRecipe>(R) || (isa(R) && all_of(cast(R)->users(), [&](VPUser *U) { diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll index 5a67b54c7a3d5..cb071f989dafa 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll @@ -3,7 +3,8 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) { ; CHECK-LABEL: add -; CHECK: LV(REG): Found max usage: 2 item +; CHECK: LV(REG): VF = 8 +; CHECK-NEXT: LV(REG): Found max usage: 2 item ; CHECK-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers ; CHECK-NEXT: LV(REG): Found invariant usage: 1 item diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll index d4909fa61b4f5..15facfc48137b 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll @@ -4,12 +4,14 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) { ; CHECK-LABEL: add -; ZVFH: LV(REG): Found max usage: 2 item +; ZVFH: LV(REG): VF = 8 +; ZVFH-NEXT: LV(REG): Found max usage: 2 item ; ZVFH-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; ZVFH-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers ; ZVFH-NEXT: LV(REG): Found invariant usage: 1 item ; ZVFH-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; ZVFHMIN: LV(REG): Found max usage: 2 item +; ZVFHMIN: LV(REG): VF = 8 +; ZVFHMIN-NEXT: LV(REG): Found max usage: 2 item ; ZVFHMIN-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; ZVFHMIN-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers ; ZVFHMIN-NEXT: LV(REG): Found invariant usage: 1 item diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll index 318ecd5934e13..870f52876c5a9 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll @@ -28,24 +28,28 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture rea ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::FPRRC, 2 registers ; CHECK-SCALAR-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL1: LV(REG): Found max usage: 2 item +; CHECK-LMUL1: LV(REG): VF = 2 +; CHECK-LMUL1-NEXT: LV(REG): Found max usage: 2 item ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers ; CHECK-LMUL1-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL2: LV(REG): Found max usage: 2 item +; CHECK-LMUL2: LV(REG): VF = 4 +; CHECK-LMUL2-NEXT: LV(REG): Found max usage: 2 item ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers +; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers ; CHECK-LMUL2-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL4: LV(REG): Found max usage: 2 item +; CHECK-LMUL4: LV(REG): VF = 8 +; CHECK-LMUL4-NEXT: LV(REG): Found max usage: 2 item ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers +; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers ; CHECK-LMUL4-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL8: LV(REG): Found max usage: 2 item +; CHECK-LMUL8: LV(REG): VF = 16 +; CHECK-LMUL8-NEXT: LV(REG): Found max usage: 2 item ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 16 registers +; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers ; CHECK-LMUL8-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers @@ -76,17 +80,21 @@ define void @goo(ptr nocapture noundef %a, i32 noundef signext %n) { ; CHECK-SCALAR: LV(REG): VF = 1 ; CHECK-SCALAR-NEXT: LV(REG): Found max usage: 1 item ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL1: LV(REG): Found max usage: 2 item -; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers -; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 1 registers -; CHECK-LMUL2: LV(REG): Found max usage: 2 item -; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers +; CHECK-LMUL1: LV(REG): VF = 2 +; CHECK-LMUL1-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers +; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers +; CHECK-LMUL2: LV(REG): VF = 4 +; CHECK-LMUL2-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers -; CHECK-LMUL4: LV(REG): Found max usage: 2 item -; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers +; CHECK-LMUL4: LV(REG): VF = 8 +; CHECK-LMUL4-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers -; CHECK-LMUL8: LV(REG): Found max usage: 2 item -; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers +; CHECK-LMUL8: LV(REG): VF = 16 +; CHECK-LMUL8-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers entry: %cmp3 = icmp sgt i32 %n, 0 From 8de22c0c664e968326cd6cb8d6014864797536e0 Mon Sep 17 00:00:00 2001 From: Samuel Tebbs Date: Wed, 4 Jun 2025 16:17:00 +0100 Subject: [PATCH 5/8] Revert "Interpret VPWidenPointerInductionRecipe as a scalar instead" This reverts commit 760eb570768fa32f36dc64be73eb770bd27b691f. --- .../Transforms/Vectorize/LoopVectorize.cpp | 47 +++++++++++++++---- .../Transforms/Vectorize/VPlanAnalysis.cpp | 2 +- .../LoopVectorize/RISCV/reg-usage-bf16.ll | 3 +- .../LoopVectorize/RISCV/reg-usage-f16.ll | 6 +-- .../LoopVectorize/RISCV/reg-usage.ll | 40 +++++++--------- 5 files changed, 57 insertions(+), 41 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 427c1460fcfc9..83e7ed57f7bc4 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -956,6 +956,10 @@ class LoopVectorizationCostModel { return expectedCost(UserVF).isValid(); } + /// \return True if maximizing vector bandwidth is enabled by the target or + /// user options. + bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind); + /// \return The size (in bits) of the smallest and widest types in the code /// that needs to be vectorized. We ignore values that remain scalar such as /// 64 bit loop indices. @@ -3918,6 +3922,14 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) { return FixedScalableVFPair::getNone(); } +bool LoopVectorizationCostModel::useMaxBandwidth( + TargetTransformInfo::RegisterKind RegKind) { + return MaximizeBandwidth || (MaximizeBandwidth.getNumOccurrences() == 0 && + (TTI.shouldMaximizeVectorBandwidth(RegKind) || + (UseWiderVFIfCallVariantsPresent && + Legal->hasVectorCallVariants()))); +} + ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget( unsigned MaxTripCount, unsigned SmallestType, unsigned WidestType, ElementCount MaxSafeVF, bool FoldTailByMasking) { @@ -3983,10 +3995,7 @@ ElementCount LoopVectorizationCostModel::getMaximizedVFForTarget( ComputeScalableMaxVF ? TargetTransformInfo::RGK_ScalableVector : TargetTransformInfo::RGK_FixedWidthVector; ElementCount MaxVF = MaxVectorElementCount; - if (MaximizeBandwidth || - (MaximizeBandwidth.getNumOccurrences() == 0 && - (TTI.shouldMaximizeVectorBandwidth(RegKind) || - (UseWiderVFIfCallVariantsPresent && Legal->hasVectorCallVariants())))) { + if (useMaxBandwidth(RegKind)) { auto MaxVectorElementCountMaxBW = ElementCount::get( llvm::bit_floor(WidestRegister.getKnownMinValue() / SmallestType), ComputeScalableMaxVF); @@ -4341,15 +4350,24 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() { for (auto &P : VPlans) { ArrayRef VFs(P->vectorFactors().begin(), P->vectorFactors().end()); - auto RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); - for (auto [VF, RU] : zip_equal(VFs, RUs)) { + + SmallVector RUs; + if (CM.useMaxBandwidth(TargetTransformInfo::RGK_ScalableVector) || + CM.useMaxBandwidth(TargetTransformInfo::RGK_FixedWidthVector)) + RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); + + for (unsigned I = 0; I < VFs.size(); I++) { + ElementCount VF = VFs[I]; // The cost for scalar VF=1 is already calculated, so ignore it. if (VF.isScalar()) continue; /// Don't consider the VF if it exceeds the number of registers for the /// target. - if (RU.exceedsMaxNumRegs(TTI)) + if (CM.useMaxBandwidth(VF.isScalable() + ? TargetTransformInfo::RGK_ScalableVector + : TargetTransformInfo::RGK_FixedWidthVector) && + RUs[I].exceedsMaxNumRegs(TTI)) continue; InstructionCost C = CM.expectedCost(VF); @@ -7096,8 +7114,14 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() { for (auto &P : VPlans) { ArrayRef VFs(P->vectorFactors().begin(), P->vectorFactors().end()); - auto RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); - for (auto [VF, RU] : zip_equal(VFs, RUs)) { + + SmallVector RUs; + if (CM.useMaxBandwidth(TargetTransformInfo::RGK_ScalableVector) || + CM.useMaxBandwidth(TargetTransformInfo::RGK_FixedWidthVector)) + RUs = calculateRegisterUsageForPlan(*P, VFs, TTI, CM.ValuesToIgnore); + + for (unsigned I = 0; I < VFs.size(); I++) { + ElementCount VF = VFs[I]; if (VF.isScalar()) continue; if (!ForceVectorization && !willGenerateVectors(*P, VF, TTI)) { @@ -7119,7 +7143,10 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() { InstructionCost Cost = cost(*P, VF); VectorizationFactor CurrentFactor(VF, Cost, ScalarCost); - if (RU.exceedsMaxNumRegs(TTI)) { + if (CM.useMaxBandwidth(VF.isScalable() + ? TargetTransformInfo::RGK_ScalableVector + : TargetTransformInfo::RGK_FixedWidthVector) && + RUs[I].exceedsMaxNumRegs(TTI)) { LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width " << VF << " because it uses too many registers\n"); continue; diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp index 23ddd74f5d2be..76da5b0314a8e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp @@ -547,7 +547,7 @@ SmallVector llvm::calculateRegisterUsageForPlan( if (VFs[J].isScalar() || isa(R) || + VPScalarIVStepsRecipe>(R) || (isa(R) && all_of(cast(R)->users(), [&](VPUser *U) { diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll index cb071f989dafa..5a67b54c7a3d5 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-bf16.ll @@ -3,8 +3,7 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) { ; CHECK-LABEL: add -; CHECK: LV(REG): VF = 8 -; CHECK-NEXT: LV(REG): Found max usage: 2 item +; CHECK: LV(REG): Found max usage: 2 item ; CHECK-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers ; CHECK-NEXT: LV(REG): Found invariant usage: 1 item diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll index 15facfc48137b..d4909fa61b4f5 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage-f16.ll @@ -4,14 +4,12 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture readonly %src2, i32 signext %size, ptr noalias nocapture writeonly %result) { ; CHECK-LABEL: add -; ZVFH: LV(REG): VF = 8 -; ZVFH-NEXT: LV(REG): Found max usage: 2 item +; ZVFH: LV(REG): Found max usage: 2 item ; ZVFH-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; ZVFH-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers ; ZVFH-NEXT: LV(REG): Found invariant usage: 1 item ; ZVFH-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; ZVFHMIN: LV(REG): VF = 8 -; ZVFHMIN-NEXT: LV(REG): Found max usage: 2 item +; ZVFHMIN: LV(REG): Found max usage: 2 item ; ZVFHMIN-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; ZVFHMIN-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers ; ZVFHMIN-NEXT: LV(REG): Found invariant usage: 1 item diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll index 870f52876c5a9..318ecd5934e13 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll @@ -28,28 +28,24 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture rea ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::FPRRC, 2 registers ; CHECK-SCALAR-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL1: LV(REG): VF = 2 -; CHECK-LMUL1-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL1: LV(REG): Found max usage: 2 item ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers ; CHECK-LMUL1-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL2: LV(REG): VF = 4 -; CHECK-LMUL2-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL2: LV(REG): Found max usage: 2 item ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers +; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers ; CHECK-LMUL2-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL4: LV(REG): VF = 8 -; CHECK-LMUL4-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL4: LV(REG): Found max usage: 2 item ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers +; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers ; CHECK-LMUL4-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL8: LV(REG): VF = 16 -; CHECK-LMUL8-NEXT: LV(REG): Found max usage: 2 item +; CHECK-LMUL8: LV(REG): Found max usage: 2 item ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers +; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 16 registers ; CHECK-LMUL8-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers @@ -80,21 +76,17 @@ define void @goo(ptr nocapture noundef %a, i32 noundef signext %n) { ; CHECK-SCALAR: LV(REG): VF = 1 ; CHECK-SCALAR-NEXT: LV(REG): Found max usage: 1 item ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL1: LV(REG): VF = 2 -; CHECK-LMUL1-NEXT: LV(REG): Found max usage: 2 item -; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers -; CHECK-LMUL2: LV(REG): VF = 4 -; CHECK-LMUL2-NEXT: LV(REG): Found max usage: 2 item -; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers +; CHECK-LMUL1: LV(REG): Found max usage: 2 item +; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers +; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 1 registers +; CHECK-LMUL2: LV(REG): Found max usage: 2 item +; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers -; CHECK-LMUL4: LV(REG): VF = 8 -; CHECK-LMUL4-NEXT: LV(REG): Found max usage: 2 item -; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers +; CHECK-LMUL4: LV(REG): Found max usage: 2 item +; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers -; CHECK-LMUL8: LV(REG): VF = 16 -; CHECK-LMUL8-NEXT: LV(REG): Found max usage: 2 item -; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers +; CHECK-LMUL8: LV(REG): Found max usage: 2 item +; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers entry: %cmp3 = icmp sgt i32 %n, 0 From ee9aa1571cc83a795fc2457b23cd1c52f489004d Mon Sep 17 00:00:00 2001 From: Samuel Tebbs Date: Tue, 10 Jun 2025 15:46:38 +0100 Subject: [PATCH 6/8] Add useMaxBandwidth for VF --- .../Transforms/Vectorize/LoopVectorize.cpp | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 83e7ed57f7bc4..d40bdd7a5accd 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -957,9 +957,13 @@ class LoopVectorizationCostModel { } /// \return True if maximizing vector bandwidth is enabled by the target or - /// user options. + /// user options, for the given register kind. bool useMaxBandwidth(TargetTransformInfo::RegisterKind RegKind); + /// \return True if maximizing vector bandwidth is enabled by the target or + /// user options, for the given vector factor. + bool useMaxBandwidth(ElementCount VF); + /// \return The size (in bits) of the smallest and widest types in the code /// that needs to be vectorized. We ignore values that remain scalar such as /// 64 bit loop indices. @@ -3922,6 +3926,12 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) { return FixedScalableVFPair::getNone(); } +bool LoopVectorizationCostModel::useMaxBandwidth(ElementCount VF) { + return useMaxBandwidth(VF.isScalable() + ? TargetTransformInfo::RGK_ScalableVector + : TargetTransformInfo::RGK_FixedWidthVector); +} + bool LoopVectorizationCostModel::useMaxBandwidth( TargetTransformInfo::RegisterKind RegKind) { return MaximizeBandwidth || (MaximizeBandwidth.getNumOccurrences() == 0 && @@ -4364,10 +4374,7 @@ VectorizationFactor LoopVectorizationPlanner::selectVectorizationFactor() { /// Don't consider the VF if it exceeds the number of registers for the /// target. - if (CM.useMaxBandwidth(VF.isScalable() - ? TargetTransformInfo::RGK_ScalableVector - : TargetTransformInfo::RGK_FixedWidthVector) && - RUs[I].exceedsMaxNumRegs(TTI)) + if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI)) continue; InstructionCost C = CM.expectedCost(VF); @@ -7143,10 +7150,7 @@ VectorizationFactor LoopVectorizationPlanner::computeBestVF() { InstructionCost Cost = cost(*P, VF); VectorizationFactor CurrentFactor(VF, Cost, ScalarCost); - if (CM.useMaxBandwidth(VF.isScalable() - ? TargetTransformInfo::RGK_ScalableVector - : TargetTransformInfo::RGK_FixedWidthVector) && - RUs[I].exceedsMaxNumRegs(TTI)) { + if (CM.useMaxBandwidth(VF) && RUs[I].exceedsMaxNumRegs(TTI)) { LLVM_DEBUG(dbgs() << "LV(REG): Not considering vector loop of width " << VF << " because it uses too many registers\n"); continue; From 97973907721e71ba96b93438ef4226854f94c93b Mon Sep 17 00:00:00 2001 From: Samuel Tebbs Date: Tue, 10 Jun 2025 16:15:41 +0100 Subject: [PATCH 7/8] Clean up tests --- .../ARM/mve-reg-pressure-vmla.ll | 53 ++++++++----------- .../LoopVectorize/RISCV/reg-usage.ll | 16 +++--- 2 files changed, 29 insertions(+), 40 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll index 667022e0470e4..4c29a3a0d1d01 100644 --- a/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll +++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reg-pressure-vmla.ll @@ -1,7 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "^scalar.ph:" --version 5 -; RUN: opt -passes=loop-vectorize < %s -S -o - | FileCheck %s +; RUN: opt -mattr=+mve -passes=loop-vectorize < %s -S -o - | FileCheck %s -source_filename = "" target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "thumbv8.1m.main-unknown-none-eabihf" @@ -12,7 +11,7 @@ define void @fn(i32 noundef %n, ptr %in, ptr %out) #0 { ; CHECK-SAME: i32 noundef [[N:%.*]], ptr [[IN:%.*]], ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[CMP46_NOT:%.*]] = icmp eq i32 [[N]], 0 -; CHECK-NEXT: br i1 [[CMP46_NOT]], [[FOR_COND_CLEANUP:label %.*]], label %[[FOR_BODY_PREHEADER:.*]] +; CHECK-NEXT: br i1 [[CMP46_NOT]], [[EXIT:label %.*]], label %[[FOR_BODY_PREHEADER:.*]] ; CHECK: [[FOR_BODY_PREHEADER]]: ; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: @@ -78,31 +77,22 @@ define void @fn(i32 noundef %n, ptr %in, ptr %out) #0 { ; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: br [[FOR_COND_CLEANUP_LOOPEXIT:label %.*]] +; CHECK-NEXT: br [[EXIT_LOOPEXIT:label %.*]] ; CHECK: [[SCALAR_PH]]: ; entry: %cmp46.not = icmp eq i32 %n, 0 - br i1 %cmp46.not, label %for.cond.cleanup, label %for.body.preheader - -for.body.preheader: ; preds = %entry - br label %for.body - -for.cond.cleanup.loopexit: ; preds = %for.body - br label %for.cond.cleanup - -for.cond.cleanup: ; preds = %for.cond.cleanup.loopexit, %entry - ret void + br i1 %cmp46.not, label %exit, label %for.body for.body: ; preds = %for.body.preheader, %for.body - %in.addr.049 = phi ptr [ %incdec.ptr2, %for.body ], [ %in, %for.body.preheader ] - %out.addr.048 = phi ptr [ %incdec.ptr34, %for.body ], [ %out, %for.body.preheader ] - %i.047 = phi i32 [ %inc, %for.body ], [ 0, %for.body.preheader ] - %incdec.ptr = getelementptr inbounds nuw i8, ptr %in.addr.049, i32 1 - %0 = load i8, ptr %in.addr.049, align 1 - %incdec.ptr1 = getelementptr inbounds nuw i8, ptr %in.addr.049, i32 2 + %ptr.iv.1 = phi ptr [ %in, %entry ], [ %ptr.iv.1.next, %for.body ] + %ptr.iv.2 = phi ptr [ %out, %entry ], [ %ptr.iv.2.next, %for.body ] + %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ] + %incdec.ptr = getelementptr inbounds nuw i8, ptr %ptr.iv.1, i32 1 + %0 = load i8, ptr %ptr.iv.1, align 1 + %incdec.ptr1 = getelementptr inbounds nuw i8, ptr %ptr.iv.1, i32 2 %1 = load i8, ptr %incdec.ptr, align 1 - %incdec.ptr2 = getelementptr inbounds nuw i8, ptr %in.addr.049, i32 3 + %ptr.iv.1.next = getelementptr inbounds nuw i8, ptr %ptr.iv.1, i32 3 %2 = load i8, ptr %incdec.ptr1, align 1 %conv = zext i8 %0 to i32 %mul = mul nuw nsw i32 %conv, 19595 @@ -131,17 +121,16 @@ for.body: ; preds = %for.body.preheader, %add29 = add nuw i32 %add28, %mul27 %shr30 = lshr i32 %add29, 16 %conv31 = trunc i32 %shr30 to i8 - %incdec.ptr32 = getelementptr inbounds nuw i8, ptr %out.addr.048, i32 1 - store i8 %conv9, ptr %out.addr.048, align 1 - %incdec.ptr33 = getelementptr inbounds nuw i8, ptr %out.addr.048, i32 2 + %incdec.ptr32 = getelementptr inbounds nuw i8, ptr %ptr.iv.2, i32 1 + store i8 %conv9, ptr %ptr.iv.2, align 1 + %incdec.ptr33 = getelementptr inbounds nuw i8, ptr %ptr.iv.2, i32 2 store i8 %conv20, ptr %incdec.ptr32, align 1 - %incdec.ptr34 = getelementptr inbounds nuw i8, ptr %out.addr.048, i32 3 + %ptr.iv.2.next = getelementptr inbounds nuw i8, ptr %ptr.iv.2, i32 3 store i8 %conv31, ptr %incdec.ptr33, align 1 - %inc = add nuw i32 %i.047, 1 - %exitcond.not = icmp eq i32 %inc, %n - br i1 %exitcond.not, label %for.cond.cleanup.loopexit, label %for.body, !llvm.loop !7 -} - -attributes #0 = { nofree norecurse nosync nounwind memory(argmem: readwrite) "target-features"="+mve" } + %iv.next = add nuw i32 %iv, 1 + %exitcond.not = icmp eq i32 %iv.next, %n + br i1 %exitcond.not, label %exit, label %for.body -!7 = !{!"llvm.loop.mustprogress"} +exit: ; preds = %for.cond.cleanup.loopexit, %entry + ret void +} diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll index 318ecd5934e13..cee0b1222b6be 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/reg-usage.ll @@ -28,22 +28,22 @@ define void @add(ptr noalias nocapture readonly %src1, ptr noalias nocapture rea ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::FPRRC, 2 registers ; CHECK-SCALAR-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL1: LV(REG): Found max usage: 2 item +; CHECK-LMUL1: LV(REG): Found max usage: 2 item ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers ; CHECK-LMUL1-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL2: LV(REG): Found max usage: 2 item +; CHECK-LMUL2: LV(REG): Found max usage: 2 item ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers ; CHECK-LMUL2-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL4: LV(REG): Found max usage: 2 item +; CHECK-LMUL4: LV(REG): Found max usage: 2 item ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers ; CHECK-LMUL4-NEXT: LV(REG): Found invariant usage: 1 item ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers -; CHECK-LMUL8: LV(REG): Found max usage: 2 item +; CHECK-LMUL8: LV(REG): Found max usage: 2 item ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 16 registers ; CHECK-LMUL8-NEXT: LV(REG): Found invariant usage: 1 item @@ -76,16 +76,16 @@ define void @goo(ptr nocapture noundef %a, i32 noundef signext %n) { ; CHECK-SCALAR: LV(REG): VF = 1 ; CHECK-SCALAR-NEXT: LV(REG): Found max usage: 1 item ; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 3 registers -; CHECK-LMUL1: LV(REG): Found max usage: 2 item +; CHECK-LMUL1: LV(REG): Found max usage: 2 item ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers ; CHECK-LMUL1-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 1 registers -; CHECK-LMUL2: LV(REG): Found max usage: 2 item +; CHECK-LMUL2: LV(REG): Found max usage: 2 item ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers ; CHECK-LMUL2-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 2 registers -; CHECK-LMUL4: LV(REG): Found max usage: 2 item +; CHECK-LMUL4: LV(REG): Found max usage: 2 item ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers ; CHECK-LMUL4-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 4 registers -; CHECK-LMUL8: LV(REG): Found max usage: 2 item +; CHECK-LMUL8: LV(REG): Found max usage: 2 item ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 2 registers ; CHECK-LMUL8-NEXT: LV(REG): RegisterClass: RISCV::VRRC, 8 registers entry: From ea866e835dadff4465d8b02b4a7cf36b659d5add Mon Sep 17 00:00:00 2001 From: Sam Tebbs Date: Wed, 11 Jun 2025 14:57:06 +0100 Subject: [PATCH 8/8] Update RISCV test --- .../RISCV/interleaved-masked-access.ll | 284 +++++++++--------- 1 file changed, 142 insertions(+), 142 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll index 1b0feef3e6664..b7c9612e57aec 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll @@ -11,44 +11,44 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali ; SCALAR_EPILOGUE-NEXT: entry: ; SCALAR_EPILOGUE-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 ; SCALAR_EPILOGUE-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_EPILOGUE-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 3 +; SCALAR_EPILOGUE-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 4 ; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024 ; SCALAR_EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; SCALAR_EPILOGUE: vector.ph: ; SCALAR_EPILOGUE-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_EPILOGUE-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 3 +; SCALAR_EPILOGUE-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 4 ; SCALAR_EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]] ; SCALAR_EPILOGUE-NEXT: [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]] ; SCALAR_EPILOGUE-NEXT: [[TMP4:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_EPILOGUE-NEXT: [[TMP5:%.*]] = shl i32 [[TMP4]], 3 -; SCALAR_EPILOGUE-NEXT: [[TMP6:%.*]] = call @llvm.stepvector.nxv8i32() -; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[CONV]], i64 0 -; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer -; SCALAR_EPILOGUE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP5]], i64 0 -; SCALAR_EPILOGUE-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer +; SCALAR_EPILOGUE-NEXT: [[TMP5:%.*]] = shl i32 [[TMP4]], 4 +; SCALAR_EPILOGUE-NEXT: [[TMP6:%.*]] = call @llvm.stepvector.nxv16i32() +; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[CONV]], i64 0 +; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; SCALAR_EPILOGUE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP5]], i64 0 +; SCALAR_EPILOGUE-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; SCALAR_EPILOGUE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALAR_EPILOGUE: vector.body: ; SCALAR_EPILOGUE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; SCALAR_EPILOGUE-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; SCALAR_EPILOGUE-NEXT: [[TMP7:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] -; SCALAR_EPILOGUE-NEXT: [[TMP8:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 1) -; SCALAR_EPILOGUE-NEXT: [[TMP9:%.*]] = zext nneg [[TMP8]] to -; SCALAR_EPILOGUE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP9]] -; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP10]], i32 1, [[TMP7]], poison) -; SCALAR_EPILOGUE-NEXT: [[TMP11:%.*]] = or disjoint [[TMP8]], splat (i32 1) -; SCALAR_EPILOGUE-NEXT: [[TMP12:%.*]] = zext nneg [[TMP11]] to -; SCALAR_EPILOGUE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP12]] -; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP13]], i32 1, [[TMP7]], poison) -; SCALAR_EPILOGUE-NEXT: [[TMP14:%.*]] = call @llvm.smax.nxv8i8( [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]) -; SCALAR_EPILOGUE-NEXT: [[TMP15:%.*]] = zext nneg [[TMP8]] to -; SCALAR_EPILOGUE-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP15]] -; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP14]], [[TMP16]], i32 1, [[TMP7]]) -; SCALAR_EPILOGUE-NEXT: [[TMP17:%.*]] = sub zeroinitializer, [[TMP14]] -; SCALAR_EPILOGUE-NEXT: [[TMP18:%.*]] = zext nneg [[TMP11]] to -; SCALAR_EPILOGUE-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP18]] -; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP17]], [[TMP19]], i32 1, [[TMP7]]) +; SCALAR_EPILOGUE-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; SCALAR_EPILOGUE-NEXT: [[TMP7:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] +; SCALAR_EPILOGUE-NEXT: [[TMP8:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 1) +; SCALAR_EPILOGUE-NEXT: [[TMP9:%.*]] = zext nneg [[TMP8]] to +; SCALAR_EPILOGUE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP9]] +; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP10]], i32 1, [[TMP7]], poison) +; SCALAR_EPILOGUE-NEXT: [[TMP11:%.*]] = or disjoint [[TMP8]], splat (i32 1) +; SCALAR_EPILOGUE-NEXT: [[TMP12:%.*]] = zext nneg [[TMP11]] to +; SCALAR_EPILOGUE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP12]] +; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP13]], i32 1, [[TMP7]], poison) +; SCALAR_EPILOGUE-NEXT: [[TMP14:%.*]] = call @llvm.smax.nxv16i8( [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]) +; SCALAR_EPILOGUE-NEXT: [[TMP15:%.*]] = zext nneg [[TMP8]] to +; SCALAR_EPILOGUE-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP15]] +; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP14]], [[TMP16]], i32 1, [[TMP7]]) +; SCALAR_EPILOGUE-NEXT: [[TMP17:%.*]] = sub zeroinitializer, [[TMP14]] +; SCALAR_EPILOGUE-NEXT: [[TMP18:%.*]] = zext nneg [[TMP11]] to +; SCALAR_EPILOGUE-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP18]] +; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP17]], [[TMP19]], i32 1, [[TMP7]]) ; SCALAR_EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]] -; SCALAR_EPILOGUE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] +; SCALAR_EPILOGUE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALAR_EPILOGUE-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; SCALAR_EPILOGUE-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; SCALAR_EPILOGUE: middle.block: @@ -63,42 +63,42 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_TAIL_FOLDING: vector.ph: ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 3 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023 ; PREDICATED_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]] ; PREDICATED_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 3 -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = call @llvm.stepvector.nxv8i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[CONV]], i64 0 -; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP3]], i64 0 -; PREDICATED_TAIL_FOLDING-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 4 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = call @llvm.stepvector.nxv16i32() +; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[CONV]], i64 0 +; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; PREDICATED_TAIL_FOLDING-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP3]], i64 0 +; PREDICATED_TAIL_FOLDING-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; PREDICATED_TAIL_FOLDING-NEXT: br label [[VECTOR_BODY:%.*]] ; PREDICATED_TAIL_FOLDING: vector.body: ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i32(i32 [[INDEX]], i32 1024) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP5]], zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 1) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = zext nneg [[TMP7]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP8]] -; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP9]], i32 1, [[TMP6]], poison) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP10:%.*]] = or disjoint [[TMP7]], splat (i32 1) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP11:%.*]] = zext nneg [[TMP10]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP11]] -; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP12]], i32 1, [[TMP6]], poison) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = call @llvm.smax.nxv8i8( [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = zext nneg [[TMP7]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP14]] -; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP13]], [[TMP15]], i32 1, [[TMP6]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = sub zeroinitializer, [[TMP13]] -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP17:%.*]] = zext nneg [[TMP10]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP17]] -; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP16]], [[TMP18]], i32 1, [[TMP6]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 1024) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP5]], zeroinitializer +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 1) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = zext nneg [[TMP7]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP8]] +; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP9]], i32 1, [[TMP6]], poison) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP10:%.*]] = or disjoint [[TMP7]], splat (i32 1) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP11:%.*]] = zext nneg [[TMP10]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP11]] +; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP12]], i32 1, [[TMP6]], poison) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = call @llvm.smax.nxv16i8( [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = zext nneg [[TMP7]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP14]] +; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP13]], [[TMP15]], i32 1, [[TMP6]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = sub zeroinitializer, [[TMP13]] +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP17:%.*]] = zext nneg [[TMP10]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP17]] +; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP16]], [[TMP18]], i32 1, [[TMP6]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]] -; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] +; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; PREDICATED_TAIL_FOLDING: middle.block: @@ -180,60 +180,60 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; SCALAR_EPILOGUE-NEXT: entry: ; SCALAR_EPILOGUE-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 ; SCALAR_EPILOGUE-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_EPILOGUE-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 3 +; SCALAR_EPILOGUE-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 4 ; SCALAR_EPILOGUE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024 ; SCALAR_EPILOGUE-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; SCALAR_EPILOGUE: vector.ph: ; SCALAR_EPILOGUE-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_EPILOGUE-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 3 +; SCALAR_EPILOGUE-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 4 ; SCALAR_EPILOGUE-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]] ; SCALAR_EPILOGUE-NEXT: [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]] ; SCALAR_EPILOGUE-NEXT: [[TMP4:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_EPILOGUE-NEXT: [[TMP5:%.*]] = shl i32 [[TMP4]], 3 -; SCALAR_EPILOGUE-NEXT: [[TMP6:%.*]] = call @llvm.stepvector.nxv8i32() -; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[CONV]], i64 0 -; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer -; SCALAR_EPILOGUE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP5]], i64 0 -; SCALAR_EPILOGUE-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer +; SCALAR_EPILOGUE-NEXT: [[TMP5:%.*]] = shl i32 [[TMP4]], 4 +; SCALAR_EPILOGUE-NEXT: [[TMP6:%.*]] = call @llvm.stepvector.nxv16i32() +; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[CONV]], i64 0 +; SCALAR_EPILOGUE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; SCALAR_EPILOGUE-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP5]], i64 0 +; SCALAR_EPILOGUE-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; SCALAR_EPILOGUE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALAR_EPILOGUE: vector.body: ; SCALAR_EPILOGUE-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; SCALAR_EPILOGUE-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; SCALAR_EPILOGUE-NEXT: [[TMP7:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] -; SCALAR_EPILOGUE-NEXT: [[TMP8:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 2) -; SCALAR_EPILOGUE-NEXT: [[TMP9:%.*]] = or disjoint [[TMP8]], splat (i32 1) -; SCALAR_EPILOGUE-NEXT: [[TMP10:%.*]] = or disjoint [[TMP8]], splat (i32 2) -; SCALAR_EPILOGUE-NEXT: [[TMP11:%.*]] = or disjoint [[TMP8]], splat (i32 3) -; SCALAR_EPILOGUE-NEXT: [[TMP12:%.*]] = zext nneg [[TMP8]] to -; SCALAR_EPILOGUE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP12]] -; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP13]], i32 1, [[TMP7]], poison) -; SCALAR_EPILOGUE-NEXT: [[TMP14:%.*]] = zext nneg [[TMP9]] to -; SCALAR_EPILOGUE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP14]] -; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP15]], i32 1, [[TMP7]], poison) -; SCALAR_EPILOGUE-NEXT: [[TMP16:%.*]] = zext nneg [[TMP10]] to -; SCALAR_EPILOGUE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP16]] -; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP17]], i32 1, [[TMP7]], poison) -; SCALAR_EPILOGUE-NEXT: [[TMP18:%.*]] = zext nneg [[TMP11]] to -; SCALAR_EPILOGUE-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP18]] -; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP19]], i32 1, [[TMP7]], poison) -; SCALAR_EPILOGUE-NEXT: [[TMP20:%.*]] = call @llvm.smax.nxv8i8( [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]) -; SCALAR_EPILOGUE-NEXT: [[TMP21:%.*]] = sub zeroinitializer, [[TMP20]] -; SCALAR_EPILOGUE-NEXT: [[TMP22:%.*]] = call @llvm.smax.nxv8i8( [[WIDE_MASKED_GATHER2]], [[WIDE_MASKED_GATHER3]]) -; SCALAR_EPILOGUE-NEXT: [[TMP23:%.*]] = sub zeroinitializer, [[TMP22]] -; SCALAR_EPILOGUE-NEXT: [[TMP24:%.*]] = zext nneg [[TMP8]] to -; SCALAR_EPILOGUE-NEXT: [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP24]] -; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP20]], [[TMP25]], i32 1, [[TMP7]]) -; SCALAR_EPILOGUE-NEXT: [[TMP26:%.*]] = zext nneg [[TMP9]] to -; SCALAR_EPILOGUE-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP26]] -; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP21]], [[TMP27]], i32 1, [[TMP7]]) -; SCALAR_EPILOGUE-NEXT: [[TMP28:%.*]] = zext nneg [[TMP10]] to -; SCALAR_EPILOGUE-NEXT: [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP28]] -; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP22]], [[TMP29]], i32 1, [[TMP7]]) -; SCALAR_EPILOGUE-NEXT: [[TMP30:%.*]] = zext nneg [[TMP11]] to -; SCALAR_EPILOGUE-NEXT: [[TMP31:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP30]] -; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP23]], [[TMP31]], i32 1, [[TMP7]]) +; SCALAR_EPILOGUE-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; SCALAR_EPILOGUE-NEXT: [[TMP7:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] +; SCALAR_EPILOGUE-NEXT: [[TMP8:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 2) +; SCALAR_EPILOGUE-NEXT: [[TMP9:%.*]] = or disjoint [[TMP8]], splat (i32 1) +; SCALAR_EPILOGUE-NEXT: [[TMP10:%.*]] = or disjoint [[TMP8]], splat (i32 2) +; SCALAR_EPILOGUE-NEXT: [[TMP11:%.*]] = or disjoint [[TMP8]], splat (i32 3) +; SCALAR_EPILOGUE-NEXT: [[TMP12:%.*]] = zext nneg [[TMP8]] to +; SCALAR_EPILOGUE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP12]] +; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP13]], i32 1, [[TMP7]], poison) +; SCALAR_EPILOGUE-NEXT: [[TMP14:%.*]] = zext nneg [[TMP9]] to +; SCALAR_EPILOGUE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP14]] +; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP15]], i32 1, [[TMP7]], poison) +; SCALAR_EPILOGUE-NEXT: [[TMP16:%.*]] = zext nneg [[TMP10]] to +; SCALAR_EPILOGUE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP16]] +; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP17]], i32 1, [[TMP7]], poison) +; SCALAR_EPILOGUE-NEXT: [[TMP18:%.*]] = zext nneg [[TMP11]] to +; SCALAR_EPILOGUE-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP18]] +; SCALAR_EPILOGUE-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP19]], i32 1, [[TMP7]], poison) +; SCALAR_EPILOGUE-NEXT: [[TMP20:%.*]] = call @llvm.smax.nxv16i8( [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]) +; SCALAR_EPILOGUE-NEXT: [[TMP21:%.*]] = sub zeroinitializer, [[TMP20]] +; SCALAR_EPILOGUE-NEXT: [[TMP22:%.*]] = call @llvm.smax.nxv16i8( [[WIDE_MASKED_GATHER2]], [[WIDE_MASKED_GATHER3]]) +; SCALAR_EPILOGUE-NEXT: [[TMP23:%.*]] = sub zeroinitializer, [[TMP22]] +; SCALAR_EPILOGUE-NEXT: [[TMP24:%.*]] = zext nneg [[TMP8]] to +; SCALAR_EPILOGUE-NEXT: [[TMP25:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP24]] +; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP20]], [[TMP25]], i32 1, [[TMP7]]) +; SCALAR_EPILOGUE-NEXT: [[TMP26:%.*]] = zext nneg [[TMP9]] to +; SCALAR_EPILOGUE-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP26]] +; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP21]], [[TMP27]], i32 1, [[TMP7]]) +; SCALAR_EPILOGUE-NEXT: [[TMP28:%.*]] = zext nneg [[TMP10]] to +; SCALAR_EPILOGUE-NEXT: [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP28]] +; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP22]], [[TMP29]], i32 1, [[TMP7]]) +; SCALAR_EPILOGUE-NEXT: [[TMP30:%.*]] = zext nneg [[TMP11]] to +; SCALAR_EPILOGUE-NEXT: [[TMP31:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP30]] +; SCALAR_EPILOGUE-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP23]], [[TMP31]], i32 1, [[TMP7]]) ; SCALAR_EPILOGUE-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]] -; SCALAR_EPILOGUE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] +; SCALAR_EPILOGUE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALAR_EPILOGUE-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; SCALAR_EPILOGUE-NEXT: br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; SCALAR_EPILOGUE: middle.block: @@ -248,58 +248,58 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali ; PREDICATED_TAIL_FOLDING: vector.ph: ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 3 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023 ; PREDICATED_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]] ; PREDICATED_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call i32 @llvm.vscale.i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 3 -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = call @llvm.stepvector.nxv8i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[CONV]], i64 0 -; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP3]], i64 0 -; PREDICATED_TAIL_FOLDING-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 4 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = call @llvm.stepvector.nxv16i32() +; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[CONV]], i64 0 +; PREDICATED_TAIL_FOLDING-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; PREDICATED_TAIL_FOLDING-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP3]], i64 0 +; PREDICATED_TAIL_FOLDING-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; PREDICATED_TAIL_FOLDING-NEXT: br label [[VECTOR_BODY:%.*]] ; PREDICATED_TAIL_FOLDING: vector.body: ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv8i1.i32(i32 [[INDEX]], i32 1024) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP5]], zeroinitializer -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 2) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = or disjoint [[TMP7]], splat (i32 1) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP9:%.*]] = or disjoint [[TMP7]], splat (i32 2) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP10:%.*]] = or disjoint [[TMP7]], splat (i32 3) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP11:%.*]] = zext nneg [[TMP7]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP11]] -; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP12]], i32 1, [[TMP6]], poison) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = zext nneg [[TMP8]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP13]] -; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP14]], i32 1, [[TMP6]], poison) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = zext nneg [[TMP9]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP15]] -; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP16]], i32 1, [[TMP6]], poison) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP17:%.*]] = zext nneg [[TMP10]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP17]] -; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call @llvm.masked.gather.nxv8i8.nxv8p0( [[TMP18]], i32 1, [[TMP6]], poison) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = call @llvm.smax.nxv8i8( [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP20:%.*]] = sub zeroinitializer, [[TMP19]] -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP21:%.*]] = call @llvm.smax.nxv8i8( [[WIDE_MASKED_GATHER2]], [[WIDE_MASKED_GATHER3]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP22:%.*]] = sub zeroinitializer, [[TMP21]] -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP23:%.*]] = zext nneg [[TMP7]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP23]] -; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP19]], [[TMP24]], i32 1, [[TMP6]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP25:%.*]] = zext nneg [[TMP8]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP25]] -; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP20]], [[TMP26]], i32 1, [[TMP6]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP27:%.*]] = zext nneg [[TMP9]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP27]] -; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP21]], [[TMP28]], i32 1, [[TMP6]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP29:%.*]] = zext nneg [[TMP10]] to -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP29]] -; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv8i8.nxv8p0( [[TMP22]], [[TMP30]], i32 1, [[TMP6]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 1024) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = icmp ugt [[VEC_IND]], [[BROADCAST_SPLAT]] +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP5]], zeroinitializer +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP7:%.*]] = shl nuw nsw [[VEC_IND]], splat (i32 2) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP8:%.*]] = or disjoint [[TMP7]], splat (i32 1) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP9:%.*]] = or disjoint [[TMP7]], splat (i32 2) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP10:%.*]] = or disjoint [[TMP7]], splat (i32 3) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP11:%.*]] = zext nneg [[TMP7]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP11]] +; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP12]], i32 1, [[TMP6]], poison) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = zext nneg [[TMP8]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP13]] +; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER1:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP14]], i32 1, [[TMP6]], poison) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = zext nneg [[TMP9]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP15]] +; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP16]], i32 1, [[TMP6]], poison) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP17:%.*]] = zext nneg [[TMP10]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP18:%.*]] = getelementptr inbounds i8, ptr [[P]], [[TMP17]] +; PREDICATED_TAIL_FOLDING-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call @llvm.masked.gather.nxv16i8.nxv16p0( [[TMP18]], i32 1, [[TMP6]], poison) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = call @llvm.smax.nxv16i8( [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_GATHER1]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP20:%.*]] = sub zeroinitializer, [[TMP19]] +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP21:%.*]] = call @llvm.smax.nxv16i8( [[WIDE_MASKED_GATHER2]], [[WIDE_MASKED_GATHER3]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP22:%.*]] = sub zeroinitializer, [[TMP21]] +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP23:%.*]] = zext nneg [[TMP7]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP23]] +; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP19]], [[TMP24]], i32 1, [[TMP6]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP25:%.*]] = zext nneg [[TMP8]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP25]] +; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP20]], [[TMP26]], i32 1, [[TMP6]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP27:%.*]] = zext nneg [[TMP9]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP27]] +; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP21]], [[TMP28]], i32 1, [[TMP6]]) +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP29:%.*]] = zext nneg [[TMP10]] to +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP30:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP29]] +; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( [[TMP22]], [[TMP30]], i32 1, [[TMP6]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP3]] -; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] +; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; PREDICATED_TAIL_FOLDING-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; PREDICATED_TAIL_FOLDING: middle.block: