diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 0baf7172e4443..59b5b83449a3f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2367,15 +2367,6 @@ void VPScalarIVStepsRecipe::execute(VPTransformState &State) { // Compute the scalar steps and save the results in State. Type *IntStepTy = IntegerType::get(BaseIVTy->getContext(), BaseIVTy->getScalarSizeInBits()); - Type *VecIVTy = nullptr; - Value *UnitStepVec = nullptr, *SplatStep = nullptr, *SplatIV = nullptr; - if (!FirstLaneOnly && State.VF.isScalable()) { - VecIVTy = VectorType::get(BaseIVTy, State.VF); - UnitStepVec = - Builder.CreateStepVector(VectorType::get(IntStepTy, State.VF)); - SplatStep = Builder.CreateVectorSplat(State.VF, Step); - SplatIV = Builder.CreateVectorSplat(State.VF, BaseIV); - } unsigned StartLane = 0; unsigned EndLane = FirstLaneOnly ? 1 : State.VF.getKnownMinValue(); @@ -2396,19 +2387,6 @@ void VPScalarIVStepsRecipe::execute(VPTransformState &State) { StartIdx0 = Builder.CreateSExtOrTrunc(StartIdx0, IntStepTy); } - if (!FirstLaneOnly && State.VF.isScalable()) { - auto *SplatStartIdx = Builder.CreateVectorSplat(State.VF, StartIdx0); - auto *InitVec = Builder.CreateAdd(SplatStartIdx, UnitStepVec); - if (BaseIVTy->isFloatingPointTy()) - InitVec = Builder.CreateSIToFP(InitVec, VecIVTy); - auto *Mul = Builder.CreateBinOp(MulOp, InitVec, SplatStep); - auto *Add = Builder.CreateBinOp(AddOp, SplatIV, Mul); - State.set(this, Add); - // It's useful to record the lane values too for the known minimum number - // of elements so we do those below. This improves the code quality when - // trying to extract the first element, for example. - } - if (BaseIVTy->isFloatingPointTy()) StartIdx0 = Builder.CreateSIToFP(StartIdx0, BaseIVTy); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index f7281283bae81..827dd4b6439ae 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -794,12 +794,19 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) { WideIV->getDebugLoc(), Builder); // Update scalar users of IV to use Step instead. - if (!HasOnlyVectorVFs) + if (!HasOnlyVectorVFs) { + assert(!Plan.hasScalableVF() && + "plans containing a scalar VF cannot also include scalable VFs"); WideIV->replaceAllUsesWith(Steps); - else - WideIV->replaceUsesWithIf(Steps, [WideIV](VPUser &U, unsigned) { - return U.usesScalars(WideIV); - }); + } else { + bool HasScalableVF = Plan.hasScalableVF(); + WideIV->replaceUsesWithIf(Steps, + [WideIV, HasScalableVF](VPUser &U, unsigned) { + if (HasScalableVF) + return U.usesFirstLaneOnly(WideIV); + return U.usesScalars(WideIV); + }); + } } } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll index 88e035ebf3be8..131b3d1b02727 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll @@ -15,25 +15,22 @@ define void @test_invar_gep(ptr %dst) #0 { ; CHECK-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call @llvm.stepvector.nxv4i64() +; CHECK-NEXT: [[TMP4:%.*]] = mul nsw [[TMP5]], splat (i64 1) +; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw zeroinitializer, [[TMP4]] +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP3]], i64 0 +; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = call @llvm.stepvector.nxv4i64() -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[INDEX]], i64 0 -; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer -; CHECK-NEXT: [[TMP10:%.*]] = add zeroinitializer, [[TMP5]] -; CHECK-NEXT: [[TMP4:%.*]] = mul [[TMP10]], splat (i64 1) -; CHECK-NEXT: [[TMP9:%.*]] = add [[DOTSPLAT]], [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP9:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vscale.i32() ; CHECK-NEXT: [[TMP16:%.*]] = mul nuw i32 [[TMP15]], 4 ; CHECK-NEXT: [[TMP17:%.*]] = sub i32 [[TMP16]], 1 ; CHECK-NEXT: [[TMP18:%.*]] = extractelement [[TMP9]], i32 [[TMP17]] ; CHECK-NEXT: store i64 [[TMP18]], ptr [[TMP14:%.*]], align 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw [[TMP9]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -60,38 +57,26 @@ define void @test_invar_gep(ptr %dst) #0 { ; IC2: vector.ph: ; IC2-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; IC2-NEXT: [[TMP11:%.*]] = mul nuw i64 [[TMP2]], 4 +; IC2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[TMP11]], i64 0 +; IC2-NEXT: [[TMP21:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; IC2-NEXT: [[TMP3:%.*]] = mul i64 [[TMP11]], 2 ; IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]] ; IC2-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]] +; IC2-NEXT: [[TMP5:%.*]] = call @llvm.stepvector.nxv4i64() +; IC2-NEXT: [[TMP12:%.*]] = mul nsw [[TMP5]], splat (i64 1) +; IC2-NEXT: [[INDUCTION:%.*]] = add nsw zeroinitializer, [[TMP12]] ; IC2-NEXT: br label [[VECTOR_BODY:%.*]] ; IC2: vector.body: ; IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; IC2-NEXT: [[BROADCAST_SPLAT:%.*]] = call @llvm.stepvector.nxv4i64() -; IC2-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[INDEX]], i64 0 -; IC2-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer -; IC2-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement poison, i64 [[TMP11]], i64 0 -; IC2-NEXT: [[VEC_IND:%.*]] = shufflevector [[DOTSPLATINSERT1]], poison, zeroinitializer -; IC2-NEXT: [[TMP5:%.*]] = add [[VEC_IND]], [[BROADCAST_SPLAT]] -; IC2-NEXT: [[TMP21:%.*]] = mul [[TMP5]], splat (i64 1) +; IC2-NEXT: [[DOTSPLAT:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; IC2-NEXT: [[TMP22:%.*]] = add [[DOTSPLAT]], [[TMP21]] -; IC2-NEXT: [[TMP23:%.*]] = add i64 [[TMP11]], 0 -; IC2-NEXT: [[TMP24:%.*]] = mul i64 [[TMP23]], 1 -; IC2-NEXT: [[TMP25:%.*]] = add i64 [[INDEX]], [[TMP24]] -; IC2-NEXT: [[TMP12:%.*]] = add i64 [[TMP11]], 1 -; IC2-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 1 -; IC2-NEXT: [[TMP14:%.*]] = add i64 [[INDEX]], [[TMP13]] -; IC2-NEXT: [[TMP15:%.*]] = add i64 [[TMP11]], 2 -; IC2-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 1 -; IC2-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], [[TMP16]] -; IC2-NEXT: [[TMP18:%.*]] = add i64 [[TMP11]], 3 -; IC2-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 1 -; IC2-NEXT: [[TMP20:%.*]] = add i64 [[INDEX]], [[TMP19]] ; IC2-NEXT: [[TMP6:%.*]] = call i32 @llvm.vscale.i32() ; IC2-NEXT: [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 4 ; IC2-NEXT: [[TMP8:%.*]] = sub i32 [[TMP7]], 1 ; IC2-NEXT: [[TMP9:%.*]] = extractelement [[TMP22]], i32 [[TMP8]] ; IC2-NEXT: store i64 [[TMP9]], ptr [[DST:%.*]], align 1 ; IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]] +; IC2-NEXT: [[VEC_IND_NEXT]] = add nsw [[TMP22]], [[TMP21]] ; IC2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; IC2-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; IC2: middle.block: @@ -139,26 +124,24 @@ define void @test_invar_gep_var_start(i64 %start, ptr %dst) #0 { ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] ; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[START]], [[N_VEC]] +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.stepvector.nxv4i64() +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[START]], i64 0 +; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer +; CHECK-NEXT: [[TMP13:%.*]] = mul nsw [[TMP6]], splat (i64 1) +; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw [[DOTSPLAT]], [[TMP13]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i64 [[TMP4]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[START]], [[INDEX]] -; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.stepvector.nxv4i64() -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[OFFSET_IDX]], i64 0 -; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer -; CHECK-NEXT: [[TMP14:%.*]] = add zeroinitializer, [[TMP6]] -; CHECK-NEXT: [[TMP15:%.*]] = mul [[TMP14]], splat (i64 1) -; CHECK-NEXT: [[TMP7:%.*]] = add [[DOTSPLAT]], [[TMP15]] -; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[OFFSET_IDX]], 0 -; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[OFFSET_IDX]], 1 -; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[OFFSET_IDX]], 2 -; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], 3 +; CHECK-NEXT: [[TMP7:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vscale.i32() ; CHECK-NEXT: [[TMP9:%.*]] = mul nuw i32 [[TMP8]], 4 ; CHECK-NEXT: [[TMP10:%.*]] = sub i32 [[TMP9]], 1 ; CHECK-NEXT: [[TMP11:%.*]] = extractelement [[TMP7]], i32 [[TMP10]] ; CHECK-NEXT: store i64 [[TMP11]], ptr [[DST:%.*]], align 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw [[TMP7]], [[BROADCAST_SPLAT2]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: @@ -187,40 +170,29 @@ define void @test_invar_gep_var_start(i64 %start, ptr %dst) #0 { ; IC2: vector.ph: ; IC2-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() ; IC2-NEXT: [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4 +; IC2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[TMP4]], i64 0 +; IC2-NEXT: [[TMP9:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; IC2-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP5]] ; IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] ; IC2-NEXT: [[TMP6:%.*]] = add i64 [[START]], [[N_VEC]] +; IC2-NEXT: [[BROADCAST_SPLAT:%.*]] = call @llvm.stepvector.nxv4i64() +; IC2-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement poison, i64 [[START]], i64 0 +; IC2-NEXT: [[VEC_IND:%.*]] = shufflevector [[DOTSPLATINSERT1]], poison, zeroinitializer +; IC2-NEXT: [[TMP8:%.*]] = mul nsw [[BROADCAST_SPLAT]], splat (i64 1) +; IC2-NEXT: [[INDUCTION:%.*]] = add nsw [[VEC_IND]], [[TMP8]] ; IC2-NEXT: br label [[VECTOR_BODY:%.*]] ; IC2: vector.body: ; IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; IC2-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[START]], [[INDEX]] -; IC2-NEXT: [[BROADCAST_SPLAT:%.*]] = call @llvm.stepvector.nxv4i64() -; IC2-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[OFFSET_IDX]], i64 0 -; IC2-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer -; IC2-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement poison, i64 [[TMP4]], i64 0 -; IC2-NEXT: [[VEC_IND:%.*]] = shufflevector [[DOTSPLATINSERT1]], poison, zeroinitializer -; IC2-NEXT: [[TMP11:%.*]] = add [[VEC_IND]], [[BROADCAST_SPLAT]] -; IC2-NEXT: [[TMP9:%.*]] = mul [[TMP11]], splat (i64 1) +; IC2-NEXT: [[DOTSPLAT:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; IC2-NEXT: [[TMP10:%.*]] = add [[DOTSPLAT]], [[TMP9]] -; IC2-NEXT: [[TMP23:%.*]] = add i64 [[TMP4]], 0 -; IC2-NEXT: [[TMP24:%.*]] = mul i64 [[TMP23]], 1 -; IC2-NEXT: [[TMP25:%.*]] = add i64 [[OFFSET_IDX]], [[TMP24]] -; IC2-NEXT: [[TMP26:%.*]] = add i64 [[TMP4]], 1 -; IC2-NEXT: [[TMP27:%.*]] = mul i64 [[TMP26]], 1 -; IC2-NEXT: [[TMP28:%.*]] = add i64 [[OFFSET_IDX]], [[TMP27]] -; IC2-NEXT: [[TMP17:%.*]] = add i64 [[TMP4]], 2 -; IC2-NEXT: [[TMP18:%.*]] = mul i64 [[TMP17]], 1 -; IC2-NEXT: [[TMP19:%.*]] = add i64 [[OFFSET_IDX]], [[TMP18]] -; IC2-NEXT: [[TMP20:%.*]] = add i64 [[TMP4]], 3 -; IC2-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 1 -; IC2-NEXT: [[TMP22:%.*]] = add i64 [[OFFSET_IDX]], [[TMP21]] ; IC2-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32() ; IC2-NEXT: [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4 ; IC2-NEXT: [[TMP14:%.*]] = sub i32 [[TMP13]], 1 ; IC2-NEXT: [[TMP15:%.*]] = extractelement [[TMP10]], i32 [[TMP14]] ; IC2-NEXT: store i64 [[TMP15]], ptr [[DST:%.*]], align 1 ; IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] +; IC2-NEXT: [[VEC_IND_NEXT]] = add nsw [[TMP10]], [[TMP9]] ; IC2-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; IC2-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; IC2: middle.block: @@ -269,36 +241,34 @@ define void @test_invar_gep_var_start_step_2(i64 %start, ptr %dst) #0 { ; CHECK-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[N_VEC]], 2 -; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[START]], [[TMP7]] -; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] -; CHECK: vector.body: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[N_VEC]], 2 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[START]], [[TMP9]] ; CHECK-NEXT: [[TMP10:%.*]] = call @llvm.stepvector.nxv4i64() -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[OFFSET_IDX]], i64 0 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[START]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer -; CHECK-NEXT: [[TMP11:%.*]] = add zeroinitializer, [[TMP10]] -; CHECK-NEXT: [[TMP18:%.*]] = mul [[TMP11]], splat (i64 2) -; CHECK-NEXT: [[TMP12:%.*]] = add [[DOTSPLAT]], [[TMP18]] -; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[OFFSET_IDX]], 0 -; CHECK-NEXT: [[TMP20:%.*]] = add i64 [[OFFSET_IDX]], 2 -; CHECK-NEXT: [[TMP21:%.*]] = add i64 [[OFFSET_IDX]], 4 -; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[OFFSET_IDX]], 6 +; CHECK-NEXT: [[TMP18:%.*]] = mul nsw [[TMP10]], splat (i64 2) +; CHECK-NEXT: [[INDUCTION:%.*]] = add nsw [[DOTSPLAT]], [[TMP18]] +; CHECK-NEXT: [[TMP11:%.*]] = mul nsw i64 2, [[TMP6]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i64 [[TMP11]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector [[BROADCAST_SPLATINSERT1]], poison, zeroinitializer +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP12:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vscale.i32() ; CHECK-NEXT: [[TMP14:%.*]] = mul nuw i32 [[TMP13]], 4 ; CHECK-NEXT: [[TMP15:%.*]] = sub i32 [[TMP14]], 1 ; CHECK-NEXT: [[TMP16:%.*]] = extractelement [[TMP12]], i32 [[TMP15]] ; CHECK-NEXT: store i64 [[TMP16]], ptr [[DST:%.*]], align 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nsw [[TMP12]], [[BROADCAST_SPLAT2]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP8]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[OFFSET_IDX]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ] ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] @@ -322,49 +292,38 @@ define void @test_invar_gep_var_start_step_2(i64 %start, ptr %dst) #0 { ; IC2: vector.ph: ; IC2-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; IC2-NEXT: [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4 +; IC2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[TMP6]], i64 0 +; IC2-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; IC2-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 2 ; IC2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP7]] ; IC2-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] -; IC2-NEXT: [[TMP8:%.*]] = mul i64 [[N_VEC]], 2 -; IC2-NEXT: [[TMP9:%.*]] = add i64 [[START]], [[TMP8]] -; IC2-NEXT: br label [[VECTOR_BODY:%.*]] -; IC2: vector.body: -; IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; IC2-NEXT: [[TMP10:%.*]] = mul i64 [[INDEX]], 2 +; IC2-NEXT: [[TMP10:%.*]] = mul i64 [[N_VEC]], 2 ; IC2-NEXT: [[OFFSET_IDX:%.*]] = add i64 [[START]], [[TMP10]] +; IC2-NEXT: [[TMP13:%.*]] = mul [[BROADCAST_SPLAT1]], splat (i64 2) ; IC2-NEXT: [[TMP11:%.*]] = call @llvm.stepvector.nxv4i64() -; IC2-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[OFFSET_IDX]], i64 0 -; IC2-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer -; IC2-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement poison, i64 [[TMP6]], i64 0 +; IC2-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement poison, i64 [[START]], i64 0 ; IC2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[DOTSPLATINSERT1]], poison, zeroinitializer -; IC2-NEXT: [[TMP16:%.*]] = add [[BROADCAST_SPLAT]], [[TMP11]] -; IC2-NEXT: [[TMP13:%.*]] = mul [[TMP16]], splat (i64 2) +; IC2-NEXT: [[TMP12:%.*]] = mul nsw [[TMP11]], splat (i64 2) +; IC2-NEXT: [[INDUCTION:%.*]] = add nsw [[BROADCAST_SPLAT]], [[TMP12]] +; IC2-NEXT: br label [[VECTOR_BODY:%.*]] +; IC2: vector.body: +; IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; IC2-NEXT: [[DOTSPLAT:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; IC2-NEXT: [[TMP14:%.*]] = add [[DOTSPLAT]], [[TMP13]] -; IC2-NEXT: [[TMP15:%.*]] = add i64 [[TMP6]], 0 -; IC2-NEXT: [[TMP27:%.*]] = mul i64 [[TMP15]], 2 -; IC2-NEXT: [[TMP28:%.*]] = add i64 [[OFFSET_IDX]], [[TMP27]] -; IC2-NEXT: [[TMP29:%.*]] = add i64 [[TMP6]], 1 -; IC2-NEXT: [[TMP30:%.*]] = mul i64 [[TMP29]], 2 -; IC2-NEXT: [[TMP31:%.*]] = add i64 [[OFFSET_IDX]], [[TMP30]] -; IC2-NEXT: [[TMP32:%.*]] = add i64 [[TMP6]], 2 -; IC2-NEXT: [[TMP22:%.*]] = mul i64 [[TMP32]], 2 -; IC2-NEXT: [[TMP23:%.*]] = add i64 [[OFFSET_IDX]], [[TMP22]] -; IC2-NEXT: [[TMP24:%.*]] = add i64 [[TMP6]], 3 -; IC2-NEXT: [[TMP25:%.*]] = mul i64 [[TMP24]], 2 -; IC2-NEXT: [[TMP26:%.*]] = add i64 [[OFFSET_IDX]], [[TMP25]] ; IC2-NEXT: [[TMP17:%.*]] = call i32 @llvm.vscale.i32() ; IC2-NEXT: [[TMP18:%.*]] = mul nuw i32 [[TMP17]], 4 ; IC2-NEXT: [[TMP19:%.*]] = sub i32 [[TMP18]], 1 ; IC2-NEXT: [[TMP20:%.*]] = extractelement [[TMP14]], i32 [[TMP19]] ; IC2-NEXT: store i64 [[TMP20]], ptr [[DST:%.*]], align 1 ; IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] +; IC2-NEXT: [[VEC_IND_NEXT]] = add nsw [[TMP14]], [[TMP13]] ; IC2-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; IC2-NEXT: br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; IC2: middle.block: ; IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] ; IC2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] ; IC2: scalar.ph: -; IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[TMP9]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ] +; IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[OFFSET_IDX]], [[MIDDLE_BLOCK]] ], [ [[START]], [[ENTRY:%.*]] ] ; IC2-NEXT: br label [[LOOP:%.*]] ; IC2: loop: ; IC2-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]