Skip to content

Commit 9c1a5df

Browse files
fhahnfhahn
committed
[VPlan] Use VPInstructionWithType for uniform casts.
Use VPInstructionWithType instead of VPReplicate recipe for uniform casts. This is a first step towards breaking up VPReplicateRecipe. Using the general VPInstructionWithType has the additional benefit that we can now apply a number of simplifications directly. This patch also adds a new IsSingleScalar field to VPInstruction, to encode the fact we know a recipe always produces a single scalar.
1 parent 1dc91a7 commit 9c1a5df

16 files changed

+88
-63
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8395,7 +8395,7 @@ VPRecipeBuilder::tryToWidenHistogram(const HistogramInfo *HI,
83958395
return new VPHistogramRecipe(Opcode, HGramOps, HI->Store->getDebugLoc());
83968396
}
83978397

8398-
VPReplicateRecipe *
8398+
VPSingleDefRecipe *
83998399
VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
84008400
VFRange &Range) {
84018401
bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
@@ -8453,6 +8453,13 @@ VPRecipeBuilder::handleReplication(Instruction *I, ArrayRef<VPValue *> Operands,
84538453
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
84548454
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
84558455
"Should not predicate a uniform recipe");
8456+
if (IsUniform && Instruction::isCast(I->getOpcode())) {
8457+
auto *Recipe = new VPInstructionWithType(I->getOpcode(), Operands,
8458+
I->getType(), VPIRFlags(*I),
8459+
I->getDebugLoc(), I->getName());
8460+
Recipe->setUnderlyingValue(I);
8461+
return Recipe;
8462+
}
84568463
auto *Recipe = new VPReplicateRecipe(I, Operands, IsUniform, BlockInMask,
84578464
VPIRMetadata(*I, LVer));
84588465
return Recipe;

‎llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ class VPRecipeBuilder {
199199
/// Build a VPReplicationRecipe for \p I using \p Operands. If it is
200200
/// predicated, add the mask as last operand. Range.End may be decreased to
201201
/// ensure same recipe behavior from \p Range.Start to \p Range.End.
202-
VPReplicateRecipe *handleReplication(Instruction *I,
202+
VPSingleDefRecipe *handleReplication(Instruction *I,
203203
ArrayRef<VPValue *> Operands,
204204
VFRange &Range);
205205

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -876,6 +876,9 @@ class VPInstruction : public VPRecipeWithIRFlags,
876876
public VPUnrollPartAccessor<1> {
877877
friend class VPlanSlp;
878878

879+
/// True if the VPInstruction produces a single scalar value.
880+
bool IsSingleScalar;
881+
879882
public:
880883
/// VPlan opcodes, extending LLVM IR with idiomatics instructions.
881884
enum {
@@ -966,7 +969,7 @@ class VPInstruction : public VPRecipeWithIRFlags,
966969

967970
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
968971
const VPIRFlags &Flags, DebugLoc DL = {},
969-
const Twine &Name = "");
972+
const Twine &Name = "", bool IsSingleScalar = false);
970973

971974
VP_CLASSOF_IMPL(VPDef::VPInstructionSC)
972975

@@ -1051,7 +1054,8 @@ class VPInstructionWithType : public VPInstruction {
10511054
VPInstructionWithType(unsigned Opcode, ArrayRef<VPValue *> Operands,
10521055
Type *ResultTy, const VPIRFlags &Flags, DebugLoc DL,
10531056
const Twine &Name = "")
1054-
: VPInstruction(Opcode, Operands, Flags, DL, Name), ResultTy(ResultTy) {}
1057+
: VPInstruction(Opcode, Operands, Flags, DL, Name, true),
1058+
ResultTy(ResultTy) {}
10551059

10561060
static inline bool classof(const VPRecipeBase *R) {
10571061
// VPInstructionWithType are VPInstructions with specific opcodes requiring
@@ -1087,10 +1091,7 @@ class VPInstructionWithType : public VPInstruction {
10871091

10881092
/// Return the cost of this VPInstruction.
10891093
InstructionCost computeCost(ElementCount VF,
1090-
VPCostContext &Ctx) const override {
1091-
// TODO: Compute accurate cost after retiring the legacy cost model.
1092-
return 0;
1093-
}
1094+
VPCostContext &Ctx) const override;
10941095

10951096
Type *getResultType() const { return ResultTy; }
10961097

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -408,9 +408,9 @@ template class VPUnrollPartAccessor<3>;
408408

409409
VPInstruction::VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands,
410410
const VPIRFlags &Flags, DebugLoc DL,
411-
const Twine &Name)
411+
const Twine &Name, bool IsSingleScalar)
412412
: VPRecipeWithIRFlags(VPDef::VPInstructionSC, Operands, Flags, DL),
413-
Opcode(Opcode), Name(Name.str()) {
413+
IsSingleScalar(IsSingleScalar), Opcode(Opcode), Name(Name.str()) {
414414
assert(flagsValidForOpcode(getOpcode()) &&
415415
"Set flags not supported for the provided opcode");
416416
}
@@ -841,7 +841,8 @@ bool VPInstruction::isVectorToScalar() const {
841841
}
842842

843843
bool VPInstruction::isSingleScalar() const {
844-
return getOpcode() == VPInstruction::ResumePhi ||
844+
// TODO: Set IsSingleScalar for ResumePhi and PHI.
845+
return IsSingleScalar || getOpcode() == VPInstruction::ResumePhi ||
845846
getOpcode() == Instruction::PHI;
846847
}
847848

@@ -1049,15 +1050,17 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent,
10491050

10501051
void VPInstructionWithType::execute(VPTransformState &State) {
10511052
State.setDebugLocFrom(getDebugLoc());
1052-
switch (getOpcode()) {
1053-
case Instruction::ZExt:
1054-
case Instruction::Trunc: {
1053+
if (Instruction::isCast(getOpcode())) {
10551054
Value *Op = State.get(getOperand(0), VPLane(0));
10561055
Value *Cast = State.Builder.CreateCast(Instruction::CastOps(getOpcode()),
10571056
Op, ResultTy);
1057+
if (auto *I = dyn_cast<Instruction>(Cast))
1058+
applyFlags(*I);
10581059
State.set(this, Cast, VPLane(0));
1059-
break;
1060+
return;
10601061
}
1062+
1063+
switch (getOpcode()) {
10611064
case VPInstruction::StepVector: {
10621065
Value *StepVector =
10631066
State.Builder.CreateStepVector(VectorType::get(ResultTy, State.VF));
@@ -1069,10 +1072,19 @@ void VPInstructionWithType::execute(VPTransformState &State) {
10691072
}
10701073
}
10711074

1075+
InstructionCost VPInstructionWithType::computeCost(ElementCount VF,
1076+
VPCostContext &Ctx) const {
1077+
// TODO: Compute cost for VPInstructions without underlying values once
1078+
// the legacy cost model has been retired.
1079+
if (!getUnderlyingValue())
1080+
return 0;
1081+
return Ctx.getLegacyCost(cast<Instruction>(getUnderlyingValue()), VF);
1082+
}
1083+
10721084
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
10731085
void VPInstructionWithType::print(raw_ostream &O, const Twine &Indent,
10741086
VPSlotTracker &SlotTracker) const {
1075-
O << Indent << "EMIT ";
1087+
O << Indent << (isSingleScalar() ? "SINGLE-SCALAR " : "EMIT ");
10761088
printAsOperand(O, SlotTracker);
10771089
O << " = ";
10781090

@@ -1598,10 +1610,11 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
15981610
return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
15991611
Opcode == Instruction::FSub || Opcode == Instruction::FNeg ||
16001612
Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
1613+
Opcode == Instruction::FPTrunc || Opcode == Instruction::FPExt ||
16011614
Opcode == Instruction::FCmp || Opcode == Instruction::Select ||
16021615
Opcode == VPInstruction::WideIVStep;
16031616
case OperationType::NonNegOp:
1604-
return Opcode == Instruction::ZExt;
1617+
return Opcode == Instruction::UIToFP || Opcode == Instruction::ZExt;
16051618
break;
16061619
case OperationType::Cmp:
16071620
return Opcode == Instruction::ICmp;

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1047,8 +1047,14 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
10471047
unsigned ExtOpcode = match(R.getOperand(0), m_SExt(m_VPValue()))
10481048
? Instruction::SExt
10491049
: Instruction::ZExt;
1050-
auto *VPC =
1051-
new VPWidenCastRecipe(Instruction::CastOps(ExtOpcode), A, TruncTy);
1050+
VPSingleDefRecipe *VPC;
1051+
if (vputils::isSingleScalar(R.getVPSingleValue()))
1052+
VPC = new VPInstructionWithType(Instruction::CastOps(ExtOpcode), {A},
1053+
TruncTy, {}, {});
1054+
else
1055+
VPC = new VPWidenCastRecipe(Instruction::CastOps(ExtOpcode), A,
1056+
TruncTy);
1057+
10521058
if (auto *UnderlyingExt = R.getOperand(0)->getUnderlyingValue()) {
10531059
// UnderlyingExt has distinct return type, used to retain legacy cost.
10541060
VPC->setUnderlyingValue(UnderlyingExt);

‎llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-vplan.ll‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ target triple = "aarch64-unknown-linux-gnu"
2929
; CHECK-NEXT: [[STEPS:vp.*]] = SCALAR-STEPS [[IV]], ir<1>, [[VF]]
3030
; CHECK-NEXT: CLONE [[GEP_IDX:.*]] = getelementptr inbounds ir<%indices>, [[STEPS]]
3131
; CHECK-NEXT: CLONE [[IDX:.*]] = load [[GEP_IDX]]
32-
; CHECK-NEXT: CLONE [[EXT_IDX:.*]] = zext [[IDX]]
32+
; CHECK-NEXT: SINGLE-SCALAR [[EXT_IDX:.*]] = zext [[IDX]]
3333
; CHECK-NEXT: CLONE [[GEP_BUCKET:.*]] = getelementptr inbounds ir<%buckets>, [[EXT_IDX]]
3434
; CHECK-NEXT: CLONE [[HISTVAL:.*]] = load [[GEP_BUCKET]]
3535
; CHECK-NEXT: CLONE [[UPDATE:.*]] = add nsw [[HISTVAL]], ir<1>

‎llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
7575
; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
7676
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
7777
; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
78-
; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
78+
; CHECK-NEXT: SINGLE-SCALAR ir<%idxprom> = zext ir<%i.0>
7979
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
8080
; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
8181
; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]>
@@ -199,7 +199,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
199199
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<[[CAN_IV_NEXT:%.+]]>, vector.body ]
200200
; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
201201
; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[DEV_IV]]>, ir<-1>
202-
; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
202+
; CHECK-NEXT: SINGLE-SCALAR ir<%idxprom> = zext ir<%i.0>
203203
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
204204
; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%arrayidx>, ir<[[VF]]>
205205
; CHECK-NEXT: WIDEN ir<[[L:%.+]]> = load vp<[[VEC_PTR]]>
@@ -323,7 +323,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
323323
; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
324324
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[DEV_IV]]>, ir<-1>
325325
; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[STEPS]]>, ir<-1>
326-
; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
326+
; CHECK-NEXT: SINGLE-SCALAR ir<%idxprom> = zext ir<%i.0>
327327
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
328328
; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%arrayidx>, vp<[[VF]]>
329329
; CHECK-NEXT: WIDEN ir<%1> = load vp<[[VEC_PTR]]>
@@ -447,7 +447,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
447447
; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = phi [ ir<0>, ir-bb<vector.ph> ], [ vp<[[CAN_IV_NEXT:%.+]]>, vector.body ]
448448
; CHECK-NEXT: vp<[[DEV_IV:%.+]]> = DERIVED-IV ir<%n> + vp<[[CAN_IV]]> * ir<-1>
449449
; CHECK-NEXT: CLONE ir<%i.0> = add nsw vp<[[DEV_IV]]>, ir<-1>
450-
; CHECK-NEXT: CLONE ir<%idxprom> = zext ir<%i.0>
450+
; CHECK-NEXT: SINGLE-SCALAR ir<%idxprom> = zext ir<%i.0>
451451
; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%B>, ir<%idxprom>
452452
; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-end-pointer inbounds ir<%arrayidx>, ir<[[VF]]>
453453
; CHECK-NEXT: WIDEN ir<[[L:%.+]]> = load vp<[[VEC_PTR]]>

‎llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-call-intrinsics.ll‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
3535
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
3636
; IF-EVL-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
3737
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[SMAX]]>, vp<[[EVL]]>
38-
; IF-EVL-NEXT: EMIT vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
38+
; IF-EVL-NEXT: SINGLE-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
3939
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
4040
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
4141
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
@@ -92,7 +92,7 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
9292
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
9393
; IF-EVL-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
9494
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[SMIN]]>, vp<[[EVL]]>
95-
; IF-EVL-NEXT: EMIT vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
95+
; IF-EVL-NEXT: SINGLE-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
9696
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
9797
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
9898
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
@@ -149,7 +149,7 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
149149
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
150150
; IF-EVL-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
151151
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[UMAX]]>, vp<[[EVL]]>
152-
; IF-EVL-NEXT: EMIT vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
152+
; IF-EVL-NEXT: SINGLE-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
153153
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
154154
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
155155
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
@@ -206,7 +206,7 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
206206
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
207207
; IF-EVL-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
208208
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[UMIN]]>, vp<[[EVL]]>
209-
; IF-EVL-NEXT: EMIT vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
209+
; IF-EVL-NEXT: SINGLE-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
210210
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
211211
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
212212
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
@@ -260,7 +260,7 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
260260
; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
261261
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
262262
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[CTLZ]]>, vp<[[EVL]]>
263-
; IF-EVL-NEXT: EMIT vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
263+
; IF-EVL-NEXT: SINGLE-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
264264
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
265265
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
266266
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
@@ -312,7 +312,7 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
312312
; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
313313
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
314314
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[CTTZ]]>, vp<[[EVL]]>
315-
; IF-EVL-NEXT: EMIT vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
315+
; IF-EVL-NEXT: SINGLE-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
316316
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
317317
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
318318
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
@@ -366,7 +366,7 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
366366
; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
367367
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
368368
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[TRUNC]]>, vp<[[EVL]]>
369-
; IF-EVL-NEXT: EMIT vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
369+
; IF-EVL-NEXT: SINGLE-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
370370
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
371371
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
372372
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
@@ -422,7 +422,7 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
422422
; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
423423
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
424424
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[TRUNC]]>, vp<[[EVL]]>
425-
; IF-EVL-NEXT: EMIT vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
425+
; IF-EVL-NEXT: SINGLE-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
426426
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
427427
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
428428
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>
@@ -476,7 +476,7 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
476476
; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
477477
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
478478
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR2]]>, ir<[[ABS]]>, vp<[[EVL]]>
479-
; IF-EVL-NEXT: EMIT vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
479+
; IF-EVL-NEXT: SINGLE-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
480480
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
481481
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add vp<[[IV]]>, vp<[[VFUF]]>
482482
; IF-EVL-NEXT: EMIT branch-on-count vp<[[IV_NEXT_EXIT]]>, vp<[[VTC]]>

0 commit comments

Comments
 (0)