From 8a8462e74b323e8cc1a7a8f5ea9bc9e2a7101e97 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Sun, 6 Oct 2024 22:16:14 -0700 Subject: [PATCH 1/7] [VPlan] Implement VPWidenCastRecipe::computeCost(). (NFCI) This patch implement VPWidenCastRecipe::computeCost() and skip more cast recipies in the in-loop reduction. --- .../Transforms/Vectorize/LoopVectorize.cpp | 19 +++++++- llvm/lib/Transforms/Vectorize/VPlan.h | 4 ++ .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 43 +++++++++++++++++++ llvm/lib/Transforms/Vectorize/VPlanValue.h | 2 +- 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 027ee21527d22..0c8e2d1a2d989 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7258,12 +7258,29 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF, const auto &ChainOps = RdxDesc.getReductionOpChain(RedPhi, OrigLoop); SetVector ChainOpsAndOperands(ChainOps.begin(), ChainOps.end()); + auto isZExtOrSExt = [](const unsigned Opcode) -> bool { + return Opcode == Instruction::ZExt || Opcode == Instruction::SExt; + }; // Also include the operands of instructions in the chain, as the cost-model // may mark extends as free. + // + // For ARM, some of the instruction can folded into the reducion + // instruction. So we need to mark all folded instructions free. + // For example: We can fold reduce(mul(ext(A), ext(B))) into one + // instruction. for (auto *ChainOp : ChainOps) { for (Value *Op : ChainOp->operands()) { - if (auto *I = dyn_cast(Op)) + if (auto *I = dyn_cast(Op)) { ChainOpsAndOperands.insert(I); + if (I->getOpcode() == Instruction::Mul) { + auto *Ext0 = dyn_cast(I->getOperand(0)); + if (Ext0 && isZExtOrSExt(Ext0->getOpcode())) + ChainOpsAndOperands.insert(Ext0); + auto *Ext1 = dyn_cast(I->getOperand(1)); + if (Ext1 && isZExtOrSExt(Ext1->getOpcode())) + ChainOpsAndOperands.insert(Ext1); + } + } } } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 68a62638b9d58..94b7ddefbbb96 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1571,6 +1571,10 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags { /// Produce widened copies of the cast. void execute(VPTransformState &State) override; + /// Return the cost of this VPWidenCastRecipe. + InstructionCost computeCost(ElementCount VF, + VPCostContext &Ctx) const override; + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the recipe. void print(raw_ostream &O, const Twine &Indent, diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 2948ecc580edc..92131e8382bb7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1462,6 +1462,49 @@ void VPWidenCastRecipe::execute(VPTransformState &State) { State.addMetadata(Cast, cast_or_null(getUnderlyingValue())); } +InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, + VPCostContext &Ctx) const { + auto *SrcTy = cast( + ToVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF)); + auto *DestTy = cast(ToVectorTy(getResultType(), VF)); + // Computes the CastContextHint from a VPWidenMemoryRecipe instruction. + auto ComputeCCH = [&](VPWidenMemoryRecipe *R) -> TTI::CastContextHint { + assert((isa(R) || isa(R)) && + "Expected a load or a store!"); + + if (VF.isScalar()) + return TTI::CastContextHint::Normal; + if (!R->isConsecutive()) + return TTI::CastContextHint::GatherScatter; + if (R->isReverse()) + return TTI::CastContextHint::Reversed; + if (R->isMasked()) + return TTI::CastContextHint::Masked; + return TTI::CastContextHint::Normal; + }; + + TTI::CastContextHint CCH = TTI::CastContextHint::None; + // For Trunc, the context is the only user, which must be a + // VPWidenStoreRecipe. + if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) { + if (!cast(this)->hasMoreThanOneUniqueUser()) + if (VPWidenMemoryRecipe *Store = + dyn_cast(*this->user_begin())) + CCH = ComputeCCH(Store); + } + // For Z/Sext, the context is the operand, which must be a VPWidenLoadRecipe. + else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt || + Opcode == Instruction::FPExt) { + if (VPWidenMemoryRecipe *Load = dyn_cast( + this->getOperand(0)->getDefiningRecipe())) + CCH = ComputeCCH(Load); + } + // Arm TTI will use the underlying instruction to determine the cost. + return Ctx.TTI.getCastInstrCost( + Opcode, DestTy, SrcTy, CCH, TTI::TCK_RecipThroughput, + dyn_cast_if_present(getUnderlyingValue())); +} + #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPWidenCastRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index f2978b0a758b6..1900182f76e07 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -135,7 +135,7 @@ class VPValue { } /// Returns true if the value has more than one unique user. - bool hasMoreThanOneUniqueUser() { + bool hasMoreThanOneUniqueUser() const { if (getNumUsers() == 0) return false; From b43e6f9ca7bce726dcaced72e1024e5674787dee Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Sun, 13 Oct 2024 23:59:06 -0700 Subject: [PATCH 2/7] Fixup! consider VPInterleaveRecipe, VPReplicateRecipe, Live-in/out --- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 92131e8382bb7..445f062773a1a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1468,10 +1468,9 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, ToVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF)); auto *DestTy = cast(ToVectorTy(getResultType(), VF)); // Computes the CastContextHint from a VPWidenMemoryRecipe instruction. - auto ComputeCCH = [&](VPWidenMemoryRecipe *R) -> TTI::CastContextHint { + auto ComputeCCH = [&](const VPWidenMemoryRecipe *R) -> TTI::CastContextHint { assert((isa(R) || isa(R)) && "Expected a load or a store!"); - if (VF.isScalar()) return TTI::CastContextHint::Normal; if (!R->isConsecutive()) @@ -1485,19 +1484,31 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, TTI::CastContextHint CCH = TTI::CastContextHint::None; // For Trunc, the context is the only user, which must be a - // VPWidenStoreRecipe. - if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) { - if (!cast(this)->hasMoreThanOneUniqueUser()) - if (VPWidenMemoryRecipe *Store = - dyn_cast(*this->user_begin())) - CCH = ComputeCCH(Store); - } - // For Z/Sext, the context is the operand, which must be a VPWidenLoadRecipe. + // VPWidenStoreRecipe, a VPInterleaveRecipe ,a VPReplicateRecipe or a live-out + // value. + if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) && + !hasMoreThanOneUniqueUser() && getNumUsers() > 0) { + auto *StoreRecipe = dyn_cast(*user_begin()); + if (isa(*user_begin()) || isa(StoreRecipe)) + CCH = TTI::CastContextHint::Normal; + else if (const VPWidenMemoryRecipe *MemoryStoreRecipe = + dyn_cast(StoreRecipe)) + CCH = ComputeCCH(MemoryStoreRecipe); + else if (isa(StoreRecipe)) + CCH = TTI::CastContextHint::Interleave; + } + // For Z/Sext, the context is the operand, which must be a VPWidenLoadRecipe, + // a VPInterleaveRecipe, a VPReplicateRecipe or a live-in value. else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt || Opcode == Instruction::FPExt) { - if (VPWidenMemoryRecipe *Load = dyn_cast( - this->getOperand(0)->getDefiningRecipe())) - CCH = ComputeCCH(Load); + const VPRecipeBase *LoadRecipe = getOperand(0)->getDefiningRecipe(); + if (getOperand(0)->isLiveIn() || isa(LoadRecipe)) + CCH = TTI::CastContextHint::Normal; + else if (const VPWidenMemoryRecipe *MemoryLoadRecipe = + dyn_cast(LoadRecipe)) + CCH = ComputeCCH(MemoryLoadRecipe); + else if (isa(LoadRecipe)) + CCH = TTI::CastContextHint::Interleave; } // Arm TTI will use the underlying instruction to determine the cost. return Ctx.TTI.getCastInstrCost( From de00e4b44357d77873c117ad37d750adcff61611 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Mon, 14 Oct 2024 17:54:52 -0700 Subject: [PATCH 3/7] Fixup! merge ComputeCCH --- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 445f062773a1a..6283da51cc3fd 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1467,17 +1467,23 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, auto *SrcTy = cast( ToVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF)); auto *DestTy = cast(ToVectorTy(getResultType(), VF)); - // Computes the CastContextHint from a VPWidenMemoryRecipe instruction. - auto ComputeCCH = [&](const VPWidenMemoryRecipe *R) -> TTI::CastContextHint { - assert((isa(R) || isa(R)) && - "Expected a load or a store!"); + // Computes the CastContextHint from a recipes that may access memory. + auto ComputeCCH = [&](const VPRecipeBase *R) -> TTI::CastContextHint { if (VF.isScalar()) return TTI::CastContextHint::Normal; - if (!R->isConsecutive()) + if (isa(R)) + return TTI::CastContextHint::Interleave; + if (const auto *ReplicateRecipe = dyn_cast(R)) + return ReplicateRecipe->isPredicated() ? TTI::CastContextHint::Masked + : TTI::CastContextHint::Normal; + const auto *WidenMemoryRecipe = dyn_cast(R); + if (WidenMemoryRecipe == nullptr) + return TTI::CastContextHint::None; + if (!WidenMemoryRecipe->isConsecutive()) return TTI::CastContextHint::GatherScatter; - if (R->isReverse()) + if (WidenMemoryRecipe->isReverse()) return TTI::CastContextHint::Reversed; - if (R->isMasked()) + if (WidenMemoryRecipe->isMasked()) return TTI::CastContextHint::Masked; return TTI::CastContextHint::Normal; }; @@ -1489,26 +1495,20 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) && !hasMoreThanOneUniqueUser() && getNumUsers() > 0) { auto *StoreRecipe = dyn_cast(*user_begin()); - if (isa(*user_begin()) || isa(StoreRecipe)) + if (isa(*user_begin())) CCH = TTI::CastContextHint::Normal; - else if (const VPWidenMemoryRecipe *MemoryStoreRecipe = - dyn_cast(StoreRecipe)) - CCH = ComputeCCH(MemoryStoreRecipe); - else if (isa(StoreRecipe)) - CCH = TTI::CastContextHint::Interleave; + else if (StoreRecipe) + CCH = ComputeCCH(StoreRecipe); } // For Z/Sext, the context is the operand, which must be a VPWidenLoadRecipe, // a VPInterleaveRecipe, a VPReplicateRecipe or a live-in value. else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt || Opcode == Instruction::FPExt) { const VPRecipeBase *LoadRecipe = getOperand(0)->getDefiningRecipe(); - if (getOperand(0)->isLiveIn() || isa(LoadRecipe)) + if (getOperand(0)->isLiveIn()) CCH = TTI::CastContextHint::Normal; - else if (const VPWidenMemoryRecipe *MemoryLoadRecipe = - dyn_cast(LoadRecipe)) - CCH = ComputeCCH(MemoryLoadRecipe); - else if (isa(LoadRecipe)) - CCH = TTI::CastContextHint::Interleave; + else if (LoadRecipe) + CCH = ComputeCCH(LoadRecipe); } // Arm TTI will use the underlying instruction to determine the cost. return Ctx.TTI.getCastInstrCost( From 5eadd23601baa4c6261a226c71833281c6590121 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Mon, 14 Oct 2024 18:02:54 -0700 Subject: [PATCH 4/7] fixup! move def close to use. --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 6283da51cc3fd..f3e309402e91b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1464,9 +1464,6 @@ void VPWidenCastRecipe::execute(VPTransformState &State) { InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, VPCostContext &Ctx) const { - auto *SrcTy = cast( - ToVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF)); - auto *DestTy = cast(ToVectorTy(getResultType(), VF)); // Computes the CastContextHint from a recipes that may access memory. auto ComputeCCH = [&](const VPRecipeBase *R) -> TTI::CastContextHint { if (VF.isScalar()) @@ -1510,6 +1507,10 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, else if (LoadRecipe) CCH = ComputeCCH(LoadRecipe); } + + auto *SrcTy = cast( + ToVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF)); + auto *DestTy = cast(ToVectorTy(getResultType(), VF)); // Arm TTI will use the underlying instruction to determine the cost. return Ctx.TTI.getCastInstrCost( Opcode, DestTy, SrcTy, CCH, TTI::TCK_RecipThroughput, From 29c4d81a90ff250194b14313736178f9b6fb1d39 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Wed, 16 Oct 2024 20:18:43 -0700 Subject: [PATCH 5/7] Fixup! Remove live-out since it's in the loop. --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index f3e309402e91b..ff1b75e90f5a3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1487,25 +1487,20 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, TTI::CastContextHint CCH = TTI::CastContextHint::None; // For Trunc, the context is the only user, which must be a - // VPWidenStoreRecipe, a VPInterleaveRecipe ,a VPReplicateRecipe or a live-out - // value. + // VPWidenStoreRecipe, a VPInterleaveRecipe ,or a VPReplicateRecipe. if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) && !hasMoreThanOneUniqueUser() && getNumUsers() > 0) { - auto *StoreRecipe = dyn_cast(*user_begin()); - if (isa(*user_begin())) - CCH = TTI::CastContextHint::Normal; - else if (StoreRecipe) + if (auto *StoreRecipe = dyn_cast(*user_begin())) CCH = ComputeCCH(StoreRecipe); } // For Z/Sext, the context is the operand, which must be a VPWidenLoadRecipe, // a VPInterleaveRecipe, a VPReplicateRecipe or a live-in value. else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt || Opcode == Instruction::FPExt) { - const VPRecipeBase *LoadRecipe = getOperand(0)->getDefiningRecipe(); if (getOperand(0)->isLiveIn()) CCH = TTI::CastContextHint::Normal; - else if (LoadRecipe) - CCH = ComputeCCH(LoadRecipe); + else if (getOperand(0)->getDefiningRecipe()) + CCH = ComputeCCH(getOperand(0)->getDefiningRecipe()); } auto *SrcTy = cast( From fe37f080bd44625cf9a406a324eefdd7ca2bec45 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Sun, 20 Oct 2024 22:16:48 -0700 Subject: [PATCH 6/7] Address comments. --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 9 +++++---- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 11 ++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 0c8e2d1a2d989..c6158d6efa505 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7258,7 +7258,7 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF, const auto &ChainOps = RdxDesc.getReductionOpChain(RedPhi, OrigLoop); SetVector ChainOpsAndOperands(ChainOps.begin(), ChainOps.end()); - auto isZExtOrSExt = [](const unsigned Opcode) -> bool { + auto IsZExtOrSExt = [](const unsigned Opcode) -> bool { return Opcode == Instruction::ZExt || Opcode == Instruction::SExt; }; // Also include the operands of instructions in the chain, as the cost-model @@ -7274,11 +7274,12 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF, ChainOpsAndOperands.insert(I); if (I->getOpcode() == Instruction::Mul) { auto *Ext0 = dyn_cast(I->getOperand(0)); - if (Ext0 && isZExtOrSExt(Ext0->getOpcode())) - ChainOpsAndOperands.insert(Ext0); auto *Ext1 = dyn_cast(I->getOperand(1)); - if (Ext1 && isZExtOrSExt(Ext1->getOpcode())) + if (Ext0 && IsZExtOrSExt(Ext0->getOpcode()) && Ext1 && + Ext0->getOpcode() == Ext1->getOpcode()) { + ChainOpsAndOperands.insert(Ext0); ChainOpsAndOperands.insert(Ext1); + } } } } diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index ff1b75e90f5a3..b3034ff74cb16 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1485,6 +1485,7 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, return TTI::CastContextHint::Normal; }; + VPValue *Operand = getOperand(0); TTI::CastContextHint CCH = TTI::CastContextHint::None; // For Trunc, the context is the only user, which must be a // VPWidenStoreRecipe, a VPInterleaveRecipe ,or a VPReplicateRecipe. @@ -1497,14 +1498,14 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, // a VPInterleaveRecipe, a VPReplicateRecipe or a live-in value. else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt || Opcode == Instruction::FPExt) { - if (getOperand(0)->isLiveIn()) + if (Operand->isLiveIn()) CCH = TTI::CastContextHint::Normal; - else if (getOperand(0)->getDefiningRecipe()) - CCH = ComputeCCH(getOperand(0)->getDefiningRecipe()); + else if (Operand->getDefiningRecipe()) + CCH = ComputeCCH(Operand->getDefiningRecipe()); } - auto *SrcTy = cast( - ToVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF)); + auto *SrcTy = + cast(ToVectorTy(Ctx.Types.inferScalarType(Operand), VF)); auto *DestTy = cast(ToVectorTy(getResultType(), VF)); // Arm TTI will use the underlying instruction to determine the cost. return Ctx.TTI.getCastInstrCost( From 230a0d2d250c9926776d0324a7f4f9805f6b09d1 Mon Sep 17 00:00:00 2001 From: Elvis Wang Date: Mon, 21 Oct 2024 18:39:47 -0700 Subject: [PATCH 7/7] Fixup! Refine comments. --- llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index b3034ff74cb16..3acd7d5e3ca4c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -1487,15 +1487,13 @@ InstructionCost VPWidenCastRecipe::computeCost(ElementCount VF, VPValue *Operand = getOperand(0); TTI::CastContextHint CCH = TTI::CastContextHint::None; - // For Trunc, the context is the only user, which must be a - // VPWidenStoreRecipe, a VPInterleaveRecipe ,or a VPReplicateRecipe. + // For Trunc/FPTrunc, get the context from the only user. if ((Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) && !hasMoreThanOneUniqueUser() && getNumUsers() > 0) { if (auto *StoreRecipe = dyn_cast(*user_begin())) CCH = ComputeCCH(StoreRecipe); } - // For Z/Sext, the context is the operand, which must be a VPWidenLoadRecipe, - // a VPInterleaveRecipe, a VPReplicateRecipe or a live-in value. + // For Z/Sext, get the context from the operand. else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt || Opcode == Instruction::FPExt) { if (Operand->isLiveIn())