Skip to content

(release 20.x non-upstream patch) [LV] Add initial support for vectorizing literal struct return values #82

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 24, 2025
Merged

(release 20.x non-upstream patch) [LV] Add initial support for vectorizing literal struct return values #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cc6fcd3
[LV] Add initial support for vectorizing literal struct return values
MacDue Sep 20, 2024
8065001
Add a load of comments
MacDue Feb 12, 2025
199a9ec
Revert "Add a load of comments"
MacDue Feb 17, 2025
545a1bd
Revert "Revert "Add a load of comments""
MacDue Feb 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions llvm/include/llvm/Analysis/TargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1473,6 +1473,14 @@ class TargetTransformInfo {
TTI::TargetCostKind CostKind,
unsigned Index = -1) const;

/* Downstream change: #87 (sincos vectorization)*/
/// \return The expected cost of aggregate inserts and extracts. This is
/// used when the instruction is not available; a typical use case is to
/// provision the cost of vectorization/scalarization in vectorizer passes.
InstructionCost getInsertExtractValueCost(unsigned Opcode,
TTI::TargetCostKind CostKind) const;
/* End downstream change: #87 */

/// \return The cost of replication shuffle of \p VF elements typed \p EltTy
/// \p ReplicationFactor times.
///
Expand Down Expand Up @@ -2205,6 +2213,11 @@ class TargetTransformInfo::Concept {
const APInt &DemandedDstElts,
TTI::TargetCostKind CostKind) = 0;

/* Downstream change: #87 (sincos vectorization)*/
virtual InstructionCost
getInsertExtractValueCost(unsigned Opcode, TTI::TargetCostKind CostKind) = 0;
/* End downstream change: #87 */

virtual InstructionCost
getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace, TTI::TargetCostKind CostKind,
Expand Down Expand Up @@ -2926,6 +2939,13 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
return Impl.getReplicationShuffleCost(EltTy, ReplicationFactor, VF,
DemandedDstElts, CostKind);
}
/* Downstream change: #87 (sincos vectorization)*/
InstructionCost
getInsertExtractValueCost(unsigned Opcode,
TTI::TargetCostKind CostKind) override {
return Impl.getInsertExtractValueCost(Opcode, CostKind);
}
/* End downstream change: #87 */
InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
Expand Down
19 changes: 18 additions & 1 deletion llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -745,6 +745,19 @@ class TargetTransformInfoImplBase {
return 1;
}

/* Downstream change: #87 (sincos vectorization)*/
InstructionCost
getInsertExtractValueCost(unsigned Opcode,
TTI::TargetCostKind CostKind) const {
// Note: The `insertvalue` cost here is chosen to match the default case of
// getInstructionCost() -- as pior to adding this helper `insertvalue` was
// not handled.
if (Opcode == Instruction::InsertValue)
return CostKind == TTI::TCK_RecipThroughput ? -1 : TTI::TCC_Basic;
return TTI::TCC_Free;
}
/* End downstream change: #87 */

InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
unsigned AddressSpace,
TTI::TargetCostKind CostKind,
Expand Down Expand Up @@ -1296,9 +1309,13 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase {
case Instruction::PHI:
case Instruction::Switch:
return TargetTTI->getCFInstrCost(Opcode, CostKind, I);
case Instruction::ExtractValue:
case Instruction::Freeze:
return TTI::TCC_Free;
/* Downstream change: #87 (sincos vectorization)*/
case Instruction::ExtractValue:
case Instruction::InsertValue:
return TargetTTI->getInsertExtractValueCost(Opcode, CostKind);
/* End downstream change: #87 */
case Instruction::Alloca:
if (cast<AllocaInst>(U)->isStaticAlloca())
return TTI::TCC_Free;
Expand Down
14 changes: 6 additions & 8 deletions llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,9 +416,9 @@ class LoopVectorizationLegality {
/// has a vectorized variant available.
bool hasVectorCallVariants() const { return VecCallVariantsFound; }

/// Returns true if there is at least one function call in the loop which
/// returns a struct type and needs to be vectorized.
bool hasStructVectorCall() const { return StructVecCallFound; }
/* Downstream change: #87 (sincos vectorization)*/
// Removed hasStructVectorCall()
/* End downstream change: #87 */

unsigned getNumStores() const { return LAI->getNumStores(); }
unsigned getNumLoads() const { return LAI->getNumLoads(); }
Expand Down Expand Up @@ -639,11 +639,9 @@ class LoopVectorizationLegality {
/// the use of those function variants.
bool VecCallVariantsFound = false;

/// If we find a call (to be vectorized) that returns a struct type, record
/// that so we can bail out until this is supported.
/// TODO: Remove this flag once vectorizing calls with struct returns is
/// supported.
bool StructVecCallFound = false;
/* Downstream change: #87 (sincos vectorization)*/
// Removed StructVecCallFound
/* End downstream change: #87 */

/// Keep track of all the countable and uncountable exiting blocks if
/// the exact backedge taken count is not computable.
Expand Down
12 changes: 12 additions & 0 deletions llvm/lib/Analysis/TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1113,6 +1113,18 @@ TargetTransformInfo::getVectorInstrCost(const Instruction &I, Type *Val,
return Cost;
}

/* Downstream change: #87 (sincos vectorization)*/
InstructionCost TargetTransformInfo::getInsertExtractValueCost(
unsigned Opcode, TTI::TargetCostKind CostKind) const {
assert((Opcode == Instruction::InsertValue ||
Opcode == Instruction::ExtractValue) &&
"Expecting Opcode to be insertvalue/extractvalue.");
InstructionCost Cost = TTIImpl->getInsertExtractValueCost(Opcode, CostKind);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
/* End downstream change: #87 */

InstructionCost TargetTransformInfo::getReplicationShuffleCost(
Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts,
TTI::TargetCostKind CostKind) const {
Expand Down
15 changes: 5 additions & 10 deletions llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -954,24 +954,19 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
if (CI && !VFDatabase::getMappings(*CI).empty())
VecCallVariantsFound = true;

auto CanWidenInstructionTy = [this](Instruction const &Inst) {
/* Downstream change: #87 (sincos vectorization)*/
auto CanWidenInstructionTy = [](Instruction const &Inst) {
Type *InstTy = Inst.getType();
if (!isa<StructType>(InstTy))
return canVectorizeTy(InstTy);

// For now, we only recognize struct values returned from calls where
// all users are extractvalue as vectorizable. All element types of the
// struct must be types that can be widened.
if (isa<CallInst>(Inst) && canWidenCallReturnType(InstTy) &&
all_of(Inst.users(), IsaPred<ExtractValueInst>)) {
// TODO: Remove the `StructVecCallFound` flag once vectorizing calls
// with struct returns is supported.
StructVecCallFound = true;
return true;
}

return false;
return isa<CallInst>(Inst) && canWidenCallReturnType(InstTy) &&
all_of(Inst.users(), IsaPred<ExtractValueInst>);
};
/* End downstream change: #87 */

// Check that the instruction return type is vectorizable.
// We can't vectorize casts from vector type to scalar type.
Expand Down
125 changes: 84 additions & 41 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2350,7 +2350,11 @@ void InnerLoopVectorizer::scalarizeInstruction(const Instruction *Instr,
VPReplicateRecipe *RepRecipe,
const VPLane &Lane,
VPTransformState &State) {
assert(!Instr->getType()->isAggregateType() && "Can't handle vectors");
/* Downstream change: #87 (sincos vectorization)*/
assert((!Instr->getType()->isAggregateType() ||
canVectorizeTy(Instr->getType())) &&
"Expected vectorizable or non-aggregate type.");
/* End downstream change: #87 */

// Does this instruction return a value ?
bool IsVoidRetTy = Instr->getType()->isVoidTy();
Expand Down Expand Up @@ -2855,11 +2859,13 @@ LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
return ScalarCallCost;
}

static Type *maybeVectorizeType(Type *Elt, ElementCount VF) {
if (VF.isScalar() || (!Elt->isIntOrPtrTy() && !Elt->isFloatingPointTy()))
return Elt;
return VectorType::get(Elt, VF);
/* Downstream change: #87 (sincos vectorization)*/
static Type *maybeVectorizeType(Type *Ty, ElementCount VF) {
if (VF.isScalar() || !canVectorizeTy(Ty))
return Ty;
return toVectorizedTy(Ty, VF);
}
/* End downstream change: #87 */

InstructionCost
LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
Expand Down Expand Up @@ -3605,14 +3611,18 @@ void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
}
}

// ExtractValue instructions must be uniform, because the operands are
// known to be loop-invariant.
/* Downstream change: #87 (sincos vectorization)*/
if (auto *EVI = dyn_cast<ExtractValueInst>(&I)) {
assert(IsOutOfScope(EVI->getAggregateOperand()) &&
"Expected aggregate value to be loop invariant");
AddToWorklistIfAllowed(EVI);
continue;
if (IsOutOfScope(EVI->getAggregateOperand())) {
AddToWorklistIfAllowed(EVI);
continue;
}
// Only ExtractValue instructions where the aggregate value comes from a
// call are allowed to be non-uniform.
assert(isa<CallInst>(EVI->getAggregateOperand()) &&
"Expected aggregate value to be call return value");
}
/* End downstream change: #87 */

// If there's no pointer operand, there's nothing to do.
auto *Ptr = getLoadStorePointerOperand(&I);
Expand Down Expand Up @@ -4492,8 +4502,8 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
llvm_unreachable("unhandled recipe");
}

auto WillWiden = [&TTI, VF](Type *ScalarTy) {
Type *VectorTy = toVectorTy(ScalarTy, VF);
/* Downstream change: #87 (sincos vectorization)*/
auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
unsigned NumLegalParts = TTI.getNumberOfParts(VectorTy);
if (!NumLegalParts)
return false;
Expand All @@ -4505,9 +4515,10 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
// explicitly ask TTI about the register class uses for each part.
return NumLegalParts <= VF.getKnownMinValue();
}
// Two or more parts that share a register - are vectorized.
// Two or more elements that share a register - are vectorized.
return NumLegalParts < VF.getKnownMinValue();
};
/* End downstream change: #87 */

// If no def nor is a store, e.g., branches, continue - no value to check.
if (R.getNumDefinedValues() == 0 &&
Expand All @@ -4524,8 +4535,11 @@ static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
Type *ScalarTy = TypeInfo.inferScalarType(ToCheck);
if (!Visited.insert({ScalarTy}).second)
continue;
if (WillWiden(ScalarTy))
Type *WideTy = toVectorizedTy(ScalarTy, VF);
/* Downstream change: #87 (sincos vectorization)*/
if (any_of(getContainedTypes(WideTy), WillGenerateTargetVectors))
return true;
/* End downstream change: #87 */
}
}

Expand Down Expand Up @@ -5481,10 +5495,15 @@ InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
// Compute the scalarization overhead of needed insertelement instructions
// and phi nodes.
if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
ScalarCost += TTI.getScalarizationOverhead(
cast<VectorType>(toVectorTy(I->getType(), VF)),
APInt::getAllOnes(VF.getFixedValue()), /*Insert*/ true,
/*Extract*/ false, CostKind);
/* Downstream change: #87 (sincos vectorization)*/
Type *WideTy = toVectorizedTy(I->getType(), VF);
for (Type *VectorTy : getContainedTypes(WideTy)) {
ScalarCost += TTI.getScalarizationOverhead(
cast<VectorType>(VectorTy), APInt::getAllOnes(VF.getFixedValue()),
/*Insert=*/true,
/*Extract=*/false, CostKind);
}
/* End downstream change: #87 */
ScalarCost +=
VF.getFixedValue() * TTI.getCFInstrCost(Instruction::PHI, CostKind);
}
Expand All @@ -5495,16 +5514,21 @@ InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
// overhead.
for (Use &U : I->operands())
if (auto *J = dyn_cast<Instruction>(U.get())) {
assert(VectorType::isValidElementType(J->getType()) &&
/* Downstream change: #87 (sincos vectorization)*/
assert(canVectorizeTy(J->getType()) &&
"Instruction has non-scalar type");
if (CanBeScalarized(J))
Worklist.push_back(J);
else if (needsExtract(J, VF)) {
ScalarCost += TTI.getScalarizationOverhead(
cast<VectorType>(toVectorTy(J->getType(), VF)),
APInt::getAllOnes(VF.getFixedValue()), /*Insert*/ false,
/*Extract*/ true, CostKind);
Type *WideTy = toVectorizedTy(J->getType(), VF);
for (Type *VectorTy : getContainedTypes(WideTy)) {
ScalarCost += TTI.getScalarizationOverhead(
cast<VectorType>(VectorTy),
APInt::getAllOnes(VF.getFixedValue()), /*Insert*/ false,
/*Extract*/ true, CostKind);
}
}
/* End downstream change: #87 */
}

// Scale the total scalar cost by block probability.
Expand Down Expand Up @@ -5982,13 +6006,19 @@ LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
return 0;

InstructionCost Cost = 0;
Type *RetTy = toVectorTy(I->getType(), VF);
/* Downstream change: #87 (sincos vectorization)*/
Type *RetTy = toVectorizedTy(I->getType(), VF);
if (!RetTy->isVoidTy() &&
(!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore()))
Cost += TTI.getScalarizationOverhead(
cast<VectorType>(RetTy), APInt::getAllOnes(VF.getKnownMinValue()),
/*Insert*/ true,
/*Extract*/ false, CostKind);
(!isa<LoadInst>(I) || !TTI.supportsEfficientVectorElementLoadStore())) {

for (Type *VectorTy : getContainedTypes(RetTy)) {
Cost += TTI.getScalarizationOverhead(
cast<VectorType>(VectorTy), APInt::getAllOnes(VF.getKnownMinValue()),
/*Insert=*/true,
/*Extract=*/false, CostKind);
}
}
/* End downstream change: #87 */

// Some targets keep addresses scalar.
if (isa<LoadInst>(I) && !TTI.prefersVectorizedAddressing())
Expand Down Expand Up @@ -6246,9 +6276,11 @@ void LoopVectorizationCostModel::setVectorizedCallDecision(ElementCount VF) {

bool MaskRequired = Legal->isMaskRequired(CI);
// Compute corresponding vector type for return value and arguments.
Type *RetTy = toVectorTy(ScalarRetTy, VF);
/* Downstream change: #87 (sincos vectorization)*/
Type *RetTy = toVectorizedTy(ScalarRetTy, VF);
for (Type *ScalarTy : ScalarTys)
Tys.push_back(toVectorTy(ScalarTy, VF));
Tys.push_back(toVectorizedTy(ScalarTy, VF));
/* End downstream change: #87 */

// An in-loop reduction using an fmuladd intrinsic is a special case;
// we don't want the normal cost for that intrinsic.
Expand Down Expand Up @@ -6438,7 +6470,8 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
HasSingleCopyAfterVectorization(I, VF));
VectorTy = RetTy;
} else
VectorTy = toVectorTy(RetTy, VF);
// Downstream change: #87 (sincos vectorization)
VectorTy = toVectorizedTy(RetTy, VF);

if (VF.isVector() && VectorTy->isVectorTy() &&
!TTI.getNumberOfParts(VectorTy))
Expand Down Expand Up @@ -8560,7 +8593,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
case Instruction::Shl:
case Instruction::Sub:
case Instruction::Xor:
case Instruction::Freeze:
case Instruction::Freeze: {
SmallVector<VPValue *> NewOps(Operands);
if (Instruction::isBinaryOp(I->getOpcode())) {
// The legacy cost model uses SCEV to check if some of the operands are
Expand All @@ -8585,6 +8618,18 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(Instruction *I,
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
}
return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
}
/* Downstream change: #87 (sincos vectorization)*/
case Instruction::ExtractValue: {
SmallVector<VPValue *> NewOps(Operands);
Type *I32Ty = IntegerType::getInt32Ty(I->getContext());
auto *EVI = cast<ExtractValueInst>(I);
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
NewOps.push_back(Plan.getOrAddLiveIn(ConstantInt::get(I32Ty, Idx, false)));
return new VPWidenRecipe(*I, make_range(NewOps.begin(), NewOps.end()));
}
/* End downstream change: #87 */
};
}

Expand Down Expand Up @@ -9865,7 +9910,8 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
VectorType::get(UI->getType(), State.VF));
State.set(this, Poison);
}
State.packScalarIntoVectorValue(this, *State.Lane);
// Downstream change: #87 (sincos vectorization)
State.packScalarIntoVectorizedValue(this, *State.Lane);
}
return;
}
Expand Down Expand Up @@ -10382,12 +10428,9 @@ bool LoopVectorizePass::processLoop(Loop *L) {
return false;
}

if (LVL.hasStructVectorCall()) {
reportVectorizationFailure("Auto-vectorization of calls that return struct "
"types is not yet supported",
"StructCallVectorizationUnsupported", ORE, L);
return false;
}
/* Downstream change: #87 (sincos vectorization)*/
// Remove StructCallVectorizationUnsupported failure.
/* End downstream change: #87 */

// Entrance to the VPlan-native vectorization path. Outer loops are processed
// here. They may require CFG and instruction level transformations before
Expand Down
Loading