@@ -1242,7 +1242,8 @@ static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
12421242/// \returns
12431243/// - The first instruction found with matching opcode
12441244/// - nullptr if no matching instruction is found
1245- Instruction *findInstructionWithOpcode(ArrayRef<Value *> VL, unsigned Opcode) {
1245+ static Instruction *findInstructionWithOpcode(ArrayRef<Value *> VL,
1246+ unsigned Opcode) {
12461247 for (Value *V : VL) {
12471248 if (isa<PoisonValue>(V))
12481249 continue;
@@ -2071,7 +2072,7 @@ class BoUpSLP {
20712072 OptimizationRemarkEmitter *getORE() { return ORE; }
20722073
20732074 /// This structure holds any data we need about the edges being traversed
2074- /// during buildTree_rec (). We keep track of:
2075+ /// during buildTreeRec (). We keep track of:
20752076 /// (i) the user TreeEntry index, and
20762077 /// (ii) the index of the edge.
20772078 struct EdgeInfo {
@@ -3491,8 +3492,8 @@ class BoUpSLP {
34913492 OrdersType &ReorderIndices) const;
34923493
34933494 /// This is the recursive part of buildTree.
3494- void buildTree_rec (ArrayRef<Value *> Roots, unsigned Depth,
3495- const EdgeInfo &EI, unsigned InterleaveFactor = 0);
3495+ void buildTreeRec (ArrayRef<Value *> Roots, unsigned Depth, const EdgeInfo &EI ,
3496+ unsigned InterleaveFactor = 0);
34963497
34973498 /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
34983499 /// be vectorized to use the original vector (or aggregate "bitcast" to a
@@ -3810,7 +3811,7 @@ class BoUpSLP {
38103811 private:
38113812 /// The operands of each instruction in each lane Operands[op_index][lane].
38123813 /// Note: This helps avoid the replication of the code that performs the
3813- /// reordering of operands during buildTree_rec () and vectorizeTree().
3814+ /// reordering of operands during buildTreeRec () and vectorizeTree().
38143815 SmallVector<ValueList, 2> Operands;
38153816
38163817 /// MainOp and AltOp are recorded inside. S should be obtained from
@@ -4590,10 +4591,10 @@ class BoUpSLP {
45904591 };
45914592
45924593#ifndef NDEBUG
4593- friend inline raw_ostream &operator<<(raw_ostream &os ,
4594+ friend inline raw_ostream &operator<<(raw_ostream &OS ,
45944595 const BoUpSLP::ScheduleData &SD) {
4595- SD.dump(os );
4596- return os ;
4596+ SD.dump(OS );
4597+ return OS ;
45974598 }
45984599#endif
45994600
@@ -4694,10 +4695,10 @@ class BoUpSLP {
46944695 };
46954696
46964697#ifndef NDEBUG
4697- friend inline raw_ostream &operator<<(raw_ostream &os ,
4698+ friend inline raw_ostream &operator<<(raw_ostream &OS ,
46984699 const BoUpSLP::ScheduleBundle &Bundle) {
4699- Bundle.dump(os );
4700- return os ;
4700+ Bundle.dump(OS );
4701+ return OS ;
47014702 }
47024703#endif
47034704
@@ -8070,14 +8071,14 @@ void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
80708071 UserIgnoreList = &UserIgnoreLst;
80718072 if (!allSameType(Roots))
80728073 return;
8073- buildTree_rec (Roots, 0, EdgeInfo());
8074+ buildTreeRec (Roots, 0, EdgeInfo());
80748075}
80758076
80768077void BoUpSLP::buildTree(ArrayRef<Value *> Roots) {
80778078 deleteTree();
80788079 if (!allSameType(Roots))
80798080 return;
8080- buildTree_rec (Roots, 0, EdgeInfo());
8081+ buildTreeRec (Roots, 0, EdgeInfo());
80818082}
80828083
80838084/// Tries to find subvector of loads and builds new vector of only loads if can
@@ -8628,7 +8629,7 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
86288629 }))
86298630 continue;
86308631 unsigned Sz = VectorizableTree.size();
8631- buildTree_rec (SubSlice, 0, EdgeInfo(), InterleaveFactor);
8632+ buildTreeRec (SubSlice, 0, EdgeInfo(), InterleaveFactor);
86328633 if (Sz == VectorizableTree.size()) {
86338634 IsVectorized = false;
86348635 // Try non-interleaved vectorization with smaller vector
@@ -8683,7 +8684,7 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
86838684 inversePermutation(E.ReorderIndices, ReorderMask);
86848685 reorderScalars(GatheredScalars, ReorderMask);
86858686 }
8686- buildTree_rec (GatheredScalars, 0, EdgeInfo());
8687+ buildTreeRec (GatheredScalars, 0, EdgeInfo());
86878688 }
86888689 // If no new entries created, consider it as no gathered loads entries must be
86898690 // handled.
@@ -9977,9 +9978,9 @@ bool BoUpSLP::isLegalToVectorizeScalars(ArrayRef<Value *> VL, unsigned Depth,
99779978 return true;
99789979}
99799980
9980- void BoUpSLP::buildTree_rec (ArrayRef<Value *> VL, unsigned Depth,
9981- const EdgeInfo &UserTreeIdx,
9982- unsigned InterleaveFactor) {
9981+ void BoUpSLP::buildTreeRec (ArrayRef<Value *> VL, unsigned Depth,
9982+ const EdgeInfo &UserTreeIdx,
9983+ unsigned InterleaveFactor) {
99839984 assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
99849985
99859986 SmallVector<int> ReuseShuffleIndices;
@@ -10022,7 +10023,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1002210023 } else {
1002310024 TE->CombinedEntriesWithIndices.emplace_back(VectorizableTree.size(),
1002410025 Idx == 0 ? 0 : Op1.size());
10025- buildTree_rec (Op, Depth, {TE, Idx});
10026+ buildTreeRec (Op, Depth, {TE, Idx});
1002610027 }
1002710028 };
1002810029 AddNode(Op1, 0);
@@ -10115,12 +10116,12 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1011510116 continue;
1011610117 InstructionsState S = getSameOpcode(Op, *TLI);
1011710118 if ((!S || S.getOpcode() != Instruction::PHI) || S.isAltShuffle())
10118- buildTree_rec (Op, Depth + 1, {TE, I});
10119+ buildTreeRec (Op, Depth + 1, {TE, I});
1011910120 else
1012010121 PHIOps.push_back(I);
1012110122 }
1012210123 for (unsigned I : PHIOps)
10123- buildTree_rec (Operands[I], Depth + 1, {TE, I});
10124+ buildTreeRec (Operands[I], Depth + 1, {TE, I});
1012410125 };
1012510126 switch (ShuffleOrOp) {
1012610127 case Instruction::PHI: {
@@ -10164,7 +10165,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1016410165 "(ExtractValueInst/ExtractElementInst).\n";
1016510166 TE->dump());
1016610167 // This is a special case, as it does not gather, but at the same time
10167- // we are not extending buildTree_rec () towards the operands.
10168+ // we are not extending buildTreeRec () towards the operands.
1016810169 TE->setOperand(*this);
1016910170 return;
1017010171 }
@@ -10197,7 +10198,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1019710198 TE->dump());
1019810199
1019910200 TE->setOperand(*this);
10200- buildTree_rec (TE->getOperand(1), Depth + 1, {TE, 1});
10201+ buildTreeRec (TE->getOperand(1), Depth + 1, {TE, 1});
1020110202 return;
1020210203 }
1020310204 case Instruction::Load: {
@@ -10253,7 +10254,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1025310254 }
1025410255 TE->setOperand(*this);
1025510256 if (State == TreeEntry::ScatterVectorize)
10256- buildTree_rec (PointerOps, Depth + 1, {TE, 0});
10257+ buildTreeRec (PointerOps, Depth + 1, {TE, 0});
1025710258 return;
1025810259 }
1025910260 case Instruction::ZExt:
@@ -10294,7 +10295,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1029410295
1029510296 TE->setOperand(*this);
1029610297 for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
10297- buildTree_rec (TE->getOperand(I), Depth + 1, {TE, I});
10298+ buildTreeRec (TE->getOperand(I), Depth + 1, {TE, I});
1029810299 if (ShuffleOrOp == Instruction::Trunc) {
1029910300 ExtraBitWidthNodes.insert(getOperandEntry(TE, 0)->Idx);
1030010301 } else if (ShuffleOrOp == Instruction::SIToFP ||
@@ -10349,8 +10350,8 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1034910350 }
1035010351 TE->setOperand(0, Left);
1035110352 TE->setOperand(1, Right);
10352- buildTree_rec (Left, Depth + 1, {TE, 0});
10353- buildTree_rec (Right, Depth + 1, {TE, 1});
10353+ buildTreeRec (Left, Depth + 1, {TE, 0});
10354+ buildTreeRec (Right, Depth + 1, {TE, 1});
1035410355 if (ShuffleOrOp == Instruction::ICmp) {
1035510356 unsigned NumSignBits0 =
1035610357 ComputeNumSignBits(VL0->getOperand(0), *DL, 0, AC, nullptr, DT);
@@ -10395,7 +10396,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1039510396
1039610397 TE->setOperand(*this, isa<BinaryOperator>(VL0) && isCommutative(VL0));
1039710398 for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
10398- buildTree_rec (TE->getOperand(I), Depth + 1, {TE, I});
10399+ buildTreeRec (TE->getOperand(I), Depth + 1, {TE, I});
1039910400 return;
1040010401 }
1040110402 case Instruction::GetElementPtr: {
@@ -10451,7 +10452,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1045110452 TE->setOperand(IndexIdx, Operands.back());
1045210453
1045310454 for (unsigned I = 0, Ops = Operands.size(); I < Ops; ++I)
10454- buildTree_rec (Operands[I], Depth + 1, {TE, I});
10455+ buildTreeRec (Operands[I], Depth + 1, {TE, I});
1045510456 return;
1045610457 }
1045710458 case Instruction::Store: {
@@ -10468,7 +10469,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1046810469 dbgs() << "SLP: added a new TreeEntry (jumbled StoreInst).\n";
1046910470 TE->dump());
1047010471 TE->setOperand(*this);
10471- buildTree_rec (TE->getOperand(0), Depth + 1, {TE, 0});
10472+ buildTreeRec (TE->getOperand(0), Depth + 1, {TE, 0});
1047210473 return;
1047310474 }
1047410475 case Instruction::Call: {
@@ -10487,7 +10488,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1048710488 // vectorize it.
1048810489 if (isVectorIntrinsicWithScalarOpAtArg(ID, I, TTI))
1048910490 continue;
10490- buildTree_rec (TE->getOperand(I), Depth + 1, {TE, I});
10491+ buildTreeRec (TE->getOperand(I), Depth + 1, {TE, I});
1049110492 }
1049210493 return;
1049310494 }
@@ -10540,14 +10541,14 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
1054010541 }
1054110542 TE->setOperand(0, Left);
1054210543 TE->setOperand(1, Right);
10543- buildTree_rec (Left, Depth + 1, {TE, 0});
10544- buildTree_rec (Right, Depth + 1, {TE, 1});
10544+ buildTreeRec (Left, Depth + 1, {TE, 0});
10545+ buildTreeRec (Right, Depth + 1, {TE, 1});
1054510546 return;
1054610547 }
1054710548
1054810549 TE->setOperand(*this, isa<BinaryOperator>(VL0) || CI);
1054910550 for (unsigned I : seq<unsigned>(VL0->getNumOperands()))
10550- buildTree_rec (TE->getOperand(I), Depth + 1, {TE, I});
10551+ buildTreeRec (TE->getOperand(I), Depth + 1, {TE, I});
1055110552 return;
1055210553 }
1055310554 default:
@@ -11626,7 +11627,7 @@ void BoUpSLP::transformNodes() {
1162611627 unsigned PrevSize = VectorizableTree.size();
1162711628 [[maybe_unused]] unsigned PrevEntriesSize =
1162811629 LoadEntriesToVectorize.size();
11629- buildTree_rec (Slice, 0, EdgeInfo(&E, UINT_MAX));
11630+ buildTreeRec (Slice, 0, EdgeInfo(&E, UINT_MAX));
1163011631 if (PrevSize + 1 == VectorizableTree.size() && !SameTE &&
1163111632 VectorizableTree[PrevSize]->isGather() &&
1163211633 VectorizableTree[PrevSize]->hasState() &&
@@ -15970,7 +15971,7 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
1597015971
1597115972 // LastInst can still be null at this point if there's either not an entry
1597215973 // for BB in BlocksSchedules or there's no ScheduleData available for
15973- // VL.back(). This can be the case if buildTree_rec aborts for various
15974+ // VL.back(). This can be the case if buildTreeRec aborts for various
1597415975 // reasons (e.g., the maximum recursion depth is reached, the maximum region
1597515976 // size is reached, etc.). ScheduleData is initialized in the scheduling
1597615977 // "dry-run".
@@ -15981,10 +15982,10 @@ Instruction &BoUpSLP::getLastInstructionInBundle(const TreeEntry *E) {
1598115982 // last instruction in program order, LastInst will be set to Front, and we
1598215983 // will visit all the remaining instructions in the block.
1598315984 //
15984- // One of the reasons we exit early from buildTree_rec is to place an upper
15985+ // One of the reasons we exit early from buildTreeRec is to place an upper
1598515986 // bound on compile-time. Thus, taking an additional compile-time hit here is
1598615987 // not ideal. However, this should be exceedingly rare since it requires that
15987- // we both exit early from buildTree_rec and that the bundle be out-of-order
15988+ // we both exit early from buildTreeRec and that the bundle be out-of-order
1598815989 // (causing us to iterate all the way to the end of the block).
1598915990 if (!Res)
1599015991 Res = FindLastInst();
@@ -21436,8 +21437,8 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
2143621437 // not be useful.
2143721438 R.getORE()->emit([&]() {
2143821439 std::string TypeStr;
21439- llvm::raw_string_ostream rso (TypeStr);
21440- Ty->print(rso );
21440+ llvm::raw_string_ostream OS (TypeStr);
21441+ Ty->print(OS );
2144121442 return OptimizationRemarkMissed(SV_NAME, "UnsupportedType", I0)
2144221443 << "Cannot SLP vectorize list: type "
2144321444 << TypeStr + " is unsupported by vectorizer";
@@ -23331,20 +23332,20 @@ static std::optional<unsigned> getAggregateSize(Instruction *InsertInst) {
2333123332 } while (true);
2333223333}
2333323334
23334- static void findBuildAggregate_rec (Instruction *LastInsertInst,
23335- TargetTransformInfo *TTI,
23336- SmallVectorImpl<Value *> &BuildVectorOpds,
23337- SmallVectorImpl<Value *> &InsertElts,
23338- unsigned OperandOffset, const BoUpSLP &R) {
23335+ static void findBuildAggregateRec (Instruction *LastInsertInst,
23336+ TargetTransformInfo *TTI,
23337+ SmallVectorImpl<Value *> &BuildVectorOpds,
23338+ SmallVectorImpl<Value *> &InsertElts,
23339+ unsigned OperandOffset, const BoUpSLP &R) {
2333923340 do {
2334023341 Value *InsertedOperand = LastInsertInst->getOperand(1);
2334123342 std::optional<unsigned> OperandIndex =
2334223343 getElementIndex(LastInsertInst, OperandOffset);
2334323344 if (!OperandIndex || R.isDeleted(LastInsertInst))
2334423345 return;
2334523346 if (isa<InsertElementInst, InsertValueInst>(InsertedOperand)) {
23346- findBuildAggregate_rec (cast<Instruction>(InsertedOperand), TTI,
23347- BuildVectorOpds, InsertElts, *OperandIndex, R);
23347+ findBuildAggregateRec (cast<Instruction>(InsertedOperand), TTI,
23348+ BuildVectorOpds, InsertElts, *OperandIndex, R);
2334823349
2334923350 } else {
2335023351 BuildVectorOpds[*OperandIndex] = InsertedOperand;
@@ -23389,8 +23390,7 @@ static bool findBuildAggregate(Instruction *LastInsertInst,
2338923390 BuildVectorOpds.resize(*AggregateSize);
2339023391 InsertElts.resize(*AggregateSize);
2339123392
23392- findBuildAggregate_rec(LastInsertInst, TTI, BuildVectorOpds, InsertElts, 0,
23393- R);
23393+ findBuildAggregateRec(LastInsertInst, TTI, BuildVectorOpds, InsertElts, 0, R);
2339423394 llvm::erase(BuildVectorOpds, nullptr);
2339523395 llvm::erase(InsertElts, nullptr);
2339623396 if (BuildVectorOpds.size() >= 2)
0 commit comments