|
60 | 60 | //===----------------------------------------------------------------------===//
|
61 | 61 |
|
62 | 62 | #include "llvm/CodeGen/ComplexDeinterleavingPass.h"
|
| 63 | +#include "llvm/ADT/AllocatorList.h" |
63 | 64 | #include "llvm/ADT/MapVector.h"
|
64 | 65 | #include "llvm/ADT/Statistic.h"
|
65 | 66 | #include "llvm/Analysis/TargetLibraryInfo.h"
|
@@ -263,6 +264,7 @@ class ComplexDeinterleavingGraph {
|
263 | 264 | };
|
264 | 265 |
|
265 | 266 | using Addend = std::pair<Value *, bool>;
|
| 267 | + using AddendList = BumpPtrList<Addend>; |
266 | 268 | using CompositeNode = ComplexDeinterleavingCompositeNode::CompositeNode;
|
267 | 269 |
|
268 | 270 | // Helper struct for holding info about potential partial multiplication
|
@@ -291,7 +293,7 @@ class ComplexDeinterleavingGraph {
|
291 | 293 | SmallPtrSet<Instruction *, 16> FinalInstructions;
|
292 | 294 |
|
293 | 295 | /// Root instructions are instructions from which complex computation starts
|
294 |
| - std::map<Instruction *, CompositeNode *> RootToNode; |
| 296 | + DenseMap<Instruction *, CompositeNode *> RootToNode; |
295 | 297 |
|
296 | 298 | /// Topologically sorted root instructions
|
297 | 299 | SmallVector<Instruction *, 1> OrderedRoots;
|
@@ -339,7 +341,7 @@ class ComplexDeinterleavingGraph {
|
339 | 341 | /// ComplexDeinterleavingOperation::ReductionPHI node replacement. It is then
|
340 | 342 | /// used in the ComplexDeinterleavingOperation::ReductionOperation node
|
341 | 343 | /// replacement process.
|
342 |
| - std::map<PHINode *, PHINode *> OldToNewPHI; |
| 344 | + DenseMap<PHINode *, PHINode *> OldToNewPHI; |
343 | 345 |
|
344 | 346 | CompositeNode *prepareCompositeNode(ComplexDeinterleavingOperation Operation,
|
345 | 347 | Value *R, Value *I) {
|
@@ -417,28 +419,28 @@ class ComplexDeinterleavingGraph {
|
417 | 419 | /// and \p ImagAddens. If \p Accumulator is not null, add the result to it.
|
418 | 420 | /// Return nullptr if it is not possible to construct a complex number.
|
419 | 421 | /// \p Flags are needed to generate symmetric Add and Sub operations.
|
420 |
| - CompositeNode *identifyAdditions(std::list<Addend> &RealAddends, |
421 |
| - std::list<Addend> &ImagAddends, |
| 422 | + CompositeNode *identifyAdditions(AddendList &RealAddends, |
| 423 | + AddendList &ImagAddends, |
422 | 424 | std::optional<FastMathFlags> Flags,
|
423 | 425 | CompositeNode *Accumulator);
|
424 | 426 |
|
425 | 427 | /// Extract one addend that have both real and imaginary parts positive.
|
426 |
| - CompositeNode *extractPositiveAddend(std::list<Addend> &RealAddends, |
427 |
| - std::list<Addend> &ImagAddends); |
| 428 | + CompositeNode *extractPositiveAddend(AddendList &RealAddends, |
| 429 | + AddendList &ImagAddends); |
428 | 430 |
|
429 | 431 | /// Determine if sum of multiplications of complex numbers can be formed from
|
430 | 432 | /// \p RealMuls and \p ImagMuls. If \p Accumulator is not null, add the result
|
431 | 433 | /// to it. Return nullptr if it is not possible to construct a complex number.
|
432 |
| - CompositeNode *identifyMultiplications(std::vector<Product> &RealMuls, |
433 |
| - std::vector<Product> &ImagMuls, |
| 434 | + CompositeNode *identifyMultiplications(SmallVectorImpl<Product> &RealMuls, |
| 435 | + SmallVectorImpl<Product> &ImagMuls, |
434 | 436 | CompositeNode *Accumulator);
|
435 | 437 |
|
436 | 438 | /// Go through pairs of multiplication (one Real and one Imag) and find all
|
437 | 439 | /// possible candidates for partial multiplication and put them into \p
|
438 | 440 | /// Candidates. Returns true if all Product has pair with common operand
|
439 |
| - bool collectPartialMuls(const std::vector<Product> &RealMuls, |
440 |
| - const std::vector<Product> &ImagMuls, |
441 |
| - std::vector<PartialMulCandidate> &Candidates); |
| 441 | + bool collectPartialMuls(ArrayRef<Product> RealMuls, |
| 442 | + ArrayRef<Product> ImagMuls, |
| 443 | + SmallVectorImpl<PartialMulCandidate> &Candidates); |
442 | 444 |
|
443 | 445 | /// If the code is compiled with -Ofast or expressions have `reassoc` flag,
|
444 | 446 | /// the order of complex computation operations may be significantly altered,
|
@@ -1255,8 +1257,8 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
|
1255 | 1257 | // Collect multiplications and addend instructions from the given instruction
|
1256 | 1258 | // while traversing it operands. Additionally, verify that all instructions
|
1257 | 1259 | // have the same fast math flags.
|
1258 |
| - auto Collect = [&Flags](Instruction *Insn, std::vector<Product> &Muls, |
1259 |
| - std::list<Addend> &Addends) -> bool { |
| 1260 | + auto Collect = [&Flags](Instruction *Insn, SmallVectorImpl<Product> &Muls, |
| 1261 | + AddendList &Addends) -> bool { |
1260 | 1262 | SmallVector<PointerIntPair<Value *, 1, bool>> Worklist = {{Insn, true}};
|
1261 | 1263 | SmallPtrSet<Value *, 8> Visited;
|
1262 | 1264 | while (!Worklist.empty()) {
|
@@ -1336,8 +1338,8 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
|
1336 | 1338 | return true;
|
1337 | 1339 | };
|
1338 | 1340 |
|
1339 |
| - std::vector<Product> RealMuls, ImagMuls; |
1340 |
| - std::list<Addend> RealAddends, ImagAddends; |
| 1341 | + SmallVector<Product> RealMuls, ImagMuls; |
| 1342 | + AddendList RealAddends, ImagAddends; |
1341 | 1343 | if (!Collect(Real, RealMuls, RealAddends) ||
|
1342 | 1344 | !Collect(Imag, ImagMuls, ImagAddends))
|
1343 | 1345 | return nullptr;
|
@@ -1371,8 +1373,8 @@ ComplexDeinterleavingGraph::identifyReassocNodes(Instruction *Real,
|
1371 | 1373 | }
|
1372 | 1374 |
|
1373 | 1375 | bool ComplexDeinterleavingGraph::collectPartialMuls(
|
1374 |
| - const std::vector<Product> &RealMuls, const std::vector<Product> &ImagMuls, |
1375 |
| - std::vector<PartialMulCandidate> &PartialMulCandidates) { |
| 1376 | + ArrayRef<Product> RealMuls, ArrayRef<Product> ImagMuls, |
| 1377 | + SmallVectorImpl<PartialMulCandidate> &PartialMulCandidates) { |
1376 | 1378 | // Helper function to extract a common operand from two products
|
1377 | 1379 | auto FindCommonInstruction = [](const Product &Real,
|
1378 | 1380 | const Product &Imag) -> Value * {
|
@@ -1423,18 +1425,18 @@ bool ComplexDeinterleavingGraph::collectPartialMuls(
|
1423 | 1425 |
|
1424 | 1426 | ComplexDeinterleavingGraph::CompositeNode *
|
1425 | 1427 | ComplexDeinterleavingGraph::identifyMultiplications(
|
1426 |
| - std::vector<Product> &RealMuls, std::vector<Product> &ImagMuls, |
| 1428 | + SmallVectorImpl<Product> &RealMuls, SmallVectorImpl<Product> &ImagMuls, |
1427 | 1429 | CompositeNode *Accumulator = nullptr) {
|
1428 | 1430 | if (RealMuls.size() != ImagMuls.size())
|
1429 | 1431 | return nullptr;
|
1430 | 1432 |
|
1431 |
| - std::vector<PartialMulCandidate> Info; |
| 1433 | + SmallVector<PartialMulCandidate> Info; |
1432 | 1434 | if (!collectPartialMuls(RealMuls, ImagMuls, Info))
|
1433 | 1435 | return nullptr;
|
1434 | 1436 |
|
1435 | 1437 | // Map to store common instruction to node pointers
|
1436 |
| - std::map<Value *, CompositeNode *> CommonToNode; |
1437 |
| - std::vector<bool> Processed(Info.size(), false); |
| 1438 | + DenseMap<Value *, CompositeNode *> CommonToNode; |
| 1439 | + SmallVector<bool> Processed(Info.size(), false); |
1438 | 1440 | for (unsigned I = 0; I < Info.size(); ++I) {
|
1439 | 1441 | if (Processed[I])
|
1440 | 1442 | continue;
|
@@ -1463,8 +1465,8 @@ ComplexDeinterleavingGraph::identifyMultiplications(
|
1463 | 1465 | }
|
1464 | 1466 | }
|
1465 | 1467 |
|
1466 |
| - std::vector<bool> ProcessedReal(RealMuls.size(), false); |
1467 |
| - std::vector<bool> ProcessedImag(ImagMuls.size(), false); |
| 1468 | + SmallVector<bool> ProcessedReal(RealMuls.size(), false); |
| 1469 | + SmallVector<bool> ProcessedImag(ImagMuls.size(), false); |
1468 | 1470 | CompositeNode *Result = Accumulator;
|
1469 | 1471 | for (auto &PMI : Info) {
|
1470 | 1472 | if (ProcessedReal[PMI.RealIdx] || ProcessedImag[PMI.ImagIdx])
|
@@ -1580,7 +1582,7 @@ ComplexDeinterleavingGraph::identifyMultiplications(
|
1580 | 1582 |
|
1581 | 1583 | ComplexDeinterleavingGraph::CompositeNode *
|
1582 | 1584 | ComplexDeinterleavingGraph::identifyAdditions(
|
1583 |
| - std::list<Addend> &RealAddends, std::list<Addend> &ImagAddends, |
| 1585 | + AddendList &RealAddends, AddendList &ImagAddends, |
1584 | 1586 | std::optional<FastMathFlags> Flags, CompositeNode *Accumulator = nullptr) {
|
1585 | 1587 | if (RealAddends.size() != ImagAddends.size())
|
1586 | 1588 | return nullptr;
|
@@ -1671,8 +1673,8 @@ ComplexDeinterleavingGraph::identifyAdditions(
|
1671 | 1673 | }
|
1672 | 1674 |
|
1673 | 1675 | ComplexDeinterleavingGraph::CompositeNode *
|
1674 |
| -ComplexDeinterleavingGraph::extractPositiveAddend( |
1675 |
| - std::list<Addend> &RealAddends, std::list<Addend> &ImagAddends) { |
| 1676 | +ComplexDeinterleavingGraph::extractPositiveAddend(AddendList &RealAddends, |
| 1677 | + AddendList &ImagAddends) { |
1676 | 1678 | for (auto ItR = RealAddends.begin(); ItR != RealAddends.end(); ++ItR) {
|
1677 | 1679 | for (auto ItI = ImagAddends.begin(); ItI != ImagAddends.end(); ++ItI) {
|
1678 | 1680 | auto [R, IsPositiveR] = *ItR;
|
|
0 commit comments