|
16 | 16 | #include "llvm/ADT/DenseMap.h" |
17 | 17 | #include "llvm/ADT/STLExtras.h" |
18 | 18 | #include "llvm/ADT/ScopeExit.h" |
19 | | -#include "llvm/ADT/SmallVector.h" |
20 | 19 | #include "llvm/ADT/Statistic.h" |
21 | 20 | #include "llvm/Analysis/AssumptionCache.h" |
22 | 21 | #include "llvm/Analysis/BasicAliasAnalysis.h" |
|
30 | 29 | #include "llvm/IR/Dominators.h" |
31 | 30 | #include "llvm/IR/Function.h" |
32 | 31 | #include "llvm/IR/IRBuilder.h" |
33 | | -#include "llvm/IR/Instructions.h" |
34 | 32 | #include "llvm/IR/PatternMatch.h" |
35 | 33 | #include "llvm/Support/CommandLine.h" |
36 | 34 | #include "llvm/Transforms/Utils/Local.h" |
37 | 35 | #include "llvm/Transforms/Utils/LoopUtils.h" |
38 | 36 | #include <numeric> |
39 | | -#include <optional> |
40 | 37 | #include <queue> |
41 | 38 | #include <set> |
42 | | -#include <tuple> |
43 | 39 |
|
44 | 40 | #define DEBUG_TYPE "vector-combine" |
45 | 41 | #include "llvm/Transforms/Utils/InstructionWorklist.h" |
@@ -141,7 +137,6 @@ class VectorCombine { |
141 | 137 | bool foldSelectShuffle(Instruction &I, bool FromReduction = false); |
142 | 138 | bool foldInterleaveIntrinsics(Instruction &I); |
143 | 139 | bool shrinkType(Instruction &I); |
144 | | - bool shrinkLoadForShuffles(Instruction &I); |
145 | 140 | bool shrinkPhiOfShuffles(Instruction &I); |
146 | 141 |
|
147 | 142 | void replaceValue(Value &Old, Value &New) { |
@@ -3868,133 +3863,6 @@ bool VectorCombine::foldInterleaveIntrinsics(Instruction &I) { |
3868 | 3863 | return true; |
3869 | 3864 | } |
3870 | 3865 |
|
3871 | | -// Attempt to shrink loads that are only used by shufflevector instructions. |
3872 | | -bool VectorCombine::shrinkLoadForShuffles(Instruction &I) { |
3873 | | - auto *OldLoad = dyn_cast<LoadInst>(&I); |
3874 | | - if (!OldLoad || !OldLoad->isSimple()) |
3875 | | - return false; |
3876 | | - |
3877 | | - auto *OldLoadTy = dyn_cast<FixedVectorType>(OldLoad->getType()); |
3878 | | - if (!OldLoadTy) |
3879 | | - return false; |
3880 | | - |
3881 | | - unsigned const OldNumElements = OldLoadTy->getNumElements(); |
3882 | | - |
3883 | | - // Search all uses of load. If all uses are shufflevector instructions, and |
3884 | | - // the second operands are all poison values, find the minimum and maximum |
3885 | | - // indices of the vector elements referenced by all shuffle masks. |
3886 | | - // Otherwise return `std::nullopt`. |
3887 | | - using IndexRange = std::pair<int, int>; |
3888 | | - auto GetIndexRangeInShuffles = [&]() -> std::optional<IndexRange> { |
3889 | | - IndexRange OutputRange = IndexRange(OldNumElements, -1); |
3890 | | - for (llvm::Use &Use : I.uses()) { |
3891 | | - // Ensure all uses match the required pattern. |
3892 | | - User *Shuffle = Use.getUser(); |
3893 | | - ArrayRef<int> Mask; |
3894 | | - |
3895 | | - if (!match(Shuffle, |
3896 | | - m_Shuffle(m_Specific(OldLoad), m_Undef(), m_Mask(Mask)))) |
3897 | | - return std::nullopt; |
3898 | | - |
3899 | | - // Ignore shufflevector instructions that have no uses. |
3900 | | - if (Shuffle->use_empty()) |
3901 | | - continue; |
3902 | | - |
3903 | | - // Find the min and max indices used by the shufflevector instruction. |
3904 | | - for (int Index : Mask) { |
3905 | | - if (Index >= 0 && Index < static_cast<int>(OldNumElements)) { |
3906 | | - OutputRange.first = std::min(Index, OutputRange.first); |
3907 | | - OutputRange.second = std::max(Index, OutputRange.second); |
3908 | | - } |
3909 | | - } |
3910 | | - } |
3911 | | - |
3912 | | - if (OutputRange.second < OutputRange.first) |
3913 | | - return std::nullopt; |
3914 | | - |
3915 | | - return OutputRange; |
3916 | | - }; |
3917 | | - |
3918 | | - // Get the range of vector elements used by shufflevector instructions. |
3919 | | - if (std::optional<IndexRange> Indices = GetIndexRangeInShuffles()) { |
3920 | | - unsigned const NewNumElements = Indices->second + 1u; |
3921 | | - |
3922 | | - // If the range of vector elements is smaller than the full load, attempt |
3923 | | - // to create a smaller load. |
3924 | | - if (NewNumElements < OldNumElements) { |
3925 | | - IRBuilder Builder(&I); |
3926 | | - Builder.SetCurrentDebugLocation(I.getDebugLoc()); |
3927 | | - |
3928 | | - // Calculate costs of old and new ops. |
3929 | | - Type *ElemTy = OldLoadTy->getElementType(); |
3930 | | - FixedVectorType *NewLoadTy = FixedVectorType::get(ElemTy, NewNumElements); |
3931 | | - Value *PtrOp = OldLoad->getPointerOperand(); |
3932 | | - |
3933 | | - InstructionCost OldCost = TTI.getMemoryOpCost( |
3934 | | - Instruction::Load, OldLoad->getType(), OldLoad->getAlign(), |
3935 | | - OldLoad->getPointerAddressSpace(), CostKind); |
3936 | | - InstructionCost NewCost = |
3937 | | - TTI.getMemoryOpCost(Instruction::Load, NewLoadTy, OldLoad->getAlign(), |
3938 | | - OldLoad->getPointerAddressSpace(), CostKind); |
3939 | | - |
3940 | | - using UseEntry = std::pair<ShuffleVectorInst *, std::vector<int>>; |
3941 | | - SmallVector<UseEntry, 4u> NewUses; |
3942 | | - unsigned const MaxIndex = NewNumElements * 2u; |
3943 | | - |
3944 | | - for (llvm::Use &Use : I.uses()) { |
3945 | | - auto *Shuffle = cast<ShuffleVectorInst>(Use.getUser()); |
3946 | | - ArrayRef<int> OldMask = Shuffle->getShuffleMask(); |
3947 | | - |
3948 | | - // Create entry for new use. |
3949 | | - NewUses.push_back({Shuffle, OldMask}); |
3950 | | - |
3951 | | - // Validate mask indices. |
3952 | | - for (int Index : OldMask) { |
3953 | | - if (Index >= static_cast<int>(MaxIndex)) |
3954 | | - return false; |
3955 | | - } |
3956 | | - |
3957 | | - // Update costs. |
3958 | | - OldCost += |
3959 | | - TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, Shuffle->getType(), |
3960 | | - OldLoadTy, OldMask, CostKind); |
3961 | | - NewCost += |
3962 | | - TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, Shuffle->getType(), |
3963 | | - NewLoadTy, OldMask, CostKind); |
3964 | | - } |
3965 | | - |
3966 | | - LLVM_DEBUG( |
3967 | | - dbgs() << "Found a load used only by shufflevector instructions: " |
3968 | | - << I << "\n OldCost: " << OldCost |
3969 | | - << " vs NewCost: " << NewCost << "\n"); |
3970 | | - |
3971 | | - if (OldCost < NewCost || !NewCost.isValid()) |
3972 | | - return false; |
3973 | | - |
3974 | | - // Create new load of smaller vector. |
3975 | | - auto *NewLoad = cast<LoadInst>( |
3976 | | - Builder.CreateAlignedLoad(NewLoadTy, PtrOp, OldLoad->getAlign())); |
3977 | | - NewLoad->copyMetadata(I); |
3978 | | - |
3979 | | - // Replace all uses. |
3980 | | - for (UseEntry &Use : NewUses) { |
3981 | | - ShuffleVectorInst *Shuffle = Use.first; |
3982 | | - std::vector<int> &NewMask = Use.second; |
3983 | | - |
3984 | | - Builder.SetInsertPoint(Shuffle); |
3985 | | - Builder.SetCurrentDebugLocation(Shuffle->getDebugLoc()); |
3986 | | - Value *NewShuffle = Builder.CreateShuffleVector( |
3987 | | - NewLoad, PoisonValue::get(NewLoadTy), NewMask); |
3988 | | - |
3989 | | - replaceValue(*Shuffle, *NewShuffle); |
3990 | | - } |
3991 | | - |
3992 | | - return true; |
3993 | | - } |
3994 | | - } |
3995 | | - return false; |
3996 | | -} |
3997 | | - |
3998 | 3866 | // Attempt to narrow a phi of shufflevector instructions where the two incoming |
3999 | 3867 | // values have the same operands but different masks. If the two shuffle masks |
4000 | 3868 | // are offsets of one another we can use one branch to rotate the incoming |
@@ -4166,9 +4034,6 @@ bool VectorCombine::run() { |
4166 | 4034 | MadeChange |= foldSelectShuffle(I); |
4167 | 4035 | MadeChange |= foldShuffleToIdentity(I); |
4168 | 4036 | break; |
4169 | | - case Instruction::Load: |
4170 | | - MadeChange |= shrinkLoadForShuffles(I); |
4171 | | - break; |
4172 | 4037 | case Instruction::BitCast: |
4173 | 4038 | MadeChange |= foldBitcastShuffle(I); |
4174 | 4039 | break; |
|
0 commit comments