Enable vectorization of i8 values.

Author: doru1004

llvm/include/llvm/Analysis/TargetTransformInfo.h

@@ -1907,6 +1907,10 @@ class TargetTransformInfo {
   /// pad to. Default is no padding.
   unsigned getNumBytesToPadGlobalArray(unsigned Size, Type *ArrayType) const;
 
+  /// \return Returns true if vectorizing 4 x i8s into an i32 is possible.
+  /// Currently only used by the SLP vectorizer.
+  bool canVectorizei8s() const;
+
   /// @}
 
   /// Collect kernel launch bounds for \p F into \p LB.
@@ -2363,6 +2367,7 @@ class TargetTransformInfo::Concept {
   virtual void collectKernelLaunchBounds(
       const Function &F,
       SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const = 0;
+  virtual bool canVectorizei8s() const = 0;
 };
 
 template <typename T>
@@ -3229,6 +3234,8 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
       SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const override {
     Impl.collectKernelLaunchBounds(F, LB);
   }
+
+  bool canVectorizei8s() const override { return Impl.canVectorizei8s(); }
 };
 
 template <typename T>

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

@@ -1063,6 +1063,8 @@ class TargetTransformInfoImplBase {
 
   unsigned getMaxNumArgs() const { return UINT_MAX; }
 
+  bool canVectorizei8s() const { return false; }
+
   unsigned getNumBytesToPadGlobalArray(unsigned Size, Type *ArrayType) const {
     return 0;
   }

llvm/lib/Analysis/TargetTransformInfo.cpp

@@ -1419,6 +1419,10 @@ unsigned TargetTransformInfo::getMaxNumArgs() const {
   return TTIImpl->getMaxNumArgs();
 }
 
+bool TargetTransformInfo::canVectorizei8s() const {
+  return TTIImpl->canVectorizei8s();
+}
+
 bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const {
   return TTIImpl->shouldExpandReduction(II);
 }

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

@@ -344,9 +344,10 @@ unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const {
 unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
   if (Opcode == Instruction::Load || Opcode == Instruction::Store)
     return 32 * 4 / ElemWidth;
-  return (ElemWidth == 16 && ST->has16BitInsts()) ? 2
-       : (ElemWidth == 32 && ST->hasPackedFP32Ops()) ? 2
-       : 1;
+  return ElemWidth == 8                                ? 4
+         : (ElemWidth == 16 && ST->has16BitInsts())    ? 2
+         : (ElemWidth == 32 && ST->hasPackedFP32Ops()) ? 2
+                                                       : 1;
 }
 
 unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,
@@ -1120,6 +1121,17 @@ Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,
   }
 }
 
+InstructionCost GCNTTIImpl::getScalarizationOverhead(
+    VectorType *InTy, const APInt &DemandedElts, bool Insert, bool Extract,
+    TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) {
+  unsigned NumVectorElts = cast<FixedVectorType>(InTy)->getNumElements();
+  if (NumVectorElts > 1 &&
+      InTy->getElementType() == IntegerType::getInt8Ty(InTy->getContext()))
+    return 0;
+  return BaseT::getScalarizationOverhead(InTy, DemandedElts, Insert, Extract,
+                                         CostKind, VL);
+}
+
 InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
                                            VectorType *VT, ArrayRef<int> Mask,
                                            TTI::TargetCostKind CostKind,
@@ -1134,6 +1146,11 @@ InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
   // Larger vector widths may require additional instructions, but are
   // typically cheaper than scalarized versions.
   unsigned NumVectorElts = cast<FixedVectorType>(VT)->getNumElements();
+
+  if (NumVectorElts > 1 &&
+      VT->getElementType() == IntegerType::getInt8Ty(VT->getContext()))
+    return 0;
+
   if (ST->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
       DL.getTypeSizeInBits(VT->getElementType()) == 16) {
     bool HasVOP3P = ST->hasVOP3PInsts();
@@ -1423,3 +1440,5 @@ void GCNTTIImpl::collectKernelLaunchBounds(
   LB.push_back({"amdgpu-waves-per-eu[0]", WavesPerEU.first});
   LB.push_back({"amdgpu-waves-per-eu[1]", WavesPerEU.second});
 }
+
+bool GCNTTIImpl::canVectorizei8s() const { return true; }

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

@@ -240,6 +240,12 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
 
   InstructionCost getVectorSplitCost() { return 0; }
 
+  InstructionCost getScalarizationOverhead(VectorType *InTy,
+                                           const APInt &DemandedElts,
+                                           bool Insert, bool Extract,
+                                           TTI::TargetCostKind CostKind,
+                                           ArrayRef<Value *> VL = {});
+
   InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
                                  ArrayRef<int> Mask,
                                  TTI::TargetCostKind CostKind, int Index,
@@ -282,6 +288,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
   void collectKernelLaunchBounds(
       const Function &F,
       SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const;
+
+  /// \return return true if we can pack 4 i8s into an i32.
+  bool canVectorizei8s() const;
 };
 
 } // end namespace llvm

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

@@ -12971,9 +12971,16 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
               LI0->getPointerAddressSpace(), CostKind);
 
         } else {
-          VecLdCost = TTI->getMemoryOpCost(
-              Instruction::Load, VecTy, LI0->getAlign(),
-              LI0->getPointerAddressSpace(), CostKind, TTI::OperandValueInfo());
+          if (VecTy->getElementType() ==
+                  IntegerType::getInt8Ty(VecTy->getContext()) &&
+              TTI->canVectorizei8s()) {
+            VecLdCost = 1;
+          } else {
+            VecLdCost =
+                TTI->getMemoryOpCost(Instruction::Load, VecTy, LI0->getAlign(),
+                                     LI0->getPointerAddressSpace(), CostKind,
+                                     TTI::OperandValueInfo());
+          }
         }
         break;
       case TreeEntry::StridedVectorize: {
@@ -20927,7 +20934,11 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
     // provided vectorization factor (i.e. the scalar type is used for vector
     // code during codegen).
     auto *VecTy = getWidenedType(ScalarTy, VF);
-    if (TTI->getNumberOfParts(VecTy) == VF)
+    unsigned NumParts = TTI->getNumberOfParts(VecTy);
+    if (TTI->canVectorizei8s() &&
+        VecTy->getElementType() == IntegerType::getInt8Ty(VecTy->getContext()))
+      NumParts = 1;
+    if (NumParts == VF)
       continue;
     for (unsigned I = NextInst; I < MaxInst; ++I) {
       unsigned ActualVF = std::min(MaxInst - I, VF);