Add TII::preferStridedLoadStore to control the behavior

Author: wangpc-pp

llvm/include/llvm/Analysis/TargetTransformInfo.h

@@ -1776,6 +1776,10 @@ class TargetTransformInfo {
   /// otherwise scalar epilogue loop.
   bool preferEpilogueVectorization() const;
 
+  /// Return true if the loop vectorizer prefer strided load/store when
+  /// vectorizing reversed load/store.
+  bool preferStridedLoadStore() const;
+
   /// \returns True if the target wants to expand the given reduction intrinsic
   /// into a shuffle sequence.
   bool shouldExpandReduction(const IntrinsicInst *II) const;
@@ -2301,6 +2305,7 @@ class TargetTransformInfo::Concept {
   virtual bool preferPredicatedReductionSelect(unsigned Opcode, Type *Ty,
                                                ReductionFlags) const = 0;
   virtual bool preferEpilogueVectorization() const = 0;
+  virtual bool preferStridedLoadStore() const = 0;
 
   virtual bool shouldExpandReduction(const IntrinsicInst *II) const = 0;
   virtual ReductionShuffle
@@ -3105,6 +3110,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
     return Impl.preferEpilogueVectorization();
   }
 
+  bool preferStridedLoadStore() const override {
+    return Impl.preferStridedLoadStore();
+  }
+
   bool shouldExpandReduction(const IntrinsicInst *II) const override {
     return Impl.shouldExpandReduction(II);
   }

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

@@ -1003,6 +1003,8 @@ class TargetTransformInfoImplBase {
     return true;
   }
 
+  bool preferStridedLoadStore() const { return false; }
+
   bool shouldExpandReduction(const IntrinsicInst *II) const { return true; }
 
   TTI::ReductionShuffle

llvm/lib/Analysis/TargetTransformInfo.cpp

@@ -1374,6 +1374,10 @@ bool TargetTransformInfo::preferEpilogueVectorization() const {
   return TTIImpl->preferEpilogueVectorization();
 }
 
+bool TargetTransformInfo::preferStridedLoadStore() const {
+  return TTIImpl->preferStridedLoadStore();
+}
+
 TargetTransformInfo::VPLegalization
 TargetTransformInfo::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
   return TTIImpl->getVPLegalizationStrategy(VPI);

llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h

@@ -118,6 +118,8 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
     return false;
   }
 
+  bool preferStridedLoadStore() const { return true; }
+
   InstructionCost getMaskedMemoryOpCost(unsigned Opcode, Type *Src,
                                         Align Alignment, unsigned AddressSpace,
                                         TTI::TargetCostKind CostKind);

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

@@ -2603,13 +2603,25 @@ void VPWidenLoadRecipe::print(raw_ostream &O, const Twine &Indent,
 }
 #endif
 
+/// Use all-true mask for reverse rather than actual mask, as it avoids a
+/// dependence w/o affecting the result.
+static Instruction *createReverseEVL(IRBuilderBase &Builder, Value *Operand,
+                                     Value *EVL, const Twine &Name) {
+  VectorType *ValTy = cast<VectorType>(Operand->getType());
+  Value *AllTrueMask =
+      Builder.CreateVectorSplat(ValTy->getElementCount(), Builder.getTrue());
+  return Builder.CreateIntrinsic(ValTy, Intrinsic::experimental_vp_reverse,
+                                 {Operand, AllTrueMask, EVL}, nullptr, Name);
+}
+
 void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
   auto *LI = cast<LoadInst>(&Ingredient);
 
   Type *ScalarDataTy = getLoadStoreType(&Ingredient);
   auto *DataTy = VectorType::get(ScalarDataTy, State.VF);
   const Align Alignment = getLoadStoreAlignment(&Ingredient);
   bool CreateGather = !isConsecutive();
+  bool UseStridedLoadStore = State.TTI->preferStridedLoadStore();
 
   auto &Builder = State.Builder;
   State.setDebugLocFrom(getDebugLoc());
@@ -2619,6 +2631,8 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
   Value *Mask = nullptr;
   if (VPValue *VPMask = getMask()) {
     Mask = State.get(VPMask);
+    if (isReverse() && !UseStridedLoadStore)
+      Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");
   } else {
     Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
   }
@@ -2627,7 +2641,7 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
     NewLI =
         Builder.CreateIntrinsic(DataTy, Intrinsic::vp_gather, {Addr, Mask, EVL},
                                 nullptr, "wide.masked.gather");
-  } else if (isReverse()) {
+  } else if (isReverse() && UseStridedLoadStore) {
     auto *EltTy = DataTy->getElementType();
     auto *PtrTy = Addr->getType();
     Value *Operands[] = {
@@ -2648,6 +2662,8 @@ void VPWidenLoadEVLRecipe::execute(VPTransformState &State) {
       0, Attribute::getWithAlignment(NewLI->getContext(), Alignment));
   State.addMetadata(NewLI, LI);
   Instruction *Res = NewLI;
+  if (isReverse() && !UseStridedLoadStore)
+    Res = createReverseEVL(Builder, Res, EVL, "vp.reverse");
   State.set(this, Res);
 }
 
@@ -2738,16 +2754,21 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
   VPValue *StoredValue = getStoredValue();
   bool CreateScatter = !isConsecutive();
   const Align Alignment = getLoadStoreAlignment(&Ingredient);
+  bool UseStridedLoadStore = State.TTI->preferStridedLoadStore();
 
   auto &Builder = State.Builder;
   State.setDebugLocFrom(getDebugLoc());
 
   CallInst *NewSI = nullptr;
   Value *StoredVal = State.get(StoredValue);
   Value *EVL = State.get(getEVL(), VPLane(0));
+  if (isReverse() && !UseStridedLoadStore)
+    StoredVal = createReverseEVL(Builder, StoredVal, EVL, "vp.reverse");
   Value *Mask = nullptr;
   if (VPValue *VPMask = getMask()) {
     Mask = State.get(VPMask);
+    if (isReverse() && !UseStridedLoadStore)
+      Mask = createReverseEVL(Builder, Mask, EVL, "vp.reverse.mask");
   } else {
     Mask = Builder.CreateVectorSplat(State.VF, Builder.getTrue());
   }
@@ -2756,7 +2777,7 @@ void VPWidenStoreEVLRecipe::execute(VPTransformState &State) {
     NewSI = Builder.CreateIntrinsic(Type::getVoidTy(EVL->getContext()),
                                     Intrinsic::vp_scatter,
                                     {StoredVal, Addr, Mask, EVL});
-  } else if (isReverse()) {
+  } else if (isReverse() && UseStridedLoadStore) {
     Type *StoredValTy = StoredVal->getType();
     auto *EltTy = cast<VectorType>(StoredValTy)->getElementType();
     auto *PtrTy = Addr->getType();