[LV] Mask off possibly aliasing vector lanes

    When vectorising a loop that uses loads and stores, those pointers could
    overlap if their difference is less than the vector factor. For example,
    if address 20 is being stored to and address 23 is being loaded from, they
    overlap when the vector factor is 4 or higher. Currently LoopVectorize
    branches to a scalar loop in these cases with a runtime check. Howver if
    we construct a mask that disables the overlapping (aliasing) lanes then
    the vectorised loop can be safely entered, as long as the loads and
    stores are masked off.

Author: SamTebbs33

llvm/include/llvm/Analysis/LoopAccessAnalysis.h

@@ -491,11 +491,12 @@ struct PointerDiffInfo {
   const SCEV *SinkStart;
   unsigned AccessSize;
   bool NeedsFreeze;
+  bool WriteAfterRead;
 
   PointerDiffInfo(const SCEV *SrcStart, const SCEV *SinkStart,
-                  unsigned AccessSize, bool NeedsFreeze)
+                  unsigned AccessSize, bool NeedsFreeze, bool WriteAfterRead)
       : SrcStart(SrcStart), SinkStart(SinkStart), AccessSize(AccessSize),
-        NeedsFreeze(NeedsFreeze) {}
+        NeedsFreeze(NeedsFreeze), WriteAfterRead(WriteAfterRead) {}
 };
 
 /// Holds information about the memory runtime legality checks to verify

llvm/include/llvm/Analysis/TargetTransformInfo.h

@@ -205,6 +205,13 @@ enum class TailFoldingStyle {
   DataWithEVL,
 };
 
+enum class RTCheckStyle {
+  /// Create runtime checks based on the difference between two pointers
+  ScalarDifference,
+  /// Form a mask based on elements which won't be a WAR or RAW hazard.
+  UseSafeEltsMask,
+};
+
 struct TailFoldingInfo {
   TargetLibraryInfo *TLI;
   LoopVectorizationLegality *LVL;
@@ -1354,6 +1361,11 @@ class TargetTransformInfo {
       PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
       TTI::TargetCostKind CostKind) const;
 
+  /// \return true if a mask should be formed that disables lanes that could
+  /// alias between two pointers. The mask is created by the
+  /// loop_dependence_{war,raw}_mask intrinsics.
+  LLVM_ABI bool useSafeEltsMask() const;
+
   /// \return The maximum interleave factor that any transform should try to
   /// perform for this target. This number depends on the level of parallelism
   /// and the number of execution units in the CPU.

llvm/include/llvm/Analysis/TargetTransformInfoImpl.h

@@ -659,6 +659,8 @@ class TargetTransformInfoImplBase {
     return InstructionCost::getInvalid();
   }
 
+  virtual bool useSafeEltsMask() const { return false; }
+
   virtual unsigned getMaxInterleaveFactor(ElementCount VF) const { return 1; }
 
   virtual InstructionCost getArithmeticInstrCost(

llvm/include/llvm/CodeGen/BasicTTIImpl.h

@@ -2164,6 +2164,53 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
       // Otherwise, fallback to default scalarization cost.
       break;
     }
+    case Intrinsic::loop_dependence_raw_mask:
+    case Intrinsic::loop_dependence_war_mask: {
+      InstructionCost Cost = 0;
+      Type *PtrTy = ICA.getArgTypes()[0];
+      bool IsReadAfterWrite = IID == Intrinsic::loop_dependence_raw_mask;
+
+      Cost +=
+          thisT()->getArithmeticInstrCost(Instruction::Sub, PtrTy, CostKind);
+      if (IsReadAfterWrite) {
+        IntrinsicCostAttributes AbsAttrs(Intrinsic::abs, PtrTy, {PtrTy}, {});
+        Cost += thisT()->getIntrinsicInstrCost(AbsAttrs, CostKind);
+      }
+
+      Cost +=
+          thisT()->getArithmeticInstrCost(Instruction::SDiv, PtrTy, CostKind);
+      Type *CmpTy =
+          getTLI()
+              ->getSetCCResultType(
+                  thisT()->getDataLayout(), RetTy->getContext(),
+                  getTLI()->getValueType(thisT()->getDataLayout(), PtrTy))
+              .getTypeForEVT(RetTy->getContext());
+      Cost += thisT()->getCmpSelInstrCost(
+          BinaryOperator::ICmp, CmpTy, PtrTy,
+          IsReadAfterWrite ? CmpInst::ICMP_EQ : CmpInst::ICMP_SLE, CostKind);
+
+      // The deconstructed active lane mask
+      VectorType *RetTyVec = cast<VectorType>(RetTy);
+      VectorType *SplatTy = cast<VectorType>(RetTyVec->getWithNewType(PtrTy));
+      Cost += thisT()->getShuffleCost(TTI::SK_Broadcast, SplatTy, SplatTy, {},
+                                      CostKind, 0, nullptr);
+      IntrinsicCostAttributes StepVecAttrs(Intrinsic::stepvector, SplatTy, {},
+                                           FMF);
+      Cost += thisT()->getIntrinsicInstrCost(StepVecAttrs, CostKind);
+      Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, SplatTy,
+                                          SplatTy, CmpInst::ICMP_ULT, CostKind);
+
+      Cost +=
+          thisT()->getCastInstrCost(Instruction::CastOps::ZExt, RetTy, SplatTy,
+                                    TTI::CastContextHint::None, CostKind);
+      Cost += thisT()->getCastInstrCost(Instruction::CastOps::ZExt,
+                                        RetTyVec->getElementType(), CmpTy,
+                                        TTI::CastContextHint::None, CostKind);
+      Cost += thisT()->getShuffleCost(TTI::SK_Broadcast, RetTyVec, RetTyVec, {},
+                                      CostKind, 0, nullptr);
+      Cost += thisT()->getArithmeticInstrCost(Instruction::Or, RetTy, CostKind);
+      return Cost;
+    }
     }
 
     // Assume that we need to scalarize this intrinsic.)

llvm/include/llvm/Transforms/Utils/LoopUtils.h

@@ -580,6 +580,11 @@ addRuntimeChecks(Instruction *Loc, Loop *TheLoop,
                  const SmallVectorImpl<RuntimePointerCheck> &PointerChecks,
                  SCEVExpander &Expander, bool HoistRuntimeChecks = false);
 
+LLVM_ABI Value *addSafeEltsRuntimeChecks(Instruction *Loc,
+                                         ArrayRef<PointerDiffInfo> Checks,
+                                         SCEVExpander &Expander,
+                                         ElementCount VF);
+
 LLVM_ABI Value *addDiffRuntimeChecks(
     Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
     function_ref<Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC);

llvm/lib/Analysis/LoopAccessAnalysis.cpp

@@ -511,11 +511,14 @@ bool RuntimePointerChecking::tryToCreateDiffCheck(
     }
   }
 
+  bool WriteAfterRead = !Src->IsWritePtr && Sink->IsWritePtr;
+
   LLVM_DEBUG(dbgs() << "LAA: Creating diff runtime check for:\n"
                     << "SrcStart: " << *SrcStartInt << '\n'
                     << "SinkStartInt: " << *SinkStartInt << '\n');
   DiffChecks.emplace_back(SrcStartInt, SinkStartInt, AllocSize,
-                          Src->NeedsFreeze || Sink->NeedsFreeze);
+                          Src->NeedsFreeze || Sink->NeedsFreeze,
+                          WriteAfterRead);
   return true;
 }
 

llvm/lib/Analysis/TargetTransformInfo.cpp

@@ -878,6 +878,10 @@ InstructionCost TargetTransformInfo::getPartialReductionCost(
                                           BinOp, CostKind);
 }
 
+bool TargetTransformInfo::useSafeEltsMask() const {
+  return TTIImpl->useSafeEltsMask();
+}
+
 unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const {
   return TTIImpl->getMaxInterleaveFactor(VF);
 }

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

@@ -1004,6 +1004,12 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
     }
     break;
   }
+  case Intrinsic::loop_dependence_raw_mask:
+  case Intrinsic::loop_dependence_war_mask:
+    // The whilewr/rw instructions require SVE2
+    if (ST->hasSVE2() || ST->hasSME())
+      return 1;
+    break;
   default:
     break;
   }
@@ -5725,6 +5731,11 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost(
   return Cost + 4;
 }
 
+bool AArch64TTIImpl::useSafeEltsMask() const {
+  // The whilewr/rw instructions require SVE2
+  return ST->hasSVE2() || ST->hasSME();
+}
+
 InstructionCost
 AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
                                VectorType *SrcTy, ArrayRef<int> Mask,

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h

@@ -398,6 +398,8 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
       TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
       TTI::TargetCostKind CostKind) const override;
 
+  bool useSafeEltsMask() const override;
+
   bool enableOrderedReductions() const override { return true; }
 
   InstructionCost getInterleavedMemoryOpCost(

llvm/lib/Transforms/Utils/LoopUtils.cpp

@@ -2094,6 +2094,58 @@ Value *llvm::addRuntimeChecks(
   return MemoryRuntimeCheck;
 }
 
+Value *llvm::addSafeEltsRuntimeChecks(Instruction *Loc,
+                                      ArrayRef<PointerDiffInfo> Checks,
+                                      SCEVExpander &Expander, ElementCount VF) {
+  IRBuilder ChkBuilder(Loc->getContext(),
+                       InstSimplifyFolder(Loc->getDataLayout()));
+  ChkBuilder.SetInsertPoint(Loc);
+  Value *MemoryRuntimeCheck = nullptr;
+
+  // Map to keep track of created compares, The key is the pair of operands for
+  // the compare, to allow detecting and re-using redundant compares.
+  DenseMap<std::pair<Value *, Value *>, Value *> SeenCompares;
+  Value *AliasLaneMask = nullptr;
+  for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze,
+                    WriteAfterRead] : Checks) {
+    Type *Ty = SinkStart->getType();
+    Value *Sink = Expander.expandCodeFor(SinkStart, Ty, Loc);
+    Value *Src = Expander.expandCodeFor(SrcStart, Ty, Loc);
+    if (SeenCompares.lookup({Sink, Src}))
+      continue;
+
+    unsigned IntOpc = WriteAfterRead ? Intrinsic::loop_dependence_war_mask
+                                     : Intrinsic::loop_dependence_raw_mask;
+    Value *SourceAsPtr = ChkBuilder.CreateCast(Instruction::IntToPtr, Src,
+                                               ChkBuilder.getPtrTy());
+    Value *SinkAsPtr = ChkBuilder.CreateCast(Instruction::IntToPtr, Sink,
+                                             ChkBuilder.getPtrTy());
+    Value *M = ChkBuilder.CreateIntrinsic(
+        IntOpc, {VectorType::get(ChkBuilder.getInt1Ty(), VF)},
+        {SourceAsPtr, SinkAsPtr, ChkBuilder.getInt64(AccessSize)}, nullptr,
+        "alias.lane.mask");
+    SeenCompares.insert({{Sink, Src}, M});
+    if (AliasLaneMask)
+      M = ChkBuilder.CreateAnd(AliasLaneMask, M);
+    else
+      AliasLaneMask = M;
+  }
+  assert(AliasLaneMask && "Expected an alias lane mask to have been created.");
+  auto *VecVT = VectorType::get(ChkBuilder.getInt1Ty(), VF);
+  // Extend to an i8 since i1 is too small to add with
+  Value *PopCount = ChkBuilder.CreateCast(
+      Instruction::ZExt, AliasLaneMask,
+      VectorType::get(ChkBuilder.getInt8Ty(), VecVT->getElementCount()));
+
+  PopCount =
+      ChkBuilder.CreateUnaryIntrinsic(Intrinsic::vector_reduce_add, PopCount);
+  PopCount = ChkBuilder.CreateCast(Instruction::ZExt, PopCount,
+                                   ChkBuilder.getInt64Ty());
+  MemoryRuntimeCheck = ChkBuilder.CreateICmpUGT(
+      PopCount, ConstantInt::get(ChkBuilder.getInt64Ty(), 0));
+  return MemoryRuntimeCheck;
+}
+
 Value *llvm::addDiffRuntimeChecks(
     Instruction *Loc, ArrayRef<PointerDiffInfo> Checks, SCEVExpander &Expander,
     function_ref<Value *(IRBuilderBase &, unsigned)> GetVF, unsigned IC) {
@@ -2108,7 +2160,8 @@ Value *llvm::addDiffRuntimeChecks(
   // Map to keep track of created compares, The key is the pair of operands for
   // the compare, to allow detecting and re-using redundant compares.
   DenseMap<std::pair<Value *, Value *>, Value *> SeenCompares;
-  for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze] : Checks) {
+  for (const auto &[SrcStart, SinkStart, AccessSize, NeedsFreeze,
+                    WriteAfterRead] : Checks) {
     Type *Ty = SinkStart->getType();
     // Compute VF * IC * AccessSize.
     auto *VFTimesICTimesSize =
@@ -2117,8 +2170,8 @@ Value *llvm::addDiffRuntimeChecks(
     Value *Diff =
         Expander.expandCodeFor(SE.getMinusSCEV(SinkStart, SrcStart), Ty, Loc);
 
-    // Check if the same compare has already been created earlier. In that case,
-    // there is no need to check it again.
+    // Check if the same compare has already been created earlier. In that
+    // case, there is no need to check it again.
     Value *IsConflict = SeenCompares.lookup({Diff, VFTimesICTimesSize});
     if (IsConflict)
       continue;