Split away the strided segmented load implementation

mshockwave · mshockwave · commit 301e64065ed7 · 2025-01-15T11:17:29.000-08:00
We're going to put it in a separate patch in the future.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3181,19 +3181,6 @@ class TargetLoweringBase {
     return false;
   }
 
-  /// Lower a deinterleave intrinsic to a target specific strided load
-  /// intrinsic. Return true on success.
-  ///
-  /// \p StridedLoad is the vp.strided.load instruction.
-  /// \p DI is the deinterleave intrinsic.
-  /// \p Factor is the interleave factor.
-  /// \p DeinterleaveRes is a list of deinterleaved results.
-  virtual bool lowerDeinterleaveIntrinsicToStridedLoad(
-      VPIntrinsic *StridedLoad, IntrinsicInst *DI, unsigned Factor,
-      ArrayRef<Value *> DeinterleaveRes) const {
-    return false;
-  }
-
   /// Lower a deinterleave intrinsic to a target specific load intrinsic.
   /// Return true on success. Currently only supports
   /// llvm.vector.deinterleave2
diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp
@@ -661,12 +661,13 @@ bool InterleavedAccessImpl::lowerInterleavedStore(
 
 bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
     IntrinsicInst *DI, SmallSetVector<Instruction *, 32> &DeadInsts) {
-  using namespace PatternMatch;
-  SmallVector<Value *, 8> DeInterleaveResults;
-  unsigned Factor = getVectorDeInterleaveFactor(DI, DeInterleaveResults);
+  if (auto *VPLoad = dyn_cast<VPIntrinsic>(DI->getOperand(0))) {
+    SmallVector<Value *, 8> DeInterleaveResults;
+    unsigned Factor = getVectorDeInterleaveFactor(DI, DeInterleaveResults);
+    if (!Factor)
+      return false;
 
-  if (auto *VPLoad = dyn_cast<VPIntrinsic>(DI->getOperand(0));
-      Factor && VPLoad) {
+    using namespace PatternMatch;
     if (!match(VPLoad, m_OneUse(m_Intrinsic<Intrinsic::vp_load>())))
       return false;
 
@@ -689,53 +690,6 @@ bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic(
     return true;
   }
 
-  // Match
-  //   %x = vp.strided.load  ;; VPStridedLoad
-  //   %y = bitcast %x       ;; BitCast
-  //   %y' = inttoptr %y
-  //   %z = deinterleave %y  ;; DI
-  if (Factor && isa<BitCastInst, IntToPtrInst>(DI->getOperand(0))) {
-    auto *BitCast = cast<Instruction>(DI->getOperand(0));
-    if (!BitCast->hasOneUse())
-      return false;
-
-    Instruction *IntToPtrCast = nullptr;
-    if (auto *BC = dyn_cast<BitCastInst>(BitCast->getOperand(0))) {
-      IntToPtrCast = BitCast;
-      BitCast = BC;
-    }
-
-    // Match the type is
-    //   <VF x (factor * elementTy)> bitcast to <(VF * factor) x elementTy>
-    Value *BitCastSrc = BitCast->getOperand(0);
-    auto *BitCastSrcTy = dyn_cast<VectorType>(BitCastSrc->getType());
-    auto *BitCastDstTy = cast<VectorType>(BitCast->getType());
-    if (!BitCastSrcTy || (BitCastSrcTy->getElementCount() * Factor !=
-                          BitCastDstTy->getElementCount()))
-      return false;
-
-    if (auto *VPStridedLoad = dyn_cast<VPIntrinsic>(BitCast->getOperand(0))) {
-      if (VPStridedLoad->getIntrinsicID() !=
-              Intrinsic::experimental_vp_strided_load ||
-          !VPStridedLoad->hasOneUse())
-        return false;
-
-      LLVM_DEBUG(dbgs() << "IA: Found a deinterleave intrinsic: " << *DI
-                        << "\n");
-
-      if (!TLI->lowerDeinterleaveIntrinsicToStridedLoad(
-              VPStridedLoad, DI, Factor, DeInterleaveResults))
-        return false;
-
-      DeadInsts.push_back(DI);
-      if (IntToPtrCast)
-        DeadInsts.push_back(IntToPtrCast);
-      DeadInsts.push_back(BitCast);
-      DeadInsts.push_back(VPStridedLoad);
-      return true;
-    }
-  }
-
   LoadInst *LI = dyn_cast<LoadInst>(DI->getOperand(0));
 
   if (!LI || !LI->hasOneUse() || !LI->isSimple())
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -22608,147 +22608,6 @@ bool RISCVTargetLowering::lowerInterleavedScalableStore(
   return true;
 }
 
-/// Lower an interleaved vp.strided.load into a vlssegN intrinsic.
-///
-/// E.g. Lower an interleaved vp.strided.load (Factor = 2):
-///   %l = call <vscale x 2 x i16>
-///           @llvm.experimental.vp.strided.load.nxv2i16.p0.i64(ptr %ptr,
-///                                                             %stride,
-///                                                             <all-true-mask>,
-///                                                             i32 %rvl)
-///   %l.cast = bitcast <vscale x 2 x i16> %l to <vscale x 4 x i8>
-///   %dl = tail call { <vscale x 2 x i8>, <vscale x 2 x i8> }
-///             @llvm.vector.deinterleave2.nxv2i8(
-///               <vscale x 4 x i8> %l.cast)
-///   %r0 = extractvalue { <vscale x 2 x i8>, <vscale x 2 x i8> } %dl, 0
-///   %r1 = extractvalue { <vscale x 2 x i8>, <vscale x 2 x i8> } %dl, 1
-///
-/// Into:
-///   %ssl = call { <vscale x 2 x i8>, <vscale x 2 x i8> }
-///              @llvm.riscv.vlseg2.nxv2i8.i64(<vscale x 32 x i8> poison,
-///                                            <vscale x 32 x i8> poison,
-///                                            %ptr,
-///                                            %stride,
-///                                            i64 %rvl)
-///   %r0 = extractvalue { <vscale x 2 x i8>, <vscale x 2 x i8> } %ssl, 0
-///   %r1 = extractvalue { <vscale x 2 x i8>, <vscale x 2 x i8> } %ssl, 1
-///
-/// NOTE: the deinterleave2 intrinsic and the bitcast instruction won't be
-/// touched and is expected to be removed by the caller
-bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToStridedLoad(
-    VPIntrinsic *StridedLoad, IntrinsicInst *DI, unsigned Factor,
-    ArrayRef<Value *> DeInterleaveResults) const {
-  using namespace llvm::PatternMatch;
-  Value *BasePtr, *Stride, *Mask, *EVL;
-  if (!match(StridedLoad, m_Intrinsic<Intrinsic::experimental_vp_strided_load>(
-                              m_Value(BasePtr), m_Value(Stride), m_Value(Mask),
-                              m_Value(EVL))))
-    return false;
-
-  [[maybe_unused]] auto *DISrcTy =
-      cast<VectorType>(DI->getOperand(0)->getType());
-  [[maybe_unused]] auto *LTy = cast<VectorType>(StridedLoad->getType());
-  auto &DL = StridedLoad->getModule()->getDataLayout();
-  assert(DL.getTypeAllocSizeInBits(DISrcTy) == DL.getTypeAllocSizeInBits(LTy) &&
-         "The primitive size of strided load and the source of deinterleave "
-         "should be the same.");
-  assert(DISrcTy->getElementCount() == LTy->getElementCount() * Factor &&
-         "ElementCount of source deinterleave should be equal to the "
-         "ElementCount of strided load multiplied by factor.");
-
-  auto *ResTy = cast<VectorType>(DeInterleaveResults[0]->getType());
-
-  Align Alignment =
-      cast<VPIntrinsic>(StridedLoad)->getPointerAlignment().valueOrOne();
-  if (!isLegalInterleavedAccessType(
-          ResTy, Factor, Alignment,
-          BasePtr->getType()->getPointerAddressSpace(), DL))
-    return false;
-
-  IRBuilder<> Builder(StridedLoad);
-  auto *XLenTy =
-      Type::getIntNTy(StridedLoad->getContext(), Subtarget.getXLen());
-  assert(Stride->getType() == XLenTy &&
-         "The type of stride must be the XLEN integer type.");
-  EVL = Builder.CreateZExtOrTrunc(EVL, XLenTy);
-
-  static const Intrinsic::ID IntrMaskIds[] = {
-      Intrinsic::riscv_vlsseg2_mask, Intrinsic::riscv_vlsseg3_mask,
-      Intrinsic::riscv_vlsseg4_mask, Intrinsic::riscv_vlsseg5_mask,
-      Intrinsic::riscv_vlsseg6_mask, Intrinsic::riscv_vlsseg7_mask,
-      Intrinsic::riscv_vlsseg8_mask,
-  };
-
-  static const Intrinsic::ID IntrIds[] = {
-      Intrinsic::riscv_vlsseg2, Intrinsic::riscv_vlsseg3,
-      Intrinsic::riscv_vlsseg4, Intrinsic::riscv_vlsseg5,
-      Intrinsic::riscv_vlsseg6, Intrinsic::riscv_vlsseg7,
-      Intrinsic::riscv_vlsseg8,
-  };
-
-  unsigned SEW = DL.getTypeSizeInBits(ResTy->getElementType());
-  unsigned NumElts = ResTy->getElementCount().getKnownMinValue();
-  Type *VecTupTy = TargetExtType::get(
-      StridedLoad->getContext(), "riscv.vector.tuple",
-      ScalableVectorType::get(Type::getInt8Ty(StridedLoad->getContext()),
-                              NumElts * SEW / 8),
-      Factor);
-
-  Value *PoisonVal = PoisonValue::get(VecTupTy);
-  SmallVector<Value *, 7> Operands;
-  Operands.append({PoisonVal, BasePtr, Stride});
-
-  Intrinsic::ID VlssegNID = IntrIds[Factor - 2];
-  bool IsMasked = !match(Mask, m_AllOnes());
-  if (IsMasked) {
-    VlssegNID = IntrMaskIds[Factor - 2];
-    Operands.push_back(Mask);
-  }
-
-  Operands.push_back(EVL);
-
-  // Set the tail policy to tail-agnostic, mask-agnostic (tama) for masked
-  // intrinsics
-  if (IsMasked)
-    Operands.push_back(ConstantInt::get(XLenTy, 3));
-
-  Operands.push_back(ConstantInt::get(XLenTy, Log2_64(SEW)));
-
-  Function *VlssegNFunc;
-  if (IsMasked) {
-    VlssegNFunc = Intrinsic::getOrInsertDeclaration(
-        StridedLoad->getModule(), VlssegNID,
-        {VecTupTy, EVL->getType(), Mask->getType()});
-  } else {
-    VlssegNFunc = Intrinsic::getOrInsertDeclaration(
-        StridedLoad->getModule(), VlssegNID, {VecTupTy, EVL->getType()});
-  }
-  CallInst *VlssegN = Builder.CreateCall(VlssegNFunc, Operands);
-
-  SmallVector<Type *, 8> AggrTypes{Factor, ResTy};
-  Value *Return =
-      PoisonValue::get(StructType::get(StridedLoad->getContext(), AggrTypes));
-  Function *VecExtractFunc = Intrinsic::getOrInsertDeclaration(
-      StridedLoad->getModule(), Intrinsic::riscv_tuple_extract,
-      {ResTy, VecTupTy});
-  for (unsigned i = 0; i < Factor; ++i) {
-    Value *VecExtract =
-        Builder.CreateCall(VecExtractFunc, {VlssegN, Builder.getInt32(i)});
-    Return = Builder.CreateInsertValue(Return, VecExtract, i);
-  }
-
-  for (auto [Idx, DIO] : enumerate(DeInterleaveResults)) {
-    // We have to create a brand new ExtractValue to replace each
-    // of these old ExtractValue instructions.
-    Value *NewEV =
-        Builder.CreateExtractValue(Return, {static_cast<unsigned>(Idx)});
-    DIO->replaceAllUsesWith(NewEV);
-  }
-  DI->replaceAllUsesWith(UndefValue::get(DI->getType()));
-
-  return true;
-}
-
 MachineInstr *
 RISCVTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
                                    MachineBasicBlock::instr_iterator &MBBI,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -916,10 +916,6 @@ class RISCVTargetLowering : public TargetLowering {
       VPIntrinsic *Store, Value *Mask, IntrinsicInst *InterleaveIntrin,
       unsigned Factor, ArrayRef<Value *> InterleaveOps) const override;
 
-  bool lowerDeinterleaveIntrinsicToStridedLoad(
-      VPIntrinsic *StridedLoad, IntrinsicInst *DI, unsigned Factor,
-      ArrayRef<Value *> DeinterleaveRes) const override;
-
   bool supportKCFIBundles() const override { return true; }
 
   SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr,
diff --git a/llvm/test/CodeGen/RISCV/rvv/scalable-vectors-strided-interleave-load-32.ll b/llvm/test/CodeGen/RISCV/rvv/scalable-vectors-strided-interleave-load-32.ll
diff --git a/llvm/test/CodeGen/RISCV/rvv/scalable-vectors-strided-interleave-load-64.ll b/llvm/test/CodeGen/RISCV/rvv/scalable-vectors-strided-interleave-load-64.ll
