1. add mask Op in VPlanTransform

LiqinWeng · LiqinWeng · commit 9f1312793650 · 2024-11-14T18:02:31.000+08:00
2. remove the vpintrinsic's implment in execute
diff --git a/llvm/include/llvm/IR/VectorBuilder.h b/llvm/include/llvm/IR/VectorBuilder.h
@@ -99,11 +99,11 @@ class VectorBuilder {
                                  const Twine &Name = Twine());
 
   /// Emit a VP reduction intrinsic call for recurrence kind.
-  /// \param ID          The intrinsic ID of call Intrinsic
+  /// \param RdxID       The intrinsic ID of llvm.vector.reduce.*
   /// \param ValTy       The type of operand which the reduction operation is
   ///                    performed.
   /// \param VecOpArray  The operand list.
-  Value *createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy,
+  Value *createSimpleReduction(Intrinsic::ID RdxID, Type *ValTy,
                                ArrayRef<Value *> VecOpArray,
                                const Twine &Name = Twine());
 };
diff --git a/llvm/lib/IR/VectorBuilder.cpp b/llvm/lib/IR/VectorBuilder.cpp
@@ -60,12 +60,13 @@ Value *VectorBuilder::createVectorInstruction(unsigned Opcode, Type *ReturnTy,
   return createVectorInstructionImpl(VPID, ReturnTy, InstOpArray, Name);
 }
 
-Value *VectorBuilder::createSimpleIntrinsic(Intrinsic::ID ID, Type *ValTy,
+Value *VectorBuilder::createSimpleReduction(Intrinsic::ID RdxID,
+                                            Type *ValTy,
                                             ArrayRef<Value *> InstOpArray,
                                             const Twine &Name) {
-  auto VPID = VPIntrinsic::getForIntrinsic(ID);
-  assert(VPIntrinsic::isVPIntrinsic(VPID) &&
-         "No VPIntrinsic for this Intrinsic");
+  auto VPID = VPIntrinsic::getForIntrinsic(RdxID);
+  assert(VPReductionIntrinsic::isVPReduction(VPID) &&
+         "No VPIntrinsic for this reduction");
   return createVectorInstructionImpl(VPID, ValTy, InstOpArray, Name);
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1118,8 +1118,6 @@ RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
   case Intrinsic::vp_udiv:
   case Intrinsic::vp_urem:
   case Intrinsic::vp_xor:
-  // TODO: add new patch for it.
-  case Intrinsic::vp_fneg:
   // vp float arithmetic ops.
   case Intrinsic::vp_fadd:
   case Intrinsic::vp_fsub:
diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp
@@ -1300,7 +1300,7 @@ Value *llvm::createSimpleReduction(VectorBuilder &VBuilder, Value *Src,
   Type *SrcEltTy = SrcTy->getElementType();
   Value *Iden = getRecurrenceIdentity(Kind, SrcEltTy, Desc.getFastMathFlags());
   Value *Ops[] = {Iden, Src};
-  return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops);
+  return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
 }
 
 Value *llvm::createReduction(IRBuilderBase &B,
@@ -1343,7 +1343,7 @@ Value *llvm::createOrderedReduction(VectorBuilder &VBuilder,
   Intrinsic::ID Id = getReductionIntrinsicID(RecurKind::FAdd);
   auto *SrcTy = cast<VectorType>(Src->getType());
   Value *Ops[] = {Start, Src};
-  return VBuilder.createSimpleIntrinsic(Id, SrcTy, Ops);
+  return VBuilder.createSimpleReduction(Id, SrcTy, Ops);
 }
 
 void llvm::propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue,
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1585,18 +1585,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags {
   bool MayHaveSideEffects;
 
 public:
-  VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
-                         ArrayRef<VPValue *> CallArguments, Type *Ty,
-                         DebugLoc DL = {})
-      : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
-        VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
-        MayReadFromMemory(CI.mayReadFromMemory()),
-        MayWriteToMemory(CI.mayWriteToMemory()),
-        MayHaveSideEffects(CI.mayHaveSideEffects()) {}
-
-  template <typename IterT>
   VPWidenIntrinsicRecipe(Instruction &I, Intrinsic::ID VectorIntrinsicID,
-                         iterator_range<IterT> Operands, Type *Ty,
+                         ArrayRef<VPValue *> Operands, Type *Ty,
                          DebugLoc DL = {})
       : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, Operands, I),
         VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -954,45 +954,31 @@ void VPWidenIntrinsicRecipe::execute(VPTransformState &State) {
     Args.push_back(Arg);
   }
 
-  if (VPIntrinsic::isVPIntrinsic(VectorIntrinsicID) &&
-      VectorIntrinsicID != Intrinsic::vp_select) {
-    VectorBuilder VBuilder(State.Builder);
-    Value *Mask =
-        State.Builder.CreateVectorSplat(State.VF, State.Builder.getTrue());
-    VBuilder.setMask(Mask).setEVL(Args.back());
-    // Remove EVL from Args
-    Args.pop_back();
-
-    Value *VPInst = VBuilder.createSimpleIntrinsic(
-        VectorIntrinsicID, TysForDecl[0], Args, "vp.call");
-
-    if (isa<FPMathOperator>(VPInst))
-      setFlags(cast<Instruction>(VPInst));
-
-    if (!VPInst->getType()->isVoidTy())
-      State.set(this, VPInst);
-    State.addMetadata(VPInst,
-                      dyn_cast_or_null<Instruction>(getUnderlyingValue()));
-  } else {
-    // Use vector version of the intrinsic.
-    Module *M = State.Builder.GetInsertBlock()->getModule();
-    Function *VectorF =
-        Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
-    assert(VectorF && "Can't retrieve vector intrinsic.");
-
-    auto *CI = cast_or_null<CallInst>(getUnderlyingValue());
-    SmallVector<OperandBundleDef, 1> OpBundles;
-    if (CI)
-      CI->getOperandBundlesAsDefs(OpBundles);
+  // Use vector version of the intrinsic.
+  Module *M = State.Builder.GetInsertBlock()->getModule();
+  bool IsVPIntrinsic = VPIntrinsic::isVPIntrinsic(VectorIntrinsicID);
+  Function *VectorF =
+      Intrinsic::getOrInsertDeclaration(M, VectorIntrinsicID, TysForDecl);
+  assert(VectorF && "Can't retrieve vector intrinsic.");
 
-    CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
+  SmallVector<OperandBundleDef, 1> OpBundles;
+  if (!IsVPIntrinsic) {
+    if (auto *CI = cast_or_null<CallInst>(getUnderlyingValue()))
+      CI->getOperandBundlesAsDefs(OpBundles);
+  }
 
-    setFlags(V);
+  Value *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
 
-    if (!V->getType()->isVoidTy())
-      State.set(this, V);
-    State.addMetadata(V, CI);
+  if (IsVPIntrinsic) {
+    if (isa<FPMathOperator>(V))
+    setFlags(cast<Instruction>(V));
+  } else {
+    setFlags(cast<Instruction>(V));
   }
+
+  if (!V->getType()->isVoidTy())
+    State.set(this, V);
+  State.addMetadata(V, dyn_cast_or_null<Instruction>(getUnderlyingValue()));
 }
 
 InstructionCost VPWidenIntrinsicRecipe::computeCost(ElementCount VF,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1469,13 +1469,20 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
                 if (!Instruction::isBinaryOp(Opcode) &&
                     !Instruction::isUnaryOp(Opcode))
                   return nullptr;
+
                 auto *I = cast<Instruction>(W->getUnderlyingInstr());
+                Intrinsic::ID VPID = VPIntrinsic::getForOpcode(Opcode);
                 SmallVector<VPValue *> Ops(W->operands());
+                assert(VPIntrinsic::getMaskParamPos(VPID) &&
+                       VPIntrinsic::getVectorLengthParamPos(VPID) &&
+                       "Expected VP intrinsic");
+                VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::getTrue(
+                    IntegerType::getInt1Ty(I->getContext())));
+                Ops.push_back(Mask);
                 Ops.push_back(&EVL);
-                Intrinsic::ID VPID = VPIntrinsic::getForOpcode(W->getOpcode());
-                return new VPWidenIntrinsicRecipe(
-                    *I, VPID, make_range(Ops.begin(), Ops.end()), I->getType(),
-                    I->getDebugLoc());
+                return new VPWidenIntrinsicRecipe(*I, VPID, Ops,
+                                                  TypeInfo.inferScalarType(W),
+                                                  W->getDebugLoc());
               })
               .Case<VPReductionRecipe>([&](VPReductionRecipe *Red) {
                 VPValue *NewMask = GetNewMask(Red->getCondOp());
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll
@@ -32,7 +32,7 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL-NEXT:    CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
 ; IF-EVL-NEXT:    vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
 ; IF-EVL-NEXT:    WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
-; IF-EVL-NEXT:    WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call nsw llvm.vp.add(ir<[[LD2]]>, ir<[[LD1]]>, vp<[[EVL]]>)
+; IF-EVL-NEXT:    WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call nsw llvm.vp.add(ir<[[LD2]]>, ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
 ; IF-EVL-NEXT:    CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
 ; IF-EVL-NEXT:    vp<[[PTR3:%[0-9]+]]> = vector-pointer ir<[[GEP3]]>
 ; IF-EVL-NEXT:    WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
@@ -28,9 +28,9 @@
  ; IF-EVL-NEXT:     vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
  ; IF-EVL-NEXT:     WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
  ; IF-EVL-NEXT:     WIDEN ir<[[CMP:%.+]]> = icmp sgt ir<[[LD1]]>, ir<[[LD2]]>
- ; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[SUB:%.+]]> = call llvm.vp.sub(ir<0>, ir<[[LD2]]>, vp<[[EVL]]>)
+ ; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[SUB:%.+]]> = call llvm.vp.sub(ir<0>, ir<[[LD2]]>, ir<true>, vp<[[EVL]]>)
  ; IF-EVL-NEXT:     WIDEN-INTRINSIC vp<[[SELECT:%.+]]> = call llvm.vp.select(ir<[[CMP]]>, ir<%1>, ir<%2>, vp<[[EVL]]>)
- ; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call llvm.vp.add(vp<[[SELECT]]>, ir<[[LD1]]>, vp<[[EVL]]>)
+ ; IF-EVL-NEXT:     WIDEN-INTRINSIC ir<[[ADD:%.+]]> = call llvm.vp.add(vp<[[SELECT]]>, ir<[[LD1]]>, ir<true>, vp<[[EVL]]>)
  ; IF-EVL-NEXT:     CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
  ; IF-EVL-NEXT:     vp<[[PTR3:%.+]]> = vector-pointer ir<[[GEP3]]>
  ; IF-EVL-NEXT:     WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
