Add subtarget feature

Author: wangpc-pp

llvm/lib/Target/RISCV/RISCVFeatures.td

@@ -1359,6 +1359,10 @@ def TuneOptimizedZeroStrideLoad
                       "true", "Optimized (perform fewer memory operations)"
                       "zero-stride vector load">;
 
+def TuneOptimizedIndexedLoadStore
+   : SubtargetFeature<"optimized-indexed-load-store", "HasOptimizedIndexedLoadStore",
+                      "true", "Optimized vector indexed load/store">;
+
 def Experimental
    : SubtargetFeature<"experimental", "HasExperimental",
                       "true", "Experimental intrinsics">;

llvm/lib/Target/RISCV/RISCVISelLowering.cpp

@@ -11136,8 +11136,9 @@ SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
   if (!VL)
     VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
 
-  SDValue Index;
-  if (!IsUnmasked && IsExpandingLoad) {
+  SDValue Result;
+  if (!IsUnmasked && IsExpandingLoad &&
+      Subtarget.hasOptimizedIndexedLoadStore()) {
     MVT IndexVT = ContainerVT;
     if (ContainerVT.isFloatingPoint())
       IndexVT = IndexVT.changeVectorElementTypeToInteger();
@@ -11147,47 +11148,98 @@ SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
       IndexVT = IndexVT.changeVectorElementType(XLenVT);
 
     // If index vector is an i8 vector and the element count exceeds 256, we
-    // should change the element type of index vector to i16 to avoid overflow.
+    // should change the element type of index vector to i16 to avoid
+    // overflow.
     if (IndexEltVT == MVT::i8 && VT.getVectorNumElements() > 256) {
       // FIXME: We need to do vector splitting manually for LMUL=8 cases.
       if (getLMUL(IndexVT) == RISCVII::LMUL_8)
         return SDValue();
       IndexVT = IndexVT.changeVectorElementType(MVT::i16);
     }
 
-    Index =
+    SDValue Index =
         DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
                     DAG.getTargetConstant(Intrinsic::riscv_viota, DL, XLenVT),
                     DAG.getUNDEF(IndexVT), Mask, VL);
     if (uint64_t EltSize = ContainerVT.getScalarSizeInBits(); EltSize > 8)
       Index = DAG.getNode(RISCVISD::SHL_VL, DL, IndexVT, Index,
                           DAG.getConstant(Log2_64(EltSize / 8), DL, IndexVT),
                           DAG.getUNDEF(IndexVT), Mask, VL);
-  }
-
-  unsigned IntID = IsUnmasked        ? Intrinsic::riscv_vle
-                   : IsExpandingLoad ? Intrinsic::riscv_vluxei_mask
-                                     : Intrinsic::riscv_vle_mask;
-  SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
-  if (IsUnmasked)
-    Ops.push_back(DAG.getUNDEF(ContainerVT));
-  else
+    unsigned IntID = Intrinsic::riscv_vluxei_mask;
+    SmallVector<SDValue, 8> Ops{Chain,
+                                DAG.getTargetConstant(IntID, DL, XLenVT)};
     Ops.push_back(PassThru);
-  Ops.push_back(BasePtr);
-  if (!IsUnmasked) {
-    if (IsExpandingLoad)
-      Ops.push_back(Index);
+    Ops.push_back(BasePtr);
+    Ops.push_back(Index);
     Ops.push_back(Mask);
-  }
-  Ops.push_back(VL);
-  if (!IsUnmasked)
+    Ops.push_back(VL);
     Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
 
-  SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
+    SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
 
-  SDValue Result =
-      DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
-  Chain = Result.getValue(1);
+    Result = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
+                                     MemVT, MMO);
+    Chain = Result.getValue(1);
+  } else {
+    SDValue ExpandingVL;
+    if (!IsUnmasked && IsExpandingLoad &&
+        !Subtarget.hasOptimizedIndexedLoadStore()) {
+      ExpandingVL = VL;
+      VL = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Mask,
+                       getAllOnesMask(Mask.getSimpleValueType(), VL, DL, DAG),
+                       VL);
+    }
+
+    unsigned IntID = IsUnmasked || (IsExpandingLoad &&
+                                    !Subtarget.hasOptimizedIndexedLoadStore())
+                         ? Intrinsic::riscv_vle
+                         : Intrinsic::riscv_vle_mask;
+    SmallVector<SDValue, 8> Ops{Chain,
+                                DAG.getTargetConstant(IntID, DL, XLenVT)};
+    if (IntID == Intrinsic::riscv_vle)
+      Ops.push_back(DAG.getUNDEF(ContainerVT));
+    else
+      Ops.push_back(PassThru);
+    Ops.push_back(BasePtr);
+    if (IntID == Intrinsic::riscv_vle_mask)
+      Ops.push_back(Mask);
+    Ops.push_back(VL);
+    if (IntID == Intrinsic::riscv_vle_mask)
+      Ops.push_back(DAG.getTargetConstant(RISCVII::TAIL_AGNOSTIC, DL, XLenVT));
+
+    SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
+
+    Result = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
+                                     MemVT, MMO);
+    Chain = Result.getValue(1);
+    if (ExpandingVL) {
+      MVT IndexVT = ContainerVT;
+      if (ContainerVT.isFloatingPoint())
+        IndexVT = ContainerVT.changeVectorElementTypeToInteger();
+
+      MVT IndexEltVT = IndexVT.getVectorElementType();
+      bool UseVRGATHEREI16 = false;
+      // If index vector is an i8 vector and the element count exceeds 256, we
+      // should change the element type of index vector to i16 to avoid
+      // overflow.
+      if (IndexEltVT == MVT::i8 && VT.getVectorNumElements() > 256) {
+        // FIXME: We need to do vector splitting manually for LMUL=8 cases.
+        if (getLMUL(IndexVT) == RISCVII::LMUL_8)
+          return SDValue();
+        IndexVT = IndexVT.changeVectorElementType(MVT::i16);
+        UseVRGATHEREI16 = true;
+      }
+
+      SDValue Iota =
+          DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
+                      DAG.getConstant(Intrinsic::riscv_viota, DL, XLenVT),
+                      DAG.getUNDEF(IndexVT), Mask, ExpandingVL);
+      Result = DAG.getNode(UseVRGATHEREI16 ? RISCVISD::VRGATHEREI16_VV_VL
+                                           : RISCVISD::VRGATHER_VV_VL,
+                           DL, ContainerVT, Result, Iota, PassThru, Mask,
+                           ExpandingVL);
+    }
+  }
 
   if (VT.isFixedLengthVector())
     Result = convertFromScalableVector(VT, Result, DAG, Subtarget);