@@ -4907,41 +4907,56 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
49074907
49084908 assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
49094909
4910- SmallVector<SDValue> MaskVals;
4911- // As a backup, shuffles can be lowered via a vrgather instruction, possibly
4912- // merged with a second vrgather.
4913- SmallVector<SDValue> GatherIndicesLHS, GatherIndicesRHS;
4914-
49154910 // By default we preserve the original operand order, and use a mask to
49164911 // select LHS as true and RHS as false. However, since RVV vector selects may
49174912 // feature splats but only on the LHS, we may choose to invert our mask and
49184913 // instead select between RHS and LHS.
49194914 bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
4920- bool InvertMask = IsSelect == SwapOps;
4915+
4916+ if (IsSelect) {
4917+ // Now construct the mask that will be used by the vselect operation.
4918+ SmallVector<SDValue> MaskVals;
4919+ for (int MaskIndex : Mask) {
4920+ bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ SwapOps;
4921+ MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
4922+ }
4923+
4924+ if (SwapOps)
4925+ std::swap(V1, V2);
4926+
4927+ assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
4928+ MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
4929+ SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
4930+ return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
4931+ }
4932+
4933+
4934+ // As a backup, shuffles can be lowered via a vrgather instruction, possibly
4935+ // merged with a second vrgather.
4936+ SmallVector<SDValue> GatherIndicesLHS, GatherIndicesRHS;
49214937
49224938 // Keep a track of which non-undef indices are used by each LHS/RHS shuffle
49234939 // half.
49244940 DenseMap<int, unsigned> LHSIndexCounts, RHSIndexCounts;
49254941
4926- // Now construct the mask that will be used by the vselect or blended
4927- // vrgather operation. For vrgathers, construct the appropriate indices into
4928- // each vector.
4942+ SmallVector<SDValue> MaskVals;
4943+
4944+ // Now construct the mask that will be used by the blended vrgather operation.
4945+ // Cconstruct the appropriate indices into each vector.
49294946 for (int MaskIndex : Mask) {
4930- bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ InvertMask ;
4947+ bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps ;
49314948 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
4932- if (!IsSelect) {
4933- bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
4934- GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
4935- ? DAG.getConstant(MaskIndex, DL, XLenVT)
4936- : DAG.getUNDEF(XLenVT));
4937- GatherIndicesRHS.push_back(
4938- IsLHSOrUndefIndex ? DAG.getUNDEF(XLenVT)
4939- : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT));
4940- if (IsLHSOrUndefIndex && MaskIndex >= 0)
4941- ++LHSIndexCounts[MaskIndex];
4942- if (!IsLHSOrUndefIndex)
4943- ++RHSIndexCounts[MaskIndex - NumElts];
4944- }
4949+ bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
4950+ GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
4951+ ? DAG.getConstant(MaskIndex, DL, XLenVT)
4952+ : DAG.getUNDEF(XLenVT));
4953+ GatherIndicesRHS.push_back(
4954+ IsLHSOrUndefIndex ? DAG.getUNDEF(XLenVT)
4955+ : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT));
4956+ if (IsLHSOrUndefIndex && MaskIndex >= 0)
4957+ ++LHSIndexCounts[MaskIndex];
4958+ if (!IsLHSOrUndefIndex)
4959+ ++RHSIndexCounts[MaskIndex - NumElts];
49454960 }
49464961
49474962 if (SwapOps) {
@@ -4953,9 +4968,6 @@ static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
49534968 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
49544969 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
49554970
4956- if (IsSelect)
4957- return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
4958-
49594971 // We might be able to express the shuffle as a bitrotate. But even if we
49604972 // don't have Zvkb and have to expand, the expanded sequence of approx. 2
49614973 // shifts and a vor will have a higher throughput than a vrgather.
0 commit comments