@@ -8251,10 +8251,28 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
82518251 visitCallBrLandingPad (I);
82528252 return ;
82538253 case Intrinsic::vector_interleave2:
8254- visitVectorInterleave (I);
8254+ visitVectorInterleave (I, 2 );
8255+ return ;
8256+ case Intrinsic::vector_interleave3:
8257+ visitVectorInterleave (I, 3 );
8258+ return ;
8259+ case Intrinsic::vector_interleave5:
8260+ visitVectorInterleave (I, 5 );
8261+ return ;
8262+ case Intrinsic::vector_interleave7:
8263+ visitVectorInterleave (I, 7 );
82558264 return ;
82568265 case Intrinsic::vector_deinterleave2:
8257- visitVectorDeinterleave (I);
8266+ visitVectorDeinterleave (I, 2 );
8267+ return ;
8268+ case Intrinsic::vector_deinterleave3:
8269+ visitVectorDeinterleave (I, 3 );
8270+ return ;
8271+ case Intrinsic::vector_deinterleave5:
8272+ visitVectorDeinterleave (I, 5 );
8273+ return ;
8274+ case Intrinsic::vector_deinterleave7:
8275+ visitVectorDeinterleave (I, 7 );
82588276 return ;
82598277 case Intrinsic::experimental_vector_compress:
82608278 setValue (&I, DAG.getNode (ISD::VECTOR_COMPRESS, sdl,
@@ -12565,59 +12583,75 @@ void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) {
1256512583 setValue (&I, DAG.getVectorShuffle (VT, DL, V, DAG.getUNDEF (VT), Mask));
1256612584}
1256712585
12568- void SelectionDAGBuilder::visitVectorDeinterleave (const CallInst &I) {
12586+ void SelectionDAGBuilder::visitVectorDeinterleave (const CallInst &I,
12587+ unsigned Factor) {
1256912588 auto DL = getCurSDLoc ();
1257012589 SDValue InVec = getValue (I.getOperand (0 ));
12571- EVT OutVT =
12572- InVec.getValueType ().getHalfNumVectorElementsVT (*DAG.getContext ());
1257312590
12591+ SmallVector<EVT, 4 > ValueVTs;
12592+ ComputeValueVTs (DAG.getTargetLoweringInfo (), DAG.getDataLayout (), I.getType (),
12593+ ValueVTs);
12594+
12595+ EVT OutVT = ValueVTs[0 ];
1257412596 unsigned OutNumElts = OutVT.getVectorMinNumElements ();
1257512597
12576- // ISD Node needs the input vectors split into two equal parts
12577- SDValue Lo = DAG.getNode (ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12578- DAG.getVectorIdxConstant (0 , DL));
12579- SDValue Hi = DAG.getNode (ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12580- DAG.getVectorIdxConstant (OutNumElts, DL));
12598+ SmallVector<SDValue, 4 > SubVecs (Factor);
12599+ for (unsigned i = 0 ; i != Factor; ++i) {
12600+ assert (ValueVTs[i] == OutVT && " Expected VTs to be the same" );
12601+ SubVecs[i] = DAG.getNode (ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec,
12602+ DAG.getVectorIdxConstant (OutNumElts * i, DL));
12603+ }
1258112604
12582- // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12583- // legalisation and combines.
12584- if (OutVT.isFixedLengthVector ()) {
12585- SDValue Even = DAG.getVectorShuffle (OutVT, DL, Lo, Hi ,
12605+ // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
12606+ // from existing legalisation and combines.
12607+ if (OutVT.isFixedLengthVector () && Factor == 2 ) {
12608+ SDValue Even = DAG.getVectorShuffle (OutVT, DL, SubVecs[ 0 ], SubVecs[ 1 ] ,
1258612609 createStrideMask (0 , 2 , OutNumElts));
12587- SDValue Odd = DAG.getVectorShuffle (OutVT, DL, Lo, Hi ,
12610+ SDValue Odd = DAG.getVectorShuffle (OutVT, DL, SubVecs[ 0 ], SubVecs[ 1 ] ,
1258812611 createStrideMask (1 , 2 , OutNumElts));
1258912612 SDValue Res = DAG.getMergeValues ({Even, Odd}, getCurSDLoc ());
1259012613 setValue (&I, Res);
1259112614 return ;
1259212615 }
1259312616
1259412617 SDValue Res = DAG.getNode (ISD::VECTOR_DEINTERLEAVE, DL,
12595- DAG.getVTList (OutVT, OutVT ), Lo, Hi );
12618+ DAG.getVTList (ValueVTs ), SubVecs );
1259612619 setValue (&I, Res);
1259712620}
1259812621
12599- void SelectionDAGBuilder::visitVectorInterleave (const CallInst &I) {
12622+ void SelectionDAGBuilder::visitVectorInterleave (const CallInst &I,
12623+ unsigned Factor) {
1260012624 auto DL = getCurSDLoc ();
12601- EVT InVT = getValue (I.getOperand (0 )).getValueType ();
12602- SDValue InVec0 = getValue (I.getOperand (0 ));
12603- SDValue InVec1 = getValue (I.getOperand (1 ));
1260412625 const TargetLowering &TLI = DAG.getTargetLoweringInfo ();
12626+ EVT InVT = getValue (I.getOperand (0 )).getValueType ();
1260512627 EVT OutVT = TLI.getValueType (DAG.getDataLayout (), I.getType ());
1260612628
12607- // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing
12608- // legalisation and combines.
12609- if (OutVT.isFixedLengthVector ()) {
12629+ SmallVector<SDValue, 8 > InVecs (Factor);
12630+ for (unsigned i = 0 ; i < Factor; ++i) {
12631+ InVecs[i] = getValue (I.getOperand (i));
12632+ assert (InVecs[i].getValueType () == InVecs[0 ].getValueType () &&
12633+ " Expected VTs to be the same" );
12634+ }
12635+
12636+ // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit
12637+ // from existing legalisation and combines.
12638+ if (OutVT.isFixedLengthVector () && Factor == 2 ) {
1261012639 unsigned NumElts = InVT.getVectorMinNumElements ();
12611- SDValue V = DAG.getNode (ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1 );
12640+ SDValue V = DAG.getNode (ISD::CONCAT_VECTORS, DL, OutVT, InVecs );
1261212641 setValue (&I, DAG.getVectorShuffle (OutVT, DL, V, DAG.getUNDEF (OutVT),
1261312642 createInterleaveMask (NumElts, 2 )));
1261412643 return ;
1261512644 }
1261612645
12617- SDValue Res = DAG.getNode (ISD::VECTOR_INTERLEAVE, DL,
12618- DAG.getVTList (InVT, InVT), InVec0, InVec1);
12619- Res = DAG.getNode (ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue (0 ),
12620- Res.getValue (1 ));
12646+ SmallVector<EVT, 8 > ValueVTs (Factor, InVT);
12647+ SDValue Res =
12648+ DAG.getNode (ISD::VECTOR_INTERLEAVE, DL, DAG.getVTList (ValueVTs), InVecs);
12649+
12650+ SmallVector<SDValue, 8 > Results (Factor);
12651+ for (unsigned i = 0 ; i < Factor; ++i)
12652+ Results[i] = Res.getValue (i);
12653+
12654+ Res = DAG.getNode (ISD::CONCAT_VECTORS, DL, OutVT, Results);
1262112655 setValue (&I, Res);
1262212656}
1262312657
0 commit comments