@@ -554,7 +554,7 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
554554 return IndexCost +
555555 getRISCVInstructionCost (RISCV::VRGATHER_VV, LT.second , CostKind);
556556 }
557- [[fallthrough]] ;
557+ break ;
558558 }
559559 case TTI::SK_Transpose:
560560 case TTI::SK_PermuteTwoSrc: {
@@ -574,55 +574,65 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
574574 LT.second , CostKind) +
575575 MaskCost;
576576 }
577- [[fallthrough]];
578- }
579- case TTI::SK_Select: {
580- // We are going to permute multiple sources and the result will be in
581- // multiple destinations. Providing an accurate cost only for splits where
582- // the element type remains the same.
583- if (!Mask.empty () && LT.first .isValid () && LT.first != 1 &&
584- LT.second .getVectorElementType ().getSizeInBits () ==
585- Tp->getElementType ()->getPrimitiveSizeInBits () &&
586- LT.second .getVectorNumElements () <
587- cast<FixedVectorType>(Tp)->getNumElements () &&
588- divideCeil (Mask.size (),
589- cast<FixedVectorType>(Tp)->getNumElements ()) ==
590- static_cast <unsigned >(*LT.first .getValue ())) {
591- unsigned NumRegs = *LT.first .getValue ();
592- unsigned VF = cast<FixedVectorType>(Tp)->getNumElements ();
593- unsigned SubVF = PowerOf2Ceil (VF / NumRegs);
594- auto *SubVecTy = FixedVectorType::get (Tp->getElementType (), SubVF);
595-
596- InstructionCost Cost = 0 ;
597- for (unsigned I = 0 , NumSrcRegs = divideCeil (Mask.size (), SubVF);
598- I < NumSrcRegs; ++I) {
599- bool IsSingleVector = true ;
600- SmallVector<int > SubMask (SubVF, PoisonMaskElem);
601- transform (
602- Mask.slice (I * SubVF,
603- I == NumSrcRegs - 1 ? Mask.size () % SubVF : SubVF),
604- SubMask.begin (), [&](int I) -> int {
605- if (I == PoisonMaskElem)
606- return PoisonMaskElem;
607- bool SingleSubVector = I / VF == 0 ;
608- IsSingleVector &= SingleSubVector;
609- return (SingleSubVector ? 0 : 1 ) * SubVF + (I % VF) % SubVF;
610- });
611- if (all_of (enumerate(SubMask), [](auto &&P) {
612- return P.value () == PoisonMaskElem ||
613- static_cast <unsigned >(P.value ()) == P.index ();
614- }))
615- continue ;
616- Cost += getShuffleCost (IsSingleVector ? TTI::SK_PermuteSingleSrc
617- : TTI::SK_PermuteTwoSrc,
618- SubVecTy, SubMask, CostKind, 0 , nullptr );
619- }
620- return Cost;
621- }
622577 break ;
623578 }
624579 }
625- };
580+
581+ auto shouldSplit = [](TTI::ShuffleKind Kind) {
582+ switch (Kind) {
583+ default :
584+ return false ;
585+ case TTI::SK_PermuteSingleSrc:
586+ case TTI::SK_Transpose:
587+ case TTI::SK_PermuteTwoSrc:
588+ case TTI::SK_Select:
589+ return true ;
590+ }
591+ };
592+ // We are going to permute multiple sources and the result will be in
593+ // multiple destinations. Providing an accurate cost only for splits where
594+ // the element type remains the same.
595+ if (!Mask.empty () && LT.first .isValid () && LT.first != 1 &&
596+ shouldSplit (Kind) &&
597+ LT.second .getVectorElementType ().getSizeInBits () ==
598+ Tp->getElementType ()->getPrimitiveSizeInBits () &&
599+ LT.second .getVectorNumElements () <
600+ cast<FixedVectorType>(Tp)->getNumElements () &&
601+ divideCeil (Mask.size (),
602+ cast<FixedVectorType>(Tp)->getNumElements ()) ==
603+ static_cast <unsigned >(*LT.first .getValue ())) {
604+ unsigned NumRegs = *LT.first .getValue ();
605+ unsigned VF = cast<FixedVectorType>(Tp)->getNumElements ();
606+ unsigned SubVF = PowerOf2Ceil (VF / NumRegs);
607+ auto *SubVecTy = FixedVectorType::get (Tp->getElementType (), SubVF);
608+
609+ InstructionCost Cost = 0 ;
610+ for (unsigned I = 0 , NumSrcRegs = divideCeil (Mask.size (), SubVF);
611+ I < NumSrcRegs; ++I) {
612+ bool IsSingleVector = true ;
613+ SmallVector<int > SubMask (SubVF, PoisonMaskElem);
614+ transform (
615+ Mask.slice (I * SubVF,
616+ I == NumSrcRegs - 1 ? Mask.size () % SubVF : SubVF),
617+ SubMask.begin (), [&](int I) -> int {
618+ if (I == PoisonMaskElem)
619+ return PoisonMaskElem;
620+ bool SingleSubVector = I / VF == 0 ;
621+ IsSingleVector &= SingleSubVector;
622+ return (SingleSubVector ? 0 : 1 ) * SubVF + (I % VF) % SubVF;
623+ });
624+ if (all_of (enumerate(SubMask), [](auto &&P) {
625+ return P.value () == PoisonMaskElem ||
626+ static_cast <unsigned >(P.value ()) == P.index ();
627+ }))
628+ continue ;
629+ Cost += getShuffleCost (IsSingleVector ? TTI::SK_PermuteSingleSrc
630+ : TTI::SK_PermuteTwoSrc,
631+ SubVecTy, SubMask, CostKind, 0 , nullptr );
632+ }
633+ return Cost;
634+ }
635+ }
626636
627637 // Handle scalable vectors (and fixed vectors legalized to scalable vectors).
628638 switch (Kind) {
0 commit comments