[RISCV] Use slideup to lower build_vector when all operand are (extract_element X, 0) #154450
+444
−17
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -4512,42 +4512,98 @@ static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, | |||||
| "Illegal type which will result in reserved encoding"); | ||||||
|
|
||||||
| const unsigned Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC; | ||||||
| auto getVSlide = [&](bool SlideUp, EVT ContainerVT, SDValue Passthru, | ||||||
| SDValue Vec, SDValue Offset, SDValue Mask, | ||||||
| SDValue VL) -> SDValue { | ||||||
| if (SlideUp) | ||||||
| return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset, | ||||||
| Mask, VL, Policy); | ||||||
| return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset, | ||||||
| Mask, VL, Policy); | ||||||
| }; | ||||||
|
|
||||||
| // General case: splat the first operand and slide other operands down one | ||||||
| // by one to form a vector. Alternatively, if the last operand is an | ||||||
| // extraction from element 0 of a vector, we can use the original vector | ||||||
| // reduction result as the start value and slide up instead of slide down. | ||||||
lukel97 marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
| // Such that we can avoid the splat. | ||||||
| SmallVector<SDValue> Operands(Op->op_begin(), Op->op_end()); | ||||||
| SDValue EVec; | ||||||
| bool SlideUp = false; | ||||||
| // Find the first first non-undef from the tail. | ||||||
| auto ItLastNonUndef = find_if(Operands.rbegin(), Operands.rend(), | ||||||
| [](SDValue V) { return !V.isUndef(); }); | ||||||
| if (ItLastNonUndef != Operands.rend()) { | ||||||
| using namespace SDPatternMatch; | ||||||
| // Check if the last non-undef operand was an extraction. | ||||||
| SlideUp = sd_match(*ItLastNonUndef, m_ExtractElt(m_Value(EVec), m_Zero())); | ||||||
| } | ||||||
|
|
||||||
| if (SlideUp) { | ||||||
| MVT EVecContainerVT = EVec.getSimpleValueType(); | ||||||
| // Make sure the original vector has scalable vector type. | ||||||
| if (EVecContainerVT.isFixedLengthVector()) { | ||||||
| EVecContainerVT = | ||||||
| getContainerForFixedLengthVector(DAG, EVecContainerVT, Subtarget); | ||||||
| EVec = convertToScalableVector(EVecContainerVT, EVec, DAG, Subtarget); | ||||||
| } | ||||||
|
|
||||||
| // Adapt EVec's type into ContainerVT. | ||||||
| if (EVecContainerVT.getVectorMinNumElements() < | ||||||
| ContainerVT.getVectorMinNumElements()) | ||||||
| EVec = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), EVec, 0); | ||||||
| else | ||||||
| EVec = DAG.getExtractSubvector(DL, ContainerVT, EVec, 0); | ||||||
|
|
||||||
| // Reverse the elements as we're going to slide up from the last element. | ||||||
| std::reverse(Operands.begin(), Operands.end()); | ||||||
|
There was a problem hiding this comment. Nit, does llvm::reverse work here?
Suggested change
There was a problem hiding this comment.
That was actually what I thought, but unfortunately, |
||||||
| } | ||||||
|
|
||||||
| SDValue Vec; | ||||||
| UndefCount = 0; | ||||||
| for (SDValue V : Operands) { | ||||||
| if (V.isUndef()) { | ||||||
| UndefCount++; | ||||||
| continue; | ||||||
| } | ||||||
|
|
||||||
| // Start our sequence with either a TA splat or a reduction result in the | ||||||
lukel97 marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
| // hopes that hardware is able to recognize there's no dependency on the | ||||||
| // prior value of our temporary register. | ||||||
| if (!Vec) { | ||||||
| if (SlideUp) { | ||||||
| Vec = EVec; | ||||||
| } else { | ||||||
| Vec = DAG.getSplatVector(VT, DL, V); | ||||||
| Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget); | ||||||
| } | ||||||
|
|
||||||
| UndefCount = 0; | ||||||
| continue; | ||||||
| } | ||||||
|
|
||||||
| if (UndefCount) { | ||||||
| const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT()); | ||||||
| Vec = getVSlide(SlideUp, ContainerVT, DAG.getUNDEF(ContainerVT), Vec, | ||||||
| Offset, Mask, VL); | ||||||
| UndefCount = 0; | ||||||
| } | ||||||
|
|
||||||
| unsigned OpCode; | ||||||
lukel97 marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
| if (VT.isFloatingPoint()) | ||||||
| OpCode = SlideUp ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VFSLIDE1DOWN_VL; | ||||||
| else | ||||||
| OpCode = SlideUp ? RISCVISD::VSLIDE1UP_VL : RISCVISD::VSLIDE1DOWN_VL; | ||||||
|
|
||||||
| if (!VT.isFloatingPoint()) | ||||||
| V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V); | ||||||
| Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec, | ||||||
| V, Mask, VL); | ||||||
| } | ||||||
| if (UndefCount) { | ||||||
| const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT()); | ||||||
| Vec = getVSlide(SlideUp, ContainerVT, DAG.getUNDEF(ContainerVT), Vec, | ||||||
|
There was a problem hiding this comment. I don't think we have test coverage for emitting a vslideup. I think we can add a test for an undef in the middle of the build_vector, and another test with an undef at the start of the build_vector? There was a problem hiding this comment. I've added tests for both cases |
||||||
| Offset, Mask, VL); | ||||||
| } | ||||||
| return convertFromScalableVector(VT, Vec, DAG, Subtarget); | ||||||
| } | ||||||
|
|
||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.