Skip to content

[InstCombine] Pull extract through broadcast #143380

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Jul 4, 2025
Merged

[InstCombine] Pull extract through broadcast #143380

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
34c1222
Initial changes
agorenstein-nvidia Jun 6, 2025
31cb512
Merge branch 'main' into instcombine-extract-through-broadcast
agorenstein-nvidia Jun 9, 2025
e315c89
PR feedback 1
agorenstein-nvidia Jun 9, 2025
274cec9
clang-format
agorenstein-nvidia Jun 9, 2025
4fda162
Fixing edit damage in test file
agorenstein-nvidia Jun 9, 2025
5d9e4d9
Removed unneeded check
agorenstein-nvidia Jun 9, 2025
a1aec2e
Improve undef/poison usage in test (per PR automation failure), and u…
agorenstein-nvidia Jun 9, 2025
70d3c01
Fixed varname convention per PR feedback; on reflection from PR feedb…
agorenstein-nvidia Jun 9, 2025
52f5c4b
Restore old tests, reframe optimization as extension of existing one.
agorenstein-nvidia Jun 10, 2025
e21bde6
fix formatting errors, sorry
agorenstein-nvidia Jun 16, 2025
09d5e4c
Improvement via optional from RKSimon's suggestion
agorenstein-nvidia Jun 24, 2025
ce809f6
Improved tests with semantic names per RKSimon's PR feedback
agorenstein-nvidia Jun 24, 2025
21b8638
Improved comments to explicitly-named tests per PR suggestion
agorenstein-nvidia Jun 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 32 additions & 20 deletions llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -542,27 +542,39 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
}
}
} else if (auto *SVI = dyn_cast<ShuffleVectorInst>(I)) {
// If this is extracting an element from a shufflevector, figure out where
// it came from and extract from the appropriate input element instead.
// Restrict the following transformation to fixed-length vector.
if (isa<FixedVectorType>(SVI->getType()) && isa<ConstantInt>(Index)) {
int SrcIdx =
SVI->getMaskValue(cast<ConstantInt>(Index)->getZExtValue());
Value *Src;
unsigned LHSWidth = cast<FixedVectorType>(SVI->getOperand(0)->getType())
->getNumElements();

if (SrcIdx < 0)
return replaceInstUsesWith(EI, PoisonValue::get(EI.getType()));
if (SrcIdx < (int)LHSWidth)
Src = SVI->getOperand(0);
else {
SrcIdx -= LHSWidth;
Src = SVI->getOperand(1);
int SplatIndex = getSplatIndex(SVI->getShuffleMask());
// We know the all-0 splat must be reading from the first operand, even
// in the case of scalable vectors (vscale is always > 0).
if (SplatIndex == 0)
return ExtractElementInst::Create(SVI->getOperand(0),
Builder.getInt64(0));

if (isa<FixedVectorType>(SVI->getType())) {
std::optional<int> SrcIdx;
// getSplatIndex returns -1 to mean not-found.
if (SplatIndex != -1)
SrcIdx = SplatIndex;
else if (ConstantInt *CI = dyn_cast<ConstantInt>(Index))
SrcIdx = SVI->getMaskValue(CI->getZExtValue());

if (SrcIdx) {
Value *Src;
unsigned LHSWidth =
cast<FixedVectorType>(SVI->getOperand(0)->getType())
->getNumElements();

if (*SrcIdx < 0)
return replaceInstUsesWith(EI, PoisonValue::get(EI.getType()));
if (*SrcIdx < (int)LHSWidth)
Src = SVI->getOperand(0);
else {
*SrcIdx -= LHSWidth;
Src = SVI->getOperand(1);
}
Type *Int64Ty = Type::getInt64Ty(EI.getContext());
return ExtractElementInst::Create(
Src, ConstantInt::get(Int64Ty, *SrcIdx, false));
}
Type *Int64Ty = Type::getInt64Ty(EI.getContext());
return ExtractElementInst::Create(
Src, ConstantInt::get(Int64Ty, SrcIdx, false));
}
} else if (auto *CI = dyn_cast<CastInst>(I)) {
// Canonicalize extractelement(cast) -> cast(extractelement).
Expand Down
52 changes: 52 additions & 0 deletions llvm/test/Transforms/InstCombine/vec_extract_through_broadcast.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
; RUN: opt -passes=instcombine -S < %s | FileCheck %s

define float @extract_from_zero_init_shuffle(<2 x float> %vec, i64 %idx) {
; CHECK-LABEL: @extract_from_zero_init_shuffle(
; CHECK-NEXT: %extract = extractelement <2 x float> %vec, i64 0
; CHECK-NEXT: ret float %extract
;
%shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> zeroinitializer
%extract = extractelement <4 x float> %shuffle, i64 %idx
ret float %extract
}


define float @extract_from_general_splat(<2 x float> %vec, i64 %idx) {
; CHECK-LABEL: @extract_from_general_splat(
; CHECK-NEXT: %extract = extractelement <2 x float> %vec, i64 1
; CHECK-NEXT: ret float %extract
;
%shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
%extract = extractelement <4 x float> %shuffle, i64 %idx
ret float %extract
}

define float @extract_from_general_scalable_splat(<vscale x 2 x float> %vec, i64 %idx) {
; CHECK-LABEL: @extract_from_general_scalable_splat(
; CHECK-NEXT: %extract = extractelement <vscale x 2 x float> %vec, i64 0
; CHECK-NEXT: ret float %extract
;
%shuffle = shufflevector <vscale x 2 x float> %vec, <vscale x 2 x float> poison, <vscale x 4 x i32> zeroinitializer
%extract = extractelement <vscale x 4 x float> %shuffle, i64 %idx
ret float %extract
}

define float @extract_from_splat_with_poison_0(<2 x float> %vec, i64 %idx) {
; CHECK-LABEL: @extract_from_splat_with_poison_0(
; CHECK-NEXT: %extract = extractelement <2 x float> %vec, i64 1
; CHECK-NEXT: ret float %extract
;
%shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> <i32 poison, i32 1, i32 1, i32 1>
%extract = extractelement <4 x float> %shuffle, i64 %idx
ret float %extract
}

define float @extract_from_splat_with_poison_1(<2 x float> %vec, i64 %idx) {
; CHECK-LABEL: @extract_from_splat_with_poison_1(
; CHECK-NEXT: %extract = extractelement <2 x float> %vec, i64 1
; CHECK-NEXT: ret float %extract
;
%shuffle = shufflevector <2 x float> %vec, <2 x float> poison, <4 x i32> <i32 1, i32 poison, i32 1, i32 1>
%extract = extractelement <4 x float> %shuffle, i64 %idx
ret float %extract
}
3 changes: 1 addition & 2 deletions llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,7 @@ define float @test6(<4 x float> %X) {

define float @testvscale6(<vscale x 4 x float> %X) {
; CHECK-LABEL: @testvscale6(
; CHECK-NEXT: [[T2:%.*]] = shufflevector <vscale x 4 x float> [[X:%.*]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x float> [[T2]], i64 0
; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x float> [[X:%.*]], i64 0
; CHECK-NEXT: ret float [[R]]
;
%X1 = bitcast <vscale x 4 x float> %X to <vscale x 4 x i32>
Expand Down
3 changes: 1 addition & 2 deletions llvm/test/Transforms/InstCombine/vec_shuffle.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,7 @@ define float @test6(<4 x float> %X) {

define float @testvscale6(<vscale x 4 x float> %X) {
; CHECK-LABEL: @testvscale6(
; CHECK-NEXT: [[T2:%.*]] = shufflevector <vscale x 4 x float> [[X:%.*]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x float> [[T2]], i64 0
; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x float> [[X:%.*]], i64 0
; CHECK-NEXT: ret float [[R]]
;
%X1 = bitcast <vscale x 4 x float> %X to <vscale x 4 x i32>
Expand Down
13 changes: 5 additions & 8 deletions llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,12 +89,12 @@ define i8 @extractelement_bitcast_insert_extra_use_bitcast(<vscale x 2 x i32> %a
ret i8 %r
}

; while it may be that the extract is out-of-bounds, any valid index
; is going to yield %v (because the mask is all-zeros).

define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
; CHECK-LABEL: @extractelement_shuffle_maybe_out_of_range(
; CHECK-NEXT: [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4
; CHECK-NEXT: ret i32 [[R]]
Copy link
Contributor Author

@agorenstein-nvidia agorenstein-nvidia Jun 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is analogous to the other comment; it may be that the extractelement ... i32 4 is out of bounds (when vscale=1, so to speak). However, we are able to know that any valid index into %splat must be %v, so we just return %v. So, similar questions as the other comment.

Copy link
Collaborator

@RKSimon RKSimon Jun 25, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

might be useful to add a comment above the test explaining the reasoning,similar to your comment here

Copy link
Contributor Author

@agorenstein-nvidia agorenstein-nvidia Jun 26, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did so! Thanks.

; CHECK-NEXT: ret i32 [[V:%.*]]
;
%in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
%splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
Expand All @@ -104,10 +104,7 @@ define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {

define i32 @extractelement_shuffle_invalid_index(i32 %v) {
; CHECK-LABEL: @extractelement_shuffle_invalid_index(
Copy link
Contributor Author

@agorenstein-nvidia agorenstein-nvidia Jun 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here we optimize, but also in doing so miss the poison "opportunity" (in that the illegal -1 index-usage is removed). So I believe this is legal, but perhaps not optimal. However, I'm a bit confused: it is the case that earlier in our transformation we explicitly look for invalid indices in extractelements:

    // InstSimplify should handle cases where the index is invalid.
    // For fixed-length vector, it's invalid to extract out-of-range element.

Without tracing through, it looks like this may "just" be an order-of-transformations issue. I'm inclined to preserve the behavior. I've simply changed the test to reflect the new output; is there a preferred way of updating/changing these tests in this sort of situation?

; CHECK-NEXT: [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4294967295
; CHECK-NEXT: ret i32 [[R]]
; CHECK-NEXT: ret i32 [[V:%.*]]
;
%in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
%splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
Expand Down
13 changes: 5 additions & 8 deletions llvm/test/Transforms/InstCombine/vscale_extractelement.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,12 @@ define i8 @extractelement_bitcast_useless_insert(<vscale x 2 x i32> %a, i32 %x)
ret i8 %r
}

; while in these tests it may be that the extract is out-of-bounds,
; any valid index is going to yield %v (because the mask is all-zeros).

define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
; CHECK-LABEL: @extractelement_shuffle_maybe_out_of_range(
; CHECK-NEXT: [[IN:%.*]] = insertelement <vscale x 4 x i32> undef, i32 [[V:%.*]], i64 0
; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4
; CHECK-NEXT: ret i32 [[R]]
; CHECK-NEXT: ret i32 [[V:%.*]]
;
%in = insertelement <vscale x 4 x i32> undef, i32 %v, i32 0
%splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
Expand All @@ -68,10 +68,7 @@ define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {

define i32 @extractelement_shuffle_invalid_index(i32 %v) {
; CHECK-LABEL: @extractelement_shuffle_invalid_index(
; CHECK-NEXT: [[IN:%.*]] = insertelement <vscale x 4 x i32> undef, i32 [[V:%.*]], i64 0
; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i64 4294967295
; CHECK-NEXT: ret i32 [[R]]
; CHECK-NEXT: ret i32 [[V:%.*]]
;
%in = insertelement <vscale x 4 x i32> undef, i32 %v, i32 0
%splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
Expand Down