Skip to content

Commit 7ea2d27

Browse files
ParkHanbumParkHanbum
committed
[VectorCombine] support mismatching extract/insert indices for foldInsExtFNeg
insertelt DestVec, (fneg (extractelt SrcVec, Index)), Index -> shuffle DestVec, (shuffle (fneg SrcVec), poison, SrcMask), Mask In previous patches, the above transform was only possible if the Extract/Insert Index was the same; this patch makes the above transform possible even if the two indexes are different. Proof: https://alive2.llvm.org/ce/z/aDfdyG Fixes: #125675
1 parent 50cdf6c commit 7ea2d27

File tree

4 files changed

+71
-67
lines changed

4 files changed

+71
-67
lines changed

‎llvm/lib/Transforms/Vectorize/VectorCombine.cpp‎

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -663,60 +663,61 @@ bool VectorCombine::foldExtractExtract(Instruction &I) {
663663
/// shuffle.
664664
bool VectorCombine::foldInsExtFNeg(Instruction &I) {
665665
// Match an insert (op (extract)) pattern.
666-
Value *DestVec;
667-
uint64_t Index;
666+
Value *DstVec;
667+
uint64_t ExtIdx, InsIdx;
668668
Instruction *FNeg;
669-
if (!match(&I, m_InsertElt(m_Value(DestVec), m_OneUse(m_Instruction(FNeg)),
670-
m_ConstantInt(Index))))
669+
if (!match(&I, m_InsertElt(m_Value(DstVec), m_OneUse(m_Instruction(FNeg)),
670+
m_ConstantInt(InsIdx))))
671671
return false;
672672

673673
// Note: This handles the canonical fneg instruction and "fsub -0.0, X".
674674
Value *SrcVec;
675675
Instruction *Extract;
676676
if (!match(FNeg, m_FNeg(m_CombineAnd(
677677
m_Instruction(Extract),
678-
m_ExtractElt(m_Value(SrcVec), m_SpecificInt(Index))))))
678+
m_ExtractElt(m_Value(SrcVec), m_ConstantInt(ExtIdx))))))
679679
return false;
680680

681-
auto *VecTy = cast<FixedVectorType>(I.getType());
682-
auto *ScalarTy = VecTy->getScalarType();
681+
auto *DstVecTy = cast<FixedVectorType>(DstVec->getType());
682+
auto *DstVecScalarTy = DstVecTy->getScalarType();
683683
auto *SrcVecTy = dyn_cast<FixedVectorType>(SrcVec->getType());
684-
if (!SrcVecTy || ScalarTy != SrcVecTy->getScalarType())
684+
if (!SrcVecTy || DstVecScalarTy != SrcVecTy->getScalarType())
685685
return false;
686686

687687
// Ignore bogus insert/extract index.
688-
unsigned NumElts = VecTy->getNumElements();
689-
if (Index >= NumElts)
688+
unsigned NumDstElts = DstVecTy->getNumElements();
689+
unsigned NumSrcElts = SrcVecTy->getNumElements();
690+
if (InsIdx >= NumDstElts || ExtIdx >= NumSrcElts || NumDstElts == 1)
690691
return false;
691692

692693
// We are inserting the negated element into the same lane that we extracted
693694
// from. This is equivalent to a select-shuffle that chooses all but the
694695
// negated element from the destination vector.
695-
SmallVector<int> Mask(NumElts);
696+
SmallVector<int> Mask(NumDstElts);
696697
std::iota(Mask.begin(), Mask.end(), 0);
697-
Mask[Index] = Index + NumElts;
698+
Mask[InsIdx] = (ExtIdx % NumDstElts) + NumDstElts;
698699
InstructionCost OldCost =
699-
TTI.getArithmeticInstrCost(Instruction::FNeg, ScalarTy, CostKind) +
700-
TTI.getVectorInstrCost(I, VecTy, CostKind, Index);
700+
TTI.getArithmeticInstrCost(Instruction::FNeg, DstVecScalarTy, CostKind) +
701+
TTI.getVectorInstrCost(I, DstVecTy, CostKind, InsIdx);
701702

702703
// If the extract has one use, it will be eliminated, so count it in the
703704
// original cost. If it has more than one use, ignore the cost because it will
704705
// be the same before/after.
705706
if (Extract->hasOneUse())
706-
OldCost += TTI.getVectorInstrCost(*Extract, VecTy, CostKind, Index);
707+
OldCost += TTI.getVectorInstrCost(*Extract, SrcVecTy, CostKind, ExtIdx);
707708

708709
InstructionCost NewCost =
709-
TTI.getArithmeticInstrCost(Instruction::FNeg, VecTy, CostKind) +
710-
TTI.getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc, VecTy, Mask,
710+
TTI.getArithmeticInstrCost(Instruction::FNeg, SrcVecTy, CostKind) +
711+
TTI.getShuffleCost(TargetTransformInfo::SK_PermuteTwoSrc, DstVecTy, Mask,
711712
CostKind);
712713

713-
bool NeedLenChg = SrcVecTy->getNumElements() != NumElts;
714+
bool NeedLenChg = SrcVecTy->getNumElements() != NumDstElts;
714715
// If the lengths of the two vectors are not equal,
715716
// we need to add a length-change vector. Add this cost.
716717
SmallVector<int> SrcMask;
717718
if (NeedLenChg) {
718-
SrcMask.assign(NumElts, PoisonMaskElem);
719-
SrcMask[Index] = Index;
719+
SrcMask.assign(NumDstElts, PoisonMaskElem);
720+
SrcMask[(ExtIdx % NumDstElts)] = ExtIdx;
720721
NewCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc,
721722
SrcVecTy, SrcMask, CostKind);
722723
}
@@ -725,15 +726,15 @@ bool VectorCombine::foldInsExtFNeg(Instruction &I) {
725726
return false;
726727

727728
Value *NewShuf;
728-
// insertelt DestVec, (fneg (extractelt SrcVec, Index)), Index
729+
// insertelt DstVec, (fneg (extractelt SrcVec, Index)), Index
729730
Value *VecFNeg = Builder.CreateFNegFMF(SrcVec, FNeg);
730731
if (NeedLenChg) {
731-
// shuffle DestVec, (shuffle (fneg SrcVec), poison, SrcMask), Mask
732+
// shuffle DstVec, (shuffle (fneg SrcVec), poison, SrcMask), Mask
732733
Value *LenChgShuf = Builder.CreateShuffleVector(VecFNeg, SrcMask);
733-
NewShuf = Builder.CreateShuffleVector(DestVec, LenChgShuf, Mask);
734+
NewShuf = Builder.CreateShuffleVector(DstVec, LenChgShuf, Mask);
734735
} else {
735-
// shuffle DestVec, (fneg SrcVec), Mask
736-
NewShuf = Builder.CreateShuffleVector(DestVec, VecFNeg, Mask);
736+
// shuffle DstVec, (fneg SrcVec), Mask
737+
NewShuf = Builder.CreateShuffleVector(DstVec, VecFNeg, Mask);
737738
}
738739

739740
replaceValue(I, *NewShuf);

‎llvm/test/Transforms/PhaseOrdering/X86/addsub-inseltpoison.ll‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,11 +104,9 @@ define void @add_aggregate_store(<2 x float> %a0, <2 x float> %a1, <2 x float> %
104104
; PR58139
105105
define <2 x double> @_mm_complexmult_pd_naive(<2 x double> %a, <2 x double> %b) {
106106
; SSE-LABEL: @_mm_complexmult_pd_naive(
107-
; SSE-NEXT: [[B1:%.*]] = extractelement <2 x double> [[B:%.*]], i64 1
108-
; SSE-NEXT: [[TMP1:%.*]] = fneg double [[B1]]
109107
; SSE-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 1>
110-
; SSE-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[B]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
111-
; SSE-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[TMP1]], i64 0
108+
; SSE-NEXT: [[TMP3:%.*]] = fneg <2 x double> [[B:%.*]]
109+
; SSE-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[B]], <2 x i32> <i32 1, i32 2>
112110
; SSE-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[TMP2]], [[TMP4]]
113111
; SSE-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[A]], <2 x double> poison, <2 x i32> zeroinitializer
114112
; SSE-NEXT: [[TMP7:%.*]] = tail call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP6]], <2 x double> [[B]], <2 x double> [[TMP5]])

‎llvm/test/Transforms/PhaseOrdering/X86/addsub.ll‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -104,11 +104,9 @@ define void @add_aggregate_store(<2 x float> %a0, <2 x float> %a1, <2 x float> %
104104
; PR58139
105105
define <2 x double> @_mm_complexmult_pd_naive(<2 x double> %a, <2 x double> %b) {
106106
; SSE-LABEL: @_mm_complexmult_pd_naive(
107-
; SSE-NEXT: [[B1:%.*]] = extractelement <2 x double> [[B:%.*]], i64 1
108-
; SSE-NEXT: [[TMP1:%.*]] = fneg double [[B1]]
109107
; SSE-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> poison, <2 x i32> <i32 1, i32 1>
110-
; SSE-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[B]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
111-
; SSE-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[TMP1]], i64 0
108+
; SSE-NEXT: [[TMP3:%.*]] = fneg <2 x double> [[B:%.*]]
109+
; SSE-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[B]], <2 x i32> <i32 1, i32 2>
112110
; SSE-NEXT: [[TMP5:%.*]] = fmul <2 x double> [[TMP2]], [[TMP4]]
113111
; SSE-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[A]], <2 x double> poison, <2 x i32> zeroinitializer
114112
; SSE-NEXT: [[TMP7:%.*]] = tail call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP6]], <2 x double> [[B]], <2 x double> [[TMP5]])

‎llvm/test/Transforms/VectorCombine/X86/extract-fneg-insert.ll‎

Lines changed: 41 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,9 @@ define <4 x float> @ext2_v4f32(<4 x float> %x, <4 x float> %y) {
4747

4848
define <4 x float> @ext2_v2f32v4f32(<2 x float> %x, <4 x float> %y) {
4949
; CHECK-LABEL: @ext2_v2f32v4f32(
50-
; CHECK-NEXT: [[TMP1:%.*]] = fneg <2 x float> [[X:%.*]]
51-
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 poison, i32 poison, i32 2, i32 poison>
52-
; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
50+
; CHECK-NEXT: [[E:%.*]] = extractelement <2 x float> [[X:%.*]], i32 2
51+
; CHECK-NEXT: [[N:%.*]] = fneg float [[E]]
52+
; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[N]], i32 2
5353
; CHECK-NEXT: ret <4 x float> [[R]]
5454
;
5555
%e = extractelement <2 x float> %x, i32 2
@@ -73,17 +73,11 @@ define <2 x double> @ext1_v2f64(<2 x double> %x, <2 x double> %y) {
7373
}
7474

7575
define <4 x double> @ext1_v2f64v4f64(<2 x double> %x, <4 x double> %y) {
76-
; SSE-LABEL: @ext1_v2f64v4f64(
77-
; SSE-NEXT: [[E:%.*]] = extractelement <2 x double> [[X:%.*]], i32 1
78-
; SSE-NEXT: [[N:%.*]] = fneg nsz double [[E]]
79-
; SSE-NEXT: [[R:%.*]] = insertelement <4 x double> [[Y:%.*]], double [[N]], i32 1
80-
; SSE-NEXT: ret <4 x double> [[R]]
81-
;
82-
; AVX-LABEL: @ext1_v2f64v4f64(
83-
; AVX-NEXT: [[TMP1:%.*]] = fneg nsz <2 x double> [[X:%.*]]
84-
; AVX-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
85-
; AVX-NEXT: [[R:%.*]] = shufflevector <4 x double> [[Y:%.*]], <4 x double> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 3>
86-
; AVX-NEXT: ret <4 x double> [[R]]
76+
; CHECK-LABEL: @ext1_v2f64v4f64(
77+
; CHECK-NEXT: [[TMP1:%.*]] = fneg nsz <2 x double> [[X:%.*]]
78+
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
79+
; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x double> [[Y:%.*]], <4 x double> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 3>
80+
; CHECK-NEXT: ret <4 x double> [[R]]
8781
;
8882
%e = extractelement <2 x double> %x, i32 1
8983
%n = fneg nsz double %e
@@ -105,9 +99,9 @@ define <8 x float> @ext7_v8f32(<8 x float> %x, <8 x float> %y) {
10599

106100
define <8 x float> @ext7_v4f32v8f32(<4 x float> %x, <8 x float> %y) {
107101
; CHECK-LABEL: @ext7_v4f32v8f32(
108-
; CHECK-NEXT: [[E:%.*]] = extractelement <4 x float> [[X:%.*]], i32 3
109-
; CHECK-NEXT: [[N:%.*]] = fneg float [[E]]
110-
; CHECK-NEXT: [[R:%.*]] = insertelement <8 x float> [[Y:%.*]], float [[N]], i32 7
102+
; CHECK-NEXT: [[TMP1:%.*]] = fneg <4 x float> [[X:%.*]]
103+
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
104+
; CHECK-NEXT: [[R:%.*]] = shufflevector <8 x float> [[Y:%.*]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 11>
111105
; CHECK-NEXT: ret <8 x float> [[R]]
112106
;
113107
%e = extractelement <4 x float> %x, i32 3
@@ -141,12 +135,20 @@ define <8 x float> @ext7_v8f32_use1(<8 x float> %x, <8 x float> %y) {
141135
}
142136

143137
define <8 x float> @ext7_v4f32v8f32_use1(<4 x float> %x, <8 x float> %y) {
144-
; CHECK-LABEL: @ext7_v4f32v8f32_use1(
145-
; CHECK-NEXT: [[E:%.*]] = extractelement <4 x float> [[X:%.*]], i32 3
146-
; CHECK-NEXT: call void @use(float [[E]])
147-
; CHECK-NEXT: [[N:%.*]] = fneg float [[E]]
148-
; CHECK-NEXT: [[R:%.*]] = insertelement <8 x float> [[Y:%.*]], float [[N]], i32 3
149-
; CHECK-NEXT: ret <8 x float> [[R]]
138+
; SSE-LABEL: @ext7_v4f32v8f32_use1(
139+
; SSE-NEXT: [[E:%.*]] = extractelement <4 x float> [[X:%.*]], i32 3
140+
; SSE-NEXT: call void @use(float [[E]])
141+
; SSE-NEXT: [[TMP1:%.*]] = fneg <4 x float> [[X]]
142+
; SSE-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <8 x i32> <i32 poison, i32 poison, i32 poison, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
143+
; SSE-NEXT: [[R:%.*]] = shufflevector <8 x float> [[Y:%.*]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 11, i32 4, i32 5, i32 6, i32 7>
144+
; SSE-NEXT: ret <8 x float> [[R]]
145+
;
146+
; AVX-LABEL: @ext7_v4f32v8f32_use1(
147+
; AVX-NEXT: [[E:%.*]] = extractelement <4 x float> [[X:%.*]], i32 3
148+
; AVX-NEXT: call void @use(float [[E]])
149+
; AVX-NEXT: [[N:%.*]] = fneg float [[E]]
150+
; AVX-NEXT: [[R:%.*]] = insertelement <8 x float> [[Y:%.*]], float [[N]], i32 3
151+
; AVX-NEXT: ret <8 x float> [[R]]
150152
;
151153
%e = extractelement <4 x float> %x, i32 3
152154
call void @use(float %e)
@@ -220,9 +222,8 @@ define <4 x double> @ext_index_var_v2f64v4f64(<2 x double> %x, <4 x double> %y,
220222

221223
define <2 x double> @ext1_v2f64_ins0(<2 x double> %x, <2 x double> %y) {
222224
; CHECK-LABEL: @ext1_v2f64_ins0(
223-
; CHECK-NEXT: [[E:%.*]] = extractelement <2 x double> [[X:%.*]], i32 1
224-
; CHECK-NEXT: [[N:%.*]] = fneg nsz double [[E]]
225-
; CHECK-NEXT: [[R:%.*]] = insertelement <2 x double> [[Y:%.*]], double [[N]], i32 0
225+
; CHECK-NEXT: [[TMP1:%.*]] = fneg nsz <2 x double> [[X:%.*]]
226+
; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x double> [[Y:%.*]], <2 x double> [[TMP1]], <2 x i32> <i32 3, i32 1>
226227
; CHECK-NEXT: ret <2 x double> [[R]]
227228
;
228229
%e = extractelement <2 x double> %x, i32 1
@@ -234,9 +235,9 @@ define <2 x double> @ext1_v2f64_ins0(<2 x double> %x, <2 x double> %y) {
234235
; Negative test - extract from an index greater than the vector width of the destination
235236
define <2 x double> @ext3_v4f64v2f64(<4 x double> %x, <2 x double> %y) {
236237
; CHECK-LABEL: @ext3_v4f64v2f64(
237-
; CHECK-NEXT: [[E:%.*]] = extractelement <4 x double> [[X:%.*]], i32 3
238-
; CHECK-NEXT: [[N:%.*]] = fneg nsz double [[E]]
239-
; CHECK-NEXT: [[R:%.*]] = insertelement <2 x double> [[Y:%.*]], double [[N]], i32 1
238+
; CHECK-NEXT: [[TMP1:%.*]] = fneg nsz <4 x double> [[X:%.*]]
239+
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <2 x i32> <i32 poison, i32 3>
240+
; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x double> [[Y:%.*]], <2 x double> [[TMP2]], <2 x i32> <i32 0, i32 3>
240241
; CHECK-NEXT: ret <2 x double> [[R]]
241242
;
242243
%e = extractelement <4 x double> %x, i32 3
@@ -246,11 +247,17 @@ define <2 x double> @ext3_v4f64v2f64(<4 x double> %x, <2 x double> %y) {
246247
}
247248

248249
define <4 x double> @ext1_v2f64v4f64_ins0(<2 x double> %x, <4 x double> %y) {
249-
; CHECK-LABEL: @ext1_v2f64v4f64_ins0(
250-
; CHECK-NEXT: [[E:%.*]] = extractelement <2 x double> [[X:%.*]], i32 1
251-
; CHECK-NEXT: [[N:%.*]] = fneg nsz double [[E]]
252-
; CHECK-NEXT: [[R:%.*]] = insertelement <4 x double> [[Y:%.*]], double [[N]], i32 0
253-
; CHECK-NEXT: ret <4 x double> [[R]]
250+
; SSE-LABEL: @ext1_v2f64v4f64_ins0(
251+
; SSE-NEXT: [[TMP1:%.*]] = fneg nsz <2 x double> [[X:%.*]]
252+
; SSE-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
253+
; SSE-NEXT: [[R:%.*]] = shufflevector <4 x double> [[Y:%.*]], <4 x double> [[TMP2]], <4 x i32> <i32 5, i32 1, i32 2, i32 3>
254+
; SSE-NEXT: ret <4 x double> [[R]]
255+
;
256+
; AVX-LABEL: @ext1_v2f64v4f64_ins0(
257+
; AVX-NEXT: [[E:%.*]] = extractelement <2 x double> [[X:%.*]], i32 1
258+
; AVX-NEXT: [[N:%.*]] = fneg nsz double [[E]]
259+
; AVX-NEXT: [[R:%.*]] = insertelement <4 x double> [[Y:%.*]], double [[N]], i32 0
260+
; AVX-NEXT: ret <4 x double> [[R]]
254261
;
255262
%e = extractelement <2 x double> %x, i32 1
256263
%n = fneg nsz double %e

0 commit comments

Comments
 (0)