Skip to content

Commit fd93dc5

Browse files
committed
[SLP]Correctly schedule standalone schedule data, which is part of tree entry
If a standalone schedule data relates to a vectorized instruction, still need to schedule it as a part of pseudo-bundle to correctly handle dependencies between its child nodes.
1 parent c2c7b7e commit fd93dc5

File tree

3 files changed

+196
-29
lines changed

3 files changed

+196
-29
lines changed

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5574,7 +5574,23 @@ class BoUpSLP {
55745574
if (auto *SD = dyn_cast<ScheduleData>(Data)) {
55755575
SD->setScheduled(/*Scheduled=*/true);
55765576
LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
5577-
ProcessBundleMember(SD, {});
5577+
SmallVector<std::unique_ptr<ScheduleBundle>> PseudoBundles;
5578+
SmallVector<ScheduleBundle *> Bundles;
5579+
Instruction *In = SD->getInst();
5580+
if (R.isVectorized(In)) {
5581+
ArrayRef<TreeEntry *> Entries = R.getTreeEntries(In);
5582+
for (TreeEntry *TE : Entries) {
5583+
if (!isa<ExtractValueInst, ExtractElementInst, CallBase>(In) &&
5584+
In->getNumOperands() != TE->getNumOperands())
5585+
continue;
5586+
auto &BundlePtr =
5587+
PseudoBundles.emplace_back(std::make_unique<ScheduleBundle>());
5588+
BundlePtr->setTreeEntry(TE);
5589+
BundlePtr->add(SD);
5590+
Bundles.push_back(BundlePtr.get());
5591+
}
5592+
}
5593+
ProcessBundleMember(SD, Bundles);
55785594
} else {
55795595
ScheduleBundle &Bundle = *cast<ScheduleBundle>(Data);
55805596
Bundle.setScheduled(/*Scheduled=*/true);
@@ -20853,23 +20869,7 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
2085320869
for (Value *V : VL) {
2085420870
if (S.isNonSchedulable(V))
2085520871
continue;
20856-
// For copybales with parent nodes, which do not need to be scheduled, the
20857-
// parents should not be commutative, otherwise may incorrectly handle deps
20858-
// because of the potential reordering of commutative operations.
20859-
if ((S.isCopyableElement(V) && EI.UserTE && !EI.UserTE->isGather() &&
20860-
EI.UserTE->hasState() && EI.UserTE->doesNotNeedToSchedule() &&
20861-
any_of(EI.UserTE->Scalars,
20862-
[&](Value *V) {
20863-
if (isa<PoisonValue>(V))
20864-
return false;
20865-
auto *I = dyn_cast<Instruction>(V);
20866-
return isCommutative(
20867-
(I && EI.UserTE->isAltShuffle())
20868-
? EI.UserTE->getMatchingMainOpOrAltOp(I)
20869-
: EI.UserTE->getMainOp(),
20870-
V);
20871-
})) ||
20872-
!extendSchedulingRegion(V, S)) {
20872+
if (!extendSchedulingRegion(V, S)) {
2087320873
// If the scheduling region got new instructions at the lower end (or it
2087420874
// is a new region for the first bundle). This makes it necessary to
2087520875
// recalculate all dependencies.

llvm/test/Transforms/SLPVectorizer/X86/copyable-with-non-scheduled-parent-node.ll

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,15 @@
44
define i64 @test(ptr %a) {
55
; CHECK-LABEL: define i64 @test(
66
; CHECK-SAME: ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
7-
; CHECK-NEXT: [[TMP1:%.*]] = add i64 0, 0
87
; CHECK-NEXT: [[TMP2:%.*]] = load i64, ptr [[A]], align 4
9-
; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], 0
10-
; CHECK-NEXT: [[TMP4:%.*]] = add i64 1, [[TMP1]]
11-
; CHECK-NEXT: [[TMP5:%.*]] = ashr i64 0, 1
12-
; CHECK-NEXT: [[TMP6:%.*]] = ashr i64 0, 0
8+
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i64> <i64 poison, i64 0, i64 0, i64 0>, i64 [[TMP2]], i32 0
9+
; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> zeroinitializer, [[TMP7]]
10+
; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> <i64 0, i64 0, i64 0, i64 1>, [[TMP3]]
11+
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
12+
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <6 x i64> [[TMP5]], <6 x i64> <i64 0, i64 0, i64 undef, i64 undef, i64 undef, i64 undef>, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
1313
; CHECK-NEXT: br label %[[BB7:.*]]
1414
; CHECK: [[BB7]]:
15-
; CHECK-NEXT: [[TMP8:%.*]] = phi i64 [ [[TMP3]], [[TMP0:%.*]] ]
16-
; CHECK-NEXT: [[TMP9:%.*]] = phi i64 [ 0, [[TMP0]] ]
17-
; CHECK-NEXT: [[TMP10:%.*]] = phi i64 [ [[TMP6]], [[TMP0]] ]
18-
; CHECK-NEXT: [[TMP11:%.*]] = phi i64 [ [[TMP5]], [[TMP0]] ]
19-
; CHECK-NEXT: [[TMP12:%.*]] = phi i64 [ 0, [[TMP0]] ]
20-
; CHECK-NEXT: [[TMP13:%.*]] = phi i64 [ [[TMP4]], [[TMP0]] ]
15+
; CHECK-NEXT: [[TMP8:%.*]] = phi <6 x i64> [ [[TMP6]], [[TMP0:%.*]] ]
2116
; CHECK-NEXT: ret i64 0
2217
;
2318
%1 = add i64 0, 0
Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
2+
; RUN: opt -S --passes=slp-vectorizer -S -mtriple=i686-unknown-linux-android29 -mattr=+sse2 < %s | FileCheck %s
3+
4+
define void @test(ptr %0, i64 %1, i64 %2, i1 %3, i64 %4, i64 %5) {
5+
; CHECK-LABEL: define void @test(
6+
; CHECK-SAME: ptr [[TMP0:%.*]], i64 [[TMP1:%.*]], i64 [[TMP2:%.*]], i1 [[TMP3:%.*]], i64 [[TMP4:%.*]], i64 [[TMP5:%.*]]) #[[ATTR0:[0-9]+]] {
7+
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[TMP0]], i32 240
8+
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[TMP0]], i32 128
9+
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i64> poison, i64 [[TMP1]], i32 0
10+
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> poison, <4 x i32> zeroinitializer
11+
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i64> <i64 1, i64 1, i64 1, i64 poison>, i64 [[TMP2]], i32 3
12+
; CHECK-NEXT: [[TMP12:%.*]] = add <4 x i64> [[TMP10]], [[TMP11]]
13+
; CHECK-NEXT: [[TMP13:%.*]] = load <2 x i64>, ptr [[TMP7]], align 4
14+
; CHECK-NEXT: [[TMP14:%.*]] = load i64, ptr null, align 4
15+
; CHECK-NEXT: [[TMP15:%.*]] = load <2 x i64>, ptr [[TMP8]], align 4
16+
; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x i64> [[TMP13]], <2 x i64> [[TMP15]], <6 x i32> <i32 0, i32 1, i32 poison, i32 3, i32 2, i32 2>
17+
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <6 x i64> poison, i64 [[TMP14]], i32 0
18+
; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <6 x i64> [[TMP17]], <6 x i64> poison, <6 x i32> <i32 poison, i32 poison, i32 0, i32 poison, i32 poison, i32 poison>
19+
; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <6 x i64> [[TMP16]], <6 x i64> [[TMP18]], <6 x i32> <i32 0, i32 1, i32 8, i32 3, i32 4, i32 5>
20+
; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 0>
21+
; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <6 x i64> [[TMP20]], <6 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 poison>, <6 x i32> <i32 6, i32 7, i32 8, i32 9, i32 10, i32 0>
22+
; CHECK-NEXT: [[TMP22:%.*]] = add <6 x i64> [[TMP19]], [[TMP21]]
23+
; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP13]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
24+
; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> [[TMP23]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
25+
; CHECK-NEXT: [[TMP25:%.*]] = sub <4 x i64> zeroinitializer, [[TMP24]]
26+
; CHECK-NEXT: [[TMP26:%.*]] = sub <6 x i64> zeroinitializer, [[TMP22]]
27+
; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <6 x i64> [[TMP19]], <6 x i64> poison, <2 x i32> <i32 2, i32 2>
28+
; CHECK-NEXT: [[TMP28:%.*]] = add <2 x i64> [[TMP27]], splat (i64 1)
29+
; CHECK-NEXT: [[TMP29:%.*]] = ashr <2 x i64> [[TMP28]], splat (i64 14)
30+
; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <6 x i64> [[TMP26]], <6 x i64> poison, <14 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison>
31+
; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <4 x i64> [[TMP12]], <4 x i64> poison, <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
32+
; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <14 x i64> [[TMP30]], <14 x i64> [[TMP31]], <14 x i32> <i32 14, i32 15, i32 16, i32 17, i32 poison, i32 poison, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison>
33+
; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <4 x i64> [[TMP25]], <4 x i64> poison, <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
34+
; CHECK-NEXT: [[TMP34:%.*]] = shufflevector <14 x i64> [[TMP32]], <14 x i64> [[TMP33]], <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 14, i32 15, i32 16, i32 17, i32 8, i32 9, i32 10, i32 11, i32 poison, i32 poison>
35+
; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <2 x i64> [[TMP29]], <2 x i64> poison, <14 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
36+
; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <14 x i64> [[TMP34]], <14 x i64> [[TMP35]], <14 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 14, i32 15>
37+
; CHECK-NEXT: br i1 [[TMP3]], label %[[BB52:.*]], label %[[BB37:.*]]
38+
; CHECK: [[BB37]]:
39+
; CHECK-NEXT: [[TMP38:%.*]] = add <4 x i64> [[TMP10]], splat (i64 1)
40+
; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> poison, <2 x i32> zeroinitializer
41+
; CHECK-NEXT: [[TMP40:%.*]] = add <2 x i64> [[TMP39]], splat (i64 1)
42+
; CHECK-NEXT: [[TMP41:%.*]] = lshr <2 x i64> [[TMP39]], splat (i64 1)
43+
; CHECK-NEXT: [[TMP42:%.*]] = add <2 x i64> [[TMP40]], [[TMP41]]
44+
; CHECK-NEXT: [[TMP43:%.*]] = shufflevector <4 x i64> [[TMP10]], <4 x i64> [[TMP11]], <10 x i32> <i32 0, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
45+
; CHECK-NEXT: [[TMP44:%.*]] = insertelement <10 x i64> [[TMP43]], i64 [[TMP4]], i32 6
46+
; CHECK-NEXT: [[TMP45:%.*]] = insertelement <10 x i64> [[TMP44]], i64 [[TMP5]], i32 7
47+
; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <4 x i64> [[TMP38]], <4 x i64> poison, <10 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
48+
; CHECK-NEXT: [[TMP47:%.*]] = shufflevector <2 x i64> [[TMP42]], <2 x i64> poison, <10 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
49+
; CHECK-NEXT: [[TMP48:%.*]] = shufflevector <10 x i64> [[TMP46]], <10 x i64> [[TMP47]], <10 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 5, i32 6, i32 7, i32 10, i32 11>
50+
; CHECK-NEXT: [[TMP49:%.*]] = shufflevector <10 x i64> [[TMP48]], <10 x i64> [[TMP45]], <10 x i32> <i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 8, i32 9>
51+
; CHECK-NEXT: [[TMP50:%.*]] = shufflevector <10 x i64> [[TMP49]], <10 x i64> poison, <14 x i32> <i32 0, i32 1, i32 0, i32 2, i32 0, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 0, i32 0>
52+
; CHECK-NEXT: [[TMP51:%.*]] = ashr <14 x i64> [[TMP50]], splat (i64 2)
53+
; CHECK-NEXT: br label %[[BB52]]
54+
; CHECK: [[BB52]]:
55+
; CHECK-NEXT: [[TMP53:%.*]] = phi <14 x i64> [ [[TMP51]], %[[BB37]] ], [ [[TMP36]], [[TMP6:%.*]] ]
56+
; CHECK-NEXT: [[TMP54:%.*]] = extractelement <14 x i64> [[TMP53]], i32 0
57+
; CHECK-NEXT: [[TMP55:%.*]] = extractelement <14 x i64> [[TMP53]], i32 13
58+
; CHECK-NEXT: [[TMP56:%.*]] = or i64 [[TMP54]], [[TMP55]]
59+
; CHECK-NEXT: [[TMP57:%.*]] = extractelement <14 x i64> [[TMP53]], i32 4
60+
; CHECK-NEXT: [[TMP58:%.*]] = extractelement <14 x i64> [[TMP53]], i32 12
61+
; CHECK-NEXT: [[TMP59:%.*]] = or i64 [[TMP57]], [[TMP58]]
62+
; CHECK-NEXT: [[TMP60:%.*]] = extractelement <14 x i64> [[TMP53]], i32 1
63+
; CHECK-NEXT: [[TMP61:%.*]] = extractelement <14 x i64> [[TMP53]], i32 2
64+
; CHECK-NEXT: [[TMP62:%.*]] = or i64 [[TMP60]], [[TMP61]]
65+
; CHECK-NEXT: [[TMP63:%.*]] = or i64 [[TMP59]], [[TMP56]]
66+
; CHECK-NEXT: [[TMP64:%.*]] = extractelement <14 x i64> [[TMP53]], i32 5
67+
; CHECK-NEXT: [[TMP65:%.*]] = extractelement <14 x i64> [[TMP53]], i32 8
68+
; CHECK-NEXT: [[TMP66:%.*]] = or i64 [[TMP64]], [[TMP65]]
69+
; CHECK-NEXT: [[TMP67:%.*]] = extractelement <14 x i64> [[TMP53]], i32 3
70+
; CHECK-NEXT: [[TMP68:%.*]] = or i64 [[TMP67]], [[TMP62]]
71+
; CHECK-NEXT: [[TMP69:%.*]] = extractelement <14 x i64> [[TMP53]], i32 9
72+
; CHECK-NEXT: [[TMP70:%.*]] = or i64 [[TMP69]], [[TMP66]]
73+
; CHECK-NEXT: [[TMP71:%.*]] = extractelement <14 x i64> [[TMP53]], i32 6
74+
; CHECK-NEXT: [[TMP72:%.*]] = or i64 [[TMP71]], [[TMP70]]
75+
; CHECK-NEXT: [[TMP73:%.*]] = or i64 [[TMP63]], [[TMP72]]
76+
; CHECK-NEXT: [[TMP74:%.*]] = extractelement <14 x i64> [[TMP53]], i32 10
77+
; CHECK-NEXT: [[TMP75:%.*]] = or i64 [[TMP74]], [[TMP73]]
78+
; CHECK-NEXT: store i64 [[TMP68]], ptr [[TMP0]], align 4
79+
; CHECK-NEXT: [[TMP76:%.*]] = extractelement <14 x i64> [[TMP53]], i32 11
80+
; CHECK-NEXT: store i64 [[TMP76]], ptr null, align 4
81+
; CHECK-NEXT: [[TMP77:%.*]] = extractelement <14 x i64> [[TMP53]], i32 7
82+
; CHECK-NEXT: store i64 [[TMP77]], ptr [[TMP0]], align 4
83+
; CHECK-NEXT: store i64 [[TMP75]], ptr null, align 4
84+
; CHECK-NEXT: ret void
85+
;
86+
%7 = getelementptr i8, ptr %0, i32 248
87+
%8 = load i64, ptr %7, align 4
88+
%9 = getelementptr i8, ptr %0, i32 240
89+
%10 = load i64, ptr %9, align 4
90+
%11 = load i64, ptr null, align 4
91+
%12 = add i64 %1, 1
92+
%13 = add i64 %1, 1
93+
%14 = add i64 %1, %2
94+
%15 = getelementptr i8, ptr %0, i32 136
95+
%16 = load i64, ptr %15, align 4
96+
%17 = getelementptr i8, ptr %0, i32 128
97+
%18 = load i64, ptr %17, align 4
98+
%19 = add i64 %18, %1
99+
%20 = sub i64 0, %18
100+
%21 = sub i64 0, %16
101+
%22 = sub i64 0, %11
102+
%23 = add i64 %1, 1
103+
%24 = sub i64 0, %1
104+
%25 = sub i64 0, %1
105+
%26 = sub i64 0, %10
106+
%27 = sub i64 0, %8
107+
%28 = sub i64 0, %19
108+
%29 = add i64 %11, 1
109+
%30 = ashr i64 %29, 14
110+
%31 = add i64 %11, 1
111+
%32 = ashr i64 %31, 14
112+
br i1 %3, label %58, label %33
113+
114+
33:
115+
%34 = ashr i64 %2, 2
116+
%35 = ashr i64 %1, 2
117+
%36 = add i64 %1, 1
118+
%37 = ashr i64 %36, 2
119+
%38 = add i64 %1, 1
120+
%39 = lshr i64 %1, 1
121+
%40 = add i64 %38, %39
122+
%41 = ashr i64 %40, 2
123+
%42 = add i64 %1, 1
124+
%43 = lshr i64 %1, 1
125+
%44 = add i64 %42, %43
126+
%45 = ashr i64 %44, 2
127+
%46 = ashr i64 %5, 2
128+
%47 = ashr i64 %4, 2
129+
%48 = ashr i64 %1, 2
130+
%49 = ashr i64 %1, 2
131+
%50 = ashr i64 %1, 2
132+
%51 = ashr i64 %1, 2
133+
%52 = add i64 %1, 1
134+
%53 = ashr i64 %52, 2
135+
%54 = add i64 %1, 1
136+
%55 = ashr i64 %54, 2
137+
%56 = add i64 %1, 1
138+
%57 = ashr i64 %56, 2
139+
br label %58
140+
141+
58:
142+
%59 = phi i64 [ %51, %33 ], [ %24, %6 ]
143+
%60 = phi i64 [ %50, %33 ], [ %32, %6 ]
144+
%61 = phi i64 [ %53, %33 ], [ %25, %6 ]
145+
%62 = phi i64 [ %55, %33 ], [ %26, %6 ]
146+
%63 = phi i64 [ %57, %33 ], [ %27, %6 ]
147+
%64 = phi i64 [ %49, %33 ], [ %30, %6 ]
148+
%65 = phi i64 [ %48, %33 ], [ %23, %6 ]
149+
%66 = phi i64 [ %47, %33 ], [ %22, %6 ]
150+
%67 = phi i64 [ %46, %33 ], [ %21, %6 ]
151+
%68 = phi i64 [ %45, %33 ], [ %20, %6 ]
152+
%69 = phi i64 [ %41, %33 ], [ %28, %6 ]
153+
%70 = phi i64 [ %34, %33 ], [ %12, %6 ]
154+
%71 = phi i64 [ %35, %33 ], [ %13, %6 ]
155+
%72 = phi i64 [ %37, %33 ], [ %14, %6 ]
156+
%73 = or i64 %65, %64
157+
%74 = or i64 %59, %60
158+
%75 = or i64 %70, %71
159+
%76 = or i64 %74, %73
160+
%77 = or i64 %61, %66
161+
%78 = or i64 %72, %75
162+
%79 = or i64 %67, %77
163+
%80 = or i64 %62, %79
164+
%81 = or i64 %76, %80
165+
%82 = or i64 %68, %81
166+
store i64 %78, ptr %0, align 4
167+
store i64 %69, ptr null, align 4
168+
store i64 %63, ptr %0, align 4
169+
store i64 %82, ptr null, align 4
170+
ret void
171+
}
172+

0 commit comments

Comments
 (0)