Skip to content

Commit 8c41859

Browse files
alexey-bataevalexey-bataev
committed
[SLP]Clear the operands deps of non-schedulable nodes, if previously all operands were copyable
If all operands of the non-schedulable nodes were previously only copyables, need to clear the dependencies of the original schedule data for such copyable operands and recalculate them to correctly handle number of dependecies. Fixes #159406
1 parent 3e0c58b commit 8c41859

File tree

2 files changed

+190
-4
lines changed

2 files changed

+190
-4
lines changed

‎llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20804,12 +20804,45 @@ BoUpSLP::BlockScheduling::tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
2080420804
const EdgeInfo &EI) {
2080520805
// No need to schedule PHIs, insertelement, extractelement and extractvalue
2080620806
// instructions.
20807-
bool HasCopyables = S.areInstructionsWithCopyableElements();
2080820807
if (isa<PHINode>(S.getMainOp()) ||
20809-
isVectorLikeInstWithConstOps(S.getMainOp()) ||
20810-
(!HasCopyables && doesNotNeedToSchedule(VL)) ||
20811-
all_of(VL, [&](Value *V) { return S.isNonSchedulable(V); }))
20808+
isVectorLikeInstWithConstOps(S.getMainOp()))
20809+
return nullptr;
20810+
bool HasCopyables = S.areInstructionsWithCopyableElements();
20811+
if (((!HasCopyables && doesNotNeedToSchedule(VL)) ||
20812+
all_of(VL, [&](Value *V) { return S.isNonSchedulable(V); }))) {
20813+
// If all operands were replaced by copyables, the operands of this node
20814+
// might be not, so need to recalculate dependencies for schedule data,
20815+
// replaced by copyable schedule data.
20816+
SmallVector<ScheduleData *> ControlDependentMembers;
20817+
for (Value *V : VL) {
20818+
auto *I = dyn_cast<Instruction>(V);
20819+
if (!I || (HasCopyables && S.isCopyableElement(V)))
20820+
continue;
20821+
SmallDenseMap<std::pair<Instruction *, Value *>, unsigned> UserOpToNumOps;
20822+
for (const Use &U : I->operands()) {
20823+
unsigned &NumOps =
20824+
UserOpToNumOps.try_emplace(std::make_pair(I, U.get()), 0)
20825+
.first->getSecond();
20826+
++NumOps;
20827+
if (auto *Op = dyn_cast<Instruction>(U.get());
20828+
Op && areAllOperandsReplacedByCopyableData(I, Op, *SLP, NumOps)) {
20829+
if (ScheduleData *OpSD = getScheduleData(Op);
20830+
OpSD && OpSD->hasValidDependencies()) {
20831+
OpSD->clearDirectDependencies();
20832+
if (RegionHasStackSave ||
20833+
!isGuaranteedToTransferExecutionToSuccessor(OpSD->getInst()))
20834+
ControlDependentMembers.push_back(OpSD);
20835+
}
20836+
}
20837+
}
20838+
}
20839+
if (!ControlDependentMembers.empty()) {
20840+
ScheduleBundle Invalid = ScheduleBundle::invalid();
20841+
calculateDependencies(Invalid, /*InsertInReadyList=*/true, SLP,
20842+
ControlDependentMembers);
20843+
}
2081220844
return nullptr;
20845+
}
2081320846

2081420847
// Initialize the instruction bundle.
2081520848
Instruction *OldScheduleEnd = ScheduleEnd;

‎llvm/test/Transforms/SLPVectorizer/X86/non-sched-inst-has-copyable-before.ll‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
2+
; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-cros-linux-gnu < %s | FileCheck %s
3+
4+
%struct.fe = type { [5 x i64] }
5+
6+
define i32 @test(i64 %0, i128 %1, i1 %2) {
7+
; CHECK-LABEL: define i32 @test(
8+
; CHECK-SAME: i64 [[TMP0:%.*]], i128 [[TMP1:%.*]], i1 [[TMP2:%.*]]) {
9+
; CHECK-NEXT: [[TMP4:%.*]] = alloca [[STRUCT_FE:%.*]], align 8
10+
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 16
11+
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 24
12+
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 32
13+
; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 8
14+
; CHECK-NEXT: br label %[[BB9:.*]]
15+
; CHECK: [[BB9]]:
16+
; CHECK-NEXT: [[TMP10:%.*]] = phi i64 [ undef, [[TMP3:%.*]] ], [ [[TMP29:%.*]], %[[BB9]] ]
17+
; CHECK-NEXT: [[TMP11:%.*]] = phi i64 [ undef, [[TMP3]] ], [ [[TMP26:%.*]], %[[BB9]] ]
18+
; CHECK-NEXT: [[TMP12:%.*]] = phi i64 [ undef, [[TMP3]] ], [ [[TMP23:%.*]], %[[BB9]] ]
19+
; CHECK-NEXT: [[TMP13:%.*]] = phi i64 [ undef, [[TMP3]] ], [ [[TMP20:%.*]], %[[BB9]] ]
20+
; CHECK-NEXT: [[TMP14:%.*]] = phi i64 [ undef, [[TMP3]] ], [ [[TMP17:%.*]], %[[BB9]] ]
21+
; CHECK-NEXT: [[DOTSROA_14_0:%.*]] = phi i64 [ undef, [[TMP3]] ], [ [[TMP52:%.*]], %[[BB9]] ]
22+
; CHECK-NEXT: [[DOTSROA_11_0:%.*]] = phi i64 [ undef, [[TMP3]] ], [ [[TMP50:%.*]], %[[BB9]] ]
23+
; CHECK-NEXT: [[DOTSROA_8_0:%.*]] = phi i64 [ undef, [[TMP3]] ], [ [[TMP57:%.*]], %[[BB9]] ]
24+
; CHECK-NEXT: [[DOTSROA_4_0:%.*]] = phi i64 [ undef, [[TMP3]] ], [ [[TMP56:%.*]], %[[BB9]] ]
25+
; CHECK-NEXT: [[DOTSROA_0_0:%.*]] = phi i64 [ undef, [[TMP3]] ], [ [[TMP54:%.*]], %[[BB9]] ]
26+
; CHECK-NEXT: [[TMP15:%.*]] = xor i64 [[DOTSROA_0_0]], [[TMP14]]
27+
; CHECK-NEXT: [[TMP16:%.*]] = and i64 [[TMP15]], [[TMP0]]
28+
; CHECK-NEXT: [[TMP17]] = xor i64 [[TMP16]], 1
29+
; CHECK-NEXT: store i64 [[TMP17]], ptr [[TMP4]], align 8
30+
; CHECK-NEXT: [[TMP18:%.*]] = xor i64 [[DOTSROA_4_0]], [[TMP13]]
31+
; CHECK-NEXT: [[TMP19:%.*]] = and i64 [[TMP18]], [[TMP0]]
32+
; CHECK-NEXT: [[TMP20]] = xor i64 [[TMP19]], 1
33+
; CHECK-NEXT: store i64 [[TMP20]], ptr [[TMP8]], align 8
34+
; CHECK-NEXT: [[TMP21:%.*]] = xor i64 [[DOTSROA_8_0]], [[TMP12]]
35+
; CHECK-NEXT: [[TMP22:%.*]] = and i64 [[TMP21]], [[TMP0]]
36+
; CHECK-NEXT: [[TMP23]] = xor i64 [[TMP22]], 1
37+
; CHECK-NEXT: store i64 [[TMP23]], ptr [[TMP5]], align 8
38+
; CHECK-NEXT: [[TMP24:%.*]] = xor i64 [[DOTSROA_11_0]], [[TMP11]]
39+
; CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], [[TMP0]]
40+
; CHECK-NEXT: [[TMP26]] = xor i64 [[TMP25]], 1
41+
; CHECK-NEXT: store i64 [[TMP26]], ptr [[TMP6]], align 8
42+
; CHECK-NEXT: [[TMP27:%.*]] = xor i64 [[DOTSROA_14_0]], [[TMP10]]
43+
; CHECK-NEXT: [[TMP28:%.*]] = and i64 [[TMP27]], [[TMP0]]
44+
; CHECK-NEXT: [[TMP29]] = xor i64 [[TMP28]], 1
45+
; CHECK-NEXT: store i64 [[TMP29]], ptr [[TMP7]], align 8
46+
; CHECK-NEXT: [[TMP30:%.*]] = load i64, ptr null, align 4294967296
47+
; CHECK-NEXT: [[TMP31:%.*]] = or i64 [[TMP19]], 1
48+
; CHECK-NEXT: [[TMP32:%.*]] = or i64 [[TMP16]], 1
49+
; CHECK-NEXT: [[TMP33:%.*]] = add i64 [[TMP17]], 1
50+
; CHECK-NEXT: [[TMP34:%.*]] = mul i64 [[TMP29]], 19
51+
; CHECK-NEXT: [[TMP35:%.*]] = zext i64 [[TMP34]] to i128
52+
; CHECK-NEXT: [[TMP36:%.*]] = mul i64 [[TMP26]], 19
53+
; CHECK-NEXT: [[TMP37:%.*]] = zext i64 [[TMP36]] to i128
54+
; CHECK-NEXT: [[TMP38:%.*]] = mul i64 [[TMP23]], 19
55+
; CHECK-NEXT: [[TMP39:%.*]] = zext i64 [[TMP38]] to i128
56+
; CHECK-NEXT: [[TMP40:%.*]] = mul nuw nsw i128 [[TMP39]], 24
57+
; CHECK-NEXT: [[TMP41:%.*]] = zext i64 [[TMP32]] to i128
58+
; CHECK-NEXT: [[TMP42:%.*]] = mul nuw i128 [[TMP37]], [[TMP41]]
59+
; CHECK-NEXT: [[TMP43:%.*]] = zext i64 [[TMP31]] to i128
60+
; CHECK-NEXT: [[TMP44:%.*]] = mul nuw i128 [[TMP35]], [[TMP43]]
61+
; CHECK-NEXT: [[TMP45:%.*]] = zext i64 [[TMP33]] to i128
62+
; CHECK-NEXT: [[TMP46:%.*]] = mul i128 [[TMP1]], [[TMP45]]
63+
; CHECK-NEXT: [[TMP47:%.*]] = add i128 [[TMP40]], [[TMP46]]
64+
; CHECK-NEXT: [[TMP48:%.*]] = add i128 [[TMP47]], [[TMP42]]
65+
; CHECK-NEXT: [[TMP49:%.*]] = add i128 [[TMP48]], [[TMP44]]
66+
; CHECK-NEXT: [[TMP50]] = and i64 [[TMP29]], 1
67+
; CHECK-NEXT: [[TMP51:%.*]] = trunc i128 [[TMP49]] to i64
68+
; CHECK-NEXT: [[TMP52]] = and i64 [[TMP30]], 1
69+
; CHECK-NEXT: [[TMP53:%.*]] = add i64 [[TMP51]], 1
70+
; CHECK-NEXT: [[TMP54]] = and i64 [[TMP53]], 1
71+
; CHECK-NEXT: [[TMP55:%.*]] = lshr i64 [[TMP53]], 1
72+
; CHECK-NEXT: [[TMP56]] = and i64 [[TMP19]], 1
73+
; CHECK-NEXT: [[TMP57]] = or i64 [[TMP55]], 1
74+
; CHECK-NEXT: br i1 [[TMP2]], label %[[BB58:.*]], label %[[BB9]]
75+
; CHECK: [[BB58]]:
76+
; CHECK-NEXT: call void @g(ptr nonnull [[TMP4]])
77+
; CHECK-NEXT: ret i32 0
78+
;
79+
%4 = alloca %struct.fe, align 8
80+
%5 = getelementptr inbounds nuw i8, ptr %4, i64 16
81+
%6 = getelementptr inbounds nuw i8, ptr %4, i64 24
82+
%7 = getelementptr inbounds nuw i8, ptr %4, i64 32
83+
%8 = getelementptr inbounds nuw i8, ptr %4, i64 8
84+
br label %9
85+
86+
9:
87+
%10 = phi i64 [ undef, %3 ], [ %29, %9 ]
88+
%11 = phi i64 [ undef, %3 ], [ %26, %9 ]
89+
%12 = phi i64 [ undef, %3 ], [ %23, %9 ]
90+
%13 = phi i64 [ undef, %3 ], [ %20, %9 ]
91+
%14 = phi i64 [ undef, %3 ], [ %17, %9 ]
92+
%.sroa.14.0 = phi i64 [ undef, %3 ], [ %52, %9 ]
93+
%.sroa.11.0 = phi i64 [ undef, %3 ], [ %50, %9 ]
94+
%.sroa.8.0 = phi i64 [ undef, %3 ], [ %57, %9 ]
95+
%.sroa.4.0 = phi i64 [ undef, %3 ], [ %56, %9 ]
96+
%.sroa.0.0 = phi i64 [ undef, %3 ], [ %54, %9 ]
97+
%15 = xor i64 %.sroa.0.0, %14
98+
%16 = and i64 %15, %0
99+
%17 = xor i64 %16, 1
100+
store i64 %17, ptr %4, align 8
101+
%18 = xor i64 %.sroa.4.0, %13
102+
%19 = and i64 %18, %0
103+
%20 = xor i64 %19, 1
104+
store i64 %20, ptr %8, align 8
105+
%21 = xor i64 %.sroa.8.0, %12
106+
%22 = and i64 %21, %0
107+
%23 = xor i64 %22, 1
108+
store i64 %23, ptr %5, align 8
109+
%24 = xor i64 %.sroa.11.0, %11
110+
%25 = and i64 %24, %0
111+
%26 = xor i64 %25, 1
112+
store i64 %26, ptr %6, align 8
113+
%27 = xor i64 %.sroa.14.0, %10
114+
%28 = and i64 %27, %0
115+
%29 = xor i64 %28, 1
116+
store i64 %29, ptr %7, align 8
117+
%30 = load i64, ptr null, align 4294967296
118+
%31 = or i64 %19, 1
119+
%32 = or i64 %16, 1
120+
%33 = add i64 %17, 1
121+
%34 = mul i64 %29, 19
122+
%35 = zext i64 %34 to i128
123+
%36 = mul i64 %26, 19
124+
%37 = zext i64 %36 to i128
125+
%38 = mul i64 %23, 19
126+
%39 = zext i64 %38 to i128
127+
%40 = mul nuw nsw i128 %39, 24
128+
%41 = zext i64 %32 to i128
129+
%42 = mul nuw i128 %37, %41
130+
%43 = zext i64 %31 to i128
131+
%44 = mul nuw i128 %35, %43
132+
%45 = zext i64 %33 to i128
133+
%46 = mul i128 %1, %45
134+
%47 = add i128 %40, %46
135+
%48 = add i128 %47, %42
136+
%49 = add i128 %48, %44
137+
%50 = and i64 %29, 1
138+
%51 = trunc i128 %49 to i64
139+
%52 = and i64 %30, 1
140+
%53 = add i64 %51, 1
141+
%54 = and i64 %53, 1
142+
%55 = lshr i64 %53, 1
143+
%56 = and i64 %19, 1
144+
%57 = or i64 %55, 1
145+
br i1 %2, label %58, label %9
146+
147+
58:
148+
call void @g(ptr nonnull %4)
149+
ret i32 0
150+
}
151+
152+
declare void @g(ptr)
153+

0 commit comments

Comments
 (0)