Skip to content

Commit 7997a79

Browse files
ElvisWang123ElvisWang123
authored
[LV] Align legacy cost model to vplan-based model for gather/scatter w/ uniform addr. (#155739)
This patch check if the addr is uniform in legacy cost model to align vplan-based cost model after #150371. This patch fixes llvm-test-suite assertion (https://lab.llvm.org/buildbot/#/builders/210/builds/1935) due to cost model misaligned after #149955 under RISCV. I've tested this patch (on top of #149955) on the llvm-test-suite locally with crashed options `rva23u64`, `rva23u64_zvl1024b` and build successfully. Since this fix will change LV, I think would be better to create a PR to fix this.
1 parent b26f424 commit 7997a79

File tree

2 files changed

+191
-2
lines changed

2 files changed

+191
-2
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5269,8 +5269,11 @@ LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
52695269
Type *ValTy = getLoadStoreType(I);
52705270
auto *VectorTy = cast<VectorType>(toVectorTy(ValTy, VF));
52715271
const Align Alignment = getLoadStoreAlignment(I);
5272-
const Value *Ptr = getLoadStorePointerOperand(I);
5273-
Type *PtrTy = toVectorTy(Ptr->getType(), VF);
5272+
Value *Ptr = getLoadStorePointerOperand(I);
5273+
Type *PtrTy = Ptr->getType();
5274+
5275+
if (!Legal->isUniform(Ptr, VF))
5276+
PtrTy = toVectorTy(PtrTy, VF);
52745277

52755278
return TTI.getAddressComputationCost(PtrTy, nullptr, nullptr, CostKind) +
52765279
TTI.getGatherScatterOpCost(I->getOpcode(), VectorTy, Ptr,

‎llvm/test/Transforms/LoopVectorize/RISCV/gather-scatter-cost.ll

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2+
; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+rva23u64 -S | FileCheck %s -check-prefixes=CHECK,RVA23
3+
; RUN: opt < %s -passes=loop-vectorize -mtriple riscv64 -mattr=+rva23u64,+zvl1024b -S | FileCheck %s -check-prefixes=CHECK,RVA23ZVL1024B
4+
5+
define void @predicated_uniform_load(ptr %src, i32 %n, ptr %dst, i1 %cond) {
6+
; CHECK-LABEL: @predicated_uniform_load(
7+
; CHECK-NEXT: entry:
8+
; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[IBOX:%.*]] to i64
9+
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1
10+
; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 0)
11+
; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[SMAX2]] to i32
12+
; CHECK-NEXT: [[TMP3:%.*]] = add nuw i32 [[TMP2]], 1
13+
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
14+
; CHECK: vector.scevcheck:
15+
; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[IBOX]] to i64
16+
; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 1
17+
; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP5]], i64 0)
18+
; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[SMAX]] to i32
19+
; CHECK-NEXT: [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0
20+
; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[SMAX]], 4294967295
21+
; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]]
22+
; CHECK-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
23+
; CHECK: vector.memcheck:
24+
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[NBRBOXES:%.*]], i64 4
25+
; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[BOXES:%.*]], i64 4
26+
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[NBRBOXES]], [[SCEVGEP1]]
27+
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[BOXES]], [[SCEVGEP]]
28+
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
29+
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
30+
; CHECK: vector.ph:
31+
; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i1> poison, i1 [[COND:%.*]], i64 0
32+
; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer
33+
; CHECK-NEXT: [[TMP13:%.*]] = xor <vscale x 4 x i1> [[BROADCAST_SPLAT1]], splat (i1 true)
34+
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[BOXES]], i64 0
35+
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
36+
; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[NBRBOXES]], i64 0
37+
; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT3]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
38+
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
39+
; CHECK: vector.body:
40+
; CHECK-NEXT: [[AVL:%.*]] = phi i32 [ [[TMP3]], [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
41+
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.experimental.get.vector.length.i32(i32 [[AVL]], i32 4, i1 true)
42+
; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.vp.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT]], <vscale x 4 x i1> [[TMP13]], i32 [[TMP10]]), !alias.scope [[META0:![0-9]+]]
43+
; CHECK-NEXT: [[PREDPHI:%.*]] = select <vscale x 4 x i1> [[BROADCAST_SPLAT1]], <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> [[WIDE_MASKED_GATHER]]
44+
; CHECK-NEXT: call void @llvm.vp.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[PREDPHI]], <vscale x 4 x ptr> align 4 [[BROADCAST_SPLAT4]], <vscale x 4 x i1> splat (i1 true), i32 [[TMP10]]), !alias.scope [[META3:![0-9]+]], !noalias [[META0]]
45+
; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i32 [[AVL]], [[TMP10]]
46+
; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i32 [[AVL_NEXT]], 0
47+
; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
48+
; CHECK: middle.block:
49+
; CHECK-NEXT: br label [[EXIT:%.*]]
50+
; CHECK: scalar.ph:
51+
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_SCEVCHECK]] ], [ 0, [[VECTOR_MEMCHECK]] ]
52+
; CHECK-NEXT: br label [[LOOP:%.*]]
53+
; CHECK: loop:
54+
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ]
55+
; CHECK-NEXT: br i1 [[COND]], label [[LOOP_THEN:%.*]], label [[LOOP_ELSE:%.*]]
56+
; CHECK: loop.then:
57+
; CHECK-NEXT: br label [[LOOP_LATCH]]
58+
; CHECK: loop.else:
59+
; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[BOXES]], align 4
60+
; CHECK-NEXT: br label [[LOOP_LATCH]]
61+
; CHECK: loop.latch:
62+
; CHECK-NEXT: [[STORE:%.*]] = phi i32 [ [[TMP17]], [[LOOP_ELSE]] ], [ 0, [[LOOP_THEN]] ]
63+
; CHECK-NEXT: store i32 [[STORE]], ptr [[NBRBOXES]], align 4
64+
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
65+
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp sgt i32 [[IV]], [[IBOX]]
66+
; CHECK-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
67+
; CHECK: exit:
68+
; CHECK-NEXT: ret void
69+
;
70+
entry:
71+
br label %loop
72+
73+
loop:
74+
%iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
75+
br i1 %cond, label %loop.then, label %loop.else
76+
77+
loop.then:
78+
br label %loop.latch
79+
80+
loop.else:
81+
%0 = load i32, ptr %src, align 4
82+
br label %loop.latch
83+
84+
loop.latch:
85+
%store = phi i32 [%0, %loop.else], [0, %loop.then]
86+
store i32 %store, ptr %dst, align 4
87+
%iv.next = add i32 %iv, 1
88+
%exitcond = icmp sgt i32 %iv, %n
89+
br i1 %exitcond, label %exit, label %loop
90+
91+
exit:
92+
ret void
93+
}
94+
95+
define void @predicated_strided_store(ptr %start) {
96+
; RVA23-LABEL: @predicated_strided_store(
97+
; RVA23-NEXT: entry:
98+
; RVA23-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
99+
; RVA23: vector.ph:
100+
; RVA23-NEXT: [[TMP0:%.*]] = call <vscale x 16 x i64> @llvm.stepvector.nxv16i64()
101+
; RVA23-NEXT: [[TMP1:%.*]] = mul <vscale x 16 x i64> [[TMP0]], splat (i64 1)
102+
; RVA23-NEXT: [[INDUCTION:%.*]] = add <vscale x 16 x i64> zeroinitializer, [[TMP1]]
103+
; RVA23-NEXT: br label [[VECTOR_BODY:%.*]]
104+
; RVA23: vector.body:
105+
; RVA23-NEXT: [[VEC_IND:%.*]] = phi <vscale x 16 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
106+
; RVA23-NEXT: [[AVL:%.*]] = phi i64 [ 586, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
107+
; RVA23-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 16, i1 true)
108+
; RVA23-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
109+
; RVA23-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i64> poison, i64 [[TMP3]], i64 0
110+
; RVA23-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i64> poison, <vscale x 16 x i32> zeroinitializer
111+
; RVA23-NEXT: [[TMP4:%.*]] = mul <vscale x 16 x i64> [[VEC_IND]], splat (i64 7)
112+
; RVA23-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START:%.*]], <vscale x 16 x i64> [[TMP4]]
113+
; RVA23-NEXT: call void @llvm.vp.scatter.nxv16i8.nxv16p0(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x ptr> align 1 [[TMP5]], <vscale x 16 x i1> splat (i1 true), i32 [[TMP2]])
114+
; RVA23-NEXT: [[TMP6:%.*]] = zext i32 [[TMP2]] to i64
115+
; RVA23-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]]
116+
; RVA23-NEXT: [[VEC_IND_NEXT]] = add <vscale x 16 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
117+
; RVA23-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
118+
; RVA23-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
119+
; RVA23: middle.block:
120+
; RVA23-NEXT: br label [[EXIT:%.*]]
121+
; RVA23: scalar.ph:
122+
; RVA23-NEXT: br label [[LOOP:%.*]]
123+
; RVA23: loop:
124+
; RVA23-NEXT: [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
125+
; RVA23-NEXT: [[TMP8:%.*]] = mul i64 [[IV]], 7
126+
; RVA23-NEXT: [[ADD_PTR:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]]
127+
; RVA23-NEXT: store i8 0, ptr [[ADD_PTR]], align 1
128+
; RVA23-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
129+
; RVA23-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV]], 585
130+
; RVA23-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
131+
; RVA23: exit:
132+
; RVA23-NEXT: ret void
133+
;
134+
; RVA23ZVL1024B-LABEL: @predicated_strided_store(
135+
; RVA23ZVL1024B-NEXT: entry:
136+
; RVA23ZVL1024B-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
137+
; RVA23ZVL1024B: vector.ph:
138+
; RVA23ZVL1024B-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
139+
; RVA23ZVL1024B-NEXT: [[TMP1:%.*]] = mul <vscale x 2 x i64> [[TMP0]], splat (i64 1)
140+
; RVA23ZVL1024B-NEXT: [[INDUCTION:%.*]] = add <vscale x 2 x i64> zeroinitializer, [[TMP1]]
141+
; RVA23ZVL1024B-NEXT: br label [[VECTOR_BODY:%.*]]
142+
; RVA23ZVL1024B: vector.body:
143+
; RVA23ZVL1024B-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
144+
; RVA23ZVL1024B-NEXT: [[AVL:%.*]] = phi i64 [ 586, [[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], [[VECTOR_BODY]] ]
145+
; RVA23ZVL1024B-NEXT: [[TMP2:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
146+
; RVA23ZVL1024B-NEXT: [[TMP3:%.*]] = zext i32 [[TMP2]] to i64
147+
; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP3]], i64 0
148+
; RVA23ZVL1024B-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
149+
; RVA23ZVL1024B-NEXT: [[TMP4:%.*]] = mul <vscale x 2 x i64> [[VEC_IND]], splat (i64 7)
150+
; RVA23ZVL1024B-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[START:%.*]], <vscale x 2 x i64> [[TMP4]]
151+
; RVA23ZVL1024B-NEXT: call void @llvm.vp.scatter.nxv2i8.nxv2p0(<vscale x 2 x i8> zeroinitializer, <vscale x 2 x ptr> align 1 [[TMP5]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP2]])
152+
; RVA23ZVL1024B-NEXT: [[TMP6:%.*]] = zext i32 [[TMP2]] to i64
153+
; RVA23ZVL1024B-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP6]]
154+
; RVA23ZVL1024B-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
155+
; RVA23ZVL1024B-NEXT: [[TMP7:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
156+
; RVA23ZVL1024B-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
157+
; RVA23ZVL1024B: middle.block:
158+
; RVA23ZVL1024B-NEXT: br label [[EXIT:%.*]]
159+
; RVA23ZVL1024B: scalar.ph:
160+
; RVA23ZVL1024B-NEXT: br label [[LOOP:%.*]]
161+
; RVA23ZVL1024B: loop:
162+
; RVA23ZVL1024B-NEXT: [[IV:%.*]] = phi i64 [ 0, [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
163+
; RVA23ZVL1024B-NEXT: [[TMP8:%.*]] = mul i64 [[IV]], 7
164+
; RVA23ZVL1024B-NEXT: [[ADD_PTR:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]]
165+
; RVA23ZVL1024B-NEXT: store i8 0, ptr [[ADD_PTR]], align 1
166+
; RVA23ZVL1024B-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
167+
; RVA23ZVL1024B-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[IV]], 585
168+
; RVA23ZVL1024B-NEXT: br i1 [[EXITCOND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
169+
; RVA23ZVL1024B: exit:
170+
; RVA23ZVL1024B-NEXT: ret void
171+
;
172+
entry:
173+
br label %loop
174+
175+
loop:
176+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
177+
%0 = mul i64 %iv, 7
178+
%add.ptr = getelementptr i8, ptr %start, i64 %0
179+
store i8 0, ptr %add.ptr, align 1
180+
%iv.next = add i64 %iv, 1
181+
%exitcond = icmp eq i64 %iv, 585
182+
br i1 %exitcond, label %exit, label %loop
183+
184+
exit:
185+
ret void
186+
}

0 commit comments

Comments
 (0)