Skip to content

Commit cd38f84

Browse files
fhahnfhahn
committed
[LV] Add argmin test for epilogue vectorization w/o wide canonical IV.
Add additional epilogue vectorization test coverage for llvm#170223. Also regenerate check lines for related tests.
1 parent 846e022 commit cd38f84

File tree

2 files changed

+265
-28
lines changed

2 files changed

+265
-28
lines changed

‎llvm/test/Transforms/LoopVectorize/epilog-iv-select-cmp.ll‎

Lines changed: 96 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -67,12 +67,12 @@ define i64 @select_icmp_const(ptr %a, i64 %n) {
6767
; CHECK-NEXT: [[CMP_N12:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
6868
; CHECK-NEXT: br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
6969
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
70-
; CHECK-NEXT: [[BC_RESUME_VAL15:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
71-
; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 3, %[[ITER_CHECK]] ]
70+
; CHECK-NEXT: [[BC_RESUME_VAL13:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
71+
; CHECK-NEXT: [[BC_MERGE_RDX14:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 3, %[[ITER_CHECK]] ]
7272
; CHECK-NEXT: br label %[[LOOP:.*]]
7373
; CHECK: [[LOOP]]:
74-
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
75-
; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
74+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
75+
; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX14]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
7676
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]]
7777
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 8
7878
; CHECK-NEXT: [[C:%.*]] = icmp eq i64 [[L]], 3
@@ -168,12 +168,12 @@ define i64 @select_fcmp_const_fast(ptr %a, i64 %n) {
168168
; CHECK-NEXT: [[CMP_N12:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
169169
; CHECK-NEXT: br i1 [[CMP_N12]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
170170
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
171-
; CHECK-NEXT: [[BC_RESUME_VAL15:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
172-
; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 2, %[[ITER_CHECK]] ]
171+
; CHECK-NEXT: [[BC_RESUME_VAL13:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
172+
; CHECK-NEXT: [[BC_MERGE_RDX14:%.*]] = phi i64 [ [[RDX_SELECT11]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 2, %[[ITER_CHECK]] ]
173173
; CHECK-NEXT: br label %[[LOOP:.*]]
174174
; CHECK: [[LOOP]]:
175-
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
176-
; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX16]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
175+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
176+
; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX14]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
177177
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]]
178178
; CHECK-NEXT: [[L:%.*]] = load float, ptr [[GEP]], align 4
179179
; CHECK-NEXT: [[C:%.*]] = fcmp fast ueq float [[L]], 3.000000e+00
@@ -279,12 +279,12 @@ define i8 @select_icmp_var_start(ptr %a, i8 %n, i8 %start) {
279279
; CHECK-NEXT: [[CMP_N14:%.*]] = icmp eq i32 [[TMP2]], [[N_VEC3]]
280280
; CHECK-NEXT: br i1 [[CMP_N14]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
281281
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
282-
; CHECK-NEXT: [[BC_RESUME_VAL15:%.*]] = phi i8 [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
283-
; CHECK-NEXT: [[BC_MERGE_RDX16:%.*]] = phi i8 [ [[RDX_SELECT13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
282+
; CHECK-NEXT: [[BC_RESUME_VAL13:%.*]] = phi i8 [ [[TMP13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
283+
; CHECK-NEXT: [[BC_MERGE_RDX14:%.*]] = phi i8 [ [[RDX_SELECT13]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ [[START]], %[[ITER_CHECK]] ]
284284
; CHECK-NEXT: br label %[[LOOP:.*]]
285285
; CHECK: [[LOOP]]:
286-
; CHECK-NEXT: [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL15]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
287-
; CHECK-NEXT: [[RDX:%.*]] = phi i8 [ [[BC_MERGE_RDX16]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
286+
; CHECK-NEXT: [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL13]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
287+
; CHECK-NEXT: [[RDX:%.*]] = phi i8 [ [[BC_MERGE_RDX14]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP]] ]
288288
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[A]], i8 [[IV]]
289289
; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP]], align 8
290290
; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 3
@@ -313,3 +313,87 @@ loop:
313313
exit:
314314
ret i8 %sel
315315
}
316+
317+
define i64 @test_vectorize_select_smin_first_idx(ptr %src, i64 %n) {
318+
; CHECK-LABEL: define i64 @test_vectorize_select_smin_first_idx(
319+
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
320+
; CHECK-NEXT: [[SCALAR_PH:.*]]:
321+
; CHECK-NEXT: br label %[[LOOP1:.*]]
322+
; CHECK: [[LOOP1]]:
323+
; CHECK-NEXT: [[IV2:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP1]] ]
324+
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP1]] ]
325+
; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[SCALAR_PH]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP1]] ]
326+
; CHECK-NEXT: [[GEP2:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV2]]
327+
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP2]], align 4
328+
; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i64 [[MIN_VAL]], [[L]]
329+
; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
330+
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV2]], i64 [[MIN_IDX]]
331+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV2]], 1
332+
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
333+
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP1]]
334+
; CHECK: [[EXIT]]:
335+
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP1]] ]
336+
; CHECK-NEXT: ret i64 [[RES]]
337+
;
338+
entry:
339+
br label %loop
340+
341+
loop:
342+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
343+
%min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
344+
%min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
345+
%gep = getelementptr i64, ptr %src, i64 %iv
346+
%l = load i64, ptr %gep
347+
%cmp = icmp sgt i64 %min.val, %l
348+
%min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
349+
%min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
350+
%iv.next = add nuw nsw i64 %iv, 1
351+
%ec = icmp eq i64 %iv.next, %n
352+
br i1 %ec, label %exit, label %loop
353+
354+
exit:
355+
%res = phi i64 [ %min.idx.next, %loop ]
356+
ret i64 %res
357+
}
358+
359+
360+
define i64 @select_argmin_iv_not_canonical(i64 %num, ptr %src) {
361+
; CHECK-LABEL: define i64 @select_argmin_iv_not_canonical(
362+
; CHECK-SAME: i64 [[NUM:%.*]], ptr [[SRC:%.*]]) {
363+
; CHECK-NEXT: [[ENTRY:.*]]:
364+
; CHECK-NEXT: br label %[[LOOP:.*]]
365+
; CHECK: [[LOOP]]:
366+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[INC_I:%.*]], %[[LOOP]] ]
367+
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
368+
; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
369+
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
370+
; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP]], align 1
371+
; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i8 @llvm.umin.i8(i8 [[L]], i8 [[MIN_VAL]])
372+
; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[L]], [[MIN_VAL]]
373+
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV]], i64 [[MIN_IDX]]
374+
; CHECK-NEXT: [[INC_I]] = add nsw i64 [[IV]], 1
375+
; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[NUM]]
376+
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
377+
; CHECK: [[EXIT]]:
378+
; CHECK-NEXT: [[MIN_IDX_NEXT_LCSSA:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
379+
; CHECK-NEXT: ret i64 [[MIN_IDX_NEXT_LCSSA]]
380+
;
381+
entry:
382+
br label %loop
383+
384+
loop:
385+
%iv = phi i64 [ 1, %entry ], [ %inc.i, %loop ]
386+
%min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
387+
%min.val = phi i8 [ 0, %entry ], [ %min.val.next, %loop ]
388+
%gep = getelementptr i64, ptr %src, i64 %iv
389+
%l = load i8, ptr %gep
390+
%min.val.next = tail call i8 @llvm.umin.i8(i8 %l, i8 %min.val)
391+
%cmp = icmp ult i8 %l, %min.val
392+
%min.idx.next = select i1 %cmp, i64 %iv, i64 %min.idx
393+
%inc.i = add nsw i64 %iv, 1
394+
%ec = icmp eq i64 %iv, %num
395+
br i1 %ec, label %exit, label %loop
396+
397+
exit:
398+
ret i64 %min.idx.next
399+
}

0 commit comments

Comments
 (0)