Skip to content

Commit cd3192a

Browse files
fhahnfhahn
committed
[VPlan] Turn IVOp assertion into early exit.
Turn assertion added in 99addbf [0] into an early exit. There are cases where the operand may not be a VPWidenIntOrFpInductionRecipe, e.g. if the IV increment is selected, as in the test cases. [0] #141431
1 parent f571293 commit cd3192a

File tree

5 files changed

+174
-3
lines changed

5 files changed

+174
-3
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1080,9 +1080,10 @@ bool VPlanTransforms::handleMultiUseReductions(VPlan &Plan) {
10801080
FindIVPhiR->getRecurrenceKind()))
10811081
return false;
10821082

1083-
assert(match(IVOp, m_TruncOrSelf(m_VPValue(IVOp))) &&
1084-
isa<VPWidenIntOrFpInductionRecipe>(IVOp) &&
1085-
"other select operand must be a (truncated) wide induction");
1083+
// TODO: Support cases where IVOp is the IV increment.
1084+
if (!match(IVOp, m_TruncOrSelf(m_VPValue(IVOp))) ||
1085+
!isa<VPWidenIntOrFpInductionRecipe>(IVOp))
1086+
return false;
10861087

10871088
CmpInst::Predicate RdxPredicate = [RdxKind]() {
10881089
switch (RdxKind) {

‎llvm/test/Transforms/LoopVectorize/select-smax-last-index.ll‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -656,5 +656,47 @@ exit:
656656
ret i64 %res
657657
}
658658

659+
define i64 @test_vectorize_select_smax_idx_inc(ptr %src, i64 %n) {
660+
; CHECK-LABEL: define i64 @test_vectorize_select_smax_idx_inc(
661+
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
662+
; CHECK-NEXT: [[ENTRY:.*]]:
663+
; CHECK-NEXT: br label %[[LOOP:.*]]
664+
; CHECK: [[LOOP]]:
665+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
666+
; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
667+
; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
668+
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
669+
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4
670+
; CHECK-NEXT: [[CMP:%.*]] = icmp sle i64 [[MAX_VAL]], [[L]]
671+
; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i64 @llvm.smax.i64(i64 [[MAX_VAL]], i64 [[L]])
672+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
673+
; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV_NEXT]], i64 [[MAX_IDX]]
674+
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
675+
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
676+
; CHECK: [[EXIT]]:
677+
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
678+
; CHECK-NEXT: ret i64 [[RES]]
679+
;
680+
entry:
681+
br label %loop
682+
683+
loop:
684+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
685+
%max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
686+
%max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
687+
%gep = getelementptr i64, ptr %src, i64 %iv
688+
%l = load i64, ptr %gep
689+
%cmp = icmp sle i64 %max.val, %l
690+
%max.val.next = tail call i64 @llvm.smax.i64(i64 %max.val, i64 %l)
691+
%iv.next = add nuw nsw i64 %iv, 1
692+
%max.idx.next = select i1 %cmp, i64 %iv.next, i64 %max.idx
693+
%exitcond.not = icmp eq i64 %iv.next, %n
694+
br i1 %exitcond.not, label %exit, label %loop
695+
696+
exit:
697+
%res = phi i64 [ %max.idx.next, %loop ]
698+
ret i64 %res
699+
}
700+
659701
declare i64 @llvm.smax.i64(i64, i64)
660702
declare i16 @llvm.smax.i16(i16, i16)

‎llvm/test/Transforms/LoopVectorize/select-smin-last-index.ll‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,5 +658,48 @@ exit:
658658
ret i64 %res
659659
}
660660

661+
define i64 @test_vectorize_select_smin_idx_inc(ptr %src, i64 %n) {
662+
; CHECK-LABEL: define i64 @test_vectorize_select_smin_idx_inc(
663+
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
664+
; CHECK-NEXT: [[ENTRY:.*]]:
665+
; CHECK-NEXT: br label %[[LOOP:.*]]
666+
; CHECK: [[LOOP]]:
667+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
668+
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
669+
; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
670+
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
671+
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4
672+
; CHECK-NEXT: [[CMP:%.*]] = icmp sge i64 [[MIN_VAL]], [[L]]
673+
; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.smin.i64(i64 [[MIN_VAL]], i64 [[L]])
674+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
675+
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV_NEXT]], i64 [[MIN_IDX]]
676+
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
677+
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
678+
; CHECK: [[EXIT]]:
679+
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
680+
; CHECK-NEXT: ret i64 [[RES]]
681+
;
682+
entry:
683+
br label %loop
684+
685+
loop:
686+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
687+
%min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
688+
%min.val = phi i64 [ 0, %entry ], [ %min.val.next, %loop ]
689+
%gep = getelementptr i64, ptr %src, i64 %iv
690+
%l = load i64, ptr %gep
691+
%cmp = icmp sge i64 %min.val, %l
692+
%min.val.next = tail call i64 @llvm.smin.i64(i64 %min.val, i64 %l)
693+
%iv.next = add nuw nsw i64 %iv, 1
694+
%min.idx.next = select i1 %cmp, i64 %iv.next, i64 %min.idx
695+
%exitcond.not = icmp eq i64 %iv.next, %n
696+
br i1 %exitcond.not, label %exit, label %loop
697+
698+
exit:
699+
%res = phi i64 [ %min.idx.next, %loop ]
700+
ret i64 %res
701+
}
702+
703+
661704
declare i64 @llvm.smin.i64(i64, i64)
662705
declare i16 @llvm.smin.i16(i16, i16)

‎llvm/test/Transforms/LoopVectorize/select-umax-last-index.ll‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -656,5 +656,47 @@ exit:
656656
ret i64 %res
657657
}
658658

659+
define i64 @test_vectorize_select_umax_idx_inc(ptr %src, i64 %n) {
660+
; CHECK-LABEL: define i64 @test_vectorize_select_umax_idx_inc(
661+
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
662+
; CHECK-NEXT: [[ENTRY:.*]]:
663+
; CHECK-NEXT: br label %[[LOOP:.*]]
664+
; CHECK: [[LOOP]]:
665+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
666+
; CHECK-NEXT: [[MAX_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_IDX_NEXT:%.*]], %[[LOOP]] ]
667+
; CHECK-NEXT: [[MAX_VAL:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MAX_VAL_NEXT:%.*]], %[[LOOP]] ]
668+
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
669+
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4
670+
; CHECK-NEXT: [[CMP:%.*]] = icmp ule i64 [[MAX_VAL]], [[L]]
671+
; CHECK-NEXT: [[MAX_VAL_NEXT]] = tail call i64 @llvm.umax.i64(i64 [[MAX_VAL]], i64 [[L]])
672+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
673+
; CHECK-NEXT: [[MAX_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV_NEXT]], i64 [[MAX_IDX]]
674+
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
675+
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
676+
; CHECK: [[EXIT]]:
677+
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MAX_IDX_NEXT]], %[[LOOP]] ]
678+
; CHECK-NEXT: ret i64 [[RES]]
679+
;
680+
entry:
681+
br label %loop
682+
683+
loop:
684+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
685+
%max.idx = phi i64 [ 0, %entry ], [ %max.idx.next, %loop ]
686+
%max.val = phi i64 [ 0, %entry ], [ %max.val.next, %loop ]
687+
%gep = getelementptr i64, ptr %src, i64 %iv
688+
%l = load i64, ptr %gep
689+
%cmp = icmp ule i64 %max.val, %l
690+
%max.val.next = tail call i64 @llvm.umax.i64(i64 %max.val, i64 %l)
691+
%iv.next = add nuw nsw i64 %iv, 1
692+
%max.idx.next = select i1 %cmp, i64 %iv.next, i64 %max.idx
693+
%exitcond.not = icmp eq i64 %iv.next, %n
694+
br i1 %exitcond.not, label %exit, label %loop
695+
696+
exit:
697+
%res = phi i64 [ %max.idx.next, %loop ]
698+
ret i64 %res
699+
}
700+
659701
declare i64 @llvm.umax.i64(i64, i64)
660702
declare i16 @llvm.umax.i16(i16, i16)

‎llvm/test/Transforms/LoopVectorize/select-umin-last-index.ll‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -737,5 +737,48 @@ exit:
737737
ret i64 %res
738738
}
739739

740+
define i64 @test_vectorize_select_umin_idx_inc(ptr %src, i64 %n) {
741+
; CHECK-LABEL: define i64 @test_vectorize_select_umin_idx_inc(
742+
; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) {
743+
; CHECK-NEXT: [[ENTRY:.*]]:
744+
; CHECK-NEXT: br label %[[LOOP:.*]]
745+
; CHECK: [[LOOP]]:
746+
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
747+
; CHECK-NEXT: [[MIN_IDX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[MIN_IDX_NEXT:%.*]], %[[LOOP]] ]
748+
; CHECK-NEXT: [[MIN_VAL:%.*]] = phi i64 [ 140, %[[ENTRY]] ], [ [[MIN_VAL_NEXT:%.*]], %[[LOOP]] ]
749+
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[SRC]], i64 [[IV]]
750+
; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP]], align 4
751+
; CHECK-NEXT: [[CMP:%.*]] = icmp uge i64 [[MIN_VAL]], [[L]]
752+
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
753+
; CHECK-NEXT: [[MIN_VAL_NEXT]] = tail call i64 @llvm.umin.i64(i64 [[MIN_VAL]], i64 [[L]])
754+
; CHECK-NEXT: [[MIN_IDX_NEXT]] = select i1 [[CMP]], i64 [[IV_NEXT]], i64 [[MIN_IDX]]
755+
; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
756+
; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT:.*]], label %[[LOOP]]
757+
; CHECK: [[EXIT]]:
758+
; CHECK-NEXT: [[RES:%.*]] = phi i64 [ [[MIN_IDX_NEXT]], %[[LOOP]] ]
759+
; CHECK-NEXT: ret i64 [[RES]]
760+
;
761+
entry:
762+
br label %loop
763+
764+
loop:
765+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
766+
%min.idx = phi i64 [ 0, %entry ], [ %min.idx.next, %loop ]
767+
%min.val = phi i64 [ 140, %entry ], [ %min.val.next, %loop ]
768+
%gep = getelementptr i64, ptr %src, i64 %iv
769+
%l = load i64, ptr %gep
770+
%cmp = icmp uge i64 %min.val, %l
771+
%iv.next = add nuw nsw i64 %iv, 1
772+
%min.val.next = tail call i64 @llvm.umin.i64(i64 %min.val, i64 %l)
773+
%min.idx.next = select i1 %cmp, i64 %iv.next, i64 %min.idx
774+
%exitcond.not = icmp eq i64 %iv.next, %n
775+
br i1 %exitcond.not, label %exit, label %loop
776+
777+
exit:
778+
%res = phi i64 [ %min.idx.next, %loop ]
779+
ret i64 %res
780+
}
781+
782+
740783
declare i64 @llvm.umin.i64(i64, i64)
741784
declare i16 @llvm.umin.i16(i16, i16)

0 commit comments

Comments
 (0)