Skip to content

Commit 8761b6c

Browse files
Mel-ChenMel-Chen
authored
[VPlan] Use VPTypeAnalysis to get the step type of widen pointer induction (#147925)
This patch uses VPTypeAnalysis to determine its type since the induction step is not always a live-in value in the VPlan and may be defined by a recipe.
1 parent 3b23fdb commit 8761b6c

File tree

2 files changed

+290
-2
lines changed

2 files changed

+290
-2
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -844,8 +844,8 @@ optimizeLatchExitInductionUser(VPlan &Plan, VPTypeAnalysis &TypeInfo,
844844
if (ScalarTy->isIntegerTy())
845845
return B.createNaryOp(Instruction::Sub, {EndValue, Step}, {}, "ind.escape");
846846
if (ScalarTy->isPointerTy()) {
847-
auto *Zero = Plan.getOrAddLiveIn(
848-
ConstantInt::get(Step->getLiveInIRValue()->getType(), 0));
847+
Type *StepTy = TypeInfo.inferScalarType(Step);
848+
auto *Zero = Plan.getOrAddLiveIn(ConstantInt::get(StepTy, 0));
849849
return B.createPtrAdd(EndValue,
850850
B.createNaryOp(Instruction::Sub, {Zero, Step}), {},
851851
"ind.escape");

‎llvm/test/Transforms/LoopVectorize/pointer-induction.ll

Lines changed: 288 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -413,3 +413,291 @@ for.body: ; preds = %for.body, %entry
413413
for.end: ; preds = %for.body
414414
ret void
415415
}
416+
417+
define i64 @ivopt_widen_ptr_indvar_1(ptr noalias %a, i64 %stride, i64 %n) {
418+
; DEFAULT-LABEL: @ivopt_widen_ptr_indvar_1(
419+
; DEFAULT-NEXT: entry:
420+
; DEFAULT-NEXT: br label [[FOR_BODY:%.*]]
421+
; DEFAULT: for.body:
422+
; DEFAULT-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
423+
; DEFAULT-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], [[FOR_BODY]] ], [ null, [[ENTRY]] ]
424+
; DEFAULT-NEXT: [[ARRAYIDX:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[INDVAR]]
425+
; DEFAULT-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
426+
; DEFAULT-NEXT: [[PTR_IV_NEXT]] = getelementptr i64, ptr [[PTR_IV]], i64 [[STRIDE:%.*]]
427+
; DEFAULT-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
428+
; DEFAULT-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVAR]], [[N:%.*]]
429+
; DEFAULT-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[FOR_BODY]]
430+
; DEFAULT: exit:
431+
; DEFAULT-NEXT: [[PTR_IV_LCSSA:%.*]] = phi ptr [ [[PTR_IV]], [[FOR_BODY]] ]
432+
; DEFAULT-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY]] ]
433+
; DEFAULT-NEXT: [[CAST_PTR:%.*]] = ptrtoint ptr [[PTR_IV_LCSSA]] to i64
434+
; DEFAULT-NEXT: [[RESULT:%.*]] = add i64 [[CAST_PTR]], [[DOTLCSSA]]
435+
; DEFAULT-NEXT: ret i64 [[RESULT]]
436+
;
437+
; STRIDED-LABEL: @ivopt_widen_ptr_indvar_1(
438+
; STRIDED-NEXT: entry:
439+
; STRIDED-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
440+
; STRIDED-NEXT: [[TMP1:%.*]] = shl i64 [[STRIDE:%.*]], 3
441+
; STRIDED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
442+
; STRIDED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
443+
; STRIDED: vector.ph:
444+
; STRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
445+
; STRIDED-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
446+
; STRIDED-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], [[TMP1]]
447+
; STRIDED-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP2]]
448+
; STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
449+
; STRIDED: vector.body:
450+
; STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
451+
; STRIDED-NEXT: [[TMP4:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[INDEX]]
452+
; STRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP4]], align 8
453+
; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
454+
; STRIDED-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
455+
; STRIDED-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
456+
; STRIDED: middle.block:
457+
; STRIDED-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i32 3
458+
; STRIDED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
459+
; STRIDED-NEXT: [[TMP8:%.*]] = sub i64 0, [[TMP1]]
460+
; STRIDED-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP8]]
461+
; STRIDED-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
462+
; STRIDED: scalar.ph:
463+
; STRIDED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
464+
; STRIDED-NEXT: [[BC_RESUME_VAL1:%.*]] = phi ptr [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ null, [[ENTRY]] ]
465+
; STRIDED-NEXT: br label [[FOR_BODY:%.*]]
466+
; STRIDED: for.body:
467+
; STRIDED-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
468+
; STRIDED-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ]
469+
; STRIDED-NEXT: [[ARRAYIDX:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDVAR]]
470+
; STRIDED-NEXT: [[TMP9:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
471+
; STRIDED-NEXT: [[PTR_IV_NEXT]] = getelementptr i64, ptr [[PTR_IV]], i64 [[STRIDE]]
472+
; STRIDED-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
473+
; STRIDED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVAR]], [[N]]
474+
; STRIDED-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
475+
; STRIDED: exit:
476+
; STRIDED-NEXT: [[PTR_IV_LCSSA:%.*]] = phi ptr [ [[PTR_IV]], [[FOR_BODY]] ], [ [[IND_ESCAPE]], [[MIDDLE_BLOCK]] ]
477+
; STRIDED-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[TMP9]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
478+
; STRIDED-NEXT: [[CAST_PTR:%.*]] = ptrtoint ptr [[PTR_IV_LCSSA]] to i64
479+
; STRIDED-NEXT: [[RESULT:%.*]] = add i64 [[CAST_PTR]], [[DOTLCSSA]]
480+
; STRIDED-NEXT: ret i64 [[RESULT]]
481+
;
482+
entry:
483+
br label %for.body
484+
485+
for.body:
486+
%indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
487+
%ptr.iv = phi ptr [ %ptr.iv.next, %for.body ], [ null, %entry ]
488+
%arrayidx = getelementptr i64, ptr %a, i64 %indvar
489+
%0 = load i64, ptr %arrayidx, align 8
490+
%ptr.iv.next = getelementptr i64, ptr %ptr.iv, i64 %stride
491+
%indvar.next = add i64 %indvar, 1
492+
%exitcond.not = icmp eq i64 %indvar, %n
493+
br i1 %exitcond.not, label %exit, label %for.body
494+
495+
exit:
496+
%cast.ptr = ptrtoint ptr %ptr.iv to i64
497+
%result = add i64 %cast.ptr, %0
498+
ret i64 %result
499+
}
500+
501+
define i64 @ivopt_widen_ptr_indvar_2(ptr noalias %a, i64 %stride, i64 %n) {
502+
; DEFAULT-LABEL: @ivopt_widen_ptr_indvar_2(
503+
; DEFAULT-NEXT: entry:
504+
; DEFAULT-NEXT: br label [[FOR_BODY:%.*]]
505+
; DEFAULT: for.body:
506+
; DEFAULT-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
507+
; DEFAULT-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], [[FOR_BODY]] ], [ null, [[ENTRY]] ]
508+
; DEFAULT-NEXT: [[ARRAYIDX:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[INDVAR]]
509+
; DEFAULT-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
510+
; DEFAULT-NEXT: store i64 [[TMP0]], ptr [[PTR_IV]], align 8
511+
; DEFAULT-NEXT: [[PTR_IV_NEXT]] = getelementptr i64, ptr [[PTR_IV]], i64 [[STRIDE:%.*]]
512+
; DEFAULT-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
513+
; DEFAULT-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVAR]], [[N:%.*]]
514+
; DEFAULT-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[FOR_BODY]]
515+
; DEFAULT: exit:
516+
; DEFAULT-NEXT: [[PTR_IV_LCSSA:%.*]] = phi ptr [ [[PTR_IV]], [[FOR_BODY]] ]
517+
; DEFAULT-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY]] ]
518+
; DEFAULT-NEXT: [[CAST_PTR:%.*]] = ptrtoint ptr [[PTR_IV_LCSSA]] to i64
519+
; DEFAULT-NEXT: [[RESULT:%.*]] = add i64 [[CAST_PTR]], [[DOTLCSSA]]
520+
; DEFAULT-NEXT: ret i64 [[RESULT]]
521+
;
522+
; STRIDED-LABEL: @ivopt_widen_ptr_indvar_2(
523+
; STRIDED-NEXT: entry:
524+
; STRIDED-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
525+
; STRIDED-NEXT: [[TMP1:%.*]] = shl i64 [[STRIDE:%.*]], 3
526+
; STRIDED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
527+
; STRIDED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
528+
; STRIDED: vector.ph:
529+
; STRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
530+
; STRIDED-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
531+
; STRIDED-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], [[TMP1]]
532+
; STRIDED-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP2]]
533+
; STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
534+
; STRIDED: vector.body:
535+
; STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
536+
; STRIDED-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[TMP1]]
537+
; STRIDED-NEXT: [[TMP8:%.*]] = mul i64 0, [[TMP1]]
538+
; STRIDED-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], [[TMP8]]
539+
; STRIDED-NEXT: [[TMP10:%.*]] = mul i64 1, [[TMP1]]
540+
; STRIDED-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], [[TMP10]]
541+
; STRIDED-NEXT: [[TMP12:%.*]] = mul i64 2, [[TMP1]]
542+
; STRIDED-NEXT: [[TMP13:%.*]] = add i64 [[OFFSET_IDX]], [[TMP12]]
543+
; STRIDED-NEXT: [[TMP14:%.*]] = mul i64 3, [[TMP1]]
544+
; STRIDED-NEXT: [[TMP15:%.*]] = add i64 [[OFFSET_IDX]], [[TMP14]]
545+
; STRIDED-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr null, i64 [[TMP9]]
546+
; STRIDED-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr null, i64 [[TMP11]]
547+
; STRIDED-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr null, i64 [[TMP13]]
548+
; STRIDED-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP15]]
549+
; STRIDED-NEXT: [[TMP21:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[INDEX]]
550+
; STRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP21]], align 8
551+
; STRIDED-NEXT: [[TMP23:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i32 0
552+
; STRIDED-NEXT: store i64 [[TMP23]], ptr [[NEXT_GEP]], align 8
553+
; STRIDED-NEXT: [[TMP24:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i32 1
554+
; STRIDED-NEXT: store i64 [[TMP24]], ptr [[NEXT_GEP1]], align 8
555+
; STRIDED-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i32 2
556+
; STRIDED-NEXT: store i64 [[TMP16]], ptr [[NEXT_GEP2]], align 8
557+
; STRIDED-NEXT: [[TMP25:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i32 3
558+
; STRIDED-NEXT: store i64 [[TMP25]], ptr [[NEXT_GEP3]], align 8
559+
; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
560+
; STRIDED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
561+
; STRIDED-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
562+
; STRIDED: middle.block:
563+
; STRIDED-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i32 3
564+
; STRIDED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
565+
; STRIDED-NEXT: [[TMP19:%.*]] = sub i64 0, [[TMP1]]
566+
; STRIDED-NEXT: [[IND_ESCAPE:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP19]]
567+
; STRIDED-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
568+
; STRIDED: scalar.ph:
569+
; STRIDED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
570+
; STRIDED-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ null, [[ENTRY]] ]
571+
; STRIDED-NEXT: br label [[FOR_BODY:%.*]]
572+
; STRIDED: for.body:
573+
; STRIDED-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
574+
; STRIDED-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
575+
; STRIDED-NEXT: [[ARRAYIDX:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDVAR]]
576+
; STRIDED-NEXT: [[TMP20:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
577+
; STRIDED-NEXT: store i64 [[TMP20]], ptr [[PTR_IV]], align 8
578+
; STRIDED-NEXT: [[PTR_IV_NEXT]] = getelementptr i64, ptr [[PTR_IV]], i64 [[STRIDE]]
579+
; STRIDED-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
580+
; STRIDED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVAR]], [[N]]
581+
; STRIDED-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
582+
; STRIDED: exit:
583+
; STRIDED-NEXT: [[PTR_IV_LCSSA:%.*]] = phi ptr [ [[PTR_IV]], [[FOR_BODY]] ], [ [[IND_ESCAPE]], [[MIDDLE_BLOCK]] ]
584+
; STRIDED-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[TMP20]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ]
585+
; STRIDED-NEXT: [[CAST_PTR:%.*]] = ptrtoint ptr [[PTR_IV_LCSSA]] to i64
586+
; STRIDED-NEXT: [[RESULT:%.*]] = add i64 [[CAST_PTR]], [[DOTLCSSA]]
587+
; STRIDED-NEXT: ret i64 [[RESULT]]
588+
;
589+
entry:
590+
br label %for.body
591+
592+
for.body:
593+
%indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
594+
%ptr.iv = phi ptr [ %ptr.iv.next, %for.body ], [ null, %entry ]
595+
%arrayidx = getelementptr i64, ptr %a, i64 %indvar
596+
%0 = load i64, ptr %arrayidx, align 8
597+
store i64 %0, ptr %ptr.iv, align 8
598+
%ptr.iv.next = getelementptr i64, ptr %ptr.iv, i64 %stride
599+
%indvar.next = add i64 %indvar, 1
600+
%exitcond.not = icmp eq i64 %indvar, %n
601+
br i1 %exitcond.not, label %exit, label %for.body
602+
603+
exit:
604+
%cast.ptr = ptrtoint ptr %ptr.iv to i64
605+
%result = add i64 %cast.ptr, %0
606+
ret i64 %result
607+
}
608+
609+
define i64 @ivopt_widen_ptr_indvar_3(ptr noalias %a, i64 %stride, i64 %n) {
610+
; DEFAULT-LABEL: @ivopt_widen_ptr_indvar_3(
611+
; DEFAULT-NEXT: entry:
612+
; DEFAULT-NEXT: br label [[FOR_BODY:%.*]]
613+
; DEFAULT: for.body:
614+
; DEFAULT-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
615+
; DEFAULT-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], [[FOR_BODY]] ], [ null, [[ENTRY]] ]
616+
; DEFAULT-NEXT: [[ARRAYIDX:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[INDVAR]]
617+
; DEFAULT-NEXT: [[TMP0:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
618+
; DEFAULT-NEXT: [[PTR_IV_NEXT]] = getelementptr i64, ptr [[PTR_IV]], i64 [[STRIDE:%.*]]
619+
; DEFAULT-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
620+
; DEFAULT-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVAR]], [[N:%.*]]
621+
; DEFAULT-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT:%.*]], label [[FOR_BODY]]
622+
; DEFAULT: exit:
623+
; DEFAULT-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY]] ]
624+
; DEFAULT-NEXT: [[PTR_IV_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_NEXT]], [[FOR_BODY]] ]
625+
; DEFAULT-NEXT: [[CAST_PTR:%.*]] = ptrtoint ptr [[PTR_IV_NEXT_LCSSA]] to i64
626+
; DEFAULT-NEXT: [[RESULT:%.*]] = add i64 [[CAST_PTR]], [[DOTLCSSA]]
627+
; DEFAULT-NEXT: ret i64 [[RESULT]]
628+
;
629+
; STRIDED-LABEL: @ivopt_widen_ptr_indvar_3(
630+
; STRIDED-NEXT: entry:
631+
; STRIDED-NEXT: [[TMP0:%.*]] = add i64 [[N:%.*]], 1
632+
; STRIDED-NEXT: [[TMP1:%.*]] = shl i64 [[STRIDE:%.*]], 3
633+
; STRIDED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4
634+
; STRIDED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
635+
; STRIDED: vector.ph:
636+
; STRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4
637+
; STRIDED-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
638+
; STRIDED-NEXT: [[TMP2:%.*]] = mul i64 [[N_VEC]], [[TMP1]]
639+
; STRIDED-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP2]]
640+
; STRIDED-NEXT: br label [[VECTOR_BODY:%.*]]
641+
; STRIDED: vector.body:
642+
; STRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
643+
; STRIDED-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], [[TMP1]]
644+
; STRIDED-NEXT: [[TMP4:%.*]] = mul i64 0, [[TMP1]]
645+
; STRIDED-NEXT: [[TMP5:%.*]] = add i64 [[OFFSET_IDX]], [[TMP4]]
646+
; STRIDED-NEXT: [[TMP6:%.*]] = mul i64 1, [[TMP1]]
647+
; STRIDED-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], [[TMP6]]
648+
; STRIDED-NEXT: [[TMP8:%.*]] = mul i64 2, [[TMP1]]
649+
; STRIDED-NEXT: [[TMP9:%.*]] = add i64 [[OFFSET_IDX]], [[TMP8]]
650+
; STRIDED-NEXT: [[TMP10:%.*]] = mul i64 3, [[TMP1]]
651+
; STRIDED-NEXT: [[TMP11:%.*]] = add i64 [[OFFSET_IDX]], [[TMP10]]
652+
; STRIDED-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr null, i64 [[TMP5]]
653+
; STRIDED-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr null, i64 [[TMP7]]
654+
; STRIDED-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr null, i64 [[TMP9]]
655+
; STRIDED-NEXT: [[NEXT_GEP3:%.*]] = getelementptr i8, ptr null, i64 [[TMP11]]
656+
; STRIDED-NEXT: [[TMP12:%.*]] = getelementptr i64, ptr [[A:%.*]], i64 [[INDEX]]
657+
; STRIDED-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP12]], align 8
658+
; STRIDED-NEXT: [[TMP14:%.*]] = getelementptr i64, ptr [[NEXT_GEP3]], i64 [[STRIDE]]
659+
; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
660+
; STRIDED-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
661+
; STRIDED-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
662+
; STRIDED: middle.block:
663+
; STRIDED-NEXT: [[TMP16:%.*]] = extractelement <4 x i64> [[WIDE_LOAD]], i32 3
664+
; STRIDED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
665+
; STRIDED-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
666+
; STRIDED: scalar.ph:
667+
; STRIDED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
668+
; STRIDED-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP3]], [[MIDDLE_BLOCK]] ], [ null, [[ENTRY]] ]
669+
; STRIDED-NEXT: br label [[FOR_BODY:%.*]]
670+
; STRIDED: for.body:
671+
; STRIDED-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
672+
; STRIDED-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL4]], [[SCALAR_PH]] ]
673+
; STRIDED-NEXT: [[ARRAYIDX:%.*]] = getelementptr i64, ptr [[A]], i64 [[INDVAR]]
674+
; STRIDED-NEXT: [[TMP17:%.*]] = load i64, ptr [[ARRAYIDX]], align 8
675+
; STRIDED-NEXT: [[PTR_IV_NEXT]] = getelementptr i64, ptr [[PTR_IV]], i64 [[STRIDE]]
676+
; STRIDED-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1
677+
; STRIDED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVAR]], [[N]]
678+
; STRIDED-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
679+
; STRIDED: exit:
680+
; STRIDED-NEXT: [[DOTLCSSA:%.*]] = phi i64 [ [[TMP17]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
681+
; STRIDED-NEXT: [[PTR_IV_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_NEXT]], [[FOR_BODY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ]
682+
; STRIDED-NEXT: [[CAST_PTR:%.*]] = ptrtoint ptr [[PTR_IV_NEXT_LCSSA]] to i64
683+
; STRIDED-NEXT: [[RESULT:%.*]] = add i64 [[CAST_PTR]], [[DOTLCSSA]]
684+
; STRIDED-NEXT: ret i64 [[RESULT]]
685+
;
686+
entry:
687+
br label %for.body
688+
689+
for.body:
690+
%indvar = phi i64 [ %indvar.next, %for.body ], [ 0, %entry ]
691+
%ptr.iv = phi ptr [ %ptr.iv.next, %for.body ], [ null, %entry ]
692+
%arrayidx = getelementptr i64, ptr %a, i64 %indvar
693+
%0 = load i64, ptr %arrayidx, align 8
694+
%ptr.iv.next = getelementptr i64, ptr %ptr.iv, i64 %stride
695+
%indvar.next = add i64 %indvar, 1
696+
%exitcond.not = icmp eq i64 %indvar, %n
697+
br i1 %exitcond.not, label %exit, label %for.body
698+
699+
exit:
700+
%cast.ptr = ptrtoint ptr %ptr.iv.next to i64
701+
%result = add i64 %cast.ptr, %0
702+
ret i64 %result
703+
}

0 commit comments

Comments
 (0)