Skip to content

Commit 87b837b

Browse files
huntergr-armhuntergr-arm
committed
Add a test case with extra arithmetic operations
1 parent 2b43298 commit 87b837b

File tree

1 file changed

+153
-0
lines changed

1 file changed

+153
-0
lines changed

‎llvm/test/Transforms/LoopVectorize/AArch64/conditional-scalar-assignment.ll‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -464,3 +464,156 @@ loop:
464464
exit:
465465
ret i32 %select.data
466466
}
467+
468+
;; Add more work to the loop besides the CSA to check cost modelling for NEON.
469+
define i32 @int_select_with_extra_arith_payload(i64 %N, ptr readonly %A, ptr readonly %B, ptr noalias %C, i32 %a) {
470+
; NEON-LABEL: define i32 @int_select_with_extra_arith_payload(
471+
; NEON-SAME: i64 [[N:%.*]], ptr readonly [[A:%.*]], ptr readonly [[B:%.*]], ptr noalias [[C:%.*]], i32 [[A:%.*]]) {
472+
; NEON-NEXT: [[ENTRY:.*]]:
473+
; NEON-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
474+
; NEON-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
475+
; NEON: [[VECTOR_PH]]:
476+
; NEON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
477+
; NEON-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
478+
; NEON-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i64 0
479+
; NEON-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
480+
; NEON-NEXT: br label %[[VECTOR_BODY:.*]]
481+
; NEON: [[VECTOR_BODY]]:
482+
; NEON-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
483+
; NEON-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
484+
; NEON-NEXT: [[TMP0:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
485+
; NEON-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
486+
; NEON-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
487+
; NEON-NEXT: [[TMP2:%.*]] = mul <4 x i32> [[WIDE_LOAD]], splat (i32 13)
488+
; NEON-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
489+
; NEON-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
490+
; NEON-NEXT: [[TMP4:%.*]] = mul <4 x i32> [[WIDE_LOAD1]], splat (i32 5)
491+
; NEON-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
492+
; NEON-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
493+
; NEON-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP6]], align 4
494+
; NEON-NEXT: [[TMP7:%.*]] = icmp slt <4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
495+
; NEON-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP7]]
496+
; NEON-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]])
497+
; NEON-NEXT: [[TMP10]] = select i1 [[TMP9]], <4 x i1> [[TMP7]], <4 x i1> [[TMP0]]
498+
; NEON-NEXT: [[TMP11]] = select i1 [[TMP9]], <4 x i32> [[WIDE_LOAD]], <4 x i32> [[VEC_PHI]]
499+
; NEON-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
500+
; NEON-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
501+
; NEON-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
502+
; NEON: [[MIDDLE_BLOCK]]:
503+
; NEON-NEXT: [[TMP13:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.v4i32(<4 x i32> [[TMP11]], <4 x i1> [[TMP10]], i32 -1)
504+
; NEON-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
505+
; NEON-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
506+
; NEON: [[SCALAR_PH]]:
507+
; NEON-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
508+
; NEON-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
509+
; NEON-NEXT: br label %[[LOOP:.*]]
510+
; NEON: [[LOOP]]:
511+
; NEON-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
512+
; NEON-NEXT: [[A_PHI:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SELECT_A:%.*]], %[[LOOP]] ]
513+
; NEON-NEXT: [[A_ADDR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
514+
; NEON-NEXT: [[LD_A:%.*]] = load i32, ptr [[A_ADDR]], align 4
515+
; NEON-NEXT: [[MUL_A:%.*]] = mul i32 [[LD_A]], 13
516+
; NEON-NEXT: [[B_ADDR:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
517+
; NEON-NEXT: [[LD_B:%.*]] = load i32, ptr [[B_ADDR]], align 4
518+
; NEON-NEXT: [[MUL_B:%.*]] = mul i32 [[LD_B]], 5
519+
; NEON-NEXT: [[ADD:%.*]] = add i32 [[MUL_A]], [[MUL_B]]
520+
; NEON-NEXT: [[C_ADDR:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
521+
; NEON-NEXT: store i32 [[ADD]], ptr [[C_ADDR]], align 4
522+
; NEON-NEXT: [[SELECT_CMP:%.*]] = icmp slt i32 [[A]], [[LD_A]]
523+
; NEON-NEXT: [[SELECT_A]] = select i1 [[SELECT_CMP]], i32 [[LD_A]], i32 [[A_PHI]]
524+
; NEON-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
525+
; NEON-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
526+
; NEON-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
527+
; NEON: [[EXIT]]:
528+
; NEON-NEXT: [[SELECT_A_LCSSA:%.*]] = phi i32 [ [[SELECT_A]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
529+
; NEON-NEXT: ret i32 [[SELECT_A_LCSSA]]
530+
;
531+
; SVE-LABEL: define i32 @int_select_with_extra_arith_payload(
532+
; SVE-SAME: i64 [[N:%.*]], ptr readonly [[A:%.*]], ptr readonly [[B:%.*]], ptr noalias [[C:%.*]], i32 [[A:%.*]]) #[[ATTR0]] {
533+
; SVE-NEXT: [[ENTRY:.*]]:
534+
; SVE-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
535+
; SVE-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
536+
; SVE-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
537+
; SVE-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
538+
; SVE: [[VECTOR_PH]]:
539+
; SVE-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
540+
; SVE-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
541+
; SVE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
542+
; SVE-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
543+
; SVE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[A]], i64 0
544+
; SVE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
545+
; SVE-NEXT: br label %[[VECTOR_BODY:.*]]
546+
; SVE: [[VECTOR_BODY]]:
547+
; SVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
548+
; SVE-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP15:%.*]], %[[VECTOR_BODY]] ]
549+
; SVE-NEXT: [[TMP4:%.*]] = phi <vscale x 4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP14:%.*]], %[[VECTOR_BODY]] ]
550+
; SVE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
551+
; SVE-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP5]], align 4
552+
; SVE-NEXT: [[TMP6:%.*]] = mul <vscale x 4 x i32> [[WIDE_LOAD]], splat (i32 13)
553+
; SVE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
554+
; SVE-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 4 x i32>, ptr [[TMP7]], align 4
555+
; SVE-NEXT: [[TMP8:%.*]] = mul <vscale x 4 x i32> [[WIDE_LOAD1]], splat (i32 5)
556+
; SVE-NEXT: [[TMP9:%.*]] = add <vscale x 4 x i32> [[TMP6]], [[TMP8]]
557+
; SVE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
558+
; SVE-NEXT: store <vscale x 4 x i32> [[TMP9]], ptr [[TMP10]], align 4
559+
; SVE-NEXT: [[TMP11:%.*]] = icmp slt <vscale x 4 x i32> [[BROADCAST_SPLAT]], [[WIDE_LOAD]]
560+
; SVE-NEXT: [[TMP12:%.*]] = freeze <vscale x 4 x i1> [[TMP11]]
561+
; SVE-NEXT: [[TMP13:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1(<vscale x 4 x i1> [[TMP12]])
562+
; SVE-NEXT: [[TMP14]] = select i1 [[TMP13]], <vscale x 4 x i1> [[TMP11]], <vscale x 4 x i1> [[TMP4]]
563+
; SVE-NEXT: [[TMP15]] = select i1 [[TMP13]], <vscale x 4 x i32> [[WIDE_LOAD]], <vscale x 4 x i32> [[VEC_PHI]]
564+
; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
565+
; SVE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
566+
; SVE-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
567+
; SVE: [[MIDDLE_BLOCK]]:
568+
; SVE-NEXT: [[TMP17:%.*]] = call i32 @llvm.experimental.vector.extract.last.active.nxv4i32(<vscale x 4 x i32> [[TMP15]], <vscale x 4 x i1> [[TMP14]], i32 -1)
569+
; SVE-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
570+
; SVE-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
571+
; SVE: [[SCALAR_PH]]:
572+
; SVE-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
573+
; SVE-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP17]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ]
574+
; SVE-NEXT: br label %[[LOOP:.*]]
575+
; SVE: [[LOOP]]:
576+
; SVE-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
577+
; SVE-NEXT: [[A_PHI:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SELECT_A:%.*]], %[[LOOP]] ]
578+
; SVE-NEXT: [[A_ADDR:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
579+
; SVE-NEXT: [[LD_A:%.*]] = load i32, ptr [[A_ADDR]], align 4
580+
; SVE-NEXT: [[MUL_A:%.*]] = mul i32 [[LD_A]], 13
581+
; SVE-NEXT: [[B_ADDR:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
582+
; SVE-NEXT: [[LD_B:%.*]] = load i32, ptr [[B_ADDR]], align 4
583+
; SVE-NEXT: [[MUL_B:%.*]] = mul i32 [[LD_B]], 5
584+
; SVE-NEXT: [[ADD:%.*]] = add i32 [[MUL_A]], [[MUL_B]]
585+
; SVE-NEXT: [[C_ADDR:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
586+
; SVE-NEXT: store i32 [[ADD]], ptr [[C_ADDR]], align 4
587+
; SVE-NEXT: [[SELECT_CMP:%.*]] = icmp slt i32 [[A]], [[LD_A]]
588+
; SVE-NEXT: [[SELECT_A]] = select i1 [[SELECT_CMP]], i32 [[LD_A]], i32 [[A_PHI]]
589+
; SVE-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
590+
; SVE-NEXT: [[EXIT_CMP:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
591+
; SVE-NEXT: br i1 [[EXIT_CMP]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
592+
; SVE: [[EXIT]]:
593+
; SVE-NEXT: [[SELECT_A_LCSSA:%.*]] = phi i32 [ [[SELECT_A]], %[[LOOP]] ], [ [[TMP17]], %[[MIDDLE_BLOCK]] ]
594+
; SVE-NEXT: ret i32 [[SELECT_A_LCSSA]]
595+
;
596+
entry:
597+
br label %loop
598+
599+
loop:
600+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
601+
%A.phi = phi i32 [ -1, %entry ], [ %select.A, %loop ]
602+
%A.addr = getelementptr inbounds i32, ptr %A, i64 %iv
603+
%ld.A = load i32, ptr %A.addr, align 4
604+
%mul.A = mul i32 %ld.A, 13
605+
%B.addr = getelementptr inbounds i32, ptr %B, i64 %iv
606+
%ld.B = load i32, ptr %B.addr, align 4
607+
%mul.B = mul i32 %ld.B, 5
608+
%add = add i32 %mul.A, %mul.B
609+
%C.addr = getelementptr inbounds i32, ptr %C, i64 %iv
610+
store i32 %add, ptr %C.addr, align 4
611+
%select.cmp = icmp slt i32 %a, %ld.A
612+
%select.A = select i1 %select.cmp, i32 %ld.A, i32 %A.phi
613+
%iv.next = add nuw nsw i64 %iv, 1
614+
%exit.cmp = icmp eq i64 %iv.next, %N
615+
br i1 %exit.cmp, label %exit, label %loop
616+
617+
exit:
618+
ret i32 %select.A
619+
}

0 commit comments

Comments
 (0)