Skip to content

Commit 778d3c8

Browse files
sushgokhsushgokh
authored
[NFC] Partial reduce test to demonstrate regression post commit #cc9c64d (llvm#162681)
We have seen performance regression for several instances of the Numba benchmark, with some ranging around 70%, on Neoverse-v2 post llvm#158641. The mentioned case is short reproducer of the same. See https://godbolt.org/z/j9Mj5WM7c for the IR differences.. A future patch will address this.
1 parent 393d57c commit 778d3c8

File tree

1 file changed

+117
-0
lines changed

1 file changed

+117
-0
lines changed

‎llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce.ll‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1093,6 +1093,123 @@ exit:
10931093
ret i32 %add.lcssa
10941094
}
10951095

1096+
define i64 @sext_reduction_i32_to_i64(ptr %arr, i64 %n) #1 {
1097+
; CHECK-INTERLEAVE1-LABEL: define i64 @sext_reduction_i32_to_i64(
1098+
; CHECK-INTERLEAVE1-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) #[[ATTR2]] {
1099+
; CHECK-INTERLEAVE1-NEXT: entry:
1100+
; CHECK-INTERLEAVE1-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
1101+
; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
1102+
; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 1
1103+
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
1104+
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
1105+
; CHECK-INTERLEAVE1: vector.ph:
1106+
; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
1107+
; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
1108+
; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP3]]
1109+
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
1110+
; CHECK-INTERLEAVE1-NEXT: br label [[VECTOR_BODY:%.*]]
1111+
; CHECK-INTERLEAVE1: vector.body:
1112+
; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1113+
; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
1114+
; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[INDEX]]
1115+
; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[TMP4]], align 4
1116+
; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = sext <vscale x 2 x i32> [[WIDE_LOAD]] to <vscale x 2 x i64>
1117+
; CHECK-INTERLEAVE1-NEXT: [[TMP6]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP5]]
1118+
; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
1119+
; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
1120+
; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
1121+
; CHECK-INTERLEAVE1: middle.block:
1122+
; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP6]])
1123+
; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
1124+
; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
1125+
; CHECK-INTERLEAVE1: scalar.ph:
1126+
;
1127+
; CHECK-INTERLEAVED-LABEL: define i64 @sext_reduction_i32_to_i64(
1128+
; CHECK-INTERLEAVED-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) #[[ATTR2]] {
1129+
; CHECK-INTERLEAVED-NEXT: entry:
1130+
; CHECK-INTERLEAVED-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
1131+
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
1132+
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
1133+
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
1134+
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
1135+
; CHECK-INTERLEAVED: vector.ph:
1136+
; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
1137+
; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
1138+
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP3]]
1139+
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
1140+
; CHECK-INTERLEAVED-NEXT: br label [[VECTOR_BODY:%.*]]
1141+
; CHECK-INTERLEAVED: vector.body:
1142+
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1143+
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ]
1144+
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
1145+
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[INDEX]]
1146+
; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
1147+
; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
1148+
; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i64 [[TMP6]]
1149+
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[TMP4]], align 4
1150+
; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 2 x i32>, ptr [[TMP7]], align 4
1151+
; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = sext <vscale x 2 x i32> [[WIDE_LOAD]] to <vscale x 2 x i64>
1152+
; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = sext <vscale x 2 x i32> [[WIDE_LOAD2]] to <vscale x 2 x i64>
1153+
; CHECK-INTERLEAVED-NEXT: [[TMP10]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP8]]
1154+
; CHECK-INTERLEAVED-NEXT: [[TMP11]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP9]]
1155+
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
1156+
; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
1157+
; CHECK-INTERLEAVED-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
1158+
; CHECK-INTERLEAVED: middle.block:
1159+
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 2 x i64> [[TMP11]], [[TMP10]]
1160+
; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[BIN_RDX]])
1161+
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
1162+
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
1163+
; CHECK-INTERLEAVED: scalar.ph:
1164+
;
1165+
; CHECK-MAXBW-LABEL: define i64 @sext_reduction_i32_to_i64(
1166+
; CHECK-MAXBW-SAME: ptr [[ARR:%.*]], i64 [[N:%.*]]) #[[ATTR2]] {
1167+
; CHECK-MAXBW-NEXT: entry:
1168+
; CHECK-MAXBW-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N]], i64 1)
1169+
; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
1170+
; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 1
1171+
; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
1172+
; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
1173+
; CHECK-MAXBW: vector.ph:
1174+
; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
1175+
; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
1176+
; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP3]]
1177+
; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
1178+
; CHECK-MAXBW-NEXT: br label [[VECTOR_BODY:%.*]]
1179+
; CHECK-MAXBW: vector.body:
1180+
; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
1181+
; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP6:%.*]], [[VECTOR_BODY]] ]
1182+
; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[ARR]], i64 [[INDEX]]
1183+
; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[TMP4]], align 4
1184+
; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = sext <vscale x 2 x i32> [[WIDE_LOAD]] to <vscale x 2 x i64>
1185+
; CHECK-MAXBW-NEXT: [[TMP6]] = add <vscale x 2 x i64> [[VEC_PHI]], [[TMP5]]
1186+
; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
1187+
; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
1188+
; CHECK-MAXBW-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]]
1189+
; CHECK-MAXBW: middle.block:
1190+
; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP6]])
1191+
; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[UMAX]], [[N_VEC]]
1192+
; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
1193+
; CHECK-MAXBW: scalar.ph:
1194+
;
1195+
entry:
1196+
br label %loop
1197+
1198+
loop:
1199+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
1200+
%acc = phi i64 [ 0, %entry ], [ %add, %loop ]
1201+
%gep = getelementptr inbounds i32, ptr %arr, i64 %iv
1202+
%load = load i32, ptr %gep
1203+
%sext = sext i32 %load to i64
1204+
%add = add i64 %acc, %sext
1205+
%iv.next = add i64 %iv, 1
1206+
%cmp = icmp ult i64 %iv.next, %n
1207+
br i1 %cmp, label %loop, label %exit
1208+
1209+
exit:
1210+
ret i64 %add
1211+
}
1212+
10961213

10971214
!0 = distinct !{!0, !1}
10981215
!1 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}

0 commit comments

Comments
 (0)