diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
index 1b22523e9f5bd..6d34964a0542f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
@@ -3419,6 +3419,7 @@ define dso_local void @not_dotp_high_register_pressure(ptr %a, ptr %b, ptr %sum,
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[WIDE_TRIP_COUNT]], [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_FOR_COND_CLEANUP_CRIT_EDGE:%.*]], label [[SCALAR_PH]]
 ; CHECK-MAXBW:       scalar.ph:
+;
 entry:
   %cmp100 = icmp sgt i32 %n, 0
   br i1 %cmp100, label %for.body.lr.ph, label %for.cond.cleanup
@@ -3530,9 +3531,495 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
   br i1 %exitcond.not, label %for.cond.for.cond.cleanup_crit_edge, label %for.body, !llvm.loop !8
 }
 
+define void @usdot_i8_to_i32(ptr %A, ptr %B, i32 %K, ptr %sum, i32 %C2) #2 {
+; CHECK-INTERLEAVE1-LABEL: define void @usdot_i8_to_i32(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[K:%.*]], ptr [[SUM:%.*]], i32 [[C2:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT:  entry:
+; CHECK-INTERLEAVE1-NEXT:    br label [[PREHEADER:%.*]]
+; CHECK-INTERLEAVE1:       preheader:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = sext i32 [[C2]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[TRIP_COUNT:%.*]] = zext i32 [[K]] to i64
+; CHECK-INTERLEAVE1-NEXT:    [[INVARIANT_GEP_B_0:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-INTERLEAVE1-NEXT:    [[INVARIANT_GEP_B_1:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 1
+; CHECK-INTERLEAVE1-NEXT:    [[INVARIANT_GEP_B_2:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 2
+; CHECK-INTERLEAVE1-NEXT:    [[INVARIANT_GEP_B_3:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 3
+; CHECK-INTERLEAVE1-NEXT:    [[INVARIANT_GEP_B_4:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 4
+; CHECK-INTERLEAVE1-NEXT:    [[INVARIANT_GEP_B_5:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 5
+; CHECK-INTERLEAVE1-NEXT:    [[INVARIANT_GEP_B_6:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 6
+; CHECK-INTERLEAVE1-NEXT:    [[INVARIANT_GEP_B_7:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 7
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TRIP_COUNT]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVE1:       vector.ph:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TRIP_COUNT]], [[TMP4]]
+; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVE1:       vector.body:
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP49:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP44:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP39:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP34:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP29:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI6:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[VEC_PHI7:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP8]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD8]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = mul nsw <vscale x 4 x i32> [[TMP12]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP14]] = add <vscale x 4 x i32> [[TMP13]], [[VEC_PHI7]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_1]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 4 x i8>, ptr [[TMP16]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD9]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = mul nsw <vscale x 4 x i32> [[TMP17]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP19]] = add <vscale x 4 x i32> [[TMP18]], [[VEC_PHI6]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_2]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i8>, ptr [[TMP21]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP22:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD10]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP23:%.*]] = mul nsw <vscale x 4 x i32> [[TMP22]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP24]] = add <vscale x 4 x i32> [[TMP23]], [[VEC_PHI5]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_3]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD11:%.*]] = load <vscale x 4 x i8>, ptr [[TMP26]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP27:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD11]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP28:%.*]] = mul nsw <vscale x 4 x i32> [[TMP27]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP29]] = add <vscale x 4 x i32> [[TMP28]], [[VEC_PHI4]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_4]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP30]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD12:%.*]] = load <vscale x 4 x i8>, ptr [[TMP31]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP32:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD12]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP33:%.*]] = mul nsw <vscale x 4 x i32> [[TMP32]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP34]] = add <vscale x 4 x i32> [[TMP33]], [[VEC_PHI3]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP35:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_5]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[TMP35]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD13:%.*]] = load <vscale x 4 x i8>, ptr [[TMP36]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP37:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD13]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP38:%.*]] = mul nsw <vscale x 4 x i32> [[TMP37]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP39]] = add <vscale x 4 x i32> [[TMP38]], [[VEC_PHI2]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP40:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_6]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP41:%.*]] = getelementptr i8, ptr [[TMP40]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD14:%.*]] = load <vscale x 4 x i8>, ptr [[TMP41]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP42:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD14]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP43:%.*]] = mul nsw <vscale x 4 x i32> [[TMP42]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP44]] = add <vscale x 4 x i32> [[TMP43]], [[VEC_PHI1]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP45:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_7]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP46:%.*]] = getelementptr i8, ptr [[TMP45]], i32 0
+; CHECK-INTERLEAVE1-NEXT:    [[WIDE_LOAD15:%.*]] = load <vscale x 4 x i8>, ptr [[TMP46]], align 1
+; CHECK-INTERLEAVE1-NEXT:    [[TMP47:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD15]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVE1-NEXT:    [[TMP48:%.*]] = mul nsw <vscale x 4 x i32> [[TMP47]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP49]] = add <vscale x 4 x i32> [[TMP48]], [[VEC_PHI]]
+; CHECK-INTERLEAVE1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-INTERLEAVE1-NEXT:    [[TMP50:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP50]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; CHECK-INTERLEAVE1:       middle.block:
+; CHECK-INTERLEAVE1-NEXT:    [[TMP51:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP49]])
+; CHECK-INTERLEAVE1-NEXT:    [[TMP52:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP44]])
+; CHECK-INTERLEAVE1-NEXT:    [[TMP53:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP39]])
+; CHECK-INTERLEAVE1-NEXT:    [[TMP54:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP34]])
+; CHECK-INTERLEAVE1-NEXT:    [[TMP55:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP29]])
+; CHECK-INTERLEAVE1-NEXT:    [[TMP56:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP24]])
+; CHECK-INTERLEAVE1-NEXT:    [[TMP57:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP19]])
+; CHECK-INTERLEAVE1-NEXT:    [[TMP58:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
+; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TRIP_COUNT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVE1:       scalar.ph:
+;
+; CHECK-INTERLEAVED-LABEL: define void @usdot_i8_to_i32(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[K:%.*]], ptr [[SUM:%.*]], i32 [[C2:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT:  entry:
+; CHECK-INTERLEAVED-NEXT:    br label [[PREHEADER:%.*]]
+; CHECK-INTERLEAVED:       preheader:
+; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = sext i32 [[C2]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[TRIP_COUNT:%.*]] = zext i32 [[K]] to i64
+; CHECK-INTERLEAVED-NEXT:    [[INVARIANT_GEP_B_0:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-INTERLEAVED-NEXT:    [[INVARIANT_GEP_B_1:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 1
+; CHECK-INTERLEAVED-NEXT:    [[INVARIANT_GEP_B_2:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 2
+; CHECK-INTERLEAVED-NEXT:    [[INVARIANT_GEP_B_3:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 3
+; CHECK-INTERLEAVED-NEXT:    [[INVARIANT_GEP_B_4:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 4
+; CHECK-INTERLEAVED-NEXT:    [[INVARIANT_GEP_B_5:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 5
+; CHECK-INTERLEAVED-NEXT:    [[INVARIANT_GEP_B_6:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 6
+; CHECK-INTERLEAVED-NEXT:    [[INVARIANT_GEP_B_7:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 7
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TRIP_COUNT]], [[TMP2]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-INTERLEAVED:       vector.ph:
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TRIP_COUNT]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-INTERLEAVED:       vector.body:
+; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP100:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP101:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP89:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP90:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI4:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP78:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP79:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI6:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP67:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI7:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP68:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI8:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP56:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI9:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP57:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI10:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP45:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI11:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI12:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP34:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI13:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP35:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI14:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP23:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[VEC_PHI15:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP24:%.*]], [[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 [[TMP10]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP8]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD16:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = zext <vscale x 4 x i8> [[WIDE_LOAD16]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[TMP14]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP17]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD17:%.*]] = load <vscale x 4 x i8>, ptr [[TMP15]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD18:%.*]] = load <vscale x 4 x i8>, ptr [[TMP18]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD17]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD18]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = mul nsw <vscale x 4 x i32> [[TMP19]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = mul nsw <vscale x 4 x i32> [[TMP20]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP23]] = add <vscale x 4 x i32> [[TMP21]], [[VEC_PHI14]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP24]] = add <vscale x 4 x i32> [[TMP22]], [[VEC_PHI15]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_1]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[TMP25]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = getelementptr i8, ptr [[TMP25]], i64 [[TMP28]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD19:%.*]] = load <vscale x 4 x i8>, ptr [[TMP26]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD20:%.*]] = load <vscale x 4 x i8>, ptr [[TMP29]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD19]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP31:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD20]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP32:%.*]] = mul nsw <vscale x 4 x i32> [[TMP30]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP33:%.*]] = mul nsw <vscale x 4 x i32> [[TMP31]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP34]] = add <vscale x 4 x i32> [[TMP32]], [[VEC_PHI12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP35]] = add <vscale x 4 x i32> [[TMP33]], [[VEC_PHI13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP36:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_2]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP37:%.*]] = getelementptr i8, ptr [[TMP36]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP38:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP39:%.*]] = mul i64 [[TMP38]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP40:%.*]] = getelementptr i8, ptr [[TMP36]], i64 [[TMP39]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD21:%.*]] = load <vscale x 4 x i8>, ptr [[TMP37]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD22:%.*]] = load <vscale x 4 x i8>, ptr [[TMP40]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP41:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD21]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP42:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD22]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP43:%.*]] = mul nsw <vscale x 4 x i32> [[TMP41]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP44:%.*]] = mul nsw <vscale x 4 x i32> [[TMP42]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP45]] = add <vscale x 4 x i32> [[TMP43]], [[VEC_PHI10]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP46]] = add <vscale x 4 x i32> [[TMP44]], [[VEC_PHI11]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP47:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_3]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP48:%.*]] = getelementptr i8, ptr [[TMP47]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP49:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP50:%.*]] = mul i64 [[TMP49]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP51:%.*]] = getelementptr i8, ptr [[TMP47]], i64 [[TMP50]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD23:%.*]] = load <vscale x 4 x i8>, ptr [[TMP48]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD24:%.*]] = load <vscale x 4 x i8>, ptr [[TMP51]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP52:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD23]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP53:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD24]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP54:%.*]] = mul nsw <vscale x 4 x i32> [[TMP52]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP55:%.*]] = mul nsw <vscale x 4 x i32> [[TMP53]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP56]] = add <vscale x 4 x i32> [[TMP54]], [[VEC_PHI8]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP57]] = add <vscale x 4 x i32> [[TMP55]], [[VEC_PHI9]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP58:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_4]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP59:%.*]] = getelementptr i8, ptr [[TMP58]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP60:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP61:%.*]] = mul i64 [[TMP60]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP62:%.*]] = getelementptr i8, ptr [[TMP58]], i64 [[TMP61]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD25:%.*]] = load <vscale x 4 x i8>, ptr [[TMP59]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD26:%.*]] = load <vscale x 4 x i8>, ptr [[TMP62]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP63:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD25]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP64:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD26]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP65:%.*]] = mul nsw <vscale x 4 x i32> [[TMP63]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP66:%.*]] = mul nsw <vscale x 4 x i32> [[TMP64]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP67]] = add <vscale x 4 x i32> [[TMP65]], [[VEC_PHI6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP68]] = add <vscale x 4 x i32> [[TMP66]], [[VEC_PHI7]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP69:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_5]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP70:%.*]] = getelementptr i8, ptr [[TMP69]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP71:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP72:%.*]] = mul i64 [[TMP71]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP73:%.*]] = getelementptr i8, ptr [[TMP69]], i64 [[TMP72]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD27:%.*]] = load <vscale x 4 x i8>, ptr [[TMP70]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD28:%.*]] = load <vscale x 4 x i8>, ptr [[TMP73]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP74:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD27]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP75:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD28]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP76:%.*]] = mul nsw <vscale x 4 x i32> [[TMP74]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP77:%.*]] = mul nsw <vscale x 4 x i32> [[TMP75]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP78]] = add <vscale x 4 x i32> [[TMP76]], [[VEC_PHI4]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP79]] = add <vscale x 4 x i32> [[TMP77]], [[VEC_PHI5]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP80:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_6]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP81:%.*]] = getelementptr i8, ptr [[TMP80]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP82:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP83:%.*]] = mul i64 [[TMP82]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP84:%.*]] = getelementptr i8, ptr [[TMP80]], i64 [[TMP83]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD29:%.*]] = load <vscale x 4 x i8>, ptr [[TMP81]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD30:%.*]] = load <vscale x 4 x i8>, ptr [[TMP84]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP85:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD29]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP86:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD30]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP87:%.*]] = mul nsw <vscale x 4 x i32> [[TMP85]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP88:%.*]] = mul nsw <vscale x 4 x i32> [[TMP86]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP89]] = add <vscale x 4 x i32> [[TMP87]], [[VEC_PHI2]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP90]] = add <vscale x 4 x i32> [[TMP88]], [[VEC_PHI3]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP91:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_7]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP92:%.*]] = getelementptr i8, ptr [[TMP91]], i32 0
+; CHECK-INTERLEAVED-NEXT:    [[TMP93:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT:    [[TMP94:%.*]] = mul i64 [[TMP93]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP95:%.*]] = getelementptr i8, ptr [[TMP91]], i64 [[TMP94]]
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD31:%.*]] = load <vscale x 4 x i8>, ptr [[TMP92]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD32:%.*]] = load <vscale x 4 x i8>, ptr [[TMP95]], align 1
+; CHECK-INTERLEAVED-NEXT:    [[TMP96:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD31]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP97:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD32]] to <vscale x 4 x i32>
+; CHECK-INTERLEAVED-NEXT:    [[TMP98:%.*]] = mul nsw <vscale x 4 x i32> [[TMP96]], [[TMP12]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP99:%.*]] = mul nsw <vscale x 4 x i32> [[TMP97]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP100]] = add <vscale x 4 x i32> [[TMP98]], [[VEC_PHI]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP101]] = add <vscale x 4 x i32> [[TMP99]], [[VEC_PHI1]]
+; CHECK-INTERLEAVED-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP102:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP102]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP43:![0-9]+]]
+; CHECK-INTERLEAVED:       middle.block:
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP101]], [[TMP100]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP103:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX33:%.*]] = add <vscale x 4 x i32> [[TMP90]], [[TMP89]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP104:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX33]])
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX34:%.*]] = add <vscale x 4 x i32> [[TMP79]], [[TMP78]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP105:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX34]])
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX35:%.*]] = add <vscale x 4 x i32> [[TMP68]], [[TMP67]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP106:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX35]])
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX36:%.*]] = add <vscale x 4 x i32> [[TMP57]], [[TMP56]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP107:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX36]])
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX37:%.*]] = add <vscale x 4 x i32> [[TMP46]], [[TMP45]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP108:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX37]])
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX38:%.*]] = add <vscale x 4 x i32> [[TMP35]], [[TMP34]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP109:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX38]])
+; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX39:%.*]] = add <vscale x 4 x i32> [[TMP24]], [[TMP23]]
+; CHECK-INTERLEAVED-NEXT:    [[TMP110:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX39]])
+; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TRIP_COUNT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-INTERLEAVED:       scalar.ph:
+;
+; CHECK-MAXBW-LABEL: define void @usdot_i8_to_i32(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i32 [[K:%.*]], ptr [[SUM:%.*]], i32 [[C2:%.*]]) #[[ATTR2:[0-9]+]] {
+; CHECK-MAXBW-NEXT:  entry:
+; CHECK-MAXBW-NEXT:    br label [[PREHEADER:%.*]]
+; CHECK-MAXBW:       preheader:
+; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = sext i32 [[C2]] to i64
+; CHECK-MAXBW-NEXT:    [[TRIP_COUNT:%.*]] = zext i32 [[K]] to i64
+; CHECK-MAXBW-NEXT:    [[INVARIANT_GEP_B_0:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP0]]
+; CHECK-MAXBW-NEXT:    [[INVARIANT_GEP_B_1:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 1
+; CHECK-MAXBW-NEXT:    [[INVARIANT_GEP_B_2:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 2
+; CHECK-MAXBW-NEXT:    [[INVARIANT_GEP_B_3:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 3
+; CHECK-MAXBW-NEXT:    [[INVARIANT_GEP_B_4:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 4
+; CHECK-MAXBW-NEXT:    [[INVARIANT_GEP_B_5:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 5
+; CHECK-MAXBW-NEXT:    [[INVARIANT_GEP_B_6:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 6
+; CHECK-MAXBW-NEXT:    [[INVARIANT_GEP_B_7:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 7
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TRIP_COUNT]], [[TMP2]]
+; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK-MAXBW:       vector.ph:
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TRIP_COUNT]], [[TMP4]]
+; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[TRIP_COUNT]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK-MAXBW:       vector.body:
+; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE22:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI1:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE20:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI2:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE18:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI3:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE16:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI4:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE14:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI5:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE12:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI6:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE10:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[VEC_PHI7:%.*]] = phi <vscale x 2 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], [[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_0]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP12:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD8]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP13:%.*]] = mul nsw <vscale x 8 x i32> [[TMP12]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI7]], <vscale x 8 x i32> [[TMP13]])
+; CHECK-MAXBW-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_1]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[TMP14]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 8 x i8>, ptr [[TMP15]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD9]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = mul nsw <vscale x 8 x i32> [[TMP16]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE10]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI6]], <vscale x 8 x i32> [[TMP17]])
+; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_2]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP18]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD11:%.*]] = load <vscale x 8 x i8>, ptr [[TMP19]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD11]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = mul nsw <vscale x 8 x i32> [[TMP20]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE12]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI5]], <vscale x 8 x i32> [[TMP21]])
+; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_3]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP23:%.*]] = getelementptr i8, ptr [[TMP22]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD13:%.*]] = load <vscale x 8 x i8>, ptr [[TMP23]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP24:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD13]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP25:%.*]] = mul nsw <vscale x 8 x i32> [[TMP24]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE14]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI4]], <vscale x 8 x i32> [[TMP25]])
+; CHECK-MAXBW-NEXT:    [[TMP26:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_4]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP27:%.*]] = getelementptr i8, ptr [[TMP26]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD15:%.*]] = load <vscale x 8 x i8>, ptr [[TMP27]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP28:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD15]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP29:%.*]] = mul nsw <vscale x 8 x i32> [[TMP28]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE16]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI3]], <vscale x 8 x i32> [[TMP29]])
+; CHECK-MAXBW-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_5]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP31:%.*]] = getelementptr i8, ptr [[TMP30]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD17:%.*]] = load <vscale x 8 x i8>, ptr [[TMP31]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP32:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD17]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP33:%.*]] = mul nsw <vscale x 8 x i32> [[TMP32]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE18]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI2]], <vscale x 8 x i32> [[TMP33]])
+; CHECK-MAXBW-NEXT:    [[TMP34:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_6]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP35:%.*]] = getelementptr i8, ptr [[TMP34]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD19:%.*]] = load <vscale x 8 x i8>, ptr [[TMP35]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP36:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD19]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP37:%.*]] = mul nsw <vscale x 8 x i32> [[TMP36]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE20]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI1]], <vscale x 8 x i32> [[TMP37]])
+; CHECK-MAXBW-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[INVARIANT_GEP_B_7]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT:    [[TMP39:%.*]] = getelementptr i8, ptr [[TMP38]], i32 0
+; CHECK-MAXBW-NEXT:    [[WIDE_LOAD21:%.*]] = load <vscale x 8 x i8>, ptr [[TMP39]], align 1
+; CHECK-MAXBW-NEXT:    [[TMP40:%.*]] = sext <vscale x 8 x i8> [[WIDE_LOAD21]] to <vscale x 8 x i32>
+; CHECK-MAXBW-NEXT:    [[TMP41:%.*]] = mul nsw <vscale x 8 x i32> [[TMP40]], [[TMP9]]
+; CHECK-MAXBW-NEXT:    [[PARTIAL_REDUCE22]] = call <vscale x 2 x i32> @llvm.experimental.vector.partial.reduce.add.nxv2i32.nxv8i32(<vscale x 2 x i32> [[VEC_PHI]], <vscale x 8 x i32> [[TMP41]])
+; CHECK-MAXBW-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]]
+; CHECK-MAXBW-NEXT:    [[TMP42:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[TMP42]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
+; CHECK-MAXBW:       middle.block:
+; CHECK-MAXBW-NEXT:    [[TMP43:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE22]])
+; CHECK-MAXBW-NEXT:    [[TMP44:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE20]])
+; CHECK-MAXBW-NEXT:    [[TMP45:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE18]])
+; CHECK-MAXBW-NEXT:    [[TMP46:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE16]])
+; CHECK-MAXBW-NEXT:    [[TMP47:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE14]])
+; CHECK-MAXBW-NEXT:    [[TMP48:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE12]])
+; CHECK-MAXBW-NEXT:    [[TMP49:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE10]])
+; CHECK-MAXBW-NEXT:    [[TMP50:%.*]] = call i32 @llvm.vector.reduce.add.nxv2i32(<vscale x 2 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TRIP_COUNT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK-MAXBW:       scalar.ph:
+;
+entry:
+  br label %preheader
+
+preheader:                                   ; preds = %entry
+  %0 = sext i32 %C2 to i64
+  %trip.count = zext nneg i32 %K to i64
+  %invariant.gep.b.0 = getelementptr i8, ptr %B, i64 %0
+  %invariant.gep.b.1 = getelementptr i8, ptr %invariant.gep.b.0, i64 1
+  %invariant.gep.b.2 = getelementptr i8, ptr %invariant.gep.b.0, i64 2
+  %invariant.gep.b.3 = getelementptr i8, ptr %invariant.gep.b.0, i64 3
+  %invariant.gep.b.4 = getelementptr i8, ptr %invariant.gep.b.0, i64 4
+  %invariant.gep.b.5 = getelementptr i8, ptr %invariant.gep.b.0, i64 5
+  %invariant.gep.b.6 = getelementptr i8, ptr %invariant.gep.b.0, i64 6
+  %invariant.gep.b.7 = getelementptr i8, ptr %invariant.gep.b.0, i64 7
+  br label %for.body
+
+for.body:                                         ; preds = %preheader, %for.body
+  %iv = phi i64 [ 0, %preheader ], [ %iv.next, %for.body ]
+  %red0 = phi i32 [ 0, %preheader ], [ %red0.next, %for.body ]
+  %red1 = phi i32 [ 0, %preheader ], [ %red1.next, %for.body ]
+  %red2 = phi i32 [ 0, %preheader ], [ %red2.next, %for.body ]
+  %red3 = phi i32 [ 0, %preheader ], [ %red3.next, %for.body ]
+  %red4 = phi i32 [ 0, %preheader ], [ %red4.next, %for.body ]
+  %red5 = phi i32 [ 0, %preheader ], [ %red5.next, %for.body ]
+  %red6 = phi i32 [ 0, %preheader ], [ %red6.next, %for.body ]
+  %red7 = phi i32 [ 0, %preheader ], [ %red7.next, %for.body ]
+  %gep.a = getelementptr inbounds nuw i8, ptr %A, i64 %iv
+  %load.a = load i8, ptr %gep.a
+  %zext.a = zext i8 %load.a to i32
+  %gep.b.0 = getelementptr i8, ptr %invariant.gep.b.0, i64 %iv
+  %load.b.0 = load i8, ptr %gep.b.0
+  %sext.b.0 = sext i8 %load.b.0 to i32
+  %mul0 = mul nsw i32 %sext.b.0, %zext.a
+  %red7.next = add nsw i32 %mul0, %red7
+  %gep.b.1 = getelementptr i8, ptr %invariant.gep.b.1, i64 %iv
+  %load.b.1 = load i8, ptr %gep.b.1
+  %sext.b.1 = sext i8 %load.b.1 to i32
+  %mul1 = mul nsw i32 %sext.b.1, %zext.a
+  %red6.next = add nsw i32 %mul1, %red6
+  %gep.b.2 = getelementptr i8, ptr %invariant.gep.b.2, i64 %iv
+  %load.b.2 = load i8, ptr %gep.b.2
+  %sext.b.2 = sext i8 %load.b.2 to i32
+  %mul2 = mul nsw i32 %sext.b.2, %zext.a
+  %red5.next = add nsw i32 %mul2, %red5
+  %gep.b.3 = getelementptr i8, ptr %invariant.gep.b.3, i64 %iv
+  %load.b.3 = load i8, ptr %gep.b.3
+  %sext.b.3 = sext i8 %load.b.3 to i32
+  %mul3 = mul nsw i32 %sext.b.3, %zext.a
+  %red4.next = add nsw i32 %mul3, %red4
+  %gep.b.4 = getelementptr i8, ptr %invariant.gep.b.4, i64 %iv
+  %load.b.4 = load i8, ptr %gep.b.4
+  %sext.b.4 = sext i8 %load.b.4 to i32
+  %mul4 = mul nsw i32 %sext.b.4, %zext.a
+  %red3.next = add nsw i32 %mul4, %red3
+  %gep.b.5 = getelementptr i8, ptr %invariant.gep.b.5, i64 %iv
+  %load.b.5 = load i8, ptr %gep.b.5
+  %sext.b.5 = sext i8 %load.b.5 to i32
+  %mul5 = mul nsw i32 %sext.b.5, %zext.a
+  %red2.next = add nsw i32 %mul5, %red2
+  %gep.b.6 = getelementptr i8, ptr %invariant.gep.b.6, i64 %iv
+  %load.b.6 = load i8, ptr %gep.b.6
+  %sext.b.6 = sext i8 %load.b.6 to i32
+  %mul6 = mul nsw i32 %sext.b.6, %zext.a
+  %red1.next = add nsw i32 %mul6, %red1
+  %gep.b.7 = getelementptr i8, ptr %invariant.gep.b.7, i64 %iv
+  %load.b.7 = load i8, ptr %gep.b.7
+  %sext.b.7 = sext i8 %load.b.7 to i32
+  %mul7 = mul nsw i32 %sext.b.7, %zext.a
+  %red0.next = add nsw i32 %mul7, %red0
+  %iv.next = add nuw nsw i64 %iv, 1
+  %cmp = icmp eq i64 %iv.next, %trip.count
+  br i1 %cmp, label %exit, label %for.body, !llvm.loop !8
+
+exit:                        ; preds = %for.body
+  store i32 %red7.next, ptr %sum
+  %idx1 = getelementptr inbounds nuw i8, ptr %sum, i64 4
+  store i32 %red6.next, ptr %idx1
+  %idx2 = getelementptr inbounds nuw i8, ptr %sum, i64 8
+  store i32 %red5.next, ptr %idx2
+  %idx3 = getelementptr inbounds nuw i8, ptr %sum, i64 12
+  store i32 %red4.next, ptr %idx3
+  %idx4 = getelementptr inbounds nuw i8, ptr %sum, i64 16
+  store i32 %red3.next, ptr %idx4
+  %idx5 = getelementptr inbounds nuw i8, ptr %sum, i64 20
+  store i32 %red2.next, ptr %idx5
+  %idx6 = getelementptr inbounds nuw i8, ptr %sum, i64 24
+  store i32 %red1.next, ptr %idx6
+  %idx7 = getelementptr inbounds nuw i8, ptr %sum, i64 28
+  store i32 %red0.next, ptr %idx7
+  ret void
+}
+
 !7 = distinct !{!7, !8, !9, !10}
 !8 = !{!"llvm.loop.mustprogress"}
 !9 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
 !10 = !{!"llvm.loop.vectorize.enable", i1 true}
 attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
 attributes #1 = { vscale_range(1,16) "target-features"="+neon,+dotprod,+sve" "cpu"="neoverse-v2" }
+attributes #2 = { vscale_range(1,16) "target-features"="+neon,+dotprod,+sve,+i8mm" "cpu"="neoverse-v2" }