@@ -268,7 +268,7 @@ define void @latch_branch_cost(ptr %dst) {
268
268
; DEFAULT: [[MIDDLE_BLOCK]]:
269
269
; DEFAULT-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
270
270
; DEFAULT: [[VEC_EPILOG_ITER_CHECK]]:
271
- ; DEFAULT-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
271
+ ; DEFAULT-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF5:![0-9]+]]
272
272
; DEFAULT: [[VEC_EPILOG_PH]]:
273
273
; DEFAULT-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
274
274
; DEFAULT-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
@@ -278,7 +278,7 @@ define void @latch_branch_cost(ptr %dst) {
278
278
; DEFAULT-NEXT: store <4 x i8> zeroinitializer, ptr [[TMP8]], align 1
279
279
; DEFAULT-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4
280
280
; DEFAULT-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 100
281
- ; DEFAULT-NEXT: br i1 [[TMP10]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP5 :![0-9]+]]
281
+ ; DEFAULT-NEXT: br i1 [[TMP10]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP6 :![0-9]+]]
282
282
; DEFAULT: [[VEC_EPILOG_MIDDLE_BLOCK]]:
283
283
; DEFAULT-NEXT: br i1 true, [[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
284
284
; DEFAULT: [[VEC_EPILOG_SCALAR_PH]]:
@@ -429,14 +429,14 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
429
429
; DEFAULT-NEXT: br label %[[VECTOR_BODY:.*]]
430
430
; DEFAULT: [[VECTOR_BODY]]:
431
431
; DEFAULT-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE37:.*]] ]
432
- ; DEFAULT-NEXT: [[TMP9:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META7 :![0-9]+]]
432
+ ; DEFAULT-NEXT: [[TMP9:%.*]] = load i32, ptr [[A]], align 4, !alias.scope [[META8 :![0-9]+]]
433
433
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT28:%.*]] = insertelement <4 x i32> poison, i32 [[TMP9]], i64 0
434
434
; DEFAULT-NEXT: [[BROADCAST_SPLAT29:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT28]], <4 x i32> poison, <4 x i32> zeroinitializer
435
- ; DEFAULT-NEXT: [[TMP19:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META10 :![0-9]+]]
435
+ ; DEFAULT-NEXT: [[TMP19:%.*]] = load i32, ptr [[B]], align 4, !alias.scope [[META11 :![0-9]+]]
436
436
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP19]], i64 0
437
437
; DEFAULT-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
438
438
; DEFAULT-NEXT: [[TMP6:%.*]] = or <4 x i32> [[BROADCAST_SPLAT]], [[BROADCAST_SPLAT29]]
439
- ; DEFAULT-NEXT: [[TMP7:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META12 :![0-9]+]]
439
+ ; DEFAULT-NEXT: [[TMP7:%.*]] = load i32, ptr [[C]], align 4, !alias.scope [[META13 :![0-9]+]]
440
440
; DEFAULT-NEXT: [[BROADCAST_SPLATINSERT30:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0
441
441
; DEFAULT-NEXT: [[BROADCAST_SPLAT31:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT30]], <4 x i32> poison, <4 x i32> zeroinitializer
442
442
; DEFAULT-NEXT: [[TMP8:%.*]] = icmp ugt <4 x i32> [[BROADCAST_SPLAT31]], [[TMP6]]
@@ -445,34 +445,34 @@ define i32 @header_mask_and_invariant_compare(ptr %A, ptr %B, ptr %C, ptr %D, pt
445
445
; DEFAULT-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
446
446
; DEFAULT: [[PRED_STORE_IF]]:
447
447
; DEFAULT-NEXT: [[TMP11:%.*]] = extractelement <4 x i32> [[TMP6]], i32 0
448
- ; DEFAULT-NEXT: store i32 [[TMP11]], ptr [[E]], align 4, !alias.scope [[META14 :![0-9]+]], !noalias [[META16 :![0-9]+]]
448
+ ; DEFAULT-NEXT: store i32 [[TMP11]], ptr [[E]], align 4, !alias.scope [[META15 :![0-9]+]], !noalias [[META17 :![0-9]+]]
449
449
; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE]]
450
450
; DEFAULT: [[PRED_STORE_CONTINUE]]:
451
451
; DEFAULT-NEXT: [[TMP12:%.*]] = extractelement <4 x i1> [[TMP8]], i32 1
452
452
; DEFAULT-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF32:.*]], label %[[PRED_STORE_CONTINUE33:.*]]
453
453
; DEFAULT: [[PRED_STORE_IF32]]:
454
454
; DEFAULT-NEXT: [[TMP13:%.*]] = extractelement <4 x i32> [[TMP6]], i32 0
455
- ; DEFAULT-NEXT: store i32 [[TMP13]], ptr [[E]], align 4, !alias.scope [[META14 ]], !noalias [[META16 ]]
455
+ ; DEFAULT-NEXT: store i32 [[TMP13]], ptr [[E]], align 4, !alias.scope [[META15 ]], !noalias [[META17 ]]
456
456
; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE33]]
457
457
; DEFAULT: [[PRED_STORE_CONTINUE33]]:
458
458
; DEFAULT-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP8]], i32 2
459
459
; DEFAULT-NEXT: br i1 [[TMP14]], label %[[PRED_STORE_IF34:.*]], label %[[PRED_STORE_CONTINUE35:.*]]
460
460
; DEFAULT: [[PRED_STORE_IF34]]:
461
461
; DEFAULT-NEXT: [[TMP15:%.*]] = extractelement <4 x i32> [[TMP6]], i32 0
462
- ; DEFAULT-NEXT: store i32 [[TMP15]], ptr [[E]], align 4, !alias.scope [[META14 ]], !noalias [[META16 ]]
462
+ ; DEFAULT-NEXT: store i32 [[TMP15]], ptr [[E]], align 4, !alias.scope [[META15 ]], !noalias [[META17 ]]
463
463
; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE35]]
464
464
; DEFAULT: [[PRED_STORE_CONTINUE35]]:
465
465
; DEFAULT-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP8]], i32 3
466
466
; DEFAULT-NEXT: br i1 [[TMP21]], label %[[PRED_STORE_IF36:.*]], label %[[PRED_STORE_CONTINUE37]]
467
467
; DEFAULT: [[PRED_STORE_IF36]]:
468
468
; DEFAULT-NEXT: [[TMP22:%.*]] = extractelement <4 x i32> [[TMP6]], i32 0
469
- ; DEFAULT-NEXT: store i32 [[TMP22]], ptr [[E]], align 4, !alias.scope [[META14 ]], !noalias [[META16 ]]
469
+ ; DEFAULT-NEXT: store i32 [[TMP22]], ptr [[E]], align 4, !alias.scope [[META15 ]], !noalias [[META17 ]]
470
470
; DEFAULT-NEXT: br label %[[PRED_STORE_CONTINUE37]]
471
471
; DEFAULT: [[PRED_STORE_CONTINUE37]]:
472
- ; DEFAULT-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> zeroinitializer, ptr [[TMP16]], i32 4, <4 x i1> [[TMP8]]), !alias.scope [[META18 :![0-9]+]], !noalias [[META19 :![0-9]+]]
472
+ ; DEFAULT-NEXT: call void @llvm.masked.store.v4i32.p0(<4 x i32> zeroinitializer, ptr [[TMP16]], i32 4, <4 x i1> [[TMP8]]), !alias.scope [[META19 :![0-9]+]], !noalias [[META20 :![0-9]+]]
473
473
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
474
474
; DEFAULT-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
475
- ; DEFAULT-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20 :![0-9]+]]
475
+ ; DEFAULT-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21 :![0-9]+]]
476
476
; DEFAULT: [[MIDDLE_BLOCK]]:
477
477
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
478
478
; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -549,7 +549,7 @@ define void @multiple_exit_conditions(ptr %src, ptr noalias %dst) #1 {
549
549
; DEFAULT-NEXT: store <8 x double> [[TMP3]], ptr [[NEXT_GEP]], align 8
550
550
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
551
551
; DEFAULT-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 256
552
- ; DEFAULT-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22 :![0-9]+]]
552
+ ; DEFAULT-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23 :![0-9]+]]
553
553
; DEFAULT: [[MIDDLE_BLOCK]]:
554
554
; DEFAULT-NEXT: br label %[[SCALAR_PH]]
555
555
; DEFAULT: [[SCALAR_PH]]:
@@ -658,15 +658,15 @@ define void @low_trip_count_fold_tail_scalarized_store(ptr %dst) {
658
658
; COMMON-NEXT: store i8 6, ptr [[TMP6]], align 1
659
659
; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE12]]
660
660
; COMMON: [[PRED_STORE_CONTINUE12]]:
661
- ; COMMON-NEXT: br i1 false, label %[[PRED_STORE_IF13:.*]], label %[[PRED_STORE_CONTINUE14 :.*]]
661
+ ; COMMON-NEXT: br i1 false, label %[[PRED_STORE_IF13:.*]], label %[[EXIT :.*]]
662
662
; COMMON: [[PRED_STORE_IF13]]:
663
663
; COMMON-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[DST]], i64 7
664
664
; COMMON-NEXT: store i8 7, ptr [[TMP7]], align 1
665
- ; COMMON-NEXT: br label %[[PRED_STORE_CONTINUE14 ]]
666
- ; COMMON: [[PRED_STORE_CONTINUE14 ]]:
665
+ ; COMMON-NEXT: br label %[[EXIT ]]
666
+ ; COMMON: [[EXIT ]]:
667
667
; COMMON-NEXT: br label %[[MIDDLE_BLOCK:.*]]
668
668
; COMMON: [[MIDDLE_BLOCK]]:
669
- ; COMMON-NEXT: br [[EXIT :label %.*]]
669
+ ; COMMON-NEXT: br [[EXIT1 :label %.*]]
670
670
; COMMON: [[SCALAR_PH]]:
671
671
;
672
672
entry:
@@ -866,7 +866,7 @@ define void @test_conditional_interleave_group (ptr noalias %src.1, ptr noalias
866
866
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
867
867
; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
868
868
; DEFAULT-NEXT: [[TMP80:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
869
- ; DEFAULT-NEXT: br i1 [[TMP80]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP25 :![0-9]+]]
869
+ ; DEFAULT-NEXT: br i1 [[TMP80]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP26 :![0-9]+]]
870
870
; DEFAULT: [[MIDDLE_BLOCK]]:
871
871
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
872
872
; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
@@ -1115,7 +1115,7 @@ define void @redundant_branch_and_tail_folding(ptr %dst, i1 %c) {
1115
1115
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
1116
1116
; DEFAULT-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[STEP_ADD]], splat (i64 4)
1117
1117
; DEFAULT-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
1118
- ; DEFAULT-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP27 :![0-9]+]]
1118
+ ; DEFAULT-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP28 :![0-9]+]]
1119
1119
; DEFAULT: [[MIDDLE_BLOCK]]:
1120
1120
; DEFAULT-NEXT: br label %[[SCALAR_PH]]
1121
1121
; DEFAULT: [[SCALAR_PH]]:
@@ -1289,7 +1289,7 @@ define void @pred_udiv_select_cost(ptr %A, ptr %B, ptr %C, i64 %n, i8 %y) #1 {
1289
1289
; DEFAULT-NEXT: store <vscale x 4 x i8> [[TMP23]], ptr [[TMP24]], align 1
1290
1290
; DEFAULT-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]]
1291
1291
; DEFAULT-NEXT: [[TMP25:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
1292
- ; DEFAULT-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP29 :![0-9]+]]
1292
+ ; DEFAULT-NEXT: br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP30 :![0-9]+]]
1293
1293
; DEFAULT: [[MIDDLE_BLOCK]]:
1294
1294
; DEFAULT-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
1295
1295
; DEFAULT-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
0 commit comments