Skip to content

Commit 7275c17

Browse files
lukel97lukel97
authored
[VPlan] Fix packed replication of struct types (#160274)
I ran into this crash when #158690 caused a loop with a struct call to be vectorized. If we have a replicate recipe in a branch-on-mask predicated region that's used by a widened recipe in another block then it will be packed together with the other lanes via a VPPredInstPHIRecipe. If we're replicating a call with a struct return type then we currently crash. The code that handles structs in packScalarIntoVectorizedValue seemed to be untested at least on test/Transforms/LoopVectorize. There's two places that need to be fixed. The poison value that the scalar is packed into needs to use toVectorizedTy to correctly handle structs (not to be confused with toVectorTy!) The other is that VPPredInstPHIRecipe expects its operand to be an InsertElementInstr when stringing together the different lanes. For structs this will be an InsertVlaueInstr, and the value for the previous lane will be at the back of a chain of InsertValueInstrs.
1 parent 8553bd2 commit 7275c17

File tree

2 files changed

+233
-6
lines changed

2 files changed

+233
-6
lines changed

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3051,7 +3051,7 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
30513051
if (State.VF.isVector() && shouldPack()) {
30523052
Value *WideValue =
30533053
State.Lane->isFirstLane()
3054-
? PoisonValue::get(VectorType::get(UI->getType(), State.VF))
3054+
? PoisonValue::get(toVectorizedTy(UI->getType(), State.VF))
30553055
: State.get(this);
30563056
State.set(this, State.packScalarIntoVectorizedValue(this, WideValue,
30573057
*State.Lane));
@@ -3267,11 +3267,22 @@ void VPPredInstPHIRecipe::execute(VPTransformState &State) {
32673267
// also do that packing, thereby "hoisting" the insert-element sequence.
32683268
// Otherwise, a phi node for the scalar value is needed.
32693269
if (State.hasVectorValue(getOperand(0))) {
3270-
Value *VectorValue = State.get(getOperand(0));
3271-
InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
3272-
PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
3273-
VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
3274-
VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
3270+
auto *VecI = cast<Instruction>(State.get(getOperand(0)));
3271+
assert((isa<InsertElementInst, InsertValueInst>(VecI)) &&
3272+
"Packed operands must generate an insertelement or insertvalue");
3273+
3274+
// If VectorI is a struct, it will be a sequence like:
3275+
// %1 = insertvalue %unmodified, %x, 0
3276+
// %2 = insertvalue %1, %y, 1
3277+
// %VectorI = insertvalue %2, %z, 2
3278+
// To get the unmodified vector we need to look through the chain.
3279+
if (auto *StructTy = dyn_cast<StructType>(VecI->getType()))
3280+
for (unsigned I = 0; I < StructTy->getNumContainedTypes() - 1; I++)
3281+
VecI = cast<InsertValueInst>(VecI->getOperand(0));
3282+
3283+
PHINode *VPhi = State.Builder.CreatePHI(VecI->getType(), 2);
3284+
VPhi->addIncoming(VecI->getOperand(0), PredicatingBB); // Unmodified vector.
3285+
VPhi->addIncoming(VecI, PredicatedBB); // New vector with inserted element.
32753286
if (State.hasVectorValue(this))
32763287
State.reset(this, VPhi);
32773288
else

‎llvm/test/Transforms/LoopVectorize/struct-return-replicate.ll‎

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,221 @@ exit:
453453
ret void
454454
}
455455

456+
define void @struct_return_2xf32_replicate_predicated(ptr %a) {
457+
; VF4-LABEL: define void @struct_return_2xf32_replicate_predicated(
458+
; VF4-SAME: ptr [[A:%.*]]) {
459+
; VF4-NEXT: [[ENTRY:.*:]]
460+
; VF4-NEXT: br label %[[VECTOR_PH:.*]]
461+
; VF4: [[VECTOR_PH]]:
462+
; VF4-NEXT: br label %[[VECTOR_BODY:.*]]
463+
; VF4: [[VECTOR_BODY]]:
464+
; VF4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE12:.*]] ]
465+
; VF4-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
466+
; VF4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 8
467+
; VF4-NEXT: [[TMP1:%.*]] = fcmp ogt <4 x float> [[WIDE_LOAD]], zeroinitializer
468+
; VF4-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
469+
; VF4-NEXT: br i1 [[TMP2]], label %[[PRED_CALL_IF:.*]], label %[[PRED_CALL_CONTINUE:.*]]
470+
; VF4: [[PRED_CALL_IF]]:
471+
; VF4-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i32 0
472+
; VF4-NEXT: [[TMP4:%.*]] = tail call { float, float } @fn2(float [[TMP3]]) #[[ATTR3:[0-9]+]]
473+
; VF4-NEXT: [[TMP5:%.*]] = extractvalue { float, float } [[TMP4]], 0
474+
; VF4-NEXT: [[TMP6:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i32 0
475+
; VF4-NEXT: [[TMP7:%.*]] = insertvalue { <4 x float>, <4 x float> } poison, <4 x float> [[TMP6]], 0
476+
; VF4-NEXT: [[TMP8:%.*]] = extractvalue { float, float } [[TMP4]], 1
477+
; VF4-NEXT: [[TMP9:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP7]], 1
478+
; VF4-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP8]], i32 0
479+
; VF4-NEXT: [[TMP11:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP7]], <4 x float> [[TMP10]], 1
480+
; VF4-NEXT: br label %[[PRED_CALL_CONTINUE]]
481+
; VF4: [[PRED_CALL_CONTINUE]]:
482+
; VF4-NEXT: [[TMP12:%.*]] = phi { <4 x float>, <4 x float> } [ poison, %[[VECTOR_BODY]] ], [ [[TMP7]], %[[PRED_CALL_IF]] ]
483+
; VF4-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
484+
; VF4-NEXT: br i1 [[TMP13]], label %[[PRED_CALL_IF1:.*]], label %[[PRED_CALL_CONTINUE2:.*]]
485+
; VF4: [[PRED_CALL_IF1]]:
486+
; VF4-NEXT: [[TMP14:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i32 1
487+
; VF4-NEXT: [[TMP15:%.*]] = tail call { float, float } @fn2(float [[TMP14]]) #[[ATTR3]]
488+
; VF4-NEXT: [[TMP16:%.*]] = extractvalue { float, float } [[TMP15]], 0
489+
; VF4-NEXT: [[TMP17:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP12]], 0
490+
; VF4-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP17]], float [[TMP16]], i32 1
491+
; VF4-NEXT: [[TMP19:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP12]], <4 x float> [[TMP18]], 0
492+
; VF4-NEXT: [[TMP20:%.*]] = extractvalue { float, float } [[TMP15]], 1
493+
; VF4-NEXT: [[TMP21:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP19]], 1
494+
; VF4-NEXT: [[TMP22:%.*]] = insertelement <4 x float> [[TMP21]], float [[TMP20]], i32 1
495+
; VF4-NEXT: [[TMP23:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP19]], <4 x float> [[TMP22]], 1
496+
; VF4-NEXT: br label %[[PRED_CALL_CONTINUE2]]
497+
; VF4: [[PRED_CALL_CONTINUE2]]:
498+
; VF4-NEXT: [[TMP24:%.*]] = phi { <4 x float>, <4 x float> } [ [[TMP12]], %[[PRED_CALL_CONTINUE]] ], [ [[TMP19]], %[[PRED_CALL_IF1]] ]
499+
; VF4-NEXT: [[TMP25:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
500+
; VF4-NEXT: br i1 [[TMP25]], label %[[PRED_CALL_IF3:.*]], label %[[PRED_CALL_CONTINUE4:.*]]
501+
; VF4: [[PRED_CALL_IF3]]:
502+
; VF4-NEXT: [[TMP26:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i32 2
503+
; VF4-NEXT: [[TMP27:%.*]] = tail call { float, float } @fn2(float [[TMP26]]) #[[ATTR3]]
504+
; VF4-NEXT: [[TMP28:%.*]] = extractvalue { float, float } [[TMP27]], 0
505+
; VF4-NEXT: [[TMP29:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP24]], 0
506+
; VF4-NEXT: [[TMP30:%.*]] = insertelement <4 x float> [[TMP29]], float [[TMP28]], i32 2
507+
; VF4-NEXT: [[TMP31:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP24]], <4 x float> [[TMP30]], 0
508+
; VF4-NEXT: [[TMP32:%.*]] = extractvalue { float, float } [[TMP27]], 1
509+
; VF4-NEXT: [[TMP33:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP31]], 1
510+
; VF4-NEXT: [[TMP34:%.*]] = insertelement <4 x float> [[TMP33]], float [[TMP32]], i32 2
511+
; VF4-NEXT: [[TMP35:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP31]], <4 x float> [[TMP34]], 1
512+
; VF4-NEXT: br label %[[PRED_CALL_CONTINUE4]]
513+
; VF4: [[PRED_CALL_CONTINUE4]]:
514+
; VF4-NEXT: [[TMP36:%.*]] = phi { <4 x float>, <4 x float> } [ [[TMP24]], %[[PRED_CALL_CONTINUE2]] ], [ [[TMP31]], %[[PRED_CALL_IF3]] ]
515+
; VF4-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
516+
; VF4-NEXT: br i1 [[TMP37]], label %[[PRED_CALL_IF5:.*]], label %[[PRED_CALL_CONTINUE6:.*]]
517+
; VF4: [[PRED_CALL_IF5]]:
518+
; VF4-NEXT: [[TMP38:%.*]] = extractelement <4 x float> [[WIDE_LOAD]], i32 3
519+
; VF4-NEXT: [[TMP39:%.*]] = tail call { float, float } @fn2(float [[TMP38]]) #[[ATTR3]]
520+
; VF4-NEXT: [[TMP40:%.*]] = extractvalue { float, float } [[TMP39]], 0
521+
; VF4-NEXT: [[TMP41:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP36]], 0
522+
; VF4-NEXT: [[TMP42:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP40]], i32 3
523+
; VF4-NEXT: [[TMP43:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP36]], <4 x float> [[TMP42]], 0
524+
; VF4-NEXT: [[TMP44:%.*]] = extractvalue { float, float } [[TMP39]], 1
525+
; VF4-NEXT: [[TMP45:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP43]], 1
526+
; VF4-NEXT: [[TMP46:%.*]] = insertelement <4 x float> [[TMP45]], float [[TMP44]], i32 3
527+
; VF4-NEXT: [[TMP47:%.*]] = insertvalue { <4 x float>, <4 x float> } [[TMP43]], <4 x float> [[TMP46]], 1
528+
; VF4-NEXT: br label %[[PRED_CALL_CONTINUE6]]
529+
; VF4: [[PRED_CALL_CONTINUE6]]:
530+
; VF4-NEXT: [[TMP48:%.*]] = phi { <4 x float>, <4 x float> } [ [[TMP36]], %[[PRED_CALL_CONTINUE4]] ], [ [[TMP43]], %[[PRED_CALL_IF5]] ]
531+
; VF4-NEXT: [[TMP49:%.*]] = extractvalue { <4 x float>, <4 x float> } [[TMP48]], 0
532+
; VF4-NEXT: [[TMP50:%.*]] = fdiv <4 x float> [[TMP49]], [[WIDE_LOAD]]
533+
; VF4-NEXT: [[TMP51:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
534+
; VF4-NEXT: br i1 [[TMP51]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
535+
; VF4: [[PRED_STORE_IF]]:
536+
; VF4-NEXT: [[TMP52:%.*]] = add i64 [[INDEX]], 0
537+
; VF4-NEXT: [[TMP53:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP52]]
538+
; VF4-NEXT: [[TMP54:%.*]] = extractelement <4 x float> [[TMP50]], i32 0
539+
; VF4-NEXT: store float [[TMP54]], ptr [[TMP53]], align 8
540+
; VF4-NEXT: br label %[[PRED_STORE_CONTINUE]]
541+
; VF4: [[PRED_STORE_CONTINUE]]:
542+
; VF4-NEXT: [[TMP55:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
543+
; VF4-NEXT: br i1 [[TMP55]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
544+
; VF4: [[PRED_STORE_IF7]]:
545+
; VF4-NEXT: [[TMP56:%.*]] = add i64 [[INDEX]], 1
546+
; VF4-NEXT: [[TMP57:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP56]]
547+
; VF4-NEXT: [[TMP58:%.*]] = extractelement <4 x float> [[TMP50]], i32 1
548+
; VF4-NEXT: store float [[TMP58]], ptr [[TMP57]], align 8
549+
; VF4-NEXT: br label %[[PRED_STORE_CONTINUE8]]
550+
; VF4: [[PRED_STORE_CONTINUE8]]:
551+
; VF4-NEXT: [[TMP59:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
552+
; VF4-NEXT: br i1 [[TMP59]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10:.*]]
553+
; VF4: [[PRED_STORE_IF9]]:
554+
; VF4-NEXT: [[TMP60:%.*]] = add i64 [[INDEX]], 2
555+
; VF4-NEXT: [[TMP61:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP60]]
556+
; VF4-NEXT: [[TMP62:%.*]] = extractelement <4 x float> [[TMP50]], i32 2
557+
; VF4-NEXT: store float [[TMP62]], ptr [[TMP61]], align 8
558+
; VF4-NEXT: br label %[[PRED_STORE_CONTINUE10]]
559+
; VF4: [[PRED_STORE_CONTINUE10]]:
560+
; VF4-NEXT: [[TMP63:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
561+
; VF4-NEXT: br i1 [[TMP63]], label %[[PRED_STORE_IF11:.*]], label %[[PRED_STORE_CONTINUE12]]
562+
; VF4: [[PRED_STORE_IF11]]:
563+
; VF4-NEXT: [[TMP64:%.*]] = add i64 [[INDEX]], 3
564+
; VF4-NEXT: [[TMP65:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP64]]
565+
; VF4-NEXT: [[TMP66:%.*]] = extractelement <4 x float> [[TMP50]], i32 3
566+
; VF4-NEXT: store float [[TMP66]], ptr [[TMP65]], align 8
567+
; VF4-NEXT: br label %[[PRED_STORE_CONTINUE12]]
568+
; VF4: [[PRED_STORE_CONTINUE12]]:
569+
; VF4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
570+
; VF4-NEXT: [[TMP67:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
571+
; VF4-NEXT: br i1 [[TMP67]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
572+
; VF4: [[MIDDLE_BLOCK]]:
573+
;
574+
; VF2IC2-LABEL: define void @struct_return_2xf32_replicate_predicated(
575+
; VF2IC2-SAME: ptr [[A:%.*]]) {
576+
; VF2IC2-NEXT: [[ENTRY:.*:]]
577+
; VF2IC2-NEXT: br label %[[VECTOR_PH:.*]]
578+
; VF2IC2: [[VECTOR_PH]]:
579+
; VF2IC2-NEXT: br label %[[VECTOR_BODY:.*]]
580+
; VF2IC2: [[VECTOR_BODY]]:
581+
; VF2IC2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE7:.*]] ]
582+
; VF2IC2-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
583+
; VF2IC2-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[TMP0]], i32 2
584+
; VF2IC2-NEXT: [[WIDE_LOAD:%.*]] = load <2 x float>, ptr [[TMP0]], align 8
585+
; VF2IC2-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x float>, ptr [[TMP1]], align 8
586+
; VF2IC2-NEXT: [[TMP2:%.*]] = fcmp ogt <2 x float> [[WIDE_LOAD]], zeroinitializer
587+
; VF2IC2-NEXT: [[TMP3:%.*]] = fcmp ogt <2 x float> [[WIDE_LOAD1]], zeroinitializer
588+
; VF2IC2-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
589+
; VF2IC2-NEXT: br i1 [[TMP4]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
590+
; VF2IC2: [[PRED_STORE_IF]]:
591+
; VF2IC2-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i32 0
592+
; VF2IC2-NEXT: [[TMP6:%.*]] = tail call { float, float } @fn2(float [[TMP5]]) #[[ATTR3:[0-9]+]]
593+
; VF2IC2-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0
594+
; VF2IC2-NEXT: [[TMP8:%.*]] = extractvalue { float, float } [[TMP6]], 0
595+
; VF2IC2-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP7]]
596+
; VF2IC2-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i32 0
597+
; VF2IC2-NEXT: [[TMP11:%.*]] = fdiv float [[TMP8]], [[TMP10]]
598+
; VF2IC2-NEXT: store float [[TMP11]], ptr [[TMP9]], align 8
599+
; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE]]
600+
; VF2IC2: [[PRED_STORE_CONTINUE]]:
601+
; VF2IC2-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
602+
; VF2IC2-NEXT: br i1 [[TMP12]], label %[[PRED_STORE_IF2:.*]], label %[[PRED_STORE_CONTINUE3:.*]]
603+
; VF2IC2: [[PRED_STORE_IF2]]:
604+
; VF2IC2-NEXT: [[TMP13:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i32 1
605+
; VF2IC2-NEXT: [[TMP14:%.*]] = tail call { float, float } @fn2(float [[TMP13]]) #[[ATTR3]]
606+
; VF2IC2-NEXT: [[TMP15:%.*]] = add i64 [[INDEX]], 1
607+
; VF2IC2-NEXT: [[TMP16:%.*]] = extractvalue { float, float } [[TMP14]], 0
608+
; VF2IC2-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]]
609+
; VF2IC2-NEXT: [[TMP18:%.*]] = extractelement <2 x float> [[WIDE_LOAD]], i32 1
610+
; VF2IC2-NEXT: [[TMP19:%.*]] = fdiv float [[TMP16]], [[TMP18]]
611+
; VF2IC2-NEXT: store float [[TMP19]], ptr [[TMP17]], align 8
612+
; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE3]]
613+
; VF2IC2: [[PRED_STORE_CONTINUE3]]:
614+
; VF2IC2-NEXT: [[TMP20:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0
615+
; VF2IC2-NEXT: br i1 [[TMP20]], label %[[PRED_STORE_IF4:.*]], label %[[PRED_STORE_CONTINUE5:.*]]
616+
; VF2IC2: [[PRED_STORE_IF4]]:
617+
; VF2IC2-NEXT: [[TMP21:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i32 0
618+
; VF2IC2-NEXT: [[TMP22:%.*]] = tail call { float, float } @fn2(float [[TMP21]]) #[[ATTR3]]
619+
; VF2IC2-NEXT: [[TMP23:%.*]] = add i64 [[INDEX]], 2
620+
; VF2IC2-NEXT: [[TMP24:%.*]] = extractvalue { float, float } [[TMP22]], 0
621+
; VF2IC2-NEXT: [[TMP25:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP23]]
622+
; VF2IC2-NEXT: [[TMP26:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i32 0
623+
; VF2IC2-NEXT: [[TMP27:%.*]] = fdiv float [[TMP24]], [[TMP26]]
624+
; VF2IC2-NEXT: store float [[TMP27]], ptr [[TMP25]], align 8
625+
; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE5]]
626+
; VF2IC2: [[PRED_STORE_CONTINUE5]]:
627+
; VF2IC2-NEXT: [[TMP28:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
628+
; VF2IC2-NEXT: br i1 [[TMP28]], label %[[PRED_STORE_IF6:.*]], label %[[PRED_STORE_CONTINUE7]]
629+
; VF2IC2: [[PRED_STORE_IF6]]:
630+
; VF2IC2-NEXT: [[TMP29:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i32 1
631+
; VF2IC2-NEXT: [[TMP30:%.*]] = tail call { float, float } @fn2(float [[TMP29]]) #[[ATTR3]]
632+
; VF2IC2-NEXT: [[TMP31:%.*]] = add i64 [[INDEX]], 3
633+
; VF2IC2-NEXT: [[TMP32:%.*]] = extractvalue { float, float } [[TMP30]], 0
634+
; VF2IC2-NEXT: [[TMP33:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP31]]
635+
; VF2IC2-NEXT: [[TMP34:%.*]] = extractelement <2 x float> [[WIDE_LOAD1]], i32 1
636+
; VF2IC2-NEXT: [[TMP35:%.*]] = fdiv float [[TMP32]], [[TMP34]]
637+
; VF2IC2-NEXT: store float [[TMP35]], ptr [[TMP33]], align 8
638+
; VF2IC2-NEXT: br label %[[PRED_STORE_CONTINUE7]]
639+
; VF2IC2: [[PRED_STORE_CONTINUE7]]:
640+
; VF2IC2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
641+
; VF2IC2-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
642+
; VF2IC2-NEXT: br i1 [[TMP36]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
643+
; VF2IC2: [[MIDDLE_BLOCK]]:
644+
;
645+
entry:
646+
br label %for.body
647+
648+
for.body:
649+
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
650+
%arrayidx = getelementptr inbounds float, ptr %a, i64 %iv
651+
%in_val = load float, ptr %arrayidx, align 8
652+
%sgt_zero = fcmp ogt float %in_val, 0.0
653+
br i1 %sgt_zero, label %if.then, label %for.inc
654+
655+
if.then:
656+
%call = tail call { float, float } @fn2(float %in_val) #3
657+
%extract_a = extractvalue { float, float } %call, 0
658+
%div = fdiv float %extract_a, %in_val
659+
store float %div, ptr %arrayidx, align 8
660+
br label %for.inc
661+
662+
for.inc:
663+
%iv.next = add nuw nsw i64 %iv, 1
664+
%exitcond.not = icmp eq i64 %iv.next, 1024
665+
br i1 %exitcond.not, label %exit, label %for.body
666+
667+
exit:
668+
ret void
669+
}
670+
456671
declare { i64 } @fn1(float)
457672
declare { float, float } @fn2(float)
458673
declare { i32, i32, i32 } @fn3(i32)
@@ -464,3 +679,4 @@ declare { <8 x i32>, <8 x i32>, <8 x i32> } @fixed_vec_fn3(<8 x i32>)
464679
attributes #0 = { nounwind "vector-function-abi-variant"="_ZGVnN8v_fn1(fixed_vec_fn1)" }
465680
attributes #1 = { nounwind "vector-function-abi-variant"="_ZGVnN8v_fn2(fixed_vec_fn2)" }
466681
attributes #2 = { nounwind "vector-function-abi-variant"="_ZGVnN8v_fn3(fixed_vec_fn3)" }
682+
attributes #3 = { nounwind "vector-function-abi-variant"="_ZGVnM8v_fn2(fixed_vec_fn2)" }

0 commit comments

Comments
 (0)