@@ -6409,8 +6409,7 @@ bool CodeGenPrepare::optimizeGatherScatterInst(Instruction *MemoryInst,
6409
6409
// not doable there, we do it here.
6410
6410
bool CodeGenPrepare::optimizeMulWithOverflow (Instruction *I, bool IsSigned,
6411
6411
ModifyDT &ModifiedDT) {
6412
- // Enable this optimization only for aarch64.
6413
- if (!TLI->getTargetMachine ().getTargetTriple ().isAArch64 ())
6412
+ if (!TLI->shouldOptimizeMulOverflowIntrinsic ())
6414
6413
return false ;
6415
6414
// If we have already seen this instruction, don't process it again.
6416
6415
if (!SeenMulWithOverflowInstrs.insert (std::make_pair (I, true )).second )
@@ -6440,29 +6439,42 @@ bool CodeGenPrepare::optimizeMulWithOverflow(Instruction *I, bool IsSigned,
6440
6439
if (!I->getType ()->isStructTy () || I->getType ()->getStructNumElements () != 2 )
6441
6440
return false ;
6442
6441
6443
- I->getParent ()->setName (" overflow.res" );
6444
- BasicBlock *OverflowResBB = I->getParent ();
6445
- BasicBlock *OverflowoEntryBB =
6446
- I->getParent ()->splitBasicBlock (I, " overflow.entry" , /* Before*/ true );
6447
- BasicBlock *NoOverflowBB = BasicBlock::Create (
6448
- I->getContext (), " overflow.no" , I->getFunction (), OverflowResBB);
6449
- BasicBlock *OverflowBB = BasicBlock::Create (I->getContext (), " overflow" ,
6450
- I->getFunction (), OverflowResBB);
6451
- // new blocks should be:
6442
+ // ----------------------------
6443
+
6444
+ // For the simple case where IR just checks the overflow flag, new blocks
6445
+ // should be:
6452
6446
// entry:
6453
6447
// if signed:
6454
6448
// (lhs_lo ^ lhs_hi) || (rhs_lo ^ rhs_hi) ? overflow, overflow_no
6455
6449
// else:
6456
6450
// (lhs_hi != 0) || (rhs_hi != 0) ? overflow, overflow_no
6451
+ // overflow_no:
6452
+ // overflow:
6457
6453
6454
+ // otherwise, new blocks should be:
6455
+ // entry:
6456
+ // if signed:
6457
+ // (lhs_lo ^ lhs_hi) || (rhs_lo ^ rhs_hi) ? overflow, overflow_no
6458
+ // else:
6459
+ // (lhs_hi != 0) || (rhs_hi != 0) ? overflow, overflow_no
6458
6460
// overflow_no:
6459
6461
// overflow:
6460
6462
// overflow.res:
6461
6463
6462
- // ----------------------------
6464
+ // New BBs:
6465
+ BasicBlock *OverflowoEntryBB =
6466
+ I->getParent ()->splitBasicBlock (I, " overflow.entry" , /* Before*/ true );
6467
+ // Remove the 'br' instruction that is generated as a result of the split:
6468
+ OverflowoEntryBB->getTerminator ()->eraseFromParent ();
6469
+ BasicBlock *NoOverflowBB =
6470
+ BasicBlock::Create (I->getContext (), " overflow.no" , I->getFunction ());
6471
+ NoOverflowBB->moveAfter (OverflowoEntryBB);
6472
+ I->getParent ()->setName (" overflow" );
6473
+ BasicBlock *OverflowBB = I->getParent ();
6474
+
6463
6475
// BB overflow.entry:
6464
- // get Lo and Hi of LHS & RHS:
6465
- IRBuilder<> Builder (OverflowoEntryBB-> getTerminator () );
6476
+ // Get Lo and Hi of LHS & RHS:
6477
+ IRBuilder<> Builder (OverflowoEntryBB);
6466
6478
Value *LoLHS = Builder.CreateTrunc (LHS, LegalTy, " lo.lhs" );
6467
6479
Value *HiLHS = Builder.CreateLShr (LHS, VTHalfBitWidth, " lhs.lsr" );
6468
6480
HiLHS = Builder.CreateTrunc (HiLHS, LegalTy, " hi.lhs" );
@@ -6479,18 +6491,16 @@ bool CodeGenPrepare::optimizeMulWithOverflow(Instruction *I, bool IsSigned,
6479
6491
Value *XorLHS = Builder.CreateXor (HiLHS, SignLoLHS);
6480
6492
Value *XorRHS = Builder.CreateXor (HiRHS, SignLoRHS);
6481
6493
Value *Or = Builder.CreateOr (XorLHS, XorRHS, " or.lhs.rhs" );
6482
- IsAnyBitTrue = Builder.CreateCmp (ICmpInst::ICMP_EQ , Or,
6483
- ConstantInt::get (Or->getType (), 1 ));
6494
+ IsAnyBitTrue = Builder.CreateCmp (ICmpInst::ICMP_NE , Or,
6495
+ ConstantInt::getNullValue (Or->getType ()));
6484
6496
} else {
6485
6497
Value *CmpLHS = Builder.CreateCmp (ICmpInst::ICMP_NE, HiLHS,
6486
6498
ConstantInt::getNullValue (LegalTy));
6487
6499
Value *CmpRHS = Builder.CreateCmp (ICmpInst::ICMP_NE, HiRHS,
6488
6500
ConstantInt::getNullValue (LegalTy));
6489
6501
IsAnyBitTrue = Builder.CreateOr (CmpLHS, CmpRHS, " or.lhs.rhs" );
6490
6502
}
6491
-
6492
6503
Builder.CreateCondBr (IsAnyBitTrue, OverflowBB, NoOverflowBB);
6493
- OverflowoEntryBB->getTerminator ()->eraseFromParent ();
6494
6504
6495
6505
// BB overflow.no:
6496
6506
Builder.SetInsertPoint (NoOverflowBB);
@@ -6503,24 +6513,95 @@ bool CodeGenPrepare::optimizeMulWithOverflow(Instruction *I, bool IsSigned,
6503
6513
ExtLoRHS = Builder.CreateZExt (LoRHS, Ty, " lo.rhs.ext" );
6504
6514
}
6505
6515
6506
- Value *Mul = Builder.CreateMul (ExtLoLHS, ExtLoRHS, " mul.no.overflow" );
6516
+ Value *Mul = Builder.CreateMul (ExtLoLHS, ExtLoRHS, " mul.overflow.no" );
6517
+
6518
+ // In overflow.no BB: we are sure that the overflow flag is false.
6519
+ // So, if we found this pattern:
6520
+ // br (extractvalue (%mul, 1)), label %if.then, label %if.end
6521
+ // then we can jump directly to %if.end as we're sure that there is no
6522
+ // overflow.
6523
+ BasicBlock *DetectNoOverflowBrBB = nullptr ;
6524
+ StructType *STy = StructType::get (
6525
+ I->getContext (), {Ty, IntegerType::getInt1Ty (I->getContext ())});
6526
+ // Look for the pattern in the users of I, and make sure that all the users
6527
+ // are either part of the pattern or NOT in the same BB as I.
6528
+ for (User *U : I->users ()) {
6529
+ if (auto *Instr = dyn_cast<Instruction>(U);
6530
+ Instr && Instr->getParent () != I->getParent ())
6531
+ continue ;
6532
+
6533
+ if (auto *ExtUser = dyn_cast<ExtractValueInst>(U)) {
6534
+ if (ExtUser->hasOneUse () && ExtUser->getNumIndices () == 1 &&
6535
+ ExtUser->getIndices ()[0 ] == 1 ) {
6536
+ if (auto *Br = dyn_cast<BranchInst>(*ExtUser->user_begin ())) {
6537
+ DetectNoOverflowBrBB = Br->getSuccessor (1 ) /* if.end*/ ;
6538
+ continue ;
6539
+ }
6540
+ }
6541
+ }
6542
+ // If we come here, it means that either the pattern doesn't exist or
6543
+ // there are multiple users in the same BB
6544
+ DetectNoOverflowBrBB = nullptr ;
6545
+ break ;
6546
+ }
6547
+ if (DetectNoOverflowBrBB) {
6548
+ // BB overflow.no: jump directly to if.end BB
6549
+ Builder.CreateBr (DetectNoOverflowBrBB);
6550
+ // BB if.end:
6551
+ Builder.SetInsertPoint (DetectNoOverflowBrBB,
6552
+ DetectNoOverflowBrBB->getFirstInsertionPt ());
6553
+ // Create PHI node to get the results of multiplication from 'overflow.no'
6554
+ // and 'overflow' BBs
6555
+ PHINode *NoOverflowPHI = Builder.CreatePHI (Ty, 2 );
6556
+ NoOverflowPHI->addIncoming (Mul, NoOverflowBB);
6557
+ // Create struct value to replace all uses of I
6558
+ Value *StructValNoOverflow = PoisonValue::get (STy);
6559
+ StructValNoOverflow =
6560
+ Builder.CreateInsertValue (StructValNoOverflow, NoOverflowPHI, {0 });
6561
+ // Overflow flag is always false as we are sure it's not overflow.
6562
+ StructValNoOverflow = Builder.CreateInsertValue (
6563
+ StructValNoOverflow, ConstantInt::getFalse (I->getContext ()), {1 });
6564
+ // Replace all uses of I, only uses dominated by the if.end BB
6565
+ I->replaceUsesOutsideBlock (StructValNoOverflow, I->getParent ());
6566
+ // BB overflow:
6567
+ Builder.SetInsertPoint (OverflowBB,
6568
+ I->getParent ()->getTerminator ()->getIterator ());
6569
+ // Extract the multiplication result to add it to the PHI node in the if.end
6570
+ // BB
6571
+ Value *IntrinsicMulRes = Builder.CreateExtractValue (I, {0 }, " mul.extract" );
6572
+ NoOverflowPHI->addIncoming (IntrinsicMulRes, OverflowBB);
6573
+ ModifiedDT = ModifyDT::ModifyBBDT;
6574
+ return true ;
6575
+ }
6576
+
6577
+ // Otherwise, we need to create the 'overflow.res' BB to merge the results of
6578
+ // the two paths.
6579
+ I->getParent ()->setName (" overflow.res" );
6580
+ BasicBlock *OverflowResBB = I->getParent ();
6581
+ OverflowBB = BasicBlock::Create (I->getContext (), " overflow" , I->getFunction (),
6582
+ OverflowResBB);
6583
+ // Initially I->getParent() was the overflow BB, now it becomes the
6584
+ // overflow.res BB. So we need to keep the old reference to the overflow BB.
6585
+ OverflowResBB->replaceAllUsesWith (OverflowBB);
6586
+
6587
+ // BB overflow.no: jump to overflow.res BB
6507
6588
Builder.CreateBr (OverflowResBB);
6508
6589
6509
6590
// BB overflow.res:
6510
6591
Builder.SetInsertPoint (OverflowResBB, OverflowResBB->getFirstInsertionPt ());
6511
- PHINode *PHINode1 = Builder.CreatePHI (Ty, 2 );
6512
- PHINode1->addIncoming (Mul, NoOverflowBB);
6513
- PHINode *PHINode2 =
6514
- Builder.CreatePHI (IntegerType::getInt1Ty (I->getContext ()), 2 );
6515
- PHINode2->addIncoming (ConstantInt::getFalse (I->getContext ()), NoOverflowBB);
6592
+ PHINode *OverflowResPHI = Builder.CreatePHI (Ty, 2 ),
6593
+ *OverflowFlagPHI =
6594
+ Builder.CreatePHI (IntegerType::getInt1Ty (I->getContext ()), 2 );
6516
6595
6517
- StructType *STy = StructType::get (
6518
- I->getContext (), {Ty, IntegerType::getInt1Ty (I->getContext ())});
6519
6596
Value *StructValOverflowRes = PoisonValue::get (STy);
6520
6597
StructValOverflowRes =
6521
- Builder.CreateInsertValue (StructValOverflowRes, PHINode1 , {0 });
6598
+ Builder.CreateInsertValue (StructValOverflowRes, OverflowResPHI , {0 });
6522
6599
StructValOverflowRes =
6523
- Builder.CreateInsertValue (StructValOverflowRes, PHINode2, {1 });
6600
+ Builder.CreateInsertValue (StructValOverflowRes, OverflowFlagPHI, {1 });
6601
+ OverflowResPHI->addIncoming (Mul, NoOverflowBB);
6602
+ OverflowFlagPHI->addIncoming (ConstantInt::getFalse (I->getContext ()),
6603
+ NoOverflowBB);
6604
+
6524
6605
// Before moving the mul.overflow intrinsic to the overflowBB, replace all its
6525
6606
// uses by StructValOverflowRes.
6526
6607
I->replaceAllUsesWith (StructValOverflowRes);
@@ -6534,9 +6615,8 @@ bool CodeGenPrepare::optimizeMulWithOverflow(Instruction *I, bool IsSigned,
6534
6615
Builder.CreateBr (OverflowResBB);
6535
6616
6536
6617
// Add The Extracted values to the PHINodes in the overflow.res block.
6537
- PHINode1->addIncoming (MulOverflow, OverflowBB);
6538
- PHINode2->addIncoming (OverflowFlag, OverflowBB);
6539
-
6618
+ OverflowResPHI->addIncoming (MulOverflow, OverflowBB);
6619
+ OverflowFlagPHI->addIncoming (OverflowFlag, OverflowBB);
6540
6620
ModifiedDT = ModifyDT::ModifyBBDT;
6541
6621
return true ;
6542
6622
}
0 commit comments