@@ -2365,8 +2365,10 @@ static Value *simplifyAndOrWithOpReplaced(Value *V, Value *Op, Value *RepOp,
23652365// / number of and/or instructions might have to be created.
23662366Value *InstCombinerImpl::reassociateBooleanAndOr (Value *LHS, Value *X, Value *Y,
23672367 Instruction &I, bool IsAnd,
2368- bool RHSIsLogical) {
2368+ bool RHSIsLogical,
2369+ bool RHSIsDisjoint) {
23692370 Instruction::BinaryOps Opcode = IsAnd ? Instruction::And : Instruction::Or;
2371+
23702372 // LHS bop (X lop Y) --> (LHS bop X) lop Y
23712373 // LHS bop (X bop Y) --> (LHS bop X) bop Y
23722374 if (Value *Res = foldBooleanAndOr (LHS, X, I, IsAnd, /* IsLogical=*/ false ))
@@ -2377,6 +2379,40 @@ Value *InstCombinerImpl::reassociateBooleanAndOr(Value *LHS, Value *X, Value *Y,
23772379 if (Value *Res = foldBooleanAndOr (LHS, Y, I, IsAnd, /* IsLogical=*/ false ))
23782380 return RHSIsLogical ? Builder.CreateLogicalOp (Opcode, X, Res)
23792381 : Builder.CreateBinOp (Opcode, X, Res);
2382+
2383+ if (RHSIsDisjoint && !IsAnd && cast<PossiblyDisjointInst>(&I)->isDisjoint ()) {
2384+ if (Value *Res = foldDisjointOr (LHS, X, I)) {
2385+ auto Disjoint = cast<PossiblyDisjointInst>(Builder.CreateOr (Res, Y));
2386+ Disjoint->setIsDisjoint (true );
2387+ return cast<Value>(Disjoint);
2388+ }
2389+ if (Value *Res = foldDisjointOr (LHS, Y, I)) {
2390+ auto Disjoint = cast<PossiblyDisjointInst>(Builder.CreateOr (Res, X));
2391+ Disjoint->setIsDisjoint (true );
2392+ return cast<Value>(Disjoint);
2393+ }
2394+ Value *X1, *Y1;
2395+ if (match (LHS, m_OneUse (m_DisjointOr (m_Value (X1), m_Value (Y1))))) {
2396+ auto TryFold = [this , &I](Value *Op0, Value *Op1, Value *Rem0,
2397+ Value *Rem1) -> Value * {
2398+ if (Value *Res = foldDisjointOr (Op0, Op1, I)) {
2399+ auto Disjoint =
2400+ cast<PossiblyDisjointInst>(Builder.CreateOr (Rem0, Rem1));
2401+ Disjoint->setIsDisjoint (true );
2402+ auto Disjoint2 =
2403+ cast<PossiblyDisjointInst>(Builder.CreateOr (Disjoint, Res));
2404+ return cast<Value>(Disjoint2);
2405+ }
2406+ return nullptr ;
2407+ };
2408+
2409+ if (Value *Res = TryFold (X, X1, Y, Y1))
2410+ return Res;
2411+
2412+ if (Value *Res = TryFold (X, Y1, Y, X1))
2413+ return Res;
2414+ }
2415+ }
23802416 return nullptr ;
23812417}
23822418
@@ -3542,55 +3578,6 @@ Value *InstCombinerImpl::foldAndOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
35423578 return foldAndOrOfICmpsUsingRanges (LHS, RHS, IsAnd);
35433579}
35443580
3545- // / If IsLogical is true, then the and/or is in select form and the transform
3546- // / must be poison-safe.
3547- Value *InstCombinerImpl::foldBooleanAndOr (Value *LHS, Value *RHS,
3548- Instruction &I, bool IsAnd,
3549- bool IsLogical) {
3550- if (!LHS->getType ()->isIntOrIntVectorTy (1 ))
3551- return nullptr ;
3552-
3553- // handle (roughly):
3554- // (icmp ne (A & B), C) | (icmp ne (A & D), E)
3555- // (icmp eq (A & B), C) & (icmp eq (A & D), E)
3556- if (Value *V = foldLogOpOfMaskedICmps (LHS, RHS, IsAnd, IsLogical, Builder,
3557- SQ.getWithInstruction (&I)))
3558- return V;
3559-
3560- if (auto *LHSCmp = dyn_cast<ICmpInst>(LHS))
3561- if (auto *RHSCmp = dyn_cast<ICmpInst>(RHS))
3562- if (Value *Res = foldAndOrOfICmps (LHSCmp, RHSCmp, I, IsAnd, IsLogical))
3563- return Res;
3564-
3565- if (auto *LHSCmp = dyn_cast<FCmpInst>(LHS))
3566- if (auto *RHSCmp = dyn_cast<FCmpInst>(RHS))
3567- if (Value *Res = foldLogicOfFCmps (LHSCmp, RHSCmp, IsAnd, IsLogical))
3568- return Res;
3569-
3570- if (Value *Res = foldEqOfParts (LHS, RHS, IsAnd))
3571- return Res;
3572-
3573- return nullptr ;
3574- }
3575-
3576- static Value *foldOrOfInversions (BinaryOperator &I,
3577- InstCombiner::BuilderTy &Builder) {
3578- assert (I.getOpcode () == Instruction::Or &&
3579- " Simplification only supports or at the moment." );
3580-
3581- Value *Cmp1, *Cmp2, *Cmp3, *Cmp4;
3582- if (!match (I.getOperand (0 ), m_And (m_Value (Cmp1), m_Value (Cmp2))) ||
3583- !match (I.getOperand (1 ), m_And (m_Value (Cmp3), m_Value (Cmp4))))
3584- return nullptr ;
3585-
3586- // Check if any two pairs of the and operations are inversions of each other.
3587- if (isKnownInversion (Cmp1, Cmp3) && isKnownInversion (Cmp2, Cmp4))
3588- return Builder.CreateXor (Cmp1, Cmp4);
3589- if (isKnownInversion (Cmp1, Cmp4) && isKnownInversion (Cmp2, Cmp3))
3590- return Builder.CreateXor (Cmp1, Cmp3);
3591-
3592- return nullptr ;
3593- }
35943581
35953582// A decomposition of ((X & Mask) * Factor). The NUW / NSW bools
35963583// track these properities for preservation. Note that we can decompose
@@ -3664,35 +3651,94 @@ static std::optional<DecomposedBitMaskMul> matchBitmaskMul(Value *V) {
36643651 return std::nullopt ;
36653652}
36663653
3667- struct CombinedBitmaskMul {
3668- std::optional<DecomposedBitMaskMul> Decomp;
3669- Value *DecompOp = nullptr ;
3670- Value *OtherOp = nullptr ;
3671- };
3654+ // (A & N) * C + (A & M) * C -> (A & (N + M)) & C
3655+ // This also accepts the equivalent select form of (A & N) * C
3656+ // expressions i.e. !(A & N) ? 0 : N * C)
3657+ static Value *foldBitmaskMul (Value *Op0, Value *Op1,
3658+ InstCombiner::BuilderTy &Builder) {
3659+ auto Decomp1 = matchBitmaskMul (Op1);
36723660
3673- static CombinedBitmaskMul matchCombinedBitmaskMul (Value *V) {
3674- auto DecompBitMaskMul = matchBitmaskMul (V);
3675- if (DecompBitMaskMul)
3676- return {DecompBitMaskMul, V, nullptr };
3661+ if (Decomp1) {
3662+ auto Decomp0 = matchBitmaskMul (Op0);
36773663
3678- // Otherwise, check the operands of V for bitmaskmul pattern
3679- auto BOp = dyn_cast<BinaryOperator>(V);
3680- if (!BOp)
3681- return CombinedBitmaskMul ();
3664+ if (Decomp0) {
3665+ // If we have independent operands in the BitmaskMul chain, then just
3666+ // reassociate to encourage combining in future iterations.
36823667
3683- auto Disj = dyn_cast<PossiblyDisjointInst>(BOp);
3684- if (!Disj || !Disj->isDisjoint ())
3685- return CombinedBitmaskMul ();
3668+ if (Decomp0->isCombineableWith (*Decomp1)) {
3669+ auto NewAnd = Builder.CreateAnd (
3670+ Decomp0->X , ConstantInt::get (Decomp0->X ->getType (),
3671+ (Decomp0->Mask + Decomp1->Mask )));
36863672
3687- auto DecompBitMaskMul0 = matchBitmaskMul (BOp->getOperand (0 ));
3688- if (DecompBitMaskMul0)
3689- return {DecompBitMaskMul0, BOp->getOperand (0 ), BOp->getOperand (1 )};
3673+ auto Res = Builder.CreateMul (
3674+ NewAnd, ConstantInt::get (NewAnd->getType (), Decomp1->Factor ), " " ,
3675+ Decomp0->NUW && Decomp1->NUW , Decomp0->NSW && Decomp1->NSW );
3676+ return Res;
3677+ }
3678+ }
3679+ }
36903680
3691- auto DecompBitMaskMul1 = matchBitmaskMul (BOp->getOperand (1 ));
3692- if (DecompBitMaskMul1)
3693- return {DecompBitMaskMul1, BOp->getOperand (1 ), BOp->getOperand (0 )};
3681+ return nullptr ;
3682+ }
3683+
3684+ // / If IsLogical is true, then the and/or is in select form and the transform
3685+ // / must be poison-safe.
3686+ Value *InstCombinerImpl::foldDisjointOr (Value *LHS, Value *RHS,
3687+ Instruction &I) {
3688+ if (Value *V = foldBitmaskMul (LHS, RHS, Builder))
3689+ return V;
36943690
3695- return CombinedBitmaskMul ();
3691+ return nullptr ;
3692+ }
3693+
3694+ // / If IsLogical is true, then the and/or is in select form and the transform
3695+ // / must be poison-safe.
3696+ Value *InstCombinerImpl::foldBooleanAndOr (Value *LHS, Value *RHS,
3697+ Instruction &I, bool IsAnd,
3698+ bool IsLogical) {
3699+ if (!LHS->getType ()->isIntOrIntVectorTy (1 ))
3700+ return nullptr ;
3701+
3702+ // handle (roughly):
3703+ // (icmp ne (A & B), C) | (icmp ne (A & D), E)
3704+ // (icmp eq (A & B), C) & (icmp eq (A & D), E)
3705+ if (Value *V = foldLogOpOfMaskedICmps (LHS, RHS, IsAnd, IsLogical, Builder,
3706+ SQ.getWithInstruction (&I)))
3707+ return V;
3708+
3709+ if (auto *LHSCmp = dyn_cast<ICmpInst>(LHS))
3710+ if (auto *RHSCmp = dyn_cast<ICmpInst>(RHS))
3711+ if (Value *Res = foldAndOrOfICmps (LHSCmp, RHSCmp, I, IsAnd, IsLogical))
3712+ return Res;
3713+
3714+ if (auto *LHSCmp = dyn_cast<FCmpInst>(LHS))
3715+ if (auto *RHSCmp = dyn_cast<FCmpInst>(RHS))
3716+ if (Value *Res = foldLogicOfFCmps (LHSCmp, RHSCmp, IsAnd, IsLogical))
3717+ return Res;
3718+
3719+ if (Value *Res = foldEqOfParts (LHS, RHS, IsAnd))
3720+ return Res;
3721+
3722+ return nullptr ;
3723+ }
3724+
3725+ static Value *foldOrOfInversions (BinaryOperator &I,
3726+ InstCombiner::BuilderTy &Builder) {
3727+ assert (I.getOpcode () == Instruction::Or &&
3728+ " Simplification only supports or at the moment." );
3729+
3730+ Value *Cmp1, *Cmp2, *Cmp3, *Cmp4;
3731+ if (!match (I.getOperand (0 ), m_And (m_Value (Cmp1), m_Value (Cmp2))) ||
3732+ !match (I.getOperand (1 ), m_And (m_Value (Cmp3), m_Value (Cmp4))))
3733+ return nullptr ;
3734+
3735+ // Check if any two pairs of the and operations are inversions of each other.
3736+ if (isKnownInversion (Cmp1, Cmp3) && isKnownInversion (Cmp2, Cmp4))
3737+ return Builder.CreateXor (Cmp1, Cmp4);
3738+ if (isKnownInversion (Cmp1, Cmp4) && isKnownInversion (Cmp2, Cmp3))
3739+ return Builder.CreateXor (Cmp1, Cmp3);
3740+
3741+ return nullptr ;
36963742}
36973743
36983744// FIXME: We use commutative matchers (m_c_*) for some, but not all, matches
@@ -3777,48 +3823,24 @@ Instruction *InstCombinerImpl::visitOr(BinaryOperator &I) {
37773823 /* NSW=*/ true , /* NUW=*/ true ))
37783824 return R;
37793825
3780- // (A & N) * C + (A & M) * C -> (A & (N + M)) & C
3781- // This also accepts the equivalent select form of (A & N) * C
3782- // expressions i.e. !(A & N) ? 0 : N * C)
3783- CombinedBitmaskMul Decomp1 = matchCombinedBitmaskMul (I.getOperand (1 ));
3784- auto BMDecomp1 = Decomp1.Decomp ;
3785-
3786- if (BMDecomp1) {
3787- CombinedBitmaskMul Decomp0 = matchCombinedBitmaskMul (I.getOperand (0 ));
3788- auto BMDecomp0 = Decomp0.Decomp ;
3789-
3790- if (BMDecomp0) {
3791- // If we have independent operands in the BitmaskMul chain, then just
3792- // reassociate to encourage combining in future iterations.
3793- if (Decomp0.OtherOp || Decomp1.OtherOp ) {
3794- Value *OtherOp = Decomp0.OtherOp ? Decomp0.OtherOp : Decomp1.OtherOp ;
3795-
3796- if (Decomp0.OtherOp && Decomp1.OtherOp ) {
3797- OtherOp = Builder.CreateOr (Decomp0.OtherOp , Decomp1.OtherOp );
3798- cast<PossiblyDisjointInst>(OtherOp)->setIsDisjoint (true );
3799- }
3800-
3801- auto CombinedOp =
3802- Builder.CreateOr (Decomp0.DecompOp , Decomp1.DecompOp );
3803- cast<PossiblyDisjointInst>(CombinedOp)->setIsDisjoint (true );
3804-
3805- return BinaryOperator::CreateDisjointOr (CombinedOp, OtherOp);
3806- }
3807-
3808- if (BMDecomp0->isCombineableWith (*BMDecomp1)) {
3809- auto NewAnd = Builder.CreateAnd (
3810- BMDecomp0->X ,
3811- ConstantInt::get (BMDecomp0->X ->getType (),
3812- (BMDecomp0->Mask + BMDecomp1->Mask )));
3813-
3814- auto *Combined = BinaryOperator::CreateMul (
3815- NewAnd, ConstantInt::get (NewAnd->getType (), BMDecomp1->Factor ));
3816-
3817- Combined->setHasNoUnsignedWrap (BMDecomp0->NUW && BMDecomp1->NUW );
3818- Combined->setHasNoSignedWrap (BMDecomp0->NSW && BMDecomp1->NSW );
3826+ if (Value *Res = foldBitmaskMul (I.getOperand (0 ), I.getOperand (1 ), Builder))
3827+ return replaceInstUsesWith (I, Res);
38193828
3820- return Combined;
3821- }
3829+ Value *X, *Y;
3830+ if (match (I.getOperand (1 ),
3831+ m_OneUse (m_DisjointOr (m_Value (X), m_Value (Y))))) {
3832+ if (auto Res = reassociateBooleanAndOr (
3833+ I.getOperand (0 ), X, Y, I, /* IsAnd=*/ false , /* RHSIsLogical=*/ true ,
3834+ /* RHSIsDisjoint*/ true )) {
3835+ return replaceInstUsesWith (I, Res);
3836+ }
3837+ }
3838+ if (match (I.getOperand (0 ),
3839+ m_OneUse (m_DisjointOr (m_Value (X), m_Value (Y))))) {
3840+ if (auto Res = reassociateBooleanAndOr (
3841+ I.getOperand (1 ), X, Y, I, /* IsAnd=*/ false , /* RHSIsLogical=*/ true ,
3842+ /* RHSIsDisjoint*/ true )) {
3843+ return replaceInstUsesWith (I, Res);
38223844 }
38233845 }
38243846 }
0 commit comments