@@ -8279,8 +8279,8 @@ static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
82798279static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
82808280 const X86Subtarget &Subtarget, SelectionDAG &DAG,
82818281 SDValue &Opnd0, SDValue &Opnd1,
8282- unsigned &NumExtracts,
8283- bool &IsSubAdd ) {
8282+ unsigned &NumExtracts, bool &IsSubAdd,
8283+ bool &HasAllowContract ) {
82848284 using namespace SDPatternMatch;
82858285
82868286 MVT VT = BV->getSimpleValueType(0);
@@ -8292,6 +8292,7 @@ static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
82928292 SDValue InVec1 = DAG.getUNDEF(VT);
82938293
82948294 NumExtracts = 0;
8295+ HasAllowContract = NumElts != 0;
82958296
82968297 // Odd-numbered elements in the input build vector are obtained from
82978298 // adding/subtracting two integer/float elements.
@@ -8350,6 +8351,7 @@ static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
83508351
83518352 // Increment the number of extractions done.
83528353 ++NumExtracts;
8354+ HasAllowContract &= Op->getFlags().hasAllowContract();
83538355 }
83548356
83558357 // Ensure we have found an opcode for both parities and that they are
@@ -8393,9 +8395,10 @@ static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
83938395/// is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit
83948396/// FMADDSUB is.
83958397static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget,
8396- SelectionDAG &DAG,
8397- SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
8398- unsigned ExpectedUses) {
8398+ SelectionDAG &DAG, SDValue &Opnd0,
8399+ SDValue &Opnd1, SDValue &Opnd2,
8400+ unsigned ExpectedUses,
8401+ bool AllowSubAddOrAddSubContract) {
83998402 if (Opnd0.getOpcode() != ISD::FMUL ||
84008403 !Opnd0->hasNUsesOfValue(ExpectedUses, 0) || !Subtarget.hasAnyFMA())
84018404 return false;
@@ -8406,7 +8409,8 @@ static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget,
84068409 // or MUL + ADDSUB to FMADDSUB.
84078410 const TargetOptions &Options = DAG.getTarget().Options;
84088411 bool AllowFusion =
8409- (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath);
8412+ Options.AllowFPOpFusion == FPOpFusion::Fast ||
8413+ (AllowSubAddOrAddSubContract && Opnd0->getFlags().hasAllowContract());
84108414 if (!AllowFusion)
84118415 return false;
84128416
@@ -8427,15 +8431,17 @@ static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
84278431 SDValue Opnd0, Opnd1;
84288432 unsigned NumExtracts;
84298433 bool IsSubAdd;
8430- if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts,
8431- IsSubAdd))
8434+ bool HasAllowContract;
8435+ if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts, IsSubAdd,
8436+ HasAllowContract))
84328437 return SDValue();
84338438
84348439 MVT VT = BV->getSimpleValueType(0);
84358440
84368441 // Try to generate X86ISD::FMADDSUB node here.
84378442 SDValue Opnd2;
8438- if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts)) {
8443+ if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts,
8444+ HasAllowContract)) {
84398445 unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
84408446 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
84418447 }
@@ -43180,7 +43186,7 @@ static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
4318043186/// the fact that they're unused.
4318143187static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
4318243188 SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
43183- bool &IsSubAdd) {
43189+ bool &IsSubAdd, bool &HasAllowContract ) {
4318443190
4318543191 EVT VT = N->getValueType(0);
4318643192 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -43231,6 +43237,8 @@ static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
4323143237 // It's a subadd if the vector in the even parity is an FADD.
4323243238 IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD
4323343239 : V2->getOpcode() == ISD::FADD;
43240+ HasAllowContract =
43241+ V1->getFlags().hasAllowContract() && V2->getFlags().hasAllowContract();
4323443242
4323543243 Opnd0 = LHS;
4323643244 Opnd1 = RHS;
@@ -43288,14 +43296,17 @@ static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N, const SDLoc &DL,
4328843296
4328943297 SDValue Opnd0, Opnd1;
4329043298 bool IsSubAdd;
43291- if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd))
43299+ bool HasAllowContract;
43300+ if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd,
43301+ HasAllowContract))
4329243302 return SDValue();
4329343303
4329443304 MVT VT = N->getSimpleValueType(0);
4329543305
4329643306 // Try to generate X86ISD::FMADDSUB node here.
4329743307 SDValue Opnd2;
43298- if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2)) {
43308+ if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2,
43309+ HasAllowContract)) {
4329943310 unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
4330043311 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
4330143312 }
0 commit comments