Do not generate S_CMP if add/sub carryout is available

Signed-off-by: John Lu <[email protected]>

Author: LU-JOHN

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

@@ -6081,9 +6081,9 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     MachineOperand &Src0 = MI.getOperand(2);
     MachineOperand &Src1 = MI.getOperand(3);
     MachineOperand &Src2 = MI.getOperand(4);
-    unsigned Opc = (MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO)
-                       ? AMDGPU::S_ADDC_U32
-                       : AMDGPU::S_SUBB_U32;
+
+    bool IsAdd = (MI.getOpcode() == AMDGPU::S_ADD_CO_PSEUDO);
+
     if (Src0.isReg() && TRI->isVectorRegister(MRI, Src0.getReg())) {
       Register RegOp0 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
       BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp0)
@@ -6103,6 +6103,7 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
       Src2.setReg(RegOp2);
     }
 
+<<<<<<< HEAD
     if (ST.isWave64()) {
       if (ST.hasScalarCompareEq64()) {
         BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U64))
@@ -6140,6 +6141,89 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
 
     unsigned SelOpc =
         ST.isWave64() ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
+=======
+    const TargetRegisterClass *Src2RC = MRI.getRegClass(Src2.getReg());
+    unsigned WaveSize = TRI->getRegSizeInBits(*Src2RC);
+    assert(WaveSize == 64 || WaveSize == 32);
+
+    unsigned SelOpc =
+        (WaveSize == 64) ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
+    unsigned AddcSubbOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32;
+    unsigned AddSubOpc = IsAdd ? AMDGPU::S_ADD_I32 : AMDGPU::S_SUB_I32;
+    //  Lowering for:
+    //
+    //    S_UADDO_PSEUDO|S_ADD_CO_PSEUDO
+    //    <no SCC def code>
+    //    S_ADD_CO_PSEUDO
+    //
+    //  produces:
+    //
+    //    S_ADD_I32|S_ADDC_U32                  ; lowered from S_UADDO_PSEUDO
+    //    SREG = S_CSELECT_B32|64 [1,-1], 0     ; lowered from S_UADDO_PSEUDO
+    //    <no SCC def code>
+    //    S_CMP32|64 SREG, 0                    ; lowered from S_ADD_CO_PSEUDO
+    //    S_ADDC_U32                            ; lowered from S_ADD_CO_PSEUDO
+    //
+    //  At this point before generating the S_CMP check if it is redundant.  If
+    //  so do not recalculate it.  Subsequent optimizations will also delete the
+    //  dead S_CSELECT*.
+
+    bool RecalculateSCC{true};
+    MachineInstr *Def = MRI.getVRegDef(Src2.getReg());
+    if (Def && Def->getParent() == BB && Def->getOpcode() == SelOpc &&
+        Def->getOperand(1).isImm() && Def->getOperand(1).getImm() != 0 &&
+        Def->getOperand(2).isImm() && Def->getOperand(2).getImm() == 0) {
+
+      auto I1 = std::next(MachineBasicBlock::reverse_iterator(Def));
+      if (I1 != BB->rend() &&
+          (I1->getOpcode() == AddSubOpc || I1->getOpcode() == AddcSubbOpc)) {
+        RecalculateSCC = false;
+        // Ensure there are no intervening definitions of SCC.
+        for (auto I2 = std::next(MachineBasicBlock::reverse_iterator(MI));
+             I2 != I1; I2++) {
+          if (I2->definesRegister(AMDGPU::SCC, TRI)) {
+            RecalculateSCC = true;
+            break;
+          }
+        }
+      }
+    }
+
+    if (RecalculateSCC) {
+      if (WaveSize == 64) {
+        if (ST.hasScalarCompareEq64()) {
+          BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U64))
+              .addReg(Src2.getReg())
+              .addImm(0);
+        } else {
+          const TargetRegisterClass *SubRC =
+              TRI->getSubRegisterClass(Src2RC, AMDGPU::sub0);
+          MachineOperand Src2Sub0 = TII->buildExtractSubRegOrImm(
+              MII, MRI, Src2, Src2RC, AMDGPU::sub0, SubRC);
+          MachineOperand Src2Sub1 = TII->buildExtractSubRegOrImm(
+              MII, MRI, Src2, Src2RC, AMDGPU::sub1, SubRC);
+          Register Src2_32 =
+              MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+
+          BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_OR_B32), Src2_32)
+              .add(Src2Sub0)
+              .add(Src2Sub1);
+
+          BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U32))
+              .addReg(Src2_32, RegState::Kill)
+              .addImm(0);
+        }
+      } else {
+        BuildMI(*BB, MII, DL, TII->get(AMDGPU::S_CMP_LG_U32))
+            .addReg(Src2.getReg())
+            .addImm(0);
+      }
+    }
+
+    BuildMI(*BB, MII, DL, TII->get(AddcSubbOpc), Dest.getReg())
+        .add(Src0)
+        .add(Src1);
+>>>>>>> 0cb43743ea30 (Do not generate S_CMP if add/sub carryout is available)
 
     BuildMI(*BB, MII, DL, TII->get(SelOpc), CarryDest.getReg())
         .addImm(-1)
@@ -16588,17 +16672,12 @@ SDValue SITargetLowering::performSetCCCombine(SDNode *N,
   // LHS = ADD i64 Y, 1            LHSlo = UADDO       i32 Ylo, 1
   // setcc LHS eq 0        ->      LHSHi = UADDO_CARRY i32 Yhi, 0
 
-  // Don't split a 64-bit add/sub into two 32-bit add/sub instructions for
-  // non-divergent operations.  This can result in lo/hi 32-bit operations
-  // being done in SGPR and VGPR with additional operations being needed
-  // to move operands and/or generate the intermediate carry.
-  if (VT == MVT::i64 && N->isDivergent() &&
-      ((CC == ISD::SETULT &&
-        sd_match(LHS, m_Add(m_Specific(RHS), m_Value()))) ||
-       (CC == ISD::SETUGT &&
-        sd_match(LHS, m_Sub(m_Specific(RHS), m_Value()))) ||
-       (CC == ISD::SETEQ && CRHS && CRHS->isZero() &&
-        sd_match(LHS, m_Add(m_Value(), m_One()))))) {
+  if (VT == MVT::i64 && ((CC == ISD::SETULT &&
+                          sd_match(LHS, m_Add(m_Specific(RHS), m_Value()))) ||
+                         (CC == ISD::SETUGT &&
+                          sd_match(LHS, m_Sub(m_Specific(RHS), m_Value()))) ||
+                         (CC == ISD::SETEQ && CRHS && CRHS->isZero() &&
+                          sd_match(LHS, m_Add(m_Value(), m_One()))))) {
     EVT TargetType = MVT::i32;
     EVT CarryVT = MVT::i1;
     bool IsAdd = LHS.getOpcode() == ISD::ADD;