[AMDGPU][AMDGPUIfConverter] First version of AMDGPUIfConversion

Author: jmmartinez

llvm/include/llvm/CodeGen/SSAIfConv.h

@@ -82,7 +82,11 @@ class SSAIfConv {
   SmallVector<MachineOperand, 4> Cond;
 
   struct PredicationStrategyBase {
-    virtual bool canConvertIf(MachineBasicBlock *Tail) { return true; }
+    virtual bool canConvertIf(MachineBasicBlock *Head, MachineBasicBlock *TBB,
+                              MachineBasicBlock *FBB, MachineBasicBlock *Tail,
+                              ArrayRef<MachineOperand> Cond) {
+      return true;
+    }
     virtual bool canPredicateInstr(const MachineInstr &I) = 0;
     /// Apply cost model and heuristics to the if-conversion in IfConv.
     /// Return true if the conversion is a good idea.

llvm/lib/CodeGen/EarlyIfConversion.cpp

@@ -76,7 +76,9 @@ struct SpeculateStrategy : SSAIfConv::PredicationStrategyBase {
                     MachineTraceMetrics *Traces = nullptr)
       : Loops(Loops), SchedModel(SchedModel), Traces(Traces) {}
 
-  bool canConvertIf(MachineBasicBlock *Tail) override {
+  bool canConvertIf(MachineBasicBlock *Head, MachineBasicBlock *TBB,
+                    MachineBasicBlock *FBB, MachineBasicBlock *Tail,
+                    ArrayRef<MachineOperand> Cond) override {
     // This is a triangle or a diamond.
     // Skip if we cannot predicate and there are no phis skip as there must
     // be side effects that can only be handled with predication.

llvm/lib/CodeGen/SSAIfConv.cpp

@@ -249,7 +249,7 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB) {
     return false;
   }
 
-  if (!Predicate.canConvertIf(Tail)) {
+  if (!Predicate.canConvertIf(Head, TBB, FBB, Tail, Cond)) {
     return false;
   }
 

llvm/lib/Target/AMDGPU/AMDGPUIfConverter.cpp

@@ -1,54 +1,158 @@
-#include <llvm/CodeGen/MachineBasicBlock.h>
-#include <llvm/CodeGen/MachineBranchProbabilityInfo.h>
-#include <llvm/CodeGen/MachineDominators.h>
-#include <llvm/CodeGen/MachineFunctionPass.h>
-#include <llvm/CodeGen/MachineLoopInfo.h>
-#include <llvm/CodeGen/SSAIfConv.h>
-#include <llvm/CodeGen/TargetInstrInfo.h>
-#include <llvm/CodeGen/TargetRegisterInfo.h>
-#include <llvm/CodeGen/TargetSchedule.h>
-#include <llvm/CodeGen/TargetSubtargetInfo.h>
-#include <llvm/InitializePasses.h>
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/SSAIfConv.h"
 
 #include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "SIMachineFunctionInfo.h"
 
 using namespace llvm;
 
-namespace {
 #define DEBUG_TYPE "amdgpu-if-cvt"
-const char PassName[] = "AMDGPU if conversion";
-
-class AMDGPUIfConverter : public MachineFunctionPass {
-  const TargetInstrInfo *TII = nullptr;
-  const TargetRegisterInfo *TRI = nullptr;
-  TargetSchedModel SchedModel;
-  MachineRegisterInfo *MRI = nullptr;
-  MachineDominatorTree *DomTree = nullptr;
-  MachineBranchProbabilityInfo *MBPI = nullptr;
-  MachineLoopInfo *Loops = nullptr;
-
-  static constexpr unsigned BlockInstrLimit = 30;
-  static constexpr bool Stress = false;
-  SSAIfConv IfConv{DEBUG_TYPE, BlockInstrLimit, Stress};
-
-public:
-  static char ID;
 
-  AMDGPUIfConverter() : MachineFunctionPass(ID) {}
+namespace {
+unsigned getReversedVCMPXOpcode(unsigned Opcode) {
+  // TODO: this is a placeholder for the real function
+  switch (Opcode) {
+  case AMDGPU::V_CMPX_LT_I32_nosdst_e64:
+    return AMDGPU::V_CMPX_GE_I32_nosdst_e64;
+  default:
+    errs() << "unhandled: " << Opcode << "\n";
+    llvm_unreachable("unhandled vcmp opcode");
+  }
+}
 
-  bool runOnMachineFunction(MachineFunction &MF) override;
+bool needsExecPredication(const SIInstrInfo *TII, const MachineInstr &I) {
+  return TII->isVALU(I) || TII->isVMEM(I);
+}
 
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
+struct ExecPredicate : SSAIfConv::PredicationStrategyBase {
+  const SIInstrInfo *TII;
+  const SIRegisterInfo *RegInfo;
+
+  MachineInstr *Cmp = nullptr;
+
+  ExecPredicate(const SIInstrInfo *TII)
+      : TII(TII), RegInfo(&TII->getRegisterInfo()) {}
+
+  bool canConvertIf(MachineBasicBlock *Head, MachineBasicBlock *TBB,
+                    MachineBasicBlock *FBB, MachineBasicBlock *Tail,
+                    ArrayRef<MachineOperand> Cond) override {
+
+    // check that the cmp is just before the branch and that it is promotable to
+    // v_cmpx
+    const unsigned SupportedBranchOpc[]{
+        AMDGPU::S_CBRANCH_SCC0, AMDGPU::S_CBRANCH_SCC1, AMDGPU::S_CBRANCH_VCCNZ,
+        AMDGPU::S_CBRANCH_VCCZ};
 
-  bool tryConvertIf(MachineBasicBlock *);
+    MachineInstr &CBranch = *Head->getFirstInstrTerminator();
+    if (!llvm::is_contained(SupportedBranchOpc, CBranch.getOpcode()))
+      return false;
+
+    auto CmpInstr = std::next(CBranch.getReverseIterator());
+    if (CmpInstr == Head->instr_rend())
+      return false;
+
+    Register SCCorVCC = Cond[1].getReg();
+    bool ModifiesConditionReg = CmpInstr->modifiesRegister(SCCorVCC, RegInfo);
+    if (!ModifiesConditionReg)
+      return false;
+
+    Cmp = &*CmpInstr;
+
+    unsigned CmpOpc = Cmp->getOpcode();
+    if (TII->isSALU(*Cmp))
+      CmpOpc = TII->getVALUOp(*Cmp);
+    if (AMDGPU::getVCMPXOpFromVCMP(CmpOpc) == -1) {
+      errs() << "unhandled branch " << *Cmp << "\n";
+      return false;
+    }
+
+    return true;
+  }
+
+  bool canPredicateInstr(const MachineInstr &I) override {
+
+    // TODO: relax this condition, if exec is masked, check that it goes back to
+    // normal
+    // TODO: what about scc or vcc ? Are they taken into acount in the MBB
+    // live-ins ?
+    MCRegister Exec = RegInfo->getExec();
+    bool ModifiesExec = I.modifiesRegister(Exec, RegInfo);
+    if (ModifiesExec)
+      return false;
+
+    if (needsExecPredication(TII, I))
+      return true;
+
+    bool DontMoveAcrossStore = true;
+    bool IsSpeculatable = I.isDereferenceableInvariantLoad() ||
+                          I.isSafeToMove(DontMoveAcrossStore);
+    if (IsSpeculatable)
+      return true;
+
+    return false;
+  }
 
+  bool shouldConvertIf(SSAIfConv &IfConv) override {
+    // TODO: cost model
+    return true;
+  }
+
+  void predicateBlock(MachineBasicBlock *MBB, ArrayRef<MachineOperand> Cond,
+                      bool Reverse) override {
+    // save exec
+    MachineFunction &MF = *MBB->getParent();
+    SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+
+    Register ExecBackup = MFI->getSGPRForEXECCopy();
+
+    const DebugLoc &CmpLoc = Cmp->getDebugLoc();
+
+    auto FirstInstruction = MBB->begin();
+    const bool IsSCCLive =
+        false; // asume not since the live-ins are supposed to be empty
+    TII->insertScratchExecCopy(MF, *MBB, FirstInstruction, CmpLoc, ExecBackup,
+                               IsSCCLive);
+
+    // mask exec
+    unsigned CmpOpc = Cmp->getOpcode();
+    if (TII->isSALU(*Cmp))
+      CmpOpc = TII->getVALUOp(*Cmp);
+
+    CmpOpc = AMDGPU::getVCMPXOpFromVCMP(CmpOpc);
+    if (Reverse)
+      CmpOpc = getReversedVCMPXOpcode(CmpOpc);
+
+    // TODO: handle this properly. The second block may kill those registers.
+    Cmp->getOperand(0).setIsKill(false);
+    Cmp->getOperand(1).setIsKill(false);
+
+    auto VCmpX = BuildMI(*MBB, FirstInstruction, CmpLoc, TII->get(CmpOpc));
+    VCmpX->addOperand(Cmp->getOperand(0));
+    VCmpX->addOperand(Cmp->getOperand(1));
+
+    // restore exec
+    TII->restoreExec(MF, *MBB, MBB->end(), DebugLoc(), ExecBackup);
+  }
+
+  ~ExecPredicate() override = default;
+};
+
+const char PassName[] = "AMDGPU If Conversion";
+
+struct AMDGPUIfConverter : MachineFunctionPass {
+  static char ID;
+  AMDGPUIfConverter() : MachineFunctionPass(ID) {}
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  bool runOnMachineFunction(MachineFunction &MF) override;
   StringRef getPassName() const override { return PassName; }
 };
 
 char AMDGPUIfConverter::ID = 0;
 
 void AMDGPUIfConverter::getAnalysisUsage(AnalysisUsage &AU) const {
-  AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
   AU.addRequired<MachineDominatorTreeWrapperPass>();
   AU.addPreserved<MachineDominatorTreeWrapperPass>();
   AU.addRequired<MachineLoopInfoWrapperPass>();
@@ -60,29 +164,19 @@ bool AMDGPUIfConverter::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
 
-  const TargetSubtargetInfo &STI = MF.getSubtarget();
-  TII = STI.getInstrInfo();
-  TRI = STI.getRegisterInfo();
-  MRI = &MF.getRegInfo();
-  SchedModel.init(&STI);
-  DomTree = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
-  Loops = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
-  MBPI = &getAnalysis<MachineBranchProbabilityInfoWrapperPass>().getMBPI();
-
-  bool Changed = false;
-  IfConv.runOnMachineFunction(MF);
+  const auto &STI = MF.getSubtarget<GCNSubtarget>();
+  if (!STI.hasGFX10_3Insts())
+    return false;
 
-  for (auto *DomNode : post_order(DomTree))
-    if (tryConvertIf(DomNode->getBlock()))
-      Changed = true;
+  const SIInstrInfo *TII = STI.getInstrInfo();
+  auto *DomTree = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
+  auto *Loops = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
 
-  return Changed;
+  ExecPredicate Predicate(TII);
+  SSAIfConv IfConv(Predicate, MF, DomTree, Loops);
+  return IfConv.run();
 }
-
-bool AMDGPUIfConverter::tryConvertIf(MachineBasicBlock *MBB) { return false; }
-
 } // namespace
-
 char &llvm::AMDGPUIfConverterID = AMDGPUIfConverter::ID;
 INITIALIZE_PASS_BEGIN(AMDGPUIfConverter, DEBUG_TYPE, PassName, false, false)
 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)

llvm/test/CodeGen/AMDGPU/amdgpu-if-cvt.ll

@@ -5,10 +5,9 @@ define amdgpu_kernel void @scalar_cmp(i32 noundef %value, ptr addrspace(8) nocap
 ; GCN-LABEL: scalar_cmp:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_load_dword s0, s[2:3], 0x4c
+; GCN-NEXT:    s_or_saveexec_b32 s105, -1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_cmp_lt_i32 s0, 1
-; GCN-NEXT:    s_cbranch_scc1 .LBB0_2
-; GCN-NEXT:  ; %bb.1: ; %if.then
+; GCN-NEXT:    v_cmpx_ge_i32_e64 s0, 1
 ; GCN-NEXT:    s_clause 0x2
 ; GCN-NEXT:    s_load_dword s4, s[2:3], 0x24
 ; GCN-NEXT:    s_load_dword s5, s[2:3], 0x44
@@ -17,7 +16,6 @@ define amdgpu_kernel void @scalar_cmp(i32 noundef %value, ptr addrspace(8) nocap
 ; GCN-NEXT:    v_mov_b32_e32 v0, s4
 ; GCN-NEXT:    v_mov_b32_e32 v1, s5
 ; GCN-NEXT:    buffer_store_dword v0, v1, s[0:3], 0 offen
-; GCN-NEXT:  .LBB0_2: ; %if.end
 ; GCN-NEXT:    s_endpgm
 entry:
   %cmp = icmp sgt i32 %flag, 0