Added code to mark divergent BB. The divergent info

jgu222 · sys_zuul · commit 2df9fa338671 · 2020-02-25T12:45:45.000-08:00
is a separate field within BB for now. It will be used
to replace InSIMDFlow field.

Change-Id: I68aba5d496314e4d776b447896f56ab2c4f7edb4
diff --git a/visa/FlowGraph.cpp b/visa/FlowGraph.cpp
@@ -933,6 +933,8 @@ void FlowGraph::constructFlowGraph(INST_LIST& instlist)
         kernelInfo->updateExitBB(BBs.back());
     }
 
+    markDivergentBBs();
+
     builder->materializeGlobalImm(getEntryBB());
     normalizeRegionDescriptors();
     localDataFlowAnalysis();
@@ -2774,6 +2776,270 @@ void FlowGraph::markSimdBlocks(std::map<std::string, G4_BB*>& labelMap, FuncInfo
     }
 }
 
+// markDivergentBBs()
+//    If BB is on the divergent path, mark it as divergent.
+// Divergent:
+//    If all active simd lanes on entry to shader/kernel are
+//    active in a BB,  that BB is NOT divergent;  otherwise,
+//    it is divergent.
+// Note: this will be used to replace inSIMDCF gradually.
+void FlowGraph::markDivergentBBs()
+{
+    // Assumption:
+    //       1.  For each function, it has a single return (for function)
+    //           or exit (for entry function). And that return/exit is the
+    //           last BB of that function.
+    //       2.  The entry function will appear first in the BB list, and
+    //           if there is a call from A to B,  A shall appear prior to B
+    //           in BB list.
+    //       3.  There is no indirect call, and no recursive call.
+    //
+    // Required:  need to set BB id.
+    //
+    // Key variables:
+    //   LastJoinBB:
+    //     LastJoinBB is the fartherest joinBB of any goto/break/if/while
+    //     so far, as described below in the algorithm.
+    //   LastJoinBBId:  Id(LastJoinBB).
+    //
+    // The algorithm initializes LastJoinBBId to -1 and scans all BBs in order.
+    // It checks control-flow instructions to see if it diverges (has join).
+    // If so, the algorithm sets LastJoinBBId to be the larger one of its join BB
+    // and LastJoinBBId; For a non-negative LastJoinBBId, it means that there is
+    // an active join in that BB, and therefore, all BBs from the current BB to
+    // that LastJoinBB (LastJoinBB not included) will be in divergent path.
+    //
+    // The algorithm checks the following cases and their join BBs are:
+    //     case 1: cf inst = goto
+    //          <currBB>    [(p)] goto L
+    //                           ...
+    //          <joinBB>    L:
+    //     case 2: cf inst = if
+    //          <currBB>    if
+    //                          ...
+    //          <joinBB>    endif
+    //
+    //     case 3:  cf inst = break
+    //          <currBB>   break
+    //                     ...
+    //                     [(p)] while
+    //          <joinBB>
+    //     case 4:
+    //          <currBB>  L:
+    //                     ....
+    //                    [(p)] while/goto L
+    //          <joinBB>
+    //
+    int LastJoinBBId;
+
+    auto pushJoin = [&](G4_BB* joinBB) {
+        LastJoinBBId = std::max(LastJoinBBId, (int)joinBB->getId());
+    };
+    auto popJoinIfMatch = [&](G4_BB* BB) {
+        if ((int)BB->getId() == LastJoinBBId) {
+            LastJoinBBId = -1;
+        }
+    };
+
+    if (BBs.empty())
+    {
+        // Sanity check
+        return;
+    }
+
+    reassignBlockIDs();
+
+    // Analyze function in topological order. As there is no recursion
+    // and no indirect call,  a function will be analyzed only if all
+    // its callers have been analyzed.
+    //
+    // If no subroutine, sortedFuncTable is empty. Here keep all functions
+    // in a vector first (it works with and without subroutines), then scan
+    // functions in topological order.
+    struct StartEndIter {
+        BB_LIST_ITER StartI;
+        BB_LIST_ITER EndI;
+        bool  InvokedFromDivergentBB;
+    };
+    int numFuncs = (int)sortedFuncTable.size();
+    std::vector<StartEndIter> allFuncs;
+    std::unordered_map<FuncInfo*, uint32_t> funcInfoIndex;
+    if (numFuncs == 0)
+    {
+        numFuncs = 1;
+        allFuncs.resize(1);
+        allFuncs[0].StartI = BBs.begin();
+        allFuncs[0].EndI = BBs.end();
+        allFuncs[0].InvokedFromDivergentBB = false;
+    }
+    else
+    {
+        allFuncs.resize(numFuncs);
+        for (int i = numFuncs; i > 0; --i)
+        {
+            uint32_t ix = (uint32_t)(i - 1);
+            FuncInfo* pFInfo = sortedFuncTable[ix];
+            G4_BB* StartBB = pFInfo->getInitBB();
+            G4_BB* EndBB = pFInfo->getExitBB();
+            uint32_t ui = (uint32_t)(numFuncs - i);
+            allFuncs[ui].StartI = std::find(BBs.begin(), BBs.end(), StartBB);
+            auto nextI = std::find(BBs.begin(), BBs.end(), EndBB);
+            assert(nextI != BBs.end() && "ICE: subroutine's end BB not found!");
+            allFuncs[ui].EndI = (++nextI);
+            allFuncs[ui].InvokedFromDivergentBB = false;
+
+            funcInfoIndex[pFInfo] = ui;
+        }
+    }
+
+    for (int i = 0; i < numFuncs; ++i)
+    {
+        // each function: [IT, IE)
+        BB_LIST_ITER& IT = allFuncs[i].StartI;
+        BB_LIST_ITER& IE = allFuncs[i].EndI;
+        if (IT == IE)
+        {
+            // sanity check
+            continue;
+        }
+
+        if (allFuncs[i].InvokedFromDivergentBB)
+        {
+            // subroutine's divergent on entry. Mark every BB as divergent
+            for (; IT != IE; ++IT)
+            {
+                G4_BB* BB = *IT;
+
+                BB->setDivergent(true);
+                // set InSIMDFlow as well, will merge two gradually
+                BB->setInSimdFlow(true);
+
+                if (BB->size() == 0)
+                {
+                    // sanity check
+                    continue;
+                }
+                if (BB->isEndWithCall() || BB->isEndWithFCall())
+                {
+                    FuncInfo* calleeFunc = BB->getCalleeInfo();
+                    if (funcInfoIndex.count(calleeFunc))
+                    {
+                        int ix = funcInfoIndex[calleeFunc];
+                        allFuncs[ix].InvokedFromDivergentBB = true;
+                    }
+                }
+            }
+            // continue for next func
+            continue;
+        }
+
+        LastJoinBBId = -1;
+        for (; IT != IE; ++IT)
+        {
+            G4_BB* BB = *IT;
+
+            // This handles cases in which BB has endif/while/join as well as others
+            // so we don't need to explicitly check whether BB has endif/while/join, etc.
+            popJoinIfMatch(BB);
+
+            // Handle loop
+            for (auto iter = BB->Preds.begin(), iterEnd = BB->Preds.end(); iter != iterEnd; ++iter)
+            {
+                G4_BB* predBB = *iter;
+                if (predBB->getId() < BB->getId())
+                    continue;
+
+                BB_LIST_ITER LoopIterEnd = std::find(BBs.begin(), BBs.end(), predBB);
+
+                // joinBB is the BB right after backward-goto/while
+                BB_LIST_ITER loopJoinIter = LoopIterEnd;
+                ++loopJoinIter;
+                assert(loopJoinIter != BBs.end() && "ICE: missing join BB!");
+                G4_BB* joinBB = *loopJoinIter;
+
+                // Scan loop to find any out-of-loop branch, set join if any
+                for (auto LoopIter = IT; LoopIter != LoopIterEnd; ++LoopIter)
+                {
+                    G4_BB* BB1 = *LoopIter;
+                    if (BB1->size() == 0)
+                    {
+                        continue;
+                    }
+                    G4_INST* lastInst = BB1->back();
+                    if ((lastInst->opcode() == G4_break || lastInst->opcode() == G4_goto) &&
+                        !lastInst->asCFInst()->isUniform())
+                    {
+                        G4_BB* newJoinBB = joinBB;
+                        if (lastInst->opcode() == G4_goto)
+                        {
+                            G4_BB* targetBB = BB1->Succs.back();
+                            if (targetBB->getId() <= predBB->getId())
+                            {
+                                continue;
+                            }
+                            newJoinBB = targetBB;
+                        }
+
+                        // Need to find a 1st join, the other joins will
+                        // be examined within the outer loop.
+                        pushJoin(newJoinBB);
+                        break;
+                    }
+                }
+
+                G4_INST* lastInst = predBB->back();
+                if (!lastInst->asCFInst()->isUniform())
+                {
+                    pushJoin(joinBB);
+                }
+            }
+
+            if ((int)BB->getId() < LastJoinBBId) {
+                BB->setDivergent(true);
+                // set InSIMDFlow as well, will merge these two fields gradually
+                BB->setInSimdFlow(true);
+            }
+
+            if (BB->size() == 0)
+            {
+                continue;
+            }
+
+            G4_INST* lastInst = BB->back();
+            if (((lastInst->opcode() == G4_goto && !lastInst->asCFInst()->isBackward()) ||
+                 lastInst->opcode() == G4_break) && !lastInst->asCFInst()->isUniform())
+            {
+                // forward goto/break : the last Succ BB is our target BB
+                // For break, it should be the BB right after while inst.
+                G4_BB* joinBB = BB->Succs.back();
+                pushJoin(joinBB);
+            }
+            else if (lastInst->opcode() == G4_if && !lastInst->asCFInst()->isUniform())
+            {
+                G4_Label* labelInst = lastInst->asCFInst()->getUip();
+                G4_BB* joinBB = findLabelBB(IT, IE, labelInst->getLabel());
+                assert(joinBB && "ICE(vISA) : missing endif label!");
+                pushJoin(joinBB);
+            }
+            else if (lastInst->opcode() == G4_call || lastInst->opcode() == G4_pseudo_fcall)
+            {
+                // If this function is already in divergent branch, the callee
+                // must be in a divergent branch!.
+                if (BB->isDivergent() || lastInst->getPredicate() != nullptr)
+                {
+                    FuncInfo* calleeFunc = BB->getCalleeInfo();
+                    if (funcInfoIndex.count(calleeFunc))
+                    {
+                        int ix = funcInfoIndex[calleeFunc];
+                        allFuncs[ix].InvokedFromDivergentBB = true;
+                    }
+                }
+            }
+        }
+    }
+    return;
+}
+
 /*
 * Insert a join at the beginning of this basic block, immediately after the label
 * If a join is already present, nothing will be done
diff --git a/visa/FlowGraph.h b/visa/FlowGraph.h
@@ -289,6 +289,12 @@ class G4_BB
     // if the block is under simd flow control
     bool inSimdFlow;
 
+    // If a BB is divergent, this field is set to true. By divergent, it means
+    // that among all active lanes on entry to shader/kernel, some lanes are not
+    // (potentially) active in this BB.
+    // Note : this field will be used to replace inSimdFlow.
+    bool divergent;
+
     // the physical pred/succ for this block (i.e., the pred/succ for this block in the BB list)
     // Note that some transformations may rearrange BB layout, so for safety it's best to recompute
     // this
@@ -374,8 +380,8 @@ class G4_BB
         traversal(0), idom(NULL), beforeCall(NULL),
         afterCall(NULL), calleeInfo(NULL), BBType(G4_BB_NONE_TYPE),
         inNaturalLoop(false), hasSendInBB(false), loopNestLevel(0), scopeID(0),
-        inSimdFlow(false), physicalPred(NULL), physicalSucc(NULL), parent(fg),
-        instList(alloc)
+        inSimdFlow(false), divergent(false), physicalPred(NULL), physicalSucc(NULL),
+        parent(fg), instList(alloc)
     {
     }
 
@@ -424,6 +430,8 @@ class G4_BB
     void     resetNestLevel()                 { loopNestLevel = 0; }
     void     setInSimdFlow(bool val)          {inSimdFlow = val;}
     bool     isInSimdFlow()                   {return inSimdFlow;}
+    void     setDivergent(bool val) { divergent = val; }
+    bool     isDivergent() const    { return divergent; }
     unsigned getScopeID() { return scopeID; }
     void setScopeID(unsigned id) { scopeID = id; }
 
@@ -1114,6 +1122,7 @@ class FlowGraph
     void addFrameSetupDeclares(IR_Builder& builder, PhyRegPool& regPool);
     void addSaveRestorePseudoDeclares(IR_Builder& builder);
     void markSimdBlocks(std::map<std::string, G4_BB*>& labelMap, FuncInfoHashTable &FuncInfoMap);
+    void markDivergentBBs();
 
     // Used for CISA 3.0
     void incrementNumBBs() { numBBId++ ; }
diff --git a/visa/Gen4_IR.hpp b/visa/Gen4_IR.hpp
@@ -1239,7 +1239,7 @@ class G4_InstCF : public G4_INST
     bool isBackwardBr;
 
     // True if this branch is a uniform (all active lanes goes to the same target)
-    // Valid for if/while/break/goto only. This could be encoded in instOpt
+    // Valid for if/while/break/goto/jmpi only. This could be encoded in instOpt
     bool isUniformBr;
 
 public:
@@ -1257,7 +1257,8 @@ class G4_InstCF : public G4_INST
         G4_INST(builder, prd, op, nullptr, false, size, nullptr, nullptr, nullptr, instOpt),
         jip(jipLabel), uip(uipLabel), isBackwardBr(false), isUniformBr(false)
     {
-        isUniformBr = (op == G4_goto && (size == 1 || prd == nullptr));
+        isUniformBr = (op == G4_jmpi ||
+                       (op == G4_goto && (size == 1 || prd == nullptr)));
     }
 
     // used by jump/call/ret
