Add FetchDelay stage (#27)

Author: andrej

CMakeLists.txt

@@ -99,6 +99,10 @@ set(_CUSTOMHW_SOURCE_FILES
     CustomHWUnits/MCADLSUnit.cpp
    )
 
+set(_CUSTOM_STAGES_SOURCE_FILES
+    CustomStages/MCADFetchDelayStage.cpp
+   )
+
 set(_BROKERS_SOURCE_FILES
     Brokers/BrokerPlugin.cpp
     Brokers/RawBytesBroker.cpp
@@ -111,6 +115,7 @@ set(_SOURCE_FILES
     llvm-mcad.cpp
     ${_MCAVIEWS_SOURCE_FILES}
     ${_CUSTOMHW_SOURCE_FILES}
+    ${_CUSTOM_STAGES_SOURCE_FILES}
     ${_BROKERS_SOURCE_FILES}
     MCAWorker.cpp
     PipelinePrinter.cpp

CustomStages/MCADFetchDelayStage.cpp

@@ -0,0 +1,55 @@
+#include <iostream>
+#include "CustomStages/MCADFetchDelayStage.h"
+
+namespace llvm {
+namespace mcad {
+
+struct MCADInstructionFetchedEvent {};
+
+bool MCADFetchDelayStage::hasWorkToComplete() const {
+    return !instrQueue.empty();
+}
+
+bool MCADFetchDelayStage::isAvailable(const llvm::mca::InstRef &IR) const {
+    return checkNextStage(IR);
+}
+
+llvm::Error MCADFetchDelayStage::forwardDueInstrs() {
+    while(!instrQueue.empty() && instrQueue.front().delayCyclesLeft == 0) {
+        llvm::mca::InstRef IR = instrQueue.front().IR;
+        if (llvm::Error Val = moveToTheNextStage(IR)) {
+            return Val;
+        }
+        instrQueue.pop_front();
+    }
+    return llvm::ErrorSuccess();
+}
+
+llvm::Error MCADFetchDelayStage::execute(llvm::mca::InstRef &IR) {
+    // We (ab-)use the LastGenericEventType to create a notification when the instruction first enters this stage.
+    // We use this elsewhere to calculate the number of cycles between when an instruction first enters the pipeline and the end of its execution.
+    notifyEvent<llvm::mca::HWInstructionEvent>(llvm::mca::HWInstructionEvent(llvm::mca::HWInstructionEvent::LastGenericEventType, IR));
+    const llvm::mca::Instruction *I = IR.getInstruction();
+    const llvm::mca::InstrDesc &ID = I->getDesc();
+    const llvm::MCInstrDesc &MCID = MCII.get(I->getOpcode());
+    bool immediatelyExecute = true;
+    unsigned delayCyclesLeft = 0;
+    if(MCID.isBranch()) {
+        // delayed, will have to wait
+        delayCyclesLeft = 100;
+    }
+    instrQueue.emplace_back(DelayedInstr { delayCyclesLeft, IR });
+    // if the instruction is not delayed, execute it immediately (it will
+    // have a delayCyclesLeft of 0 and be at the top of the queue)
+    return forwardDueInstrs();
+}
+
+llvm::Error MCADFetchDelayStage::cycleStart() {
+    if(!instrQueue.empty()) {
+        instrQueue.front().delayCyclesLeft--;
+    }
+    return forwardDueInstrs();
+}
+
+}
+}

CustomStages/MCADFetchDelayStage.h

@@ -0,0 +1,50 @@
+// This class does not model a real hardware stage. It is used to block the
+// pipeline for a number of cycles to prevent further instructions from being
+// fetched. We use this to model the cost of branch mispredictions.
+
+#ifndef LLVM_MCAD_FETCH_DELAY_STAGE_H
+#define LLVM_MCAD_FETCH_DELAY_STAGE_H
+
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/MCA/SourceMgr.h"
+#include "llvm/MCA/Stages/Stage.h"
+
+#include <vector>
+#include <queue>
+
+namespace llvm {
+namespace mcad {
+
+class MCADFetchDelayStage : public llvm::mca::Stage {
+
+    struct DelayedInstr {
+        unsigned delayCyclesLeft;
+        llvm::mca::InstRef IR;
+    };
+
+    const llvm::MCInstrInfo &MCII;
+    std::deque<DelayedInstr> instrQueue = {};
+
+public:
+    MCADFetchDelayStage(const llvm::MCInstrInfo &MCII) : MCII(MCII) {}
+
+    bool hasWorkToComplete() const override;
+    bool isAvailable(const llvm::mca::InstRef &IR) const override;
+    llvm::Error execute(llvm::mca::InstRef &IR) override;
+
+    //llvm::Error cycleStart() override;
+    llvm::Error cycleStart() override;
+
+    llvm::Error forwardDueInstrs();
+
+    ///// Called after the pipeline is resumed from pausing state.
+    //virtual Error cycleResume() { return ErrorSuccess(); }
+
+    ///// Called once at the end of each cycle.
+
+};
+
+}
+}
+
+#endif

MCAWorker.cpp

@@ -37,6 +37,7 @@
 #include <unistd.h>
 
 #include "CustomHWUnits/MCADLSUnit.h"
+#include "CustomStages/MCADFetchDelayStage.h"
 #include "MCAViews/SummaryView.h"
 #include "MCAViews/TimelineView.h"
 #include "MCAWorker.h"
@@ -183,6 +184,7 @@ std::unique_ptr<mca::Pipeline> MCAWorker::createDefaultPipeline() {
 
   // Create the pipeline stages.
   auto Fetch = std::make_unique<EntryStage>(SrcMgr);
+  auto FetchDelay = std::make_unique<MCADFetchDelayStage>(MCII);
   auto Dispatch = std::make_unique<DispatchStage>(STI, MRI, MCAPO.DispatchWidth,
                                                   *RCU, *PRF);
   auto Execute =
@@ -198,6 +200,7 @@ std::unique_ptr<mca::Pipeline> MCAWorker::createDefaultPipeline() {
   // Build the pipeline.
   auto StagePipeline = std::make_unique<Pipeline>();
   StagePipeline->appendStage(std::move(Fetch));
+  StagePipeline->appendStage(std::move(FetchDelay));
   if (MCAPO.MicroOpQueueSize)
     StagePipeline->appendStage(std::make_unique<MicroOpQueueStage>(
         MCAPO.MicroOpQueueSize, MCAPO.DecodersThroughput));
@@ -224,6 +227,7 @@ std::unique_ptr<mca::Pipeline> MCAWorker::createInOrderPipeline() {
 
   // Create the pipeline stages.
   auto Entry = std::make_unique<EntryStage>(SrcMgr);
+  auto FetchDelay = std::make_unique<MCADFetchDelayStage>(MCII);
   auto InOrderIssue = std::make_unique<InOrderIssueStage>(STI, *PRF, *CB, *LSU);
   auto StagePipeline = std::make_unique<Pipeline>();
 
@@ -233,6 +237,7 @@ std::unique_ptr<mca::Pipeline> MCAWorker::createInOrderPipeline() {
 
   // Build the pipeline.
   StagePipeline->appendStage(std::move(Entry));
+  StagePipeline->appendStage(std::move(FetchDelay));
   StagePipeline->appendStage(std::move(InOrderIssue));
 
   for (auto *listener : Listeners) {
@@ -420,6 +425,9 @@ Error MCAWorker::run() {
                                 << " has Token " << MDTok << "\n");
               NewInst->setIdentifier(MDTok);
             }
+
+            // Add this instruction to be processed by the pipeline
+            // (Entry stage will consume from source manager.)
             SrcMgr.addInst(std::move(NewInst));
           }
         }

plugins/binja-broker/Broker.cpp

@@ -150,7 +150,12 @@ class RawListener : public mca::HWEventListener {
         }
 
         switch (Event.Type) {
-        case mca::HWInstructionEvent::GenericEventType::Ready: {
+        //case mca::HWInstructionEvent::GenericEventType::Dispatched: {
+        // We (ab-)use the LastGenericEventType in the FetchDelay stage to
+        // notify of an event whenever the instruction is fetch. This way, the
+        // cycle count in Binja shows the total instruction cycle count 
+        // including the fetch and dispatch cost.
+        case mca::HWInstructionEvent::GenericEventType::LastGenericEventType : {
             BridgeRef.CountStore[index].CycleReady = CurrentCycle;
         }
         case mca::HWInstructionEvent::GenericEventType::Executed: {