feat(CustomL1Hint): new version CustomL1Hint to improve performance

src/main/scala/coupledL2/Common.scala

@@ -353,6 +353,7 @@ class PrefetchRecv extends Bundle {
 class L2ToL1Hint(implicit p: Parameters) extends Bundle {
   val sourceId = UInt(32.W)    // tilelink sourceID
   val isKeyword = Bool()       // miss entry keyword
+  val isGrantData = Bool()
 }
 
 // custom l2 - l1 tlb

src/main/scala/coupledL2/CoupledL2.scala

@@ -383,17 +383,12 @@ abstract class CoupledL2Base(implicit p: Parameters) extends LazyModule with Has
 
     // ** WARNING:TODO: this depends on where the latch is
     // ** if Hint latched in slice, while D-Channel latched in XSTile
-    // ** we need only [hintCycleAhead - 1] later
-    val sliceAhead = hintCycleAhead - 1
+    // ** we need only [hintCycleAhead - 2] later
+    val sliceAhead = hintCycleAhead - 2
 
     val hintChosen = Wire(UInt(banks.W))
     val hintFire = Wire(Bool())
-
-    // if Hint indicates that this slice should fireD, yet no D resp comes out of this slice
-    // then we releaseSourceD, enabling io.d.ready for other slices
-    // TODO: if Hint for single slice is 100% accurate, may consider remove this
-    val releaseSourceD = Wire(Vec(banks, Bool()))
-    val allCanFire = (RegNextN(!hintFire, sliceAhead) && RegNextN(!hintFire, sliceAhead + 1)) || Cat(releaseSourceD).orR
+    val hintWithData = Wire(Bool())
 
     val slices = node.in.zip(node.out).zipWithIndex.map {
       case (((in, edgeIn), (out, edgeOut)), i) =>
@@ -423,12 +418,15 @@ abstract class CoupledL2Base(implicit p: Parameters) extends LazyModule with Has
           // If slice X has no grant then, it means that the hint at cycle T is wrong,
           // so we relax the restriction on grant selection.
           val sliceCanFire = RegNextN(hintFire && i.U === hintChosen, sliceAhead) ||
-            RegNextN(hintFire && i.U === hintChosen, sliceAhead + 1)
+            RegNextN(hintFire && i.U === hintChosen && hintWithData, sliceAhead + 1)
 
-          releaseSourceD(i) := sliceCanFire && !slice.io.in.d.valid
+          when(sliceCanFire) {
+            assert(slice.io.in.d.valid)
+          }
 
-          in.d.valid := slice.io.in.d.valid && (sliceCanFire || allCanFire)
-          slice.io.in.d.ready := in.d.ready && (sliceCanFire || allCanFire)
+          in.d.valid := slice.io.in.d.valid && sliceCanFire
+          assert(in.d.ready)
+          slice.io.in.d.ready := in.d.ready && sliceCanFire
         }
         in.b.bits.address := restoreAddress(slice.io.in.b.bits.address, i)
         slice.io.sliceId := i.U
@@ -497,7 +495,7 @@ abstract class CoupledL2Base(implicit p: Parameters) extends LazyModule with Has
     if (enableHintGuidedGrant) {
       // for timing consideration, hint should latch one cycle before sending to L1
       // instead of adding a Pipeline/Queue to latch here, we just set hintQueue in GrantBuf & CustomL1Hint "flow=false"
-      val l1HintArb = Module(new Arbiter(new L2ToL1Hint(), slices.size))
+      val l1HintArb = Module(new FastArbiter(new L2ToL1Hint(), slices.size))
       val slices_l1Hint = slices.zipWithIndex.map {
         case (s, i) => s.io.l1Hint
       }
@@ -512,11 +510,13 @@ abstract class CoupledL2Base(implicit p: Parameters) extends LazyModule with Has
       io.l2_hint.valid := l1HintArb.io.out.fire && sourceIsDcache
       io.l2_hint.bits.sourceId := l1HintArb.io.out.bits.sourceId - dcacheSourceIdStart
       io.l2_hint.bits.isKeyword := l1HintArb.io.out.bits.isKeyword
-      // continuous hints can only be sent every two cycle, since GrantData takes two cycles
-      l1HintArb.io.out.ready := !RegNext(io.l2_hint.valid, false.B)
+      io.l2_hint.bits.isGrantData := l1HintArb.io.out.bits.isGrantData
 
       hintChosen := l1HintArb.io.chosen // ! THIS IS NOT ONE-HOT !
-      hintFire := io.l2_hint.valid
+      hintFire := l1HintArb.io.out.fire
+      hintWithData := l1HintArb.io.out.bits.isGrantData
+      // continuous hints can only be sent every two cycle, since GrantData takes two cycles
+      l1HintArb.io.out.ready := !RegNext(hintFire && hintWithData, false.B)
     }
 
     // Outer interface connection

src/main/scala/coupledL2/CustomL1Hint.scala

@@ -30,12 +30,17 @@ class HintQueueEntry(implicit p: Parameters) extends L2Bundle {
   val isKeyword = Bool()
 }
 
+class RetryQueueEntry(implicit p: Parameters) extends L2Bundle {
+  val source = UInt(sourceIdBits.W)
+}
+
 class CustomL1HintIOBundle(implicit p: Parameters) extends L2Bundle {
   // input information
   val s1 = Flipped(ValidIO(new TaskBundle()))
   val s3 = new L2Bundle {
       val task      = Flipped(ValidIO(new TaskBundle()))
       val need_mshr = Input(Bool())
+      val retry     = Input(Bool())
   }
 
   // output hint
@@ -58,16 +63,22 @@ class CustomL1Hint(implicit p: Parameters) extends L2Module {
   def isGrant(t: TaskBundle):      Bool = t.fromA && t.opcode === Grant
   def isHintAck(t: TaskBundle):    Bool = t.fromA && t.opcode === HintAck // HintAck has no effect on Hint
   def isRelease(t: TaskBundle):    Bool = t.fromC && (t.opcode === Release || t.opcode === ReleaseData)
+  def isReleaseAck(t: TaskBundle): Bool = t.fromC && t.opcode === ReleaseAck
+  def isCBOAck(t: TaskBundle):     Bool = t.fromA && t.opcode === CBOAck
   def isMergeGrantData(t: TaskBundle): Bool = t.fromA && t.mergeA && t.aMergeTask.opcode === GrantData
   def isMergeGrant(t: TaskBundle):     Bool = t.fromA && t.mergeA && t.aMergeTask.opcode === Grant
+  def isAccessAckData(t: TaskBundle):  Bool = t.fromA && t.opcode === AccessAckData
 
   // ==================== Hint Generation ====================
   // Hint for "MSHRTask and ReleaseAck" will fire@s1
-  val mshr_GrantData_s1 = task_s1.valid &&  mshrReq_s1 && (isGrantData(task_s1.bits) || isMergeGrantData(task_s1.bits))
-  val mshr_Grant_s1     = task_s1.valid &&  mshrReq_s1 && (isGrant(task_s1.bits) || isMergeGrant(task_s1.bits))
-  val chn_Release_s1    = task_s1.valid && !mshrReq_s1 && isRelease(task_s1.bits)
+  val mshr_GrantData_s1    = task_s1.valid &&  mshrReq_s1 && (isGrantData(task_s1.bits) || isMergeGrantData(task_s1.bits))
+  val mshr_Grant_s1        = task_s1.valid &&  mshrReq_s1 && (isGrant(task_s1.bits) || isMergeGrant(task_s1.bits))
+  val chn_CBOAck_s1        = task_s1.valid && mshrReq_s1 && isCBOAck(task_s1.bits)
+  val chn_Release_s1       = task_s1.valid && !mshrReq_s1 && isRelease(task_s1.bits)
+  val chn_ReleaseAck_s1    = task_s1.valid && !mshrReq_s1 && isReleaseAck(task_s1.bits)
+  val chn_AccessAckData_s1 = task_s1.valid && mshrReq_s1 && isAccessAckData(task_s1.bits)
 
-  val enqValid_s1 = mshr_GrantData_s1 || mshr_Grant_s1 || chn_Release_s1
+  val enqValid_s1 = mshr_GrantData_s1 || mshr_Grant_s1 || chn_Release_s1 || chn_AccessAckData_s1 || chn_CBOAck_s1
   val enqSource_s1 = Mux(task_s1.bits.mergeA, task_s1.bits.aMergeTask.sourceId, task_s1.bits.sourceId)
   val enqKeyWord_s1 = Mux(task_s1.bits.mergeA,
     task_s1.bits.aMergeTask.isKeyword.getOrElse(false.B),
@@ -77,27 +88,33 @@ class CustomL1Hint(implicit p: Parameters) extends L2Module {
     Seq(
       mshr_Grant_s1 -> Grant,
       mshr_GrantData_s1 -> GrantData,
-      chn_Release_s1 -> ReleaseAck
+      chn_Release_s1 -> ReleaseAck,
+      chn_AccessAckData_s1 -> AccessAckData,
+      chn_CBOAck_s1 -> CBOAck
     )
   )
 
   // Hint for "chnTask Hit" will fire@s3
   val chn_Grant_s3     = task_s3.valid && !mshrReq_s3 && !need_mshr_s3 && isGrant(task_s3.bits)
   val chn_GrantData_s3 = task_s3.valid && !mshrReq_s3 && !need_mshr_s3 && isGrantData(task_s3.bits)
-  val enqValid_s3 = chn_Grant_s3 || chn_GrantData_s3
+  val chn_AccessAckData_s3 = task_s3.valid && !mshrReq_s3 && !need_mshr_s3 && isAccessAckData(task_s3.bits)
+
+  val enqValid_s3 = chn_Grant_s3 || chn_GrantData_s3 || chn_AccessAckData_s3
   val enqSource_s3 = task_s3.bits.sourceId
   val enqKeyWord_s3 = task_s3.bits.isKeyword.getOrElse(false.B)
   val enqOpcode_s3 = ParallelPriorityMux(
     Seq(
       chn_Grant_s3 -> Grant,
-      chn_GrantData_s3 -> GrantData
+      chn_GrantData_s3 -> GrantData,
+      chn_AccessAckData_s3 -> AccessAckData
     )
   )
 
   // ==================== Hint Queue ====================
   val hintEntries = mshrsAll
   val hintEntriesWidth = log2Ceil(hintEntries)
   val hintQueue = Module(new Queue(new HintQueueEntry, hintEntries))
+  val hintDropQueue = Module(new Queue(new RetryQueueEntry, hintEntries, flow = true))
 
   // this will have at most 2 entries
   val hint_s1Queue = Module(new Queue(new HintQueueEntry, 4, flow = true))
@@ -106,17 +123,29 @@ class CustomL1Hint(implicit p: Parameters) extends L2Module {
   hint_s1Queue.io.enq.bits.source := enqSource_s1
   hint_s1Queue.io.enq.bits.isKeyword := enqKeyWord_s1
   hint_s1Queue.io.deq.ready := hintQueue.io.enq.ready && !enqValid_s3
-  // WARNING:TODO: ensure queue will never overflow
+
   assert(hint_s1Queue.io.enq.ready, "hint_s1Queue should never be full")
-  assert(hintQueue.io.enq.ready, "hintQueue should never be full")
+  // *NOTICE: 'hintQueue' is now possible to be full and backpressing 'hint_s1Queue'.
+  //          Hence, this assertion here was currently unnecessary and overkilled.
+  //assert(hintQueue.io.enq.ready, "hintQueue should never be full")
+
+  val hintDropValid = hintQueue.io.deq.valid && hintDropQueue.io.deq.valid && hintQueue.io.deq.bits.source === hintDropQueue.io.deq.bits.source
+
+  hintDropQueue.io.enq.valid := io.s3.retry && RegNextN(enqValid_s1, 2)
+  hintDropQueue.io.enq.bits.source := task_s3.bits.sourceId
+  hintDropQueue.io.deq.ready := hintDropValid
+
+  val deqLatency = RegNext(io.l1Hint.fire && io.l1Hint.bits.isGrantData)
+  val hintEnqValid = enqValid_s3 || hint_s1Queue.io.deq.valid
 
-  hintQueue.io.enq.valid := enqValid_s3 || hint_s1Queue.io.deq.valid
-  hintQueue.io.enq.bits.opcode := Mux(enqValid_s3, enqOpcode_s3, hint_s1Queue.io.deq.bits.opcode)
-  hintQueue.io.enq.bits.source := Mux(enqValid_s3, enqSource_s3, hint_s1Queue.io.deq.bits.source)
-  hintQueue.io.enq.bits.isKeyword := Mux(enqValid_s3, enqKeyWord_s3, hint_s1Queue.io.deq.bits.isKeyword)
-  hintQueue.io.deq.ready := io.l1Hint.ready
+  hintQueue.io.enq.valid := RegNext(hintEnqValid)
+  hintQueue.io.enq.bits.opcode := RegEnable(Mux(enqValid_s3, enqOpcode_s3, hint_s1Queue.io.deq.bits.opcode), hintEnqValid)
+  hintQueue.io.enq.bits.source := RegEnable(Mux(enqValid_s3, enqSource_s3, hint_s1Queue.io.deq.bits.source), hintEnqValid)
+  hintQueue.io.enq.bits.isKeyword := RegEnable(Mux(enqValid_s3, enqKeyWord_s3, hint_s1Queue.io.deq.bits.isKeyword), hintEnqValid)
+  hintQueue.io.deq.ready := io.l1Hint.ready && !deqLatency || hintDropValid
 
-  io.l1Hint.valid := hintQueue.io.deq.valid && hintQueue.io.deq.bits.opcode === GrantData
+  io.l1Hint.valid := hintQueue.io.deq.valid && !deqLatency && !hintDropValid
   io.l1Hint.bits.sourceId := hintQueue.io.deq.bits.source
   io.l1Hint.bits.isKeyword := hintQueue.io.deq.bits.isKeyword
+  io.l1Hint.bits.isGrantData := hintQueue.io.deq.bits.opcode === GrantData || hintQueue.io.deq.bits.opcode === AccessAckData
 }

src/main/scala/coupledL2/tl2chi/MainPipe.scala

@@ -757,6 +757,7 @@ class MainPipe(implicit p: Parameters) extends TL2CHIL2Module with HasCHIOpcodes
   customL1Hint.io.s3.task.bits.opcode := Mux(sink_resp_s3.valid, sink_resp_s3.bits.opcode, task_s3.bits.opcode)
   // customL1Hint.io.s3.d         := d_s3.valid
   customL1Hint.io.s3.need_mshr := need_mshr_s3
+  customL1Hint.io.s3.retry := task_s3.valid && mshr_refill_s3 && retry
 
   // customL1Hint.io.s4.task                  := task_s4
   // customL1Hint.io.s4.d                     := d_s4.valid

src/main/scala/coupledL2/tl2tl/MainPipe.scala

@@ -550,6 +550,7 @@ class MainPipe(implicit p: Parameters) extends L2Module with HasPerfEvents {
   // overwrite opcode: if sinkReq can respond, use sink_resp_s3.bits.opcode = Grant/GrantData
   customL1Hint.io.s3.task.bits.opcode := Mux(sink_resp_s3.valid, sink_resp_s3.bits.opcode, task_s3.bits.opcode)
   customL1Hint.io.s3.need_mshr := need_mshr_s3
+  customL1Hint.io.s3.retry := task_s3.valid && mshr_refill_s3 && retry
 
   customL1Hint.io.l1Hint <> io.l1Hint
 

src/main/scala/coupledL2/tl2tl/Slice.scala

@@ -127,9 +127,7 @@ class Slice()(implicit p: Parameters) extends BaseSlice[OuterBundle] {
   sourceC.io.in <> mainPipe.io.toSourceC
   sourceC.io.pipeStatusVec := reqArb.io.status_vec ++ mainPipe.io.status_vec_toC
 
-  io.l1Hint.valid := mainPipe.io.l1Hint.valid
-  io.l1Hint.bits.sourceId := mainPipe.io.l1Hint.bits.sourceId
-  io.l1Hint.bits.isKeyword := mainPipe.io.l1Hint.bits.isKeyword
+  io.l1Hint <> mainPipe.io.l1Hint
   mainPipe.io.l1Hint.ready := io.l1Hint.ready
   mshrCtl.io.grantStatus := grantBuf.io.grantStatus