Merge pull request #2316 from ucb-bar/ctc-bridge

Add CTC link

Author: schwarz-em

build.sbt

@@ -160,7 +160,7 @@ lazy val rocketLibDeps = (rocketchip / Keys.libraryDependencies)
 
 // -- Chipyard-managed External Projects --
 
-lazy val testchipip = withInitCheck((project in file("generators/testchipip")), "testchipip")
+lazy val testchipip = withInitCheck(freshProject("testchipip", file("generators/testchipip")), "testchipip")
   .dependsOn(rocketchip, rocketchip_blocks)
   .settings(libraryDependencies ++= rocketLibDeps.value)
   .settings(commonSettings)

docs/Generators/TestChipIP.rst

@@ -104,3 +104,18 @@ The pin can be added to a system with the ``testchipip.soc.WithChipIdPin`` confi
 width and MMIO address are parameterizable and can be set by passing ``ChipIdPinParams`` as an 
 argument to the config. The width can additionally be set using the ``testchipip.soc.WithChipIdPinWidth``
 config. 
+
+CTC
+---------------
+
+The CTC (Chip-To-Chip) link converts TileLink requests to a simple 32-bit protocol, 
+similar to TSI. This link is intended to connect chiplets in a Chipyard design, and can be 
+added to a multichip config using ``chipyard.harness.WithMultiChipCTC``. To add CTC ports to 
+a single SoC, use ``testchipip.ctc.WithCTC``. Refer to 
+`ChipletConfigs.scala <https://github.com/ucb-bar/chipyard/blob/main/generators/chipyard/src/main/scala/config/ChipletConfigs.scala>`_ 
+for examples of configs with CTC.
+
+By default, CTC uses the credited PHY provided with the Serial TileLink interface (see 
+:ref:`Generators/TestChipIP:TileLink SERDES`). CTC can also be instantiated without a PHY. The 
+intended use for this mode is for multi-chiplet FireSim simulations using the CTC bridge. For 
+more information on FireSim, see :ref:`Simulation/FPGA-Accelerated-Simulation:FireSim`.

generators/chipyard/src/main/scala/DigitalTop.scala

@@ -36,6 +36,7 @@ class DigitalTop(implicit p: Parameters) extends ChipyardSystem
   with chipyard.clocking.CanHaveClockTap // Enables optionally adding a clock tap output port
   with constellation.soc.CanHaveGlobalNoC // Support instantiating a global NoC interconnect
   with rerocc.CanHaveReRoCCTiles // Support tiles that instantiate rerocc-attached accelerators
+  with testchipip.ctc.CanHavePeripheryCTC // Support optional CTC link
 {
   override lazy val module = new DigitalTopModule(this)
 }

generators/chipyard/src/main/scala/config/AbstractConfig.scala

@@ -49,6 +49,7 @@ class AbstractConfig extends Config(
   new chipyard.iobinders.WithExtInterruptIOCells ++
   new chipyard.iobinders.WithChipIdIOCells ++
   new chipyard.iobinders.WithCustomBootPin ++
+  new chipyard.iobinders.WithCTCIOCells ++
   // The "punchthrough" IOBInders below don't generate IOCells, as these interfaces shouldn't really be mapped to ASIC IO
   // Instead, they directly pass through the DigitalTop ports to ports in the ChipTop
   new chipyard.iobinders.WithI2CPunchthrough ++

generators/chipyard/src/main/scala/config/ChipletConfigs.scala

@@ -142,3 +142,34 @@ class MultiSimLLCChipletRocketConfig extends Config(
   new chipyard.harness.WithMultiChip(0, new RocketCoreChipletConfig) ++
   new chipyard.harness.WithMultiChip(1, new LLCChipletConfig)
 )
+
+class CTCRocketConfig extends Config(
+  new chipyard.harness.WithCTCLoopback ++
+  new testchipip.ctc.WithCTC(Seq(new testchipip.ctc.CTCParams(onchipAddr = 0x1000000000L, offchipAddr = 0x0L, size = ((1L << 32) - 1), noPhy=true))) ++ 
+  new RocketConfig
+)
+
+class DoubleCTCRocketConfig extends Config(
+  new chipyard.harness.WithCTCLoopback ++
+  new testchipip.ctc.WithCTC(Seq(
+    new testchipip.ctc.CTCParams(onchipAddr = 0x1000000000L, offchipAddr = 0x0L, size = ((1L << 32) - 1), noPhy=false),
+    new testchipip.ctc.CTCParams(onchipAddr = 0x2000000000L, offchipAddr = 0x0L, size = ((1L << 32) - 1), noPhy=true)
+  )) ++ 
+  new RocketConfig
+)
+
+class MultiCTCRocketConfig extends Config(
+  new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++
+  new chipyard.harness.WithMultiChipCTC(chip0=0, chip1=1, chip0portId=0, chip1portId=0) ++ // connect CTC port 0 of chip 0 and CTC port 0 of chip 1
+  new chipyard.harness.WithMultiChip(0, new CTCRocketConfig) ++
+  new chipyard.harness.WithMultiChip(1, new CTCRocketConfig)
+)
+
+class MultiDoubleCTCRocketConfig extends Config(
+  new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++
+  new chipyard.harness.WithMultiChipCTC(chip0=0, chip1=1, chip0portId=0, chip1portId=0) ++ // connect CTC port 0 of chip 0 and CTC port 0 of chip 1
+  new chipyard.harness.WithMultiChipCTC(chip0=0, chip1=1, chip0portId=1, chip1portId=1) ++ // connect CTC port 1 of chip 0 and CTC port 1 of chip 1
+  new chipyard.harness.WithMultiChip(0, new DoubleCTCRocketConfig) ++
+  new chipyard.harness.WithMultiChip(1, new DoubleCTCRocketConfig)
+)
+

generators/chipyard/src/main/scala/harness/HarnessBinders.scala

@@ -21,6 +21,7 @@ import testchipip.uart.{UARTAdapter, UARTToSerial}
 import testchipip.serdes._
 import testchipip.iceblk.{SimBlockDevice, BlockDeviceModel}
 import testchipip.cosim.{SpikeCosim}
+import testchipip.ctc.{CTCBridgeIO}
 import icenet.{NicLoopback, SimNetwork}
 import chipyard._
 import chipyard.clocking.{HasChipyardPRCI}
@@ -341,3 +342,38 @@ class WithOffchipBusSelPlusArg extends HarnessBinder({
   }
 })
 
+class WithCTCTiedOff extends HarnessBinder({
+  case (th: HasHarnessInstantiators, port: CTCPort, chipId: Int) => {
+    port.io match {
+      case io: CreditedSourceSyncPhitIO => {
+        io.clock_in := false.B.asClock
+        io.reset_in := false.B.asAsyncReset
+        io.in := DontCare
+      }
+      case io: CTCBridgeIO => {
+        io.manager_flit := DontCare
+        io.manager_flit.in.valid := false.B
+        io.manager_flit.out.ready := false.B
+        io.client_flit := DontCare
+        io.client_flit.in.valid := false.B
+        io.client_flit.out.ready := false.B
+      }
+    }
+  }
+})
+
+class WithCTCLoopback extends HarnessBinder({
+  case (th: HasHarnessInstantiators, port: CTCPort, chipId: Int) => {
+    port.io match {
+      case io: CreditedSourceSyncPhitIO => {
+        io.clock_in := io.clock_out
+        io.reset_in := io.reset_out
+        io.in := io.out
+      }
+      case io: CTCBridgeIO => {
+        io.client_flit.in <> io.manager_flit.out
+        io.manager_flit.in <> io.client_flit.out
+      }
+    }
+  }
+})

generators/chipyard/src/main/scala/harness/MultiHarnessBinders.scala

@@ -10,9 +10,10 @@ import freechips.rocketchip.subsystem._
 import freechips.rocketchip.util._
 
 import testchipip.serdes._
+import testchipip.ctc.{CTCBridgeIO}
 
 import chipyard._
-import chipyard.iobinders.{GetSystemParameters, JTAGChipIO, HasChipyardPorts, Port, SerialTLPort}
+import chipyard.iobinders.{GetSystemParameters, JTAGChipIO, HasChipyardPorts, Port, SerialTLPort, CTCPort}
 
 import scala.reflect.{ClassTag}
 
@@ -78,3 +79,30 @@ class WithMultiChipSerialTL(chip0: Int, chip1: Int, chip0portId: Int = 0, chip1p
     }
   }
 )
+
+class WithMultiChipCTC(chip0: Int, chip1: Int, chip0portId: Int = 0, chip1portId: Int = 0) extends MultiHarnessBinder(
+  chip0, chip1,
+  (p0: CTCPort) => p0.portId == chip0portId,
+  (p1: CTCPort) => p1.portId == chip1portId,
+  (th: HasHarnessInstantiators, p0: CTCPort, p1: CTCPort) => {
+    (p0.io, p1.io) match {
+      case(io0: CreditedSourceSyncPhitIO, io1: CreditedSourceSyncPhitIO) => {
+        io0.clock_in := io1.clock_out
+        io1.clock_in := io0.clock_out
+        io0.reset_in := io1.reset_out
+        io1.reset_in := io0.reset_out
+        io0.in := io1.out
+        io1.in := io0.out
+      }
+      case(io0: CTCBridgeIO, io1: CTCBridgeIO) => {
+        // io0 client to io1 manager
+        io0.client_flit.in <> io1.manager_flit.out
+        io1.manager_flit.in <> io0.client_flit.out
+
+        // io1 client to io0 manager
+        io1.client_flit.in <> io0.manager_flit.out
+        io0.manager_flit.in <> io1.client_flit.out
+      }
+    }
+  }
+)

generators/chipyard/src/main/scala/iobinders/IOBinders.scala

@@ -37,6 +37,7 @@ import testchipip.util.{ClockedIO}
 import testchipip.iceblk.{CanHavePeripheryBlockDevice, BlockDeviceKey, BlockDeviceIO}
 import testchipip.cosim.{CanHaveTraceIO, TraceOutputTop, SpikeCosimConfig}
 import testchipip.tsi.{CanHavePeripheryUARTTSI, UARTTSIIO}
+import testchipip.ctc.{CanHavePeripheryCTC}
 import icenet.{CanHavePeripheryIceNIC, SimNetwork, NicLoopback, NICKey, NICIOvonly}
 import chipyard.{CanHaveMasterTLMemPort, ChipyardSystem, ChipyardSystemModule}
 import chipyard.example.{CanHavePeripheryGCD}
@@ -607,3 +608,25 @@ class WithOffchipBusSel extends OverrideIOBinder({
     }.getOrElse(Nil, Nil)
   }
 })
+
+class WithCTCIOCells extends OverrideIOBinder({
+  (system: CanHavePeripheryCTC) => {
+    val (ports, cells) = system.ctc_ios.zipWithIndex.map ({ case (c, id) =>
+      val sys = system.asInstanceOf[BaseSubsystem]
+      val (port, cells) = IOCell.generateIOFromSignal(c.getWrappedValue, s"ctc${id}_port", sys.p(IOCellKey), abstractResetAsAsync = true)
+      (CTCPort(() => port, id), cells)
+    }).unzip
+    (ports.toSeq, cells.flatten.toSeq)
+  }
+})
+
+class WithCTCPunchthrough extends OverrideIOBinder({
+  (system: CanHavePeripheryCTC) => {
+    val (ports, cells) = system.ctc_ios.zipWithIndex.map ({ case (c, id) =>
+      val port = IO(chiselTypeOf(c.getWrappedValue)).suggestName(s"ctc${id}_port") // Since CTC IO varies depending on params
+      port <> c.getWrappedValue
+      (CTCPort(() => port, id), Nil)
+    }).unzip
+    (ports.toSeq, cells.flatten.toSeq)
+  }
+})

generators/chipyard/src/main/scala/iobinders/Ports.scala

@@ -11,6 +11,7 @@ import testchipip.spi.{SPIChipIO}
 import testchipip.cosim.{TraceOutputTop, SpikeCosimConfig}
 import testchipip.iceblk.{BlockDeviceIO, BlockDeviceConfig}
 import testchipip.tsi.{UARTTSIIO}
+import testchipip.ctc.{CTCBridgeIO}
 import icenet.{NICIOvonly, NICConfig}
 import org.chipsalliance.cde.config.{Parameters}
 import freechips.rocketchip.amba.axi4.{AXI4Bundle, AXI4EdgeParameters}
@@ -115,3 +116,5 @@ case class GCDBusyPort     (val getIO: () => Bool)
 case class OffchipSelPort  (val getIO: () => UInt)
     extends Port[UInt]
 
+case class CTCPort (val getIO: () => Data, val portId: Int) 
+    extends Port[Data]

generators/firechip/bridgeinterfaces/src/main/scala/CTC.scala

@@ -0,0 +1,27 @@
+// // See LICENSE for license details.
+
+package firechip.bridgeinterfaces
+
+import chisel3._
+import chisel3.util._
+
+class DecoupledFlitIO(val flitWidth: Int) extends Bundle {
+  val in = Flipped(Decoupled(UInt(flitWidth.W)))
+  val out = Decoupled(UInt(flitWidth.W))
+}
+
+class CTCBridgeIO(val w: Int) extends Bundle {
+  val client_flit = new DecoupledFlitIO(w) 
+  val manager_flit = new DecoupledFlitIO(w) 
+}
+
+// NOTE: CTCBridgeIO in ctc is CTC.INNER_WIDTH=32b
+object CTC {
+  val WIDTH = 32
+}
+
+class CTCBridgeTargetIO extends Bundle {
+  val clock = Input(Clock())
+  val reset = Input(Bool())
+  val ctc_io = Flipped(new CTCBridgeIO(CTC.WIDTH))
+}