Add auto center feature to elastic buffers

In #1135 we employ a "simple" trick to allow ring buffers to be aligned
and UGNs to remain valid while clock control is still stabilizing the
system. We do all this in software, which critically relies on the
software being "quick enough". This is not always the case, as seen in
 #1168. Although we've more or less fixed the issue by making EB control
much faster (#1178), this commit adds the option to entirely offload
this responsibility to hardware.

Closes #1179

Author: martijnbastiaan

CLAUDE.md

@@ -37,6 +37,10 @@ too much, use `OverloadedRecordDot`. Both of these should be enabled by default.
 a new function has "too many" arguments or results, consider grouping them into
 records.
 
+## Executing commands
+When you execute a command you **must** wait for it to finish. Don't assume that no output
+for a while means that everything is okay.
+
 # Rust
 Before building a Rust crate, you **must** ensure that `bittide-instances` has
 been built. You can do this by running:

bittide-instances/src/Bittide/Instances/Tests/ElasticBufferWb.hs

@@ -97,5 +97,5 @@ dut = withBittideByteOrder $ withClockResetEnable clockGen (resetGenN d2) enable
         , includeIlaWb = False
         }
 
-type IMemWords = DivRU (64 * 1024) 4
+type IMemWords = DivRU (128 * 1024) 4
 type DMemWords = DivRU (32 * 1024) 4

bittide/bittide.cabal

@@ -210,6 +210,7 @@ library
     Bittide.Df
     Bittide.DoubleBufferedRam
     Bittide.ElasticBuffer
+    Bittide.ElasticBuffer.AutoCenter
     Bittide.Ethernet.Mac
     Bittide.Jtag
     Bittide.MetaPeConfig

bittide/src/Bittide/ElasticBuffer.hs

@@ -4,17 +4,21 @@
 
 module Bittide.ElasticBuffer where
 
+import Clash.Prelude
+import Protocols
+
 import Bittide.ClockControl (RelDataCount, targetDataCount)
-import Bittide.Df
+import Bittide.Df (unsafeFromDf)
+import Bittide.ElasticBuffer.AutoCenter (autoCenter)
 import Bittide.Extra.Maybe (orNothing)
 import Bittide.SharedTypes (BitboneMm)
 import Clash.Class.BitPackC (ByteOrder)
 import Clash.Cores.Xilinx.DcFifo
 import Clash.Cores.Xilinx.Xpm.Cdc.Extra (safeXpmCdcHandshake)
 import Clash.Cores.Xilinx.Xpm.Cdc.Pulse (xpmCdcPulse)
-import Clash.Prelude
-import GHC.Stack
-import Protocols (Ack (..), CSignal, Circuit (..), applyC)
+import Data.Maybe (isJust)
+import GHC.Stack (HasCallStack)
+import Protocols.Df (CollectMode (..), roundrobinCollect)
 import Protocols.Df.Extra (ackWhen, skid)
 import Protocols.MemoryMap (Access (..))
 import Protocols.MemoryMap.Registers.WishboneStandard (
@@ -24,6 +28,7 @@ import Protocols.MemoryMap.Registers.WishboneStandard (
   registerConfig,
   registerWbDfI,
   registerWbI,
+  registerWbI_,
  )
 
 import qualified Clash.Explicit.Prelude as E
@@ -251,6 +256,7 @@ The registers provided are:
 xilinxElasticBufferWb ::
   forall n readDom writeDom addrW a.
   ( HasCallStack
+  , HasSynchronousReset readDom
   , KnownDomain readDom
   , KnownDomain writeDom
   , NFDataX a
@@ -280,6 +286,11 @@ xilinxElasticBufferWb clkRead rstRead SNat clkWrite wdata =
       , wbDataCount
       , wbUnderflow
       , wbOverflow
+      , wbAutoCenterReset
+      , wbAutoCenterEnable
+      , wbAutoCenterMargin
+      , wbAutoCenterIsIdle
+      , wbAutoCenterTotalAdjustments
       ] <-
       deviceWb "ElasticBuffer" -< wb
 
@@ -312,10 +323,70 @@ xilinxElasticBufferWb clkRead rstRead SNat clkWrite wdata =
     (ebAdjustmentDf1, Fwd ebReady) <- skid -< ebAdjustmentDf0
     ackWhen ebReady -< ebAdjustmentWaitDfActivity
 
-    let (dataCount, underflow, overflow0, readData, adjustmentAck) =
-          xilinxElasticBuffer @n clkRead clkWrite ebAdjustmentSig wdata
+    -- Auto-centering state machine
+    (_autoCenterReset, Fwd autoCenterResetActivity) <-
+      registerWbI
+        (registerConfig "auto_center_reset_unchecked")
+          { access = WriteOnly
+          , description =
+              "Clear total adjustments. You must disable the state machine and wait for it to be idle before resetting it. After resetting, you must also wait for the state machine to become 'idle' again to make sure the registers are cleared."
+          }
+        ()
+        -< (wbAutoCenterReset, Fwd (pure Nothing))
+
+    (Fwd autoCenterEnable, _autoCenterEnableActivity) <-
+      registerWbI
+        (registerConfig "auto_center_enable")
+          { access = ReadWrite
+          , description = "Enable auto-centering state machine"
+          }
+        False
+        -< (wbAutoCenterEnable, Fwd (pure Nothing))
 
-    Fwd ebAdjustmentSig <- unsafeFromDf -< (ebAdjustmentDf1, Fwd adjustmentAck)
+    (Fwd autoCenterMargin, _autoCenterMarginActivity) <-
+      registerWbI
+        (registerConfig "auto_center_margin")
+          { access = ReadWrite
+          , description = "Margin for auto-centering"
+          }
+        (2 :: Unsigned 16)
+        -< (wbAutoCenterMargin, Fwd (pure Nothing))
+
+    registerWbI_
+      (registerConfig "auto_center_is_idle")
+        { access = ReadOnly
+        , description = "Whether the auto-centering state machine is idle"
+        }
+      True
+      -< (wbAutoCenterIsIdle, Fwd (Just <$> autoCenterIsIdle))
+
+    registerWbI_
+      (registerConfig "auto_center_total_adjustments")
+        { access = ReadOnly
+        , description = "Total adjustments applied by the auto-centering state machine"
+        }
+      (0 :: Signed 32)
+      -< (wbAutoCenterTotalAdjustments, Fwd (Just <$> autoCenterTotalAdjustments))
+
+    let
+      autoCenterReset = unsafeFromActiveHigh (isJust . busActivityWrite <$> autoCenterResetActivity)
+
+      (dataCount, underflow, overflow0, readData, adjustmentAck) =
+        xilinxElasticBuffer @n clkRead clkWrite ebAdjustmentSig wdata
+
+    (autoCenterAdjustmentDf, Fwd autoCenterTotalAdjustments, Fwd autoCenterIsIdle) <-
+      autoCenter
+        (autoCenterReset `E.orReset` rstRead)
+        (toEnable autoCenterEnable)
+        autoCenterMargin
+        dataCount
+        -< ()
+
+    -- Multiplex manual and auto-center adjustments using round-robin collection
+    ebAdjustmentDfMuxed <-
+      roundrobinCollect @2 Parallel -< [ebAdjustmentDf1, autoCenterAdjustmentDf]
+
+    Fwd ebAdjustmentSig <- unsafeFromDf -< (ebAdjustmentDfMuxed, Fwd adjustmentAck)
 
     -- Synchronize overflow pulse from write domain to read domain
     let overflow1 = xpmCdcPulse clkWrite clkRead overflow0

bittide/src/Bittide/ElasticBuffer/AutoCenter.hs

@@ -0,0 +1,124 @@
+-- SPDX-FileCopyrightText: 2026 Google LLC
+--
+-- SPDX-License-Identifier: Apache-2.0
+
+module Bittide.ElasticBuffer.AutoCenter where
+
+import Clash.Prelude
+import Protocols
+
+import Bittide.ClockControl (RelDataCount)
+
+data State
+  = InReset
+  | Idle
+      { adjustments :: Signed 32
+      -- ^ Cumulative adjustments that have been submitted to the elastic buffer
+      }
+  | Adjusting
+      { adjustments :: Signed 32
+      -- ^ Cumulative adjustments that have been submitted to the elastic buffer
+      , adjustment :: Signed 32
+      -- ^ The adjustment that is currently being submitted to the elastic buffer
+      }
+  | Waiting
+      { adjustments :: Signed 32
+      -- ^ Cumulative adjustments that have been submitted to the elastic buffer
+      , wait :: Index 256
+      -- ^ Number of cycles to wait before moving to Idle state. This should be long enough
+      -- to ensure that the adjustment has taken an effect on the buffer's occupancy count.
+      -- It is currently hardcoded to 256 cycles, which should be well above the time it
+      -- takes for an adjustment to take effect. At the same time, it should be more than
+      -- quick enough to not be too slow even for systems that are wildly out of balance.
+      }
+  deriving (Generic, NFDataX, Show)
+
+{- | If 'True', there are no pending adjustments to the elastic buffer. This means that the
+accumulated adjustments have been applied (though not necessarily reflected in the occupancy
+count yet) and that they can be used for, e.g., UGN adjustments.
+-}
+type IsIdle = Bool
+
+{- | Whether the auto-centering state machine is enabled. If 'False', the state machine will
+stay in the Idle state and will not submit any adjustments to the elastic buffer. Disabling
+the state machine while it is active is perfectly safe -- the state machine will handle
+its pending submissions and then move and stay in its 'Idle' state.
+-}
+type IsEnabled = Bool
+
+{- | State machine that tries to keep the buffer centered around 0 by submitting adjustments
+to the elastic buffer. Note that adjustments to the elastic buffer are destructive, and
+should therefore only be used during bittide initialization.
+-}
+autoCenter ::
+  (HiddenClock dom, KnownNat n, n <= 32) =>
+  Reset dom ->
+  Enable dom ->
+  -- | Correction margin. If set to 1, the circuit will try to keep the buffer at
+  --   [-1, 0, 1]. Note that if set to 0, the circuit will try to keep the buffer at exactly
+  --   0, which will cause it to submit many small adjustments, which may not be desirable.
+  Signal dom (Unsigned 16) ->
+  -- | The current number of items in the buffer. The circuit will try to keep this at 0
+  --   (or within the margin if set).
+  Signal dom (RelDataCount n) ->
+  Circuit
+    ()
+    ( Df dom (Signed 32)
+    , CSignal dom (Signed 32)
+    , CSignal dom IsIdle
+    )
+autoCenter reset enable margin relDataCount = Circuit go
+ where
+  go (_, (ack, _, _)) = ((), (adjustment, output, isIdle))
+   where
+    (adjustment, output, isIdle) =
+      withEnable enableGen
+        $ withReset reset
+        $ mooreB
+          goState
+          goOutput
+          InReset
+          ( fromEnable enable
+          , numConvert <$> relDataCount
+          , numConvert <$> margin
+          , ack
+          )
+
+-- XXX: These functions are outside of the circuit definition to avoid shadowing :-/
+
+goState :: State -> (IsEnabled, RelDataCount 32, Signed 32, Ack) -> State
+goState InReset _ = Idle 0
+goState s@Idle{} (False, _relDataCount, _margin, _ack) = s
+goState s@Idle{adjustments} (_isEnabled, relDataCount, margin, _ack)
+  | negate margin <= relDataCount && relDataCount <= margin = s -- within margin
+  | otherwise = Adjusting adjustments (negate relDataCount) -- outside margin
+goState s@Adjusting{adjustments, adjustment} (_isEnabled, _relDataCount, _margin, ~(Ack ack))
+  | ack = Waiting{adjustments = adjustments + adjustment, wait = 0}
+  | otherwise = s
+goState Waiting{adjustments, wait} (_isEnabled, _relDataCount, _margin, _ack)
+  | wait == maxBound = Idle adjustments
+  | otherwise = Waiting{adjustments, wait = wait + 1}
+
+goOutput :: State -> (Maybe (Signed 32), Signed 32, IsIdle)
+goOutput s = (adjustment_, adjustments_, isIdle)
+ where
+  -- Note! The "idle" output reflects the state of submissions, not the state of the
+  -- state machine. I.e., idle means that there are no pending adjustments to the elastic
+  -- buffer.
+  isIdle = case s of
+    InReset -> False
+    Idle{} -> True
+    Adjusting{} -> False
+    Waiting{} -> True
+
+  adjustments_ = case s of
+    InReset -> 0
+    Idle{adjustments} -> adjustments
+    Adjusting{adjustments} -> adjustments
+    Waiting{adjustments} -> adjustments
+
+  adjustment_ = case s of
+    InReset -> Nothing
+    Idle{} -> Nothing
+    Adjusting{adjustment} -> Just adjustment
+    Waiting{} -> Nothing