[WIP] Optimize Peras weight calculation

cabal.project

@@ -56,6 +56,15 @@ allow-newer:
     , fin:QuickCheck
     , bin:QuickCheck
 
+source-repository-package
+   type: git
+   location: https://github.com/IntersectMBO/ouroboros-network
+   tag: b07a86ed853b63881b5a83e57508902f1562ac01
+   --sha256: sha256-n/XX0+cQegq2a1cAfmGx30T64eix4oEXzpVEFCKqmg0=
+   subdir:
+     ouroboros-network-api
+     ouroboros-network
+
 source-repository-package
   type: git
   location: https://github.com/IntersectMBO/cardano-ledger

docs/website/contents/for-developers/Benchmarks.md

@@ -1,11 +1,32 @@
 # Consensus benchmarks
 
 We are in the process of adding component level microbenchmarks for Consensus.
+
+We check for regressions in performance on CI.
+
+## Mempool Benchmark
+
 We started with microbenchmarks for adding transactions to the mempool. The
 mempool benchmarks can be run using the following command.
 
 ```sh
 cabal new-run ouroboros-consensus:mempool-bench
 ```
 
-We check for regressions in performance on CI. We might publish benchmark results in this site shortly.
+## ChainSync Client Benchmark
+
+To aid the refactoring of the ChainSync client, we added a benchmark for it in [PR#823](https://github.com/IntersectMBO/ouroboros-consensus/pull/823). The benchmark could be invoked as follows:
+
+```sh
+cabal new-run ouroboros-consensus:ChainSync-client-bench -- 10 10
+```
+
+## PerasCertDB Benchmark
+
+We have a microbenchmark for the boosted chain fragment weight calculation, which could be run as follows:
+
+```sh
+cabal run ouroboros-consensus:PerasCertDB-bench -- +RTS -T -A32m -RTS
+```
+
+We request GHC runtime system statistics with `-T` to get a memory usage estimate, and also request a large nursery with `-A32m` to minimise garbage collection. See `tasty-bench` [documentation](https://github.com/Bodigrim/tasty-bench?tab=readme-ov-file#troubleshooting) for more tips.

docs/website/contents/for-developers/Glossary.md

@@ -473,6 +473,19 @@ These kinds are maintained by the Networking layer:
 - [Public root peers](#public-root-peers).
 - [Shared peers](#shared-peers).
 
+## ;Peras ;weight ;boost
+
+Peras is an extension of Praos enabling faster settlement under optimistic conditions.
+To this end, Peras can result in a block `B` receiving a *boost*, which means that any chain containing `B` gets additional weight when being compared to other chains.
+
+Consider a chain fragment `F`:
+
+- Its ;*weight boost* is the sum of all boosts received by points on this fragment (excluding the anchor). Note that the same point can be boosted multiple times.
+
+- Its ;*total weight* is its tip block number plus its weight boost.
+
+Note that these notions are always relative to a particular anchor, so different chain fragments must have the same anchor when their total weight is to be compared.
+
 ## ;Phases
 
 Byron, Shelley, Goguen (current one as of August 2023), Basho, Voltaire.

ouroboros-consensus-cardano/src/unstable-cardano-tools/Cardano/Tools/DBAnalyser/Run.hs

@@ -71,6 +71,7 @@ openLedgerDB [email protected]{LedgerDB.lgrFlavorArgs = LedgerDB.L
           lgrDbArgs
           bss
           (\_ -> error "no replay")
+          (LedgerDB.praosGetVolatileSuffix $ LedgerDB.ledgerDbCfgSecParam $ LedgerDB.lgrConfig lgrDbArgs)
       )
       emptyStream
       genesisPoint
@@ -83,6 +84,7 @@ openLedgerDB [email protected]{LedgerDB.lgrFlavorArgs = LedgerDB.L
           lgrDbArgs
           args
           (\_ -> error "no replay")
+          (LedgerDB.praosGetVolatileSuffix $ LedgerDB.ledgerDbCfgSecParam $ LedgerDB.lgrConfig lgrDbArgs)
       )
       emptyStream
       genesisPoint

ouroboros-consensus-diffusion/src/ouroboros-consensus-diffusion/Ouroboros/Consensus/Node/GSM.hs

@@ -104,10 +104,16 @@ data GsmView m upstreamPeer selection chainSyncState = GsmView
   -- thundering herd phenomenon.
   --
   -- 'Nothing' should only be used for testing.
-  , candidateOverSelection ::
-      selection ->
-      chainSyncState ->
-      CandidateVersusSelection
+  , getCandidateOverSelection ::
+      STM
+        m
+        ( selection ->
+          chainSyncState ->
+          CandidateVersusSelection
+        )
+  -- ^ Whether the candidate from the @chainSyncState@ is preferable to the
+  -- selection. This can depend on external state (Peras certificates boosting
+  -- blocks).
   , peerIsIdle :: chainSyncState -> Bool
   , durationUntilTooOld :: Maybe (selection -> m DurationFromNow)
   -- ^ How long from now until the selection will be so old that the node
@@ -234,7 +240,7 @@ realGsmEntryPoints tracerArgs gsmView =
 
   GsmView
     { antiThunderingHerd
-    , candidateOverSelection
+    , getCandidateOverSelection
     , peerIsIdle
     , durationUntilTooOld
     , equivalent
@@ -383,6 +389,7 @@ realGsmEntryPoints tracerArgs gsmView =
     -- long.
     selection <- getCurrentSelection
     candidates <- traverse StrictSTM.readTVar varsState
+    candidateOverSelection <- getCandidateOverSelection
     let ok candidate =
           WhetherCandidateIsBetter False
             == candidateOverSelection selection candidate

ouroboros-consensus-diffusion/src/ouroboros-consensus-diffusion/Ouroboros/Consensus/NodeKernel.hs

@@ -271,15 +271,18 @@ initNodeKernel
               gsmTracerArgs
               GSM.GsmView
                 { GSM.antiThunderingHerd = Just gsmAntiThunderingHerd
-                , GSM.candidateOverSelection = \(headers, _lst) state ->
-                    case AF.intersectionPoint headers (csCandidate state) of
-                      Nothing -> GSM.CandidateDoesNotIntersect
-                      Just{} ->
-                        GSM.WhetherCandidateIsBetter $ -- precondition requires intersection
-                          preferAnchoredCandidate
-                            (configBlock cfg)
-                            headers
-                            (csCandidate state)
+                , GSM.getCandidateOverSelection = do
+                    weights <- ChainDB.getPerasWeightSnapshot chainDB
+                    pure $ \(headers, _lst) state ->
+                      case AF.intersectionPoint headers (csCandidate state) of
+                        Nothing -> GSM.CandidateDoesNotIntersect
+                        Just{} ->
+                          GSM.WhetherCandidateIsBetter $ -- precondition requires intersection
+                            preferAnchoredCandidate
+                              (configBlock cfg)
+                              (forgetFingerprint weights)
+                              headers
+                              (csCandidate state)
                 , GSM.peerIsIdle = csIdling
                 , GSM.durationUntilTooOld =
                     gsmDurationUntilTooOld

ouroboros-consensus-diffusion/test/consensus-test/Test/Consensus/GSM.hs

@@ -137,7 +137,8 @@ setupGsm isHaaSatisfied vars = do
     (id, tracer)
     GSM.GsmView
       { GSM.antiThunderingHerd = Nothing
-      , GSM.candidateOverSelection = \s (PeerState c _) -> candidateOverSelection s c
+      , GSM.getCandidateOverSelection = pure $ \s (PeerState c _) ->
+          candidateOverSelection s c
       , GSM.peerIsIdle = isIdling
       , GSM.durationUntilTooOld = Just durationUntilTooOld
       , GSM.equivalent = (==) -- unsound, but harmless in this test

ouroboros-consensus-diffusion/test/consensus-test/Test/Consensus/Genesis/Tests/LoE/CaughtUp.hs

@@ -58,6 +58,7 @@ import qualified Ouroboros.Consensus.Node.GSM as GSM
 import Ouroboros.Consensus.Node.Genesis (setGetLoEFragment)
 import Ouroboros.Consensus.Node.GsmState
 import Ouroboros.Consensus.NodeId
+import Ouroboros.Consensus.Peras.Weight (emptyPerasWeightSnapshot)
 import qualified Ouroboros.Consensus.Storage.ChainDB as ChainDB
 import Ouroboros.Consensus.Storage.ChainDB.API (ChainDB)
 import qualified Ouroboros.Consensus.Storage.ChainDB.API.Types.InvalidBlockPunishment as Punishment
@@ -279,7 +280,7 @@ mkGsmEntryPoints varChainSyncHandles chainDB writeGsmState =
   GSM.realGsmEntryPoints
     (id, nullTracer)
     GSM.GsmView
-      { GSM.candidateOverSelection
+      { GSM.getCandidateOverSelection = pure candidateOverSelection
       , GSM.peerIsIdle = csIdling
       , GSM.equivalent = (==) `on` AF.headPoint
       , GSM.getChainSyncStates = fmap cschState <$> cschcMap varChainSyncHandles
@@ -301,10 +302,13 @@ mkGsmEntryPoints varChainSyncHandles chainDB writeGsmState =
       Just{} ->
         -- precondition requires intersection
         GSM.WhetherCandidateIsBetter $
-          preferAnchoredCandidate (configBlock cfg) selection candFrag
+          preferAnchoredCandidate (configBlock cfg) weights selection candFrag
    where
     candFrag = csCandidate candidateState
 
+    -- TODO https://github.com/tweag/cardano-peras/issues/67
+    weights = emptyPerasWeightSnapshot
+
 forkGDD ::
   forall m.
   IOLike m =>

ouroboros-consensus-diffusion/test/consensus-test/Test/Consensus/PeerSimulator/Trace.hs

@@ -418,8 +418,8 @@ traceChainDBEventTestBlockWith tracer = \case
         trace $ "Switched to a fork; now: " ++ terseHFragment newFragment
       StoreButDontChange point ->
         trace $ "Did not select block due to LoE: " ++ terseRealPoint point
-      IgnoreBlockOlderThanK point ->
-        trace $ "Ignored block older than k: " ++ terseRealPoint point
+      IgnoreBlockOlderThanImmTip point ->
+        trace $ "Ignored block older than imm tip: " ++ terseRealPoint point
       ChainSelectionLoEDebug curChain (LoEEnabled loeFrag0) -> do
         trace $ "Current chain: " ++ terseHFragment curChain
         trace $ "LoE fragment: " ++ terseHFragment loeFrag0

ouroboros-consensus/bench/PerasCertDB-bench/Main.hs

@@ -0,0 +1,102 @@
+{-# LANGUAGE ImportQualifiedPost #-}
+{-# LANGUAGE LambdaCase #-}
+
+-- | This module contains benchmarks for Peras chain weight calculation as
+--   implemented by the by the
+--   'Ouroboros.Consensus.Peras.Weight.weightBoostOfFragment' function.
+--
+--   We benchmark the calculation on a static sequence of chain fragments of increasing
+--   length, ranging from 0 to around 8640, with a sampling rate of 100. The chain fragments
+--   are instantiated with 'TestBlock', and every 5 blocks there is a booster block with
+--   weight 15. All parameters are set in 'benchmarkParams'.
+module Main (main) where
+
+import Data.List (iterate')
+import Numeric.Natural (Natural)
+import Ouroboros.Consensus.Block (PerasWeight (PerasWeight), SlotNo (..))
+import Ouroboros.Consensus.Peras.Weight
+  ( PerasWeightSnapshot
+  , mkPerasWeightSnapshot
+  , weightBoostOfFragment
+  )
+import Ouroboros.Network.AnchoredFragment qualified as AF
+import Test.Ouroboros.Storage.TestBlock (TestBlock (..), TestBody (..), TestHeader (..))
+import Test.Ouroboros.Storage.TestBlock qualified as TestBlock
+import Test.Tasty.Bench
+
+data BenchmarkParams = BenchmarkParams
+  { blockRate :: SlotNo
+  -- ^ How often the fragments will contain blocks, in slots
+  , fragmentLenghtSamplingRate :: Natural
+  -- ^ The rate of length increase for generate chain fragments
+  , fragmentMaxLenght :: Natural
+  -- ^ the maximum length of a fragment
+  , boostedBlockRate :: Natural
+  -- ^ How often boosted blocks occur, in blocks
+  , boostWeight :: PerasWeight
+  -- ^ The weight of the boost
+  }
+
+benchmarkParams :: BenchmarkParams
+benchmarkParams =
+  BenchmarkParams
+    { blockRate = 20
+    , fragmentLenghtSamplingRate = 100
+    , fragmentMaxLenght = 2160 + 3 * 2160
+    , boostedBlockRate = 5
+    , boostWeight = PerasWeight 15
+    }
+
+main :: IO ()
+main =
+  Test.Tasty.Bench.defaultMain $ map benchWeightBoostOfFragment inputs
+ where
+  -- NOTE: we do not use the 'env' combinator to set up the test data since
+  --       it requires 'NFData' for 'AF.AnchoredFragment'. While the necessary
+  --       instances could be provided, we do not think is necessary for this
+  --       benchmark, as the input data is rather small.
+  inputs :: [(Natural, (PerasWeightSnapshot TestBlock, AF.AnchoredFragment TestBlock))]
+  inputs =
+    getEveryN (fragmentLenghtSamplingRate benchmarkParams) $
+      take (fromIntegral $ fragmentMaxLenght benchmarkParams) $
+        zip [0 ..] $
+          zip (map uniformWeightSnapshot fragments) fragments
+
+benchWeightBoostOfFragment ::
+  (Natural, (PerasWeightSnapshot TestBlock, AF.AnchoredFragment TestBlock)) -> Benchmark
+benchWeightBoostOfFragment (i, (weightSnapshot, fragment)) =
+  bench ("weightBoostOfFragment of length " <> show i) $
+    whnf (weightBoostOfFragment weightSnapshot) fragment
+
+-- | An infinite list of chain fragments
+fragments :: [AF.AnchoredFragment TestBlock]
+fragments = iterate' addSuccessorBlock genesisFragment
+ where
+  genesisFragment :: AF.AnchoredFragment TestBlock
+  genesisFragment = AF.Empty AF.AnchorGenesis
+
+  addSuccessorBlock :: AF.AnchoredFragment TestBlock -> AF.AnchoredFragment TestBlock
+  addSuccessorBlock = \case
+    AF.Empty _ -> (AF.Empty AF.AnchorGenesis) AF.:> (TestBlock.firstBlock 0 dummyBody)
+    (xs AF.:> x) ->
+      let nextBlockSlot = blockRate benchmarkParams + (thSlotNo . testHeader $ x)
+       in (xs AF.:> x) AF.:> TestBlock.mkNextBlock x nextBlockSlot dummyBody
+
+  dummyBody :: TestBody
+  dummyBody = TestBody{tbForkNo = 0, tbIsValid = True}
+
+-- | Given a chain fragment, construct a weight snapshot where there's a boosted block every 90 slots
+uniformWeightSnapshot :: AF.AnchoredFragment TestBlock -> PerasWeightSnapshot TestBlock
+uniformWeightSnapshot fragment =
+  let pointsToBoost =
+        map snd
+          . getEveryN (boostedBlockRate benchmarkParams)
+          . zip [0 ..]
+          . map AF.blockPoint
+          . AF.toOldestFirst
+          $ fragment
+      weights = repeat (boostWeight benchmarkParams)
+   in mkPerasWeightSnapshot $ pointsToBoost `zip` weights
+
+getEveryN :: Natural -> [(Natural, a)] -> [(Natural, a)]
+getEveryN n = filter (\(i, _) -> (i `mod` n) == 0)

ouroboros-consensus/changelog.d/20250811_130239_alexander.esgen_decouple_immutability.md

@@ -0,0 +1,4 @@
+### Breaking
+
+- Renamed `IgnoreBlockOlderThanK` to `IgnoreBlockOlderThanImmTip` for future-proofing.
+- Renamed and simplified `olderThanK` to `olderThanImmTip`.

ouroboros-consensus/changelog.d/20250811_150947_alexander.esgen_decouple_immutability.md

@@ -0,0 +1,7 @@
+### Breaking
+
+- LedgerDB: generalized over the criterion used to determine which states are
+  volatile/immutable, in preparation for Ouroboros Peras.
+
+  Concretely, `LedgerDB.openDB` takes a new argument, `GetVolatileSuffix m blk`.
+  For Praos behavior, use `praosGetVolatileSuffix`.

ouroboros-consensus/ouroboros-consensus.cabal

@@ -75,6 +75,7 @@ library
     Ouroboros.Consensus.Block.RealPoint
     Ouroboros.Consensus.Block.SupportsDiffusionPipelining
     Ouroboros.Consensus.Block.SupportsMetrics
+    Ouroboros.Consensus.Block.SupportsPeras
     Ouroboros.Consensus.Block.SupportsProtocol
     Ouroboros.Consensus.Block.SupportsSanityCheck
     Ouroboros.Consensus.BlockchainTime
@@ -189,6 +190,8 @@ library
     Ouroboros.Consensus.Node.Run
     Ouroboros.Consensus.Node.Serialisation
     Ouroboros.Consensus.NodeId
+    Ouroboros.Consensus.Peras.SelectView
+    Ouroboros.Consensus.Peras.Weight
     Ouroboros.Consensus.Protocol.Abstract
     Ouroboros.Consensus.Protocol.BFT
     Ouroboros.Consensus.Protocol.LeaderSchedule
@@ -254,6 +257,9 @@ library
     Ouroboros.Consensus.Storage.LedgerDB.V2.Forker
     Ouroboros.Consensus.Storage.LedgerDB.V2.InMemory
     Ouroboros.Consensus.Storage.LedgerDB.V2.LedgerSeq
+    Ouroboros.Consensus.Storage.PerasCertDB
+    Ouroboros.Consensus.Storage.PerasCertDB.API
+    Ouroboros.Consensus.Storage.PerasCertDB.Impl
     Ouroboros.Consensus.Storage.Serialisation
     Ouroboros.Consensus.Storage.VolatileDB
     Ouroboros.Consensus.Storage.VolatileDB.API
@@ -587,6 +593,7 @@ test-suite consensus-test
     Test.Consensus.MiniProtocol.ChainSync.CSJ
     Test.Consensus.MiniProtocol.ChainSync.Client
     Test.Consensus.MiniProtocol.LocalStateQuery.Server
+    Test.Consensus.Peras.WeightSnapshot
     Test.Consensus.Util.MonadSTM.NormalForm
     Test.Consensus.Util.Versioned
 
@@ -708,6 +715,9 @@ test-suite storage-test
     Test.Ouroboros.Storage.LedgerDB.V1.DbChangelog
     Test.Ouroboros.Storage.LedgerDB.V1.LMDB
     Test.Ouroboros.Storage.Orphans
+    Test.Ouroboros.Storage.PerasCertDB
+    Test.Ouroboros.Storage.PerasCertDB.Model
+    Test.Ouroboros.Storage.PerasCertDB.StateMachine
     Test.Ouroboros.Storage.VolatileDB
     Test.Ouroboros.Storage.VolatileDB.Mock
     Test.Ouroboros.Storage.VolatileDB.Model
@@ -817,6 +827,19 @@ benchmark ChainSync-client-bench
     unstable-consensus-testlib,
     with-utf8,
 
+benchmark PerasCertDB-bench
+  import: common-bench
+  type: exitcode-stdio-1.0
+  hs-source-dirs: bench/PerasCertDB-bench
+  main-is: Main.hs
+  other-modules:
+  build-depends:
+    base,
+    ouroboros-consensus,
+    ouroboros-network-api,
+    tasty-bench,
+    unstable-consensus-testlib,
+
 test-suite doctest
   import: common-test
   main-is: doctest.hs

ouroboros-consensus/src/ouroboros-consensus/Ouroboros/Consensus/Block.hs

@@ -8,5 +8,6 @@ import Ouroboros.Consensus.Block.NestedContent as X
 import Ouroboros.Consensus.Block.RealPoint as X
 import Ouroboros.Consensus.Block.SupportsDiffusionPipelining as X
 import Ouroboros.Consensus.Block.SupportsMetrics as X
+import Ouroboros.Consensus.Block.SupportsPeras as X
 import Ouroboros.Consensus.Block.SupportsProtocol as X
 import Ouroboros.Consensus.Block.SupportsSanityCheck as X