WIP Linear Leios running simulations

- Add and respect eb-body-avg-size-bytes config file parameter.
- Prune VTs 30 seconds after their EB's slot onset instead of as Short Leios would.
- Fixup bug in submitLinearEB that wasn't adopting it if the certificate arrived first.
- Include rb_ref in GenEB output for Linear EBs.
- Partially fill in `error` stub for size calculation of certificates.
- Avoid pipeline calculation when logging VBs in the shared format.

Author: nfrisby

leios-trace-hs/src/LeiosConfig.hs

@@ -134,6 +134,7 @@ data Config = Config
   , ebValidationCpuTimeMs :: DurationMs
   , ebSizeBytesConstant :: SizeBytes
   , ebSizeBytesPerIb :: SizeBytes
+  , ebBodyAvgSizeBytes :: SizeBytes
   , ebDiffusionStrategy :: DiffusionStrategy
   , ebDiffusionMaxWindowSize :: Word16
   , ebDiffusionMaxHeadersToRequest :: Word16
@@ -210,6 +211,7 @@ instance Default Config where
       , ebValidationCpuTimeMs = 1.0
       , ebSizeBytesConstant = 240
       , ebSizeBytesPerIb = 32
+      , ebBodyAvgSizeBytes = 5000000
       , ebDiffusionStrategy = PeerOrder
       , ebDiffusionMaxWindowSize = 100
       , ebDiffusionMaxHeadersToRequest = 100
@@ -288,6 +290,7 @@ configToKVsWith getter cfg =
     , get @"ebValidationCpuTimeMs" getter cfg
     , get @"ebSizeBytesConstant" getter cfg
     , get @"ebSizeBytesPerIb" getter cfg
+    , get @"ebBodyAvgSizeBytes" getter cfg
     , get @"ebDiffusionStrategy" getter cfg
     , get @"ebDiffusionMaxWindowSize" getter cfg
     , get @"ebDiffusionMaxHeadersToRequest" getter cfg
@@ -378,6 +381,7 @@ instance FromJSON Config where
     ebValidationCpuTimeMs <- parseFieldOrDefault @Config @"ebValidationCpuTimeMs" obj
     ebSizeBytesConstant <- parseFieldOrDefault @Config @"ebSizeBytesConstant" obj
     ebSizeBytesPerIb <- parseFieldOrDefault @Config @"ebSizeBytesPerIb" obj
+    ebBodyAvgSizeBytes <- parseFieldOrDefault @Config @"ebBodyAvgSizeBytes" obj
     ebDiffusionStrategy <- parseFieldOrDefault @Config @"ebDiffusionStrategy" obj
     ebDiffusionMaxWindowSize <- parseFieldOrDefault @Config @"ebDiffusionMaxWindowSize" obj
     ebDiffusionMaxHeadersToRequest <- parseFieldOrDefault @Config @"ebDiffusionMaxHeadersToRequest" obj

simulation/docs/SimulatorModel.md

@@ -422,6 +422,8 @@ The Linear Leios simulator adds the following new variables, some of which also
   A new custom thread monitors this variable in addition to the clock, so that it can avoid issugin a vote too early or too late.
 - `linearEbOfRb`.
   A mapping from RB to its announced Linear EB _that has been validated_, which is needed when issuing an RB.
+- `pruneExpiredLinearVotes`.
+  This thread prunes votes 30 seconds after the onset of their EB's slot.
 
 TODO RB Diffusion is not pipelined
 

simulation/src/LeiosProtocol/Short.hs

@@ -45,6 +45,8 @@ data SizesConfig = SizesConfig
   , inputBlockBodyAvgSize :: !Bytes
   -- ^ as we do not model transactions we just use a fixed size for bodies.
   , inputBlockBodyMaxSize :: !Bytes
+  , endorseBlockBodyAvgSize :: !Bytes
+  -- ^ unused by Short/Full Leios, used by Linear Leios eg
   , endorseBlock :: !(EndorseBlock -> Bytes)
   , voteMsg :: !(VoteMsg -> Bytes)
   , certificate :: !(Certificate -> Bytes)
@@ -232,6 +234,7 @@ convertConfig disk =
       { inputBlockHeader = fromIntegral disk.ibHeadSizeBytes
       , inputBlockBodyAvgSize = fromIntegral disk.ibBodyAvgSizeBytes
       , inputBlockBodyMaxSize = fromIntegral disk.ibBodyMaxSizeBytes
+      , endorseBlockBodyAvgSize = fromIntegral disk.ebBodyAvgSizeBytes
       , endorseBlock = \eb ->
           fromIntegral $
             disk.ebSizeBytesConstant
@@ -242,7 +245,10 @@ convertConfig disk =
           fromIntegral $
             disk.voteBundleSizeBytesConstant
               + disk.voteBundleSizeBytesPerEb `forEach` vt.endorseBlocks
-      , certificate = const $ error "certificate size config already included in PraosConfig{bodySize}"
+      , certificate = \_cert ->
+          fromIntegral $
+            assert (0 == disk.certSizeBytesPerNode) $   -- TODO
+              disk.certSizeBytesConstant
       , rankingBlockLegacyPraosPayloadAvgSize = fromIntegral disk.rbBodyLegacyPraosPayloadAvgSizeBytes
       }
   certificateGeneration (Certificate votesMap) =
@@ -357,6 +363,7 @@ delaysAndSizesAsFull cfg@LeiosConfig{pipeline, voteSendStage} =
       { inputBlockHeader = cfg.sizes.inputBlockHeader :: Bytes
       , inputBlockBodyAvgSize = cfg.sizes.inputBlockBodyAvgSize :: Bytes
       , inputBlockBodyMaxSize = cfg.sizes.inputBlockBodyMaxSize :: Bytes
+      , endorseBlockBodyAvgSize = cfg.sizes.endorseBlockBodyAvgSize :: Bytes
       , endorseBlock = const @Bytes $ cfg.sizes.endorseBlock $ mockFullEndorseBlock cfg
       , voteMsg = const @Bytes $ cfg.sizes.voteMsg $ mockFullVoteMsg cfg
       , certificate = const @Bytes $ cfg.sizes.certificate $ mockFullCertificate cfg

simulation/src/LeiosProtocol/Short/Node.hs

@@ -20,7 +20,7 @@ import Control.Category ((>>>))
 import Control.Concurrent.Class.MonadMVar
 import Control.Concurrent.Class.MonadSTM.TSem
 import Control.Exception (assert)
-import Control.Monad (forever, guard, replicateM, unless, when)
+import Control.Monad (forever, guard, replicateM, unless, when, void)
 import Control.Monad.Class.MonadAsync
 import Control.Monad.Class.MonadFork
 import Control.Monad.Class.MonadThrow
@@ -692,7 +692,10 @@ leiosNode tracer cfg followers peers = do
   let pruningThreads =
         concat
           [ [ pruneExpiredVotes tracer cfg leiosState
-            | CleanupExpiredVote `isEnabledIn` cfg.leios.cleanupPolicies
+            | Linear /= cfg.leios.variant && CleanupExpiredVote `isEnabledIn` cfg.leios.cleanupPolicies
+            ]
+          , [ pruneExpiredLinearVotes tracer cfg leiosState
+            | Linear == cfg.leios.variant && CleanupExpiredVote `isEnabledIn` cfg.leios.cleanupPolicies
             ]
           , [ pruneExpiredUncertifiedEBs tracer cfg leiosState
             | CleanupExpiredUncertifiedEb `isEnabledIn` cfg.leios.cleanupPolicies
@@ -879,6 +882,28 @@ pruneExpiredVotes _tracer LeiosNodeConfig{leios = leios@LeiosConfig{pipeline = _
     --   traceWith tracer $! LeiosNodeEvent Pruned (EventVote $ snd vt)
     go (succ p)
 
+-- | Prune votes 30 seconds after the supported EB.   TODO magic number
+pruneExpiredLinearVotes ::
+  (Monad m, MonadDelay m, MonadTime m, MonadSTM m) =>
+  Tracer m LeiosNodeEvent ->
+  LeiosNodeConfig ->
+  LeiosNodeState m ->
+  m ()
+pruneExpiredLinearVotes _tracer cfg st = go (SlotNo 0)
+ where
+  go pruneTo = do
+    _ <- waitNextSlot cfg.slotConfig (SlotNo $ unSlotNo pruneTo + 30)   -- TODO magic number
+    _votesPruned <- atomically $ do
+      writeTVar st.prunedVoteStateToVar $! pruneTo
+      partitionRBVar st.relayVoteState.relayBufferVar $
+        \voteEntry ->
+          let voteSlot = (snd voteEntry.value).slot
+           in voteSlot < pruneTo
+    -- TODO: batch these, too many events.
+    -- for_ votesPruned $ \vt -> do
+    --   traceWith tracer $! LeiosNodeEvent Pruned (EventVote $ snd vt)
+    go (succ pruneTo)
+
 referencedEBs :: MonadSTM m => LeiosConfig -> LeiosNodeState m -> Set EndorseBlockId -> STM m [EndorseBlockId]
 referencedEBs cfg st ebIds0
   | null ebIds0 = return []
@@ -1149,15 +1174,15 @@ dispatchValidationSTM tracer cfg leiosState req =
       -- NOTE: block references are only inspected during voting.
       return [valEB eb completion | eb <- ebs]
     ValidateLinearEBs ibs completion -> do
-      let ifNoCert :: InputBlockId -> STM m () -> STM m ()
+      let ifNoCert :: InputBlockId -> (Bool -> STM m a) -> STM m ()
           ifNoCert ibId k = do
             votesForEB <- readTVar leiosState.votesForEBVar
-            case Map.lookup (convertLinearId ibId) votesForEB of
-              Just Certified{} -> pure ()
-              _ -> k
+            void $ k $ case Map.lookup (convertLinearId ibId) votesForEB of
+              Just Certified{} -> True
+              _ -> False
       waitFor
         leiosState.waitingForTipVar
-        [ (rbHash, [ifNoCert ib.id $ queue [valLinearEB ib False (const (pure ()))]])
+        [ (rbHash, [ifNoCert ib.id $ \alreadyCertified -> queue [valLinearEB ib alreadyCertified (const (pure ()))]])
         | ib <- ibs
         , BlockHash rbHash <- [ib.header.rankingBlock]
         ]
@@ -1365,7 +1390,11 @@ mkBuffersView cfg st = BuffersView{..}
             -- TODO: start from votesForEB, would allow to drop EBs from relayBuffer as soon as Endorse ends.
             $ bufferEB
       }
-  newIBData = do
+  newIBData
+    | Linear <- cfg.leios.variant = do
+    let txsPayload = cfg.leios.sizes.endorseBlockBodyAvgSize
+    return $ NewInputBlockData{referenceRankingBlock = GenesisHash {- dummy value, ignored -}, txsPayload}
+    | otherwise = do
     ledgerState <- readTVar st.ledgerStateVar
     referenceRankingBlock <-
       Chain.headHash . Chain.dropUntil (flip Map.member ledgerState . blockHash)

simulation/src/LeiosProtocol/Short/Sim.hs

@@ -155,6 +155,7 @@ logLeiosEvent nodeNames loudness e = case e of
             mconcat
               [ "slot" .= eb.slot
               , "size_bytes" .= fromBytes (messageSizeBytes eb)
+              , "rb_ref" .= rbRef (eb.header.rankingBlock)
               ]
           EventVote vt ->
             mconcat
@@ -380,7 +381,7 @@ sharedEvent leios nodeNames e = case e of
         Shared.VTBundleGenerated
           { bytes = fromIntegral (messageSizeBytes vt)
           , votes = Map.fromList $ map ((,vt.votes) . T.pack . mkStringId) vt.endorseBlocks
-          , pipeline = coerce $ voteMsgPipeline leios vt
+          , pipeline = if Linear == leios.variant then 0 else coerce $ voteMsgPipeline leios vt
           , ..
           }
   sharedEnterState :: T.Text -> String -> Word64 -> LeiosEventBlock -> Shared.Event
@@ -463,6 +464,7 @@ traceRelayLink1 connectionOptions =
                     { inputBlockHeader = kibibytes 1
                     , inputBlockBodyAvgSize = kibibytes 95
                     , inputBlockBodyMaxSize = kibibytes 100
+                    , endorseBlockBodyAvgSize = megabytes 5
                     , endorseBlock = \eb -> coerce (length eb.inputBlocks) * 32 + 32 + 128
                     , voteMsg = \v -> fromIntegral v.votes * 32 + 32 + 128
                     , certificate = const (50 * 1024)