Support leios-late-ib-inclusion

Author: nfrisby

data/simulation/config.d.ts

@@ -39,10 +39,10 @@ export interface Config {
    * Only supported by Haskell simulation. */
   "leios-vote-send-recv-stages": boolean;
   /**
-   * Extends Leios so that EB producers include IBs directly from previous pipelines
-   * where no certified EB was observed.
-   * 
-   * Only supported by Rust simulation. */
+   * Extends Leios so that EB producers include IBs directly from previous pipelines.
+   * Due to casuality, the EB must always include them, even if those IBs end up being
+   * certified in their own pipeline.
+   */
   "leios-late-ib-inclusion": boolean;
   /**
    * The expected time it takes a header to fully diffuse across the network.

leios-trace-hs/src/LeiosConfig.hs

@@ -99,6 +99,7 @@ data Config = Config
   , leiosStageActiveVotingSlots :: Word
   , leiosVoteSendRecvStages :: Bool
   , leiosVariant :: LeiosVariant
+  , leiosLateIbInclusion :: Bool
   , leiosHeaderDiffusionTimeMs :: DurationMs
   , praosChainQuality :: Double
   , txGenerationDistribution :: Distribution
@@ -171,6 +172,7 @@ instance Default Config where
       , leiosStageActiveVotingSlots = 1
       , leiosVoteSendRecvStages = False
       , leiosVariant = Short
+      , leiosLateIbInclusion = True
       , leiosHeaderDiffusionTimeMs = 1000
       , praosChainQuality = 40
       , txGenerationDistribution = Exp{lambda = 0.85, scale = Just 1000}
@@ -243,6 +245,7 @@ configToKVsWith getter cfg =
     , get @"treatBlocksAsFull" getter cfg
     , get @"cleanupPolicies" getter cfg
     , get @"leiosVariant" getter cfg
+    , get @"leiosLateIbInclusion" getter cfg
     , get @"leiosHeaderDiffusionTimeMs" getter cfg
     , get @"praosChainQuality" getter cfg
     , get @"simulateTransactions" getter cfg
@@ -329,6 +332,7 @@ instance FromJSON Config where
     treatBlocksAsFull <- parseFieldOrDefault @Config @"treatBlocksAsFull" obj
     cleanupPolicies <- parseFieldOrDefault @Config @"cleanupPolicies" obj
     leiosVariant <- parseFieldOrDefault @Config @"leiosVariant" obj
+    leiosLateIbInclusion <- parseFieldOrDefault @Config @"leiosLateIbInclusion" obj
     leiosHeaderDiffusionTimeMs <- parseFieldOrDefault @Config @"leiosHeaderDiffusionTimeMs" obj
     praosChainQuality <- parseFieldOrDefault @Config @"praosChainQuality" obj
     simulateTransactions <- parseFieldOrDefault @Config @"simulateTransactions" obj

leios-trace-verifier/hs-src/test/Spec/Scenario.hs

@@ -19,7 +19,7 @@ import qualified Data.Map.Strict as M
 import qualified Data.Set as S
 
 config :: Config
-config = Config{relayStrategy = RequestFromFirst, tcpCongestionControl = True, multiplexMiniProtocols = True, treatBlocksAsFull = False, cleanupPolicies = CleanupPolicies (S.fromList [CleanupExpiredVote]), simulateTransactions = True, leiosStageLengthSlots = 2, leiosStageActiveVotingSlots = 1, leiosVoteSendRecvStages = False, leiosVariant = Short, leiosHeaderDiffusionTimeMs = 1000.0, praosChainQuality = 20.0, txGenerationDistribution = Exp{lambda = 0.85, scale = pure 1000.0}, txSizeBytesDistribution = LogNormal{mu = 6.833, sigma = 1.127}, txValidationCpuTimeMs = 1.5, txMaxSizeBytes = 16384, rbGenerationProbability = 5.0e-2, rbGenerationCpuTimeMs = 1.0, rbHeadValidationCpuTimeMs = 1.0, rbHeadSizeBytes = 1024, rbBodyMaxSizeBytes = 90112, rbBodyLegacyPraosPayloadValidationCpuTimeMsConstant = 50.0, rbBodyLegacyPraosPayloadValidationCpuTimeMsPerByte = 5.0e-4, rbBodyLegacyPraosPayloadAvgSizeBytes = 0, ibGenerationProbability = 5.0, ibGenerationCpuTimeMs = 130.0, ibHeadSizeBytes = 304, ibHeadValidationCpuTimeMs = 1.0, ibBodyValidationCpuTimeMsConstant = 50.0, ibBodyValidationCpuTimeMsPerByte = 5.0e-4, ibBodyMaxSizeBytes = 327680, ibBodyAvgSizeBytes = 98304, ibDiffusionStrategy = FreshestFirst, ibDiffusionMaxWindowSize = 100, ibDiffusionMaxHeadersToRequest = 100, ibDiffusionMaxBodiesToRequest = 1, ibShards = 50, ebGenerationProbability = 1.5, ebGenerationCpuTimeMs = 75.0, ebValidationCpuTimeMs = 1.0, ebSizeBytesConstant = 240, ebSizeBytesPerIb = 32, ebDiffusionStrategy = PeerOrder, ebDiffusionMaxWindowSize = 100, ebDiffusionMaxHeadersToRequest = 100, ebDiffusionMaxBodiesToRequest = 1, ebMaxAgeSlots = 100, ebMaxAgeForRelaySlots = 40, voteGenerationProbability = 500.0, voteGenerationCpuTimeMsConstant = 0.164, voteGenerationCpuTimeMsPerIb = 0.0, voteValidationCpuTimeMs = 0.816, voteThreshold = 300, voteBundleSizeBytesConstant = 0, voteBundleSizeBytesPerEb = 105, voteDiffusionStrategy = PeerOrder, voteDiffusionMaxWindowSize = 100, voteDiffusionMaxHeadersToRequest = 100, voteDiffusionMaxBodiesToRequest = 1, certGenerationCpuTimeMsConstant = 90.0, certGenerationCpuTimeMsPerNode = 0.0, certValidationCpuTimeMsConstant = 130.0, certValidationCpuTimeMsPerNode = 0.0, certSizeBytesConstant = 7168, certSizeBytesPerNode = 0}
+config = Config{relayStrategy = RequestFromFirst, tcpCongestionControl = True, multiplexMiniProtocols = True, treatBlocksAsFull = False, cleanupPolicies = CleanupPolicies (S.fromList [CleanupExpiredVote]), simulateTransactions = True, leiosStageLengthSlots = 2, leiosStageActiveVotingSlots = 1, leiosVoteSendRecvStages = False, leiosVariant = Short, leiosLateIbInclusion = False, leiosHeaderDiffusionTimeMs = 1000.0, praosChainQuality = 20.0, txGenerationDistribution = Exp{lambda = 0.85, scale = pure 1000.0}, txSizeBytesDistribution = LogNormal{mu = 6.833, sigma = 1.127}, txValidationCpuTimeMs = 1.5, txMaxSizeBytes = 16384, rbGenerationProbability = 5.0e-2, rbGenerationCpuTimeMs = 1.0, rbHeadValidationCpuTimeMs = 1.0, rbHeadSizeBytes = 1024, rbBodyMaxSizeBytes = 90112, rbBodyLegacyPraosPayloadValidationCpuTimeMsConstant = 50.0, rbBodyLegacyPraosPayloadValidationCpuTimeMsPerByte = 5.0e-4, rbBodyLegacyPraosPayloadAvgSizeBytes = 0, ibGenerationProbability = 5.0, ibGenerationCpuTimeMs = 130.0, ibHeadSizeBytes = 304, ibHeadValidationCpuTimeMs = 1.0, ibBodyValidationCpuTimeMsConstant = 50.0, ibBodyValidationCpuTimeMsPerByte = 5.0e-4, ibBodyMaxSizeBytes = 327680, ibBodyAvgSizeBytes = 98304, ibDiffusionStrategy = FreshestFirst, ibDiffusionMaxWindowSize = 100, ibDiffusionMaxHeadersToRequest = 100, ibDiffusionMaxBodiesToRequest = 1, ibShards = 50, ebGenerationProbability = 1.5, ebGenerationCpuTimeMs = 75.0, ebValidationCpuTimeMs = 1.0, ebSizeBytesConstant = 240, ebSizeBytesPerIb = 32, ebDiffusionStrategy = PeerOrder, ebDiffusionMaxWindowSize = 100, ebDiffusionMaxHeadersToRequest = 100, ebDiffusionMaxBodiesToRequest = 1, ebMaxAgeSlots = 100, ebMaxAgeForRelaySlots = 40, voteGenerationProbability = 500.0, voteGenerationCpuTimeMsConstant = 0.164, voteGenerationCpuTimeMsPerIb = 0.0, voteValidationCpuTimeMs = 0.816, voteThreshold = 300, voteBundleSizeBytesConstant = 0, voteBundleSizeBytesPerEb = 105, voteDiffusionStrategy = PeerOrder, voteDiffusionMaxWindowSize = 100, voteDiffusionMaxHeadersToRequest = 100, voteDiffusionMaxBodiesToRequest = 1, certGenerationCpuTimeMsConstant = 90.0, certGenerationCpuTimeMsPerNode = 0.0, certValidationCpuTimeMsConstant = 130.0, certValidationCpuTimeMsPerNode = 0.0, certSizeBytesConstant = 7168, certSizeBytesPerNode = 0}
 
 topology :: Topology 'COORD2D
 topology = Topology{nodes = M.fromList [(NodeName "node-0", Node{nodeInfo = NodeInfo{stake = 500, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.12000040231003672, _2 = 0.1631004621065356}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-1", LinkInfo{latencyMs = 141.01364015418432, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000}), (NodeName "node-2", LinkInfo{latencyMs = 254.6249782835189, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]}), (NodeName "node-1", Node{nodeInfo = NodeInfo{stake = 200, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.34276660615051174, _2 = 0.2636899791034371}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-2", LinkInfo{latencyMs = 175.32530255486685, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000}), (NodeName "node-3", LinkInfo{latencyMs = 379.1167948193313, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]}), (NodeName "node-2", Node{nodeInfo = NodeInfo{stake = 100, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.5150493264153491, _2 = 0.27873594531347595}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-3", LinkInfo{latencyMs = 248.31457793649423, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]}), (NodeName "node-3", Node{nodeInfo = NodeInfo{stake = 0, cpuCoreCount = CpuCoreCount mzero, location = LocCoord2D{coord2D = Point{_1 = 0.3503537969220088, _2 = 0.13879558055660354}}, adversarial = mzero}, producers = M.fromList [(NodeName "node-0", LinkInfo{latencyMs = 140.19739576271448, bandwidthBytesPerSecond = BandwidthBps $ pure 1024000})]})]}

simulation/ouroboros-leios-sim.cabal

@@ -218,6 +218,7 @@ test-suite ols-test
     , bytestring
   other-modules:
     Paths_ouroboros_leios_sim
-    Test.Topology
     Test.Config
+    Test.ShortToFull
+    Test.Topology
   default-language: Haskell2010

simulation/src/LeiosProtocol/Short.hs

@@ -111,6 +111,10 @@ data LeiosConfig = forall p. IsPipeline p => LeiosConfig
   , variant :: LeiosVariant
   , headerDiffusionTime :: NominalDiffTime
   -- ^ Δ_{hdr}.
+  , lateIbInclusion :: Bool
+  -- ^ Whether an EB also includes IBs from the two previous iterations.
+  --
+  -- TODO Merely one previous iteration if 'pipeline' is 'SingleVote'?
   , pipelinesToReferenceFromEB :: Int
   -- ^ how many older pipelines to reference from an EB when `variant = Full`.
   , votingFrequencyPerStage :: Double
@@ -147,6 +151,7 @@ convertConfig disk =
           , cleanupPolicies = disk.cleanupPolicies
           , variant = disk.leiosVariant
           , headerDiffusionTime = realToFrac $ durationMsToDiffTime disk.leiosHeaderDiffusionTimeMs
+          , lateIbInclusion = disk.leiosLateIbInclusion
           , pipelinesToReferenceFromEB =
               if disk.leiosVariant == Full
                 then
@@ -285,6 +290,7 @@ delaysAndSizesAsFull cfg@LeiosConfig{pipeline, voteSendStage} =
     , cleanupPolicies = cfg.cleanupPolicies
     , variant = cfg.variant
     , headerDiffusionTime = cfg.headerDiffusionTime
+    , lateIbInclusion = cfg.lateIbInclusion
     , pipelinesToReferenceFromEB = cfg.pipelinesToReferenceFromEB
     , activeVotingStageLength = cfg.activeVotingStageLength
     , votingFrequencyPerStage = cfg.votingFrequencyPerStage
@@ -664,13 +670,13 @@ data InputBlocksQuery = InputBlocksQuery
   -- ^ This is checked against time the body is downloaded, before validation.
   }
 
-inputBlocksToEndorse ::
+inputBlocksToEndorse1 ::
   LeiosConfig ->
   -- | current slot
   SlotNo ->
   InputBlocksSnapshot ->
   [InputBlockId]
-inputBlocksToEndorse cfg@LeiosConfig{pipeline = _ :: SingPipeline p} current buffer = fromMaybe [] $ do
+inputBlocksToEndorse1 cfg@LeiosConfig{pipeline = _ :: SingPipeline p} current buffer = fromMaybe [] $ do
   generatedBetween <- stageRange @p cfg Endorse current Propose
   receivedBy <- stageEnd @p cfg Endorse current Deliver2
   pure $
@@ -680,6 +686,26 @@ inputBlocksToEndorse cfg@LeiosConfig{pipeline = _ :: SingPipeline p} current buf
         , receivedBy
         }
 
+-- | Invokes 'inputBlocksToEndorse1' as many times as 'lateIbInclusion'
+-- requires
+inputBlocksToEndorse ::
+  LeiosConfig ->
+  -- | current slot
+  SlotNo ->
+  InputBlocksSnapshot ->
+  [InputBlockId]
+inputBlocksToEndorse cfg current buffer =
+  concatMap each iterations
+ where
+  each sl = inputBlocksToEndorse1 cfg sl buffer
+  capL = fromIntegral cfg.sliceLength
+  iterations =
+    if not cfg.lateIbInclusion
+      then [current]
+      else
+        -- discard underflows
+        dropWhile (> current) [current - 2 * capL, current - capL, current]
+
 -- | Returns possible EBs to reference from current pipeline EB.
 endorseBlocksToReference ::
   LeiosConfig ->

simulation/src/LeiosProtocol/Short/Sim.hs

@@ -426,6 +426,7 @@ traceRelayLink1 connectionOptions =
               , cleanupPolicies = def
               , variant = Short
               , headerDiffusionTime = 1
+              , lateIbInclusion = False
               , pipelinesToReferenceFromEB = 0
               , activeVotingStageLength = 1
               , pipeline = SingSingleVote

simulation/test/Main.hs

@@ -1,6 +1,7 @@
 module Main where
 
 import qualified Test.Config
+import qualified Test.ShortToFull
 import Test.Tasty (defaultMain, testGroup)
 import qualified Test.Topology
 
@@ -9,4 +10,5 @@ main =
   defaultMain . testGroup "ouroboros-leios-sim" $
     [ Test.Topology.tests
     , Test.Config.tests
+    , Test.ShortToFull.tests
     ]

simulation/test/Test/ShortToFull.hs

@@ -0,0 +1,79 @@
+{-# LANGUAGE OverloadedRecordDot #-}
+{-# LANGUAGE RecordWildCards #-}
+
+module Test.ShortToFull where
+
+import Data.Default (def)
+import LeiosProtocol.Common (InputBlockId (..), NodeId (..), SlotNo)
+import LeiosProtocol.Config (Config)
+import qualified LeiosProtocol.Short as Short
+import Test.QuickCheck as Q
+import Test.Tasty (TestTree, testGroup)
+import Test.Tasty.QuickCheck (testProperty)
+
+tests :: TestTree
+tests =
+  testGroup
+    "ShortToFull"
+    [ testProperty "test_leiosLateIbInclusion" test_leiosLateIbInclusion
+    ]
+
+data Test_leiosLateIbInclusion = Test_leiosLateIbInclusion
+  { ibTable :: [(Int, SlotNo, SlotNo)] -- the Int is @zip [0..]@, /before shrinking/
+  , current :: SlotNo
+  }
+  deriving (Show)
+
+instance Q.Arbitrary Test_leiosLateIbInclusion where
+  arbitrary = do
+    let cfg = Short.convertConfig def
+    let arbSlot = Q.choose (0, 10 * cfg.sliceLength)
+    let toSlot :: Int -> SlotNo
+        toSlot = toEnum
+    current <- toSlot <$> arbSlot
+    ibTable <- do
+      n <- Q.choose (0, 1000)
+      flip mapM [0 .. n - 1] $ \i -> do
+        gen <- Q.choose (0, fromEnum current)
+        delay <- Q.choose (0, 3 * cfg.sliceLength)
+        pure (i, toSlot gen, toSlot (gen + delay))
+    pure Test_leiosLateIbInclusion{..}
+
+  shrink testSetup =
+    [ testSetup{ibTable = ibTable'}
+    | ibTable' <- shrinkList (const []) testSetup.ibTable
+    ]
+
+test_leiosLateIbInclusion :: Test_leiosLateIbInclusion -> Property
+test_leiosLateIbInclusion testSetup =
+  Q.counterexample (show (cfg.sliceLength, iterations))
+    $ Q.counterexample ("on " <> show (map (.num) on))
+    $ Q.counterexample
+      (unlines $ ("off" :) $ map (show . map (.num) . off) $ iterations)
+    $ on Q.=== concatMap off iterations
+ where
+  Test_leiosLateIbInclusion{..} = testSetup
+  cfg = Short.convertConfig def
+  ibSnapshot =
+    Short.InputBlocksSnapshot
+      { validInputBlocks = \q ->
+          [ InputBlockId{node = NodeId 0, num = i}
+          | (i, gen, recv) <- ibTable
+          , gen `Short.inRange` q.generatedBetween
+          , recv <= q.receivedBy
+          ]
+      }
+  on =
+    Short.inputBlocksToEndorse
+      cfg{Short.lateIbInclusion = True}
+      current
+      ibSnapshot
+  off sl =
+    Short.inputBlocksToEndorse
+      cfg{Short.lateIbInclusion = False}
+      sl
+      ibSnapshot
+  iterations =
+    let capL = fromIntegral cfg.sliceLength
+     in -- detect underflow
+        dropWhile (> current) [current - 2 * capL, current - capL, current]