Time step influence upon accuracy of the Rust simulation (#447)

* Designed experiment for high TPS of small transactions
* Completed first 1000 tps run
* Completed timestep analysis
* Updated logbook
---------

Co-authored-by: Brian W Bush <[email protected]>

Author: bwbush

Logbook.md

@@ -2,6 +2,12 @@
 
 ## 2025-07-19
 
+### Study of timestep effects in simulators
+
+We compared the 1000 TPS Full Leios simulation results (Rust) at two time resolutions, 0.100 ms and 0.025 ms. No significant differences in results were found. This means that we can safely run simulations at the coarser time step; such enable greater parallelism in the simulator and shorter wallclock times. See [the Jupyter notebook](analysis/sims/2025w27/analysis.ipynb) for comparisons, evidence, and details.
+
+## 2025-07-18
+
 ### 100 TPS experiment for Stracciatella and Linear Leios
 
 Experiments using the Rust simulation demonstrated the viability of Stracciatella and Linear Leios at 100 TPS if appropriate stage lengths and maximum EB sizes are chosen.
@@ -28,8 +34,6 @@ Artifacts:
 - [Jupyter notebook](analysis/sims/2025w29b/analysis)
 - [Configurations](analysis/sims/2025w29b/)
 
-## 2025-07-18
-
 ### Rust simulation
 
 Revised Linear Leios model in response to analysis: in particular we discovered that sufficient throughput depends on only partially validating EBs before propagating them to peers.

analysis/sims/2025w27/analysis.ipynb

@@ -1,32 +1,41 @@
 #!/usr/bin/env nix-shell
-#!nix-shell -i bash -p gzip
+#!nix-shell -i bash -p gnused gzip pigz "rWrapper.override { packages = with rPackages; [ data_table R_utils bit64 ggplot2 magrittr stringr ]; }"
 
 set -e
 
-for d in tsr
+mkdir -p results
+for f in lifecycle resources receipts cpus
 do
-  mkdir -p results/$d
-  for f in cpus lifecycle resources receipts
-  do
-    DIR=$(find $d -type f -name $f.csv.gz \( -not -empty \) -printf %h\\n -quit)
-    HL=$(sed -n -e '1p' "$DIR/case.csv")
-    HR=$(zcat "$DIR/$f.csv.gz" | sed -n -e '1p')
-    (
-      echo "$HL,$HR"
-      for g in $(find $d -type f -name $f.csv.gz \( -not -empty \) -printf %h\\n)
-      do
-        if [ ! -e "$g/stderr" ]
-        then
-          echo "Skipping $g because it has no stderr." >> /dev/stderr
-        elif [ -s "$g/stderr" ]
-        then
-          echo "Skipping $g because its stderr is not empty." >> /dev/stderr
-        else
-          BL=$(sed -n -e '2p' "$g/case.csv")
-          zcat "$g/$f.csv.gz" | sed -e "1d;s/^/$BL,/;s/null/NA/g"
-        fi
-      done
-    ) | pigz -p 3 -9c > results/$d/$f.csv.gz &
-  done
+  DIR=$(find tsr -type f -name $f.csv.gz \( -not -empty \) -printf %h\\n -quit)
+  HL=$(sed -n -e '1p' "$DIR/case.csv")
+  HR=$(zcat "$DIR/$f.csv.gz" | sed -n -e '1p')
+  if [[ "$f" == "lifecycle" || "$f" == "resources" ]]
+  then
+    FRACT=1.00
+  else
+    FRACT=0.20
+  fi
+  (
+    echo "$HL,$HR"
+    for g in $(find tsr -type f -name $f.csv.gz \( -not -empty \) -printf %h\\n)
+    do
+      if [ ! -e "$g/stderr" ]
+      then
+        echo "Skipping $g because it has no stderr." >> /dev/stderr
+      elif [ -s "$g/stderr" ]
+      then
+        echo "Skipping $g because its stderr is not empty." >> /dev/stderr
+      else
+        BL=$(sed -n -e '2p' "$g/case.csv")
+        zcat "$g/$f.csv.gz" | gawk 'FNR > 1 && rand() <= '"$FRACT"' { print "'"$BL"'" "," $0}'
+      fi
+    done
+  ) | pigz -p 3 -9c > results/$f.csv.gz
+R --vanilla << EOI > /dev/null
+require(data.table)
+sampleSize <- $FRACT
+print(sampleSize)
+$f <- fread("results/$f.csv.gz", stringsAsFactors=TRUE)
+save($f, sampleSize, file="results/$f.Rdata", compression_level=9)
+EOI
 done
-wait

analysis/sims/2025w27/combine-results.sh

@@ -0,0 +1,222 @@
+# yaml-language-server: $schema=./config.schema.json
+
+################################################################################
+# Simulation Configuration File
+################################################################################
+#
+# This file contains the default configuration for running Leios simulations in
+# the Haskell simulation (`simulation/`) and the Rust simulation (`sim-rs/`).
+#
+################################################################################
+# Simulation Configuration
+################################################################################
+
+relay-strategy: "request-from-first"
+tcp-congestion-control: true
+multiplex-mini-protocols: true
+simulate-transactions: true
+treat-blocks-as-full: false
+cleanup-policies: ["cleanup-expired-vote"]
+timestamp-resolution-ms: 0.1
+
+################################################################################
+# Leios Protocol Configuration
+################################################################################
+
+leios-variant: full-without-ibs
+leios-stage-length-slots: 5
+leios-stage-active-voting-slots: 1
+leios-vote-send-recv-stages: false
+leios-late-ib-inclusion: true
+leios-header-diffusion-time-ms: 1000.0
+leios-mempool-sampling-strategy: ordered-by-id
+leios-mempool-aggressive-pruning: true
+# TODO: revise default
+praos-chain-quality: 100
+praos-fallback-enabled: false
+
+################################################################################
+# Transaction Configuration
+################################################################################
+
+tx-generation-distribution:
+  distribution: exp
+  lambda: 0.85
+  scale:  1.20
+tx-size-bytes-distribution:
+  distribution: constant
+  value: 300
+tx-conflict-fraction: 0
+tx-overcollateralization-factor-distribution:
+  distribution: constant
+  value: 0
+tx-validation-cpu-time-ms: 0.065
+tx-max-size-bytes: 16384
+tx-start-time: 60
+tx-stop-time: 360
+
+################################################################################
+# Ranking Block Configuration
+################################################################################
+
+# 1/leios-stage-length-slots, targeting one RB per pipeline.
+# Also 20s is current rate of praos blocks.
+rb-generation-probability: 5.0e-2
+# Eng. team targets 1kB as worst case upper bound.
+# Actual size fairly close.
+rb-head-size-bytes: 1024
+rb-body-max-size-bytes: 90112
+# Note: certificate generation/validation is not included in the
+# timings here, see cert-* fields.
+rb-generation-cpu-time-ms: 1.0
+rb-head-validation-cpu-time-ms: 1.0
+
+# On average, no Txs directly embedded in blocks.
+rb-body-legacy-praos-payload-avg-size-bytes: 0
+rb-body-legacy-praos-payload-validation-cpu-time-ms-constant: 50.0
+# the -per-byte component is meant to be using size as a (bad)
+# proxy for the complexity of the Txs included.
+rb-body-legacy-praos-payload-validation-cpu-time-ms-per-byte: 0.0005
+
+################################################################################
+# Input Block Configuration
+################################################################################
+
+ib-generation-probability: 2.0
+ib-shards: 1
+ib-shard-period-length-slots: 1
+ib-shard-group-count: 1
+
+# ProducerId  32
+# SlotNo      64
+# VRF proof   80
+# Body hash   32
+# RB Ref      32
+# Signature   64
+# Total       304
+#
+# NOTE: using a KES Signature (like for Praos headers)
+#       would instead more than double the total to 668.
+#       And even 828 including Op Cert.
+ib-head-size-bytes: 304
+# 98KB to optimize for 3 TCP round trips
+ib-body-avg-size-bytes: 98304
+ib-body-max-size-bytes: 262144
+# Here we also use praos blocks as ballpark estimate.
+# Sec 2.3 Forging, of the benchmark cluster report, lists
+#   * Slot start to announced: 0.12975s
+ib-generation-cpu-time-ms: 130.0
+ib-head-validation-cpu-time-ms: 1.0
+ib-body-validation-cpu-time-ms-constant: 50.0
+ib-body-validation-cpu-time-ms-per-byte: 0.0005
+ib-diffusion-strategy: "freshest-first"
+
+# Haskell prototype relay mini-protocol parameters.
+ib-diffusion-max-bodies-to-request: 1
+ib-diffusion-max-headers-to-request: 100
+ib-diffusion-max-window-size: 100
+
+################################################################################
+# Endorsement Block Configuration
+################################################################################
+
+# We want one per pipeline, but not too many.
+eb-generation-probability: 2.5
+# ProducerId  32
+# SlotNo      64
+# VRF proof   80
+# Signature   64
+# Total       240
+#
+# See Note about signatures on ib-head-size-bytes.
+eb-size-bytes-constant: 240
+# IB hash
+eb-size-bytes-per-ib: 32
+# Collecting the IBs to reference and cryptography are the main tasks.
+# A comparable task is maybe mempool snapshotting.
+# Sec 2.3 Forging, of the benchmark cluster report, lists
+#   * Mempool snapshotting: 0.07252s
+# 75ms then seems a generous estimate for eb generation.
+eb-generation-cpu-time-ms: 75.0
+# Validating signature and vrf proof, as in other headers.
+eb-validation-cpu-time-ms: 1.0
+
+eb-diffusion-strategy: "peer-order"
+
+# Haskell prototype relay mini-protocol parameters.
+eb-diffusion-max-bodies-to-request: 1
+eb-diffusion-max-headers-to-request: 100
+eb-diffusion-max-window-size: 100
+
+# The maximum age of EBs included in RBs.
+# A an EB from slot `s` can only be included in RBs
+# up to slot `s+eb-max-age-slots`.
+# In short leios we expect votes to diffuse within 3 stages lengths of
+# EB generation, we allow for 2 more stage lengths to account for
+# variance in the interval within RBs.
+eb-max-age-slots: 240
+
+# The maximum age of EBs to be relayed.
+# An EB from slot `s` will only be relayed
+# up to slot `s+eb-max-age-for-relay-slots`.
+eb-max-age-for-relay-slots: 40
+
+# The maximum size of transactions (in bytes) which an EB can reference.
+# Only relevant when running with the "full-without-ibs" variant.
+eb-referenced-txs-max-size-bytes: 16384000
+
+################################################################################
+# Vote Configuration
+################################################################################
+
+# Cryptography related values taken from [vote-spec](crypto-benchmarks.rs/Specification.md)
+# using weighted averages of 80% persistent and 20% non-persistent.
+
+# vote-spec#Committe and quorum size
+#
+# Note: this is used as the expected amount of total weight of
+# generated votes in the sims.
+vote-generation-probability: 500.0
+# vote-spec#"Committe and quorum size"
+# 60% of `vote-generation-probability`
+vote-threshold: 300
+# vote-spec#"Generate vote" 0.8*135e-3 + 0.2*280e-3
+vote-generation-cpu-time-ms-constant: 164.0e-3
+# No benchmark yet.
+vote-generation-cpu-time-ms-per-ib: 0
+# vote-spec#"Verify vote" 0.8*670e-3 + 0.2*1.4
+vote-validation-cpu-time-ms: 816.0e-3
+# The `Vote` structure counted in the -per-eb already identifies slot
+# (in Eid) and voter. We can assume a vote bundle is all for the same
+# voter and slot, so for non-persistent voters we could factor their
+# PoolKeyHash (28bytes) here, but that is for 20% of cases.
+# More relevant if EB generation is very high.
+vote-bundle-size-bytes-constant: 0
+# vote-spec#Votes 0.8*90 + 0.2*164
+vote-bundle-size-bytes-per-eb: 105
+
+vote-diffusion-strategy: "peer-order"
+
+# Haskell prototype relay mini-protocol parameters.
+vote-diffusion-max-bodies-to-request: 1
+vote-diffusion-max-headers-to-request: 100
+vote-diffusion-max-window-size: 100
+
+################################################################################
+# Certificate Configuration
+################################################################################
+
+# vote-spec - certificate size plot.
+# Realistic stake distributions need about 7 kilobytes for the certificate.
+cert-size-bytes-constant: 7168
+cert-size-bytes-per-node: 0
+
+# For certificate timings we have bulk figures for realistic scenarios,
+# so we do not attempt to give -per-node (i.e. per-voter) timings.
+#
+# vote-spec#"Generate certificate"
+cert-generation-cpu-time-ms-constant: 90.0
+cert-generation-cpu-time-ms-per-node: 0
+# vote-spec#"Verify certificate"
+cert-validation-cpu-time-ms-constant: 130.0
+cert-validation-cpu-time-ms-per-node: 0

analysis/sims/2025w27/quick/config.yaml

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+set -e
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+BW=50
+
+ulimit -S -m 64000000 -v 64000000
+
+mkfifo sim.log
+
+sed -e 's/"bandwidth-bytes-per-second":125000000/"bandwidth-bytes-per-second":'"$((125000 * BW))"'/g' \
+  ../../../../data/simulation/pseudo-mainnet/topology-v2.yaml \
+  > network.yaml
+
+function cleanup() {
+  rm sim.log
+  rm network.yaml
+}
+trap cleanup EXIT
+
+grep -E -v '(Slot|No.*Generated|CpuTask|Lottery)' sim.log | pigz -p 3 -9c > sim.log.gz &
+
+../sim-cli --parameters config.yaml network.yaml --slots 900 --conformance-events sim.log > stdout 2> stderr
+
+wait
+
+cat << EOI > case.csv
+Simulator,Variant,Sharding
+Rust,full-without-ibs,unsharded
+EOI
+
+zcat sim.log.gz \
+| ../leios-trace-processor \
+  +RTS -N5 -RTS \
+  --trace-file /dev/stdin \
+  --lifecycle-file lifecycle.csv \
+  --cpu-file cpus.csv \
+  --resource-file resources.csv \
+  --receipt-file receipts.csv
+  
+pigz -p 3 -9f {lifecycle,cpus,resources,receipts}.csv
+
+cat case.csv

analysis/sims/2025w27/quick/run.sh

@@ -0,0 +1,2 @@
+Simulator,timestamp-resolution-ms
+Rust,dt = 0.025 ms

analysis/sims/2025w27/tsr/0.025/case.csv

@@ -17,7 +17,7 @@ multiplex-mini-protocols: true
 simulate-transactions: true
 treat-blocks-as-full: false
 cleanup-policies: ["cleanup-expired-vote"]
-timestamp-resolution-ms: 0.010
+timestamp-resolution-ms: 0.025
 
 ################################################################################
 # Leios Protocol Configuration

analysis/sims/2025w27/tsr/0.010/config.yaml renamed to analysis/sims/2025w27/tsr/0.025/config.yaml

@@ -7,7 +7,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")"
 LABEL="$(basename "$PWD")"
 BW=50
 
-ulimit -S -m 64000000 -v 64000000
+ulimit -S -m 144000000 -v 144000000
 
 mkfifo sim.log