Skip to content

Commit 2d6493d

Browse files
bwbushbwbush
committed
Designed experiment to compare network topologies
1 parent 8467433 commit 2d6493d

File tree

15 files changed

+283
-0
lines changed

15 files changed

+283
-0
lines changed

‎analysis/sims/micro-mainnet/combine-results.sh‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/usr/bin/env nix-shell
2+
#!nix-shell -i bash -p gnused gzip pigz "rWrapper.override { packages = with rPackages; [ data_table R_utils bit64 ggplot2 magrittr stringr ]; }"
3+
4+
set -e
5+
6+
mkdir -p results/$d
7+
for f in lifecycle resources receipts cpus sizes
8+
do
9+
DIR=$(find experiments -type f -name $f.csv.gz \( -not -empty \) -printf %h\\n -quit)
10+
HL=$(sed -n -e '1p' "$DIR/case.csv")
11+
HR=$(zcat "$DIR/$f.csv.gz" | sed -n -e '1p')
12+
if [[ "$f" == "lifecycle" || "$f" == "resources" || "$f" == "sizes" ]]
13+
then
14+
FRACT=1.00
15+
else
16+
FRACT=0.25
17+
fi
18+
(
19+
echo "$HL,$HR"
20+
for g in $(find experiments -type f -name $f.csv.gz \( -not -empty \) -printf %h\\n)
21+
do
22+
if [ ! -e "$g/stderr" ]
23+
then
24+
echo "Skipping $g because it has no stderr." >> /dev/stderr
25+
elif [ -s "$g/stderr" ]
26+
then
27+
echo "Skipping $g because its stderr is not empty." >> /dev/stderr
28+
else
29+
BL=$(sed -n -e '2p' "$g/case.csv")
30+
zcat "$g/$f.csv.gz" | gawk 'FNR > 1 && rand() <= '"$FRACT"' { print "'"$BL"'" "," $0}'
31+
fi
32+
done
33+
) | pigz -p 3 -9c > results/$f.csv.gz
34+
R --vanilla << EOI > /dev/null
35+
require(data.table)
36+
sampleSize <- $FRACT
37+
print(sampleSize)
38+
$f <- fread("results/$f.csv.gz", stringsAsFactors=TRUE)
39+
save($f, sampleSize, file="results/$f.Rdata", compression_level=9)
40+
EOI
41+
done

‎analysis/sims/micro-mainnet/experiments.list‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
experiments/topology-v2,0.100/run.sh
2+
experiments/topology-v3,0.100/run.sh
3+
experiments/topology-v2,0.150/run.sh
4+
experiments/topology-v3,0.150/run.sh
5+
experiments/topology-v2,0.200/run.sh
6+
experiments/topology-v3,0.200/run.sh
7+
experiments/topology-v2,0.250/run.sh
8+
experiments/topology-v2,0.300/run.sh
9+
experiments/topology-v3,0.250/run.sh
10+
experiments/topology-v3,0.300/run.sh

‎analysis/sims/micro-mainnet/experiments/config.yaml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
2+
# Simulation parameters.
3+
timestamp-resolution-ms: 0.1
4+
simulate-transactions: true
5+
cleanup-policies:
6+
- cleanup-expired-vote
7+
8+
# Protocol parameters for Linear Leios.
9+
leios-variant: linear-with-tx-references
10+
linear-eb-propagation-criteria: eb-received
11+
linear-vote-stage-length-slots: 4
12+
linear-diffuse-stage-length-slots: 7
13+
praos-fallback-enabled: true
14+
leios-stage-active-voting-slots: 1
15+
leios-mempool-sampling-strategy: ordered-by-id
16+
relay-strategy: request-from-first
17+
treat-blocks-as-full: false
18+
eb-diffusion-strategy: peer-order
19+
eb-referenced-txs-max-size-bytes: 12000000
20+
eb-size-bytes-constant: 240
21+
eb-size-bytes-per-ib: 32
22+
23+
# A conservative upper bound based on empiricial data of empty Praos blocks on
24+
# Cardano mainnet.
25+
leios-header-diffusion-time-ms: 1000.0
26+
27+
# Scenario-specific parameters for transaction generation.
28+
tx-start-time: 60
29+
tx-stop-time: 960
30+
tx-generation-distribution:
31+
distribution: constant
32+
value: 10.0
33+
tx-size-bytes-distribution:
34+
distribution: constant
35+
value: 1500
36+
tx-conflict-fraction: 0
37+
tx-overcollateralization-factor-distribution:
38+
distribution: constant
39+
value: 0
40+
41+
# Based on a linear model fit from `db-analyser` measurements of mainnet blocks
42+
# for the `Apply` operation, either with or without a term for Plutus execution
43+
# steps.
44+
#
45+
# `Apply CPU [ms]` / `Tx count` ~ (0.6201 ms/tx)
46+
#
47+
# `Apply CPU [ms]` ~ 0 + (0.2624 ms/tx) * `Tx count`
48+
# + (0.9487 ms/Gstep) * `Tx exec [Gstep]`
49+
#
50+
tx-validation-cpu-time-ms: 0.6201
51+
52+
# Inherited from Cardano mainnet protocol parameters for Praos.
53+
tx-max-size-bytes: 16384
54+
rb-generation-probability: 0.05
55+
rb-head-size-bytes: 1024
56+
rb-body-max-size-bytes: 90112
57+
58+
# Worst case based on benchmark cluster "Forging: Slot start to announced".
59+
rb-generation-cpu-time-ms: 71.02
60+
61+
# Based on `apply` statistics for empty blocks using `db-analyser` on blocks
62+
# from the Cardano mainnet.
63+
rb-head-validation-cpu-time-ms: 0.4438
64+
eb-header-validation-cpu-time-ms: 0.4438
65+
66+
# Based on a linear model fit from `db-analyser` measurements of mainnet blocks
67+
# for the `Reapply` operation, either with or without a term for Plutus
68+
# execution steps.
69+
#
70+
# `Reapply CPU [ms]` ~ (0.3539 ms) + (0.02151 ms/kB) * `Block size [kB]`
71+
#
72+
# `Reapply CPU [ms]` ~ (0.3478 ms) + (0.01943 ms/kB) * `Block size [kB]`
73+
# + (0.02127 ms/Gstep) * `Block exec [Gstep]`
74+
#
75+
rb-body-legacy-praos-payload-validation-cpu-time-ms-constant: 0.3539
76+
rb-body-legacy-praos-payload-validation-cpu-time-ms-per-byte: 0.00002151
77+
eb-body-validation-cpu-time-ms-constant: 0.3539
78+
eb-body-validation-cpu-time-ms-per-byte: 0.00002151
79+
80+
# Based on analysis of wFA+LS sortition for realistic stake distribution similar
81+
# to that of Cardano mainnet, and on the security requirement.
82+
vote-generation-probability: 600
83+
vote-threshold: 450
84+
85+
# Based on CDDL for votes, worst case for non-persistent voters.
86+
vote-bundle-size-bytes-constant: 0
87+
vote-bundle-size-bytes-per-eb: 164
88+
89+
# Worst case based on benchmarks from cryptography prototype.
90+
vote-generation-cpu-time-ms-constant: 0.280
91+
vote-generation-cpu-time-ms-per-tx: 0
92+
vote-validation-cpu-time-ms: 2.9
93+
94+
# Vote diffusion settings.
95+
vote-diffusion-strategy: peer-order
96+
vote-diffusion-max-bodies-to-request: 1
97+
vote-diffusion-max-headers-to-request: 100
98+
vote-diffusion-max-window-size: 100
99+
100+
# Worst case based on CDDL for certificates and on empirical study of wFA+LS
101+
# sortition.
102+
cert-size-bytes-constant: 8000
103+
cert-size-bytes-per-node: 0
104+
105+
# Worst case based on benchmarks from cryptography prototype.
106+
cert-generation-cpu-time-ms-constant: 92.5
107+
cert-generation-cpu-time-ms-per-node: 0
108+
cert-validation-cpu-time-ms-constant: 157.2
109+
cert-validation-cpu-time-ms-per-node: 0
110+

‎analysis/sims/micro-mainnet/experiments/run.sh‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/usr/bin/env nix-shell
2+
#!nix-shell -i bash -p ansifilter gnugrep gnused gzip pigz bc
3+
4+
set -eo pipefail
5+
6+
cd "$(dirname "${BASH_SOURCE[0]}")"
7+
8+
TX_START=60
9+
TX_STOP=960
10+
SIM_STOP=1500
11+
BW=10
12+
CPU_COUNT=4
13+
SIM=Rust
14+
VARIANT=linear-with-tx-references
15+
BLOCK_SIZE=12
16+
TX_SIZE=1500
17+
LABEL=$(basename "$PWD")
18+
PROPAGATION=eb-received
19+
STAGE_LENGTH_DIFF=7
20+
STAGE_LENGTH_VOTE=4
21+
NETWORK=$(echo -n "$LABEL" | sed -e 's/^\(.*\),\(.*\)/\1/')
22+
THROUGHPUT=$(echo -n "$LABEL" | sed -e 's/^\(.*\),\(.*\)/\2/')
23+
TX_SPACING_HONEST=$(echo "scale=3; $TX_SIZE / $THROUGHPUT / 1000" | bc)
24+
25+
ulimit -S -m 48000000 -v 48000000
26+
27+
if [[ -e sim.log ]]
28+
then
29+
rm sim.log
30+
fi
31+
32+
mkfifo sim.log
33+
34+
sed -e 's/"bandwidth-bytes-per-second":125000000/"bandwidth-bytes-per-second":'"$((125000 * BW))"'/g' \
35+
-e 's/"cpu-core-count":6,/"cpu-core-count":'"$CPU_COUNT"',/g' \
36+
"../../../../../data/simulation/pseudo-mainnet/$NETWORK.yaml" \
37+
> network.yaml
38+
39+
yaml2json ../config.yaml \
40+
| jq '. +
41+
{
42+
"leios-variant": "'"$VARIANT"'"
43+
, "linear-eb-propagation-criteria": "'"$PROPAGATION"'"
44+
, "linear-diffuse-stage-length-slots": '"$STAGE_LENGTH_DIFF"'
45+
, "linear-vote-stage-length-slots": '"$STAGE_LENGTH_VOTE"'
46+
, "leios-stage-length-slots": '"$STAGE_LENGTH_VOTE"'
47+
, "eb-referenced-txs-max-size-bytes": ('"$BLOCK_SIZE"' * 1000000)
48+
, "eb-body-avg-size-bytes": ('"$BLOCK_SIZE"' * 1000000)
49+
, "tx-size-bytes-distribution": {distribution: "constant", value: '"$TX_SIZE"'}
50+
, "tx-generation-distribution": {distribution: "constant", value: '"$TX_SPACING_HONEST"'}
51+
, "tx-start-time": '"$TX_START"'
52+
, "tx-stop-time": '"$TX_STOP"'
53+
}
54+
' > config.yaml
55+
56+
function cleanup() {
57+
rm sim.log
58+
rm network.yaml
59+
}
60+
trap cleanup EXIT
61+
62+
grep -E -v '(Slot|CpuTask|Lottery)' sim.log | pigz -p 3 -9c > sim.log.gz &
63+
64+
case "$SIM" in
65+
Rust)
66+
../../sim-cli --parameters config.yaml network.yaml --slots "$SIM_STOP" --conformance-events sim.log > stdout 2> stderr
67+
;;
68+
Haskell)
69+
../../ols sim leios --leios-config-file config.yaml --topology-file network.yaml --shared-log-format JSON --conformance-events --output-file sim.log --output-seconds "$SIM_STOP" > stdout 2> stderr
70+
;;
71+
*)
72+
false
73+
esac
74+
75+
wait
76+
77+
cat << EOI > case.csv
78+
Network,Bandwidth,CPU,Diffusion duration,Voting duration,Max EB size,Tx size,Throughput,Tx start [s],Tx stop [s],Sim stop [s]
79+
$NETWORK,$BW Mb/s,$CPU_COUNT vCPU/node,L_diff = $STAGE_LENGTH_DIFF slots,L_vote = $STAGE_LENGTH_VOTE slots,$BLOCK_SIZE MB/EB,$TX_SIZE B/Tx,$THROUGHPUT TxMB/s,$TX_START,$TX_STOP,$SIM_STOP
80+
EOI
81+
82+
zcat sim.log.gz \
83+
| ../../leios-trace-processor \
84+
+RTS -N5 -RTS \
85+
--trace-file /dev/stdin \
86+
--lifecycle-file lifecycle.csv \
87+
--cpu-file cpus.csv \
88+
--resource-file resources.csv \
89+
--receipt-file receipts.csv
90+
91+
(
92+
echo 'Kind,Item,Generated [s],Transactions,Endorses'
93+
zcat sim.log.gz \
94+
| grep -E '(EB|RB)Generated' \
95+
| jq -r '
96+
.message.type[0:2]
97+
+ "," + .message.id
98+
+ "," + (.time_s | tostring)
99+
+ "," + (.message.transactions | length | tostring)
100+
+ "," + (if .message.endorsement then .message.endorsement.eb.id else "NA" end)
101+
'
102+
) > sizes.csv
103+
104+
pigz -p 3 -9f {cpus,lifecycle,receipts,resources,sizes}.csv
105+
106+
cat case.csv
107+
108+
if [[ "$SIM" == "Rust" ]]
109+
then
110+
ansifilter stdout | grep -E '^ INFO (praos|leios|network): ' > summary.txt
111+
fi

‎analysis/sims/micro-mainnet/experiments/topology-v2,0.100/run.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../run.sh

‎analysis/sims/micro-mainnet/experiments/topology-v2,0.150/run.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../run.sh

‎analysis/sims/micro-mainnet/experiments/topology-v2,0.200/run.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../run.sh

‎analysis/sims/micro-mainnet/experiments/topology-v2,0.250/run.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../run.sh

‎analysis/sims/micro-mainnet/experiments/topology-v2,0.300/run.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../run.sh

‎analysis/sims/micro-mainnet/experiments/topology-v3,0.100/run.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../run.sh

0 commit comments

Comments
 (0)