diff --git a/Logbook.md b/Logbook.md index 51f055b59..8087f3940 100644 --- a/Logbook.md +++ b/Logbook.md @@ -1,5 +1,24 @@ # Leios logbook +## 2025-02-13 + +### Certificate CPU benchmarks as a function of number of voters + +In support of the Haskell and Rust simulations, we've benchmarked certificate operations as a function of the number of voters. A realistic distribution of stake is used in these measurements. + +| Number of pools | Number of committee seats | Generate certificate | Verify certificate | Weigh certificate | +|----------------:|--------------------------:|---------------------:|-------------------:|------------------:| +| 2500 | 500 | 63.4 ms | 104.8 ms | 10.6 ms | +| 2500 | 600 | 71.1 ms | 116.9 ms | 12.0 ms | +| 2500 | 700 | 77.4 ms | 125.5 ms | 12.3 ms | +| 2500 | 800 | 83.5 ms | 134.4 ms | 12.8 ms | +| 2500 | 900 | 88.2 ms | 141.1 ms | 12.4 ms | +| 2500 | 1000 | 92.5 ms | 144.9 ms | 12.3 ms | + +Serialization and deserialization likely also exhibit the same trend. + +A recipe for parallelizing parts of the certificate operations has been added to the [Specification for BLS certificates](crypto-benchmarks.rs/Specification.md). + ## 2025-02-12 ### Added BLS crypto to CI diff --git a/crypto-benchmarks.rs/Specification.md b/crypto-benchmarks.rs/Specification.md index 724171158..d64bfe59f 100644 --- a/crypto-benchmarks.rs/Specification.md +++ b/crypto-benchmarks.rs/Specification.md @@ -142,6 +142,17 @@ but not including any minor overhead arising from CBOR serialization. As noted p As a general rule of thumb, assume that 80% of votes are persistent and 20% are non-persistent. +Here are details for how certificate operations vary with committee size. + +| Number of pools | Number of committee seats | Generate certificate | Verify certificate | Weigh certificate | +|----------------:|--------------------------:|---------------------:|-------------------:|------------------:| +| 2500 | 500 | 63.4 ms | 104.8 ms | 10.6 ms | +| 2500 | 600 | 71.1 ms | 116.9 ms | 12.0 ms | +| 2500 | 700 | 77.4 ms | 125.5 ms | 12.3 ms | +| 2500 | 800 | 83.5 ms | 134.4 ms | 12.8 ms | +| 2500 | 900 | 88.2 ms | 141.1 ms | 12.4 ms | +| 2500 | 1000 | 92.5 ms | 144.9 ms | 12.3 ms | + ## Certificate size for realistic stake distributions diff --git a/crypto-benchmarks.rs/benches/vote_bench.rs b/crypto-benchmarks.rs/benches/vote_bench.rs index 6741257ea..d19b9ff9f 100644 --- a/crypto-benchmarks.rs/benches/vote_bench.rs +++ b/crypto-benchmarks.rs/benches/vote_bench.rs @@ -2,6 +2,7 @@ use criterion::{criterion_group, criterion_main, Criterion}; use leios_crypto_benchmarks::cert::*; use leios_crypto_benchmarks::registry::{arbitrary_pools, PersistentId, Registry}; use quickcheck::{Arbitrary, Gen}; +use std::env; use leios_crypto_benchmarks::key::{check_pop, key_gen, SecKey}; use leios_crypto_benchmarks::primitive::{ @@ -101,14 +102,22 @@ fn benchmark_verify_vote_nonpersistent(c: &mut Criterion) { }); } -fn benchmark_gen_cert(c: &mut Criterion) { +fn benchmark_gen_cert_impl(c: &mut Criterion, name: &'static str, n_pools: usize, n_voters: usize) { let g = &mut Gen::new(10); - c.bench_function("cert::gen_cert", |b| { + c.bench_function(name, |b| { b.iter_batched( || { let total = realistic_total_stake(g); - let n = realistic_pool_count(g); - let voters = realistic_voters(g, n); + let n = if n_pools > 0 { + n_pools + } else { + realistic_pool_count(g) + }; + let voters = if n_voters > 0 { + n_voters + } else { + realistic_voters(g, n) + }; let stake = arbitrary_stake_distribution(g, total, n, 11., 1.); let pools = arbitrary_pools(g, &stake); let reg = Registry::make(&pools, voters); @@ -121,14 +130,69 @@ fn benchmark_gen_cert(c: &mut Criterion) { }); } -fn benchmark_verify_cert(c: &mut Criterion) { +fn benchmark_gen_cert(c: &mut Criterion) { + benchmark_gen_cert_impl(c, "cert::gen_cert", 0, 0); + if env::var("BENCHMARK_CERT_EXTRA").map(|x| x == "1") == Result::Ok(true) { + benchmark_gen_cert_impl( + c, + "cert::gen_cert, n_pools = 2500, n_voters = 500", + 2500, + 500, + ); + benchmark_gen_cert_impl( + c, + "cert::gen_cert, n_pools = 2500, n_voters = 600", + 2500, + 600, + ); + benchmark_gen_cert_impl( + c, + "cert::gen_cert, n_pools = 2500, n_voters = 700", + 2500, + 700, + ); + benchmark_gen_cert_impl( + c, + "cert::gen_cert, n_pools = 2500, n_voters = 800", + 2500, + 800, + ); + benchmark_gen_cert_impl( + c, + "cert::gen_cert, n_pools = 2500, n_voters = 900", + 2500, + 900, + ); + benchmark_gen_cert_impl( + c, + "cert::gen_cert, n_pools = 2500, n_voters = 1000", + 2500, + 1000, + ); + } +} + +fn benchmark_verify_cert_impl( + c: &mut Criterion, + name: &'static str, + n_pools: usize, + n_voters: usize, +) { let g = &mut Gen::new(10); - c.bench_function("cert::verify_cert", |b| { + c.bench_function(name, |b| { b.iter_batched( || { let total = realistic_total_stake(g); - let n = realistic_pool_count(g); - let voters = realistic_voters(g, n); + let n = if n_pools > 0 { + n_pools + } else { + realistic_pool_count(g) + }; + let voters = if n_voters > 0 { + n_voters + } else { + realistic_voters(g, n) + }; let stake = arbitrary_stake_distribution(g, total, n, 11., 1.); let pools = arbitrary_pools(g, &stake); let reg = Registry::make(&pools, voters); @@ -142,14 +206,69 @@ fn benchmark_verify_cert(c: &mut Criterion) { }); } -fn benchmark_weigh_cert(c: &mut Criterion) { +fn benchmark_verify_cert(c: &mut Criterion) { + benchmark_verify_cert_impl(c, "cert::verify_cert", 0, 0); + if env::var("BENCHMARK_CERT_EXTRA").map(|x| x == "1") == Result::Ok(true) { + benchmark_verify_cert_impl( + c, + "cert::verify_cert, n_pools = 2500, n_voters = 500", + 2500, + 500, + ); + benchmark_verify_cert_impl( + c, + "cert::verify_cert, n_pools = 2500, n_voters = 600", + 2500, + 600, + ); + benchmark_verify_cert_impl( + c, + "cert::verify_cert, n_pools = 2500, n_voters = 700", + 2500, + 700, + ); + benchmark_verify_cert_impl( + c, + "cert::verify_cert, n_pools = 2500, n_voters = 800", + 2500, + 800, + ); + benchmark_verify_cert_impl( + c, + "cert::verify_cert, n_pools = 2500, n_voters = 900", + 2500, + 900, + ); + benchmark_verify_cert_impl( + c, + "cert::verify_cert, n_pools = 2500, n_voters = 1000", + 2500, + 1000, + ); + } +} + +fn benchmark_weigh_cert_impl( + c: &mut Criterion, + name: &'static str, + n_pools: usize, + n_voters: usize, +) { let g = &mut Gen::new(10); - c.bench_function("cert::weigh_cert", |b| { + c.bench_function(name, |b| { b.iter_batched( || { let total = realistic_total_stake(g); - let n = realistic_pool_count(g); - let voters = realistic_voters(g, n); + let n = if n_pools > 0 { + n_pools + } else { + realistic_pool_count(g) + }; + let voters = if n_voters > 0 { + n_voters + } else { + realistic_voters(g, n) + }; let stake = arbitrary_stake_distribution(g, total, n, 11., 1.); let pools = arbitrary_pools(g, &stake); let reg = Registry::make(&pools, voters); @@ -162,6 +281,47 @@ fn benchmark_weigh_cert(c: &mut Criterion) { ) }); } +fn benchmark_weigh_cert(c: &mut Criterion) { + benchmark_weigh_cert_impl(c, "cert::weigh_cert", 0, 0); + if env::var("BENCHMARK_CERT_EXTRA").map(|x| x == "1") == Result::Ok(true) { + benchmark_weigh_cert_impl( + c, + "cert::weigh_cert, n_pools = 2500, n_voters = 500", + 2500, + 500, + ); + benchmark_weigh_cert_impl( + c, + "cert::weigh_cert, n_pools = 2500, n_voters = 600", + 2500, + 600, + ); + benchmark_weigh_cert_impl( + c, + "cert::weigh_cert, n_pools = 2500, n_voters = 700", + 2500, + 700, + ); + benchmark_weigh_cert_impl( + c, + "cert::weigh_cert, n_pools = 2500, n_voters = 800", + 2500, + 800, + ); + benchmark_weigh_cert_impl( + c, + "cert::weigh_cert, n_pools = 2500, n_voters = 900", + 2500, + 900, + ); + benchmark_weigh_cert_impl( + c, + "cert::weigh_cert, n_pools = 2500, n_voters = 1000", + 2500, + 1000, + ); + } +} criterion_group!( benches, diff --git a/crypto-benchmarks.rs/src/main.rs b/crypto-benchmarks.rs/src/main.rs index d6e007713..f9dc57c54 100644 --- a/crypto-benchmarks.rs/src/main.rs +++ b/crypto-benchmarks.rs/src/main.rs @@ -15,8 +15,7 @@ use leios_crypto_benchmarks::key::{ check_pop, key_gen, sign_message, verify_message, PoP, PubKey, Reg, SecKey, Sig, }; use leios_crypto_benchmarks::primitive::{ - arbitrary_poolkeyhash, arbitrary_stake_distribution, Coin, EbHash, Eid, KesSig, - PoolKeyhash, + arbitrary_poolkeyhash, arbitrary_stake_distribution, Coin, EbHash, Eid, KesSig, PoolKeyhash, }; use leios_crypto_benchmarks::registry::{arbitrary_pools, PoolInfo, Registry}; use leios_crypto_benchmarks::sortition::voter_check; diff --git a/crypto-benchmarks.rs/src/primitive.rs b/crypto-benchmarks.rs/src/primitive.rs index b6cb451fc..c36491cba 100644 --- a/crypto-benchmarks.rs/src/primitive.rs +++ b/crypto-benchmarks.rs/src/primitive.rs @@ -5,14 +5,10 @@ use num_traits::FromPrimitive; use pallas::ledger::primitives::{byron::Blake2b256, Hash}; use pallas::ledger::traverse::time::Slot; use quickcheck::{Arbitrary, Gen}; -use rand::prelude::Distribution; -use rand::rngs::StdRng; -use rand::SeedableRng; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use statrs::distribution::ContinuousCDF; -use statrs::distribution::{Beta, Uniform}; use std::collections::BTreeMap; +use crate::realism::realistic_stake_dist; use crate::util::{arbitrary_fixed_bytes, deserialize_fixed_bytes, serialize_fixed_bytes}; pub use pallas::ledger::primitives::PoolKeyhash; @@ -27,28 +23,6 @@ pub fn arbitrary_coin(g: &mut Gen) -> Coin { u64::arbitrary(g) % 999999 + 1 } -pub(crate) fn realistic_stake_dist( - g: &mut Gen, - total: u64, - n: usize, - alpha: f64, - beta: f64, -) -> Vec { - let rng = &mut StdRng::seed_from_u64(u64::arbitrary(g)); - let noise = Uniform::new(0.75, 1.25).unwrap(); - let curve = Beta::new(alpha, beta).unwrap(); - let cum: Vec = (0..n) - .map(|i| curve.cdf((i as f64) / (total as f64))) - .collect(); - let dif: Vec = (1..n) - .map(|i| (cum[i] - cum[i - 1]) * noise.sample(rng)) - .collect(); - let scale: f64 = (total as f64) / dif.iter().sum::(); - dif.iter() - .map(|coin| (scale * *coin).round() as Coin) - .collect() -} - pub fn arbitrary_stake_distribution( g: &mut Gen, total: u64, diff --git a/crypto-benchmarks.rs/src/realism.rs b/crypto-benchmarks.rs/src/realism.rs index ae77b8d18..a058b97ca 100644 --- a/crypto-benchmarks.rs/src/realism.rs +++ b/crypto-benchmarks.rs/src/realism.rs @@ -1,6 +1,10 @@ +use pallas::ledger::primitives::Coin; use quickcheck::{Arbitrary, Gen}; - -use crate::primitive::Coin; +use rand::prelude::Distribution; +use rand::rngs::StdRng; +use rand::SeedableRng; +use statrs::distribution::ContinuousCDF; +use statrs::distribution::{Beta, Uniform}; pub fn realistic_pool_count(g: &mut Gen) -> usize { usize::arbitrary(g) % 1500 + 1500 @@ -13,3 +17,25 @@ pub fn realistic_total_stake(g: &mut Gen) -> Coin { pub fn realistic_voters(g: &mut Gen, pools: usize) -> usize { pools * (usize::arbitrary(g) % 500 + 500) / 1000 } + +pub(crate) fn realistic_stake_dist( + g: &mut Gen, + total: u64, + n: usize, + alpha: f64, + beta: f64, +) -> Vec { + let rng = &mut StdRng::seed_from_u64(u64::arbitrary(g)); + let noise = Uniform::new(0.75, 1.25).unwrap(); + let curve = Beta::new(alpha, beta).unwrap(); + let cum: Vec = (0..n) + .map(|i| curve.cdf((i as f64) / (total as f64))) + .collect(); + let dif: Vec = (1..n) + .map(|i| (cum[i] - cum[i - 1]) * noise.sample(rng)) + .collect(); + let scale: f64 = (total as f64) / dif.iter().sum::(); + dif.iter() + .map(|coin| (scale * *coin).round() as Coin) + .collect() +}