|
| 1 | +// Copyright 2025 Contributors to the Parsec project. |
| 2 | +// SPDX-License-Identifier: Apache-2.0 |
| 3 | +//! Benchmark example comparing get_attributes_old vs get_attributes |
| 4 | +//! |
| 5 | +//! This example demonstrates the performance difference between the original |
| 6 | +//! and optimized implementations for retrieving object attributes. |
| 7 | +
|
| 8 | +use cryptoki::context::{CInitializeArgs, Pkcs11}; |
| 9 | +use cryptoki::mechanism::Mechanism; |
| 10 | +use cryptoki::object::{Attribute, AttributeType, ObjectHandle}; |
| 11 | +use cryptoki::session::{Session, UserType}; |
| 12 | +use cryptoki::types::AuthPin; |
| 13 | +use std::env; |
| 14 | +use std::time::Instant; |
| 15 | + |
| 16 | +/// Statistics for a benchmark run |
| 17 | +/// API calls are typically log-normally distributed, so we use that distribution |
| 18 | +/// to compute geometric mean and percentiles. |
| 19 | +struct BenchmarkStats { |
| 20 | + mean: f64, |
| 21 | + stddev: f64, |
| 22 | + p50: f64, |
| 23 | + p95: f64, |
| 24 | + p99: f64, |
| 25 | +} |
| 26 | + |
| 27 | +impl BenchmarkStats { |
| 28 | + fn from_timings(mut timings: Vec<f64>) -> Self { |
| 29 | + let iterations = timings.len(); |
| 30 | + timings.sort_by(|a, b| a.partial_cmp(b).unwrap()); |
| 31 | + |
| 32 | + let p50 = timings[iterations / 2]; |
| 33 | + let p95 = timings[(iterations * 95) / 100]; |
| 34 | + let p99 = timings[(iterations * 99) / 100]; |
| 35 | + |
| 36 | + // Geometric mean (appropriate for log-normal distribution) |
| 37 | + let mean = (timings.iter().map(|x| x.ln()).sum::<f64>() / iterations as f64).exp(); |
| 38 | + |
| 39 | + // Standard deviation in log-space (geometric standard deviation) |
| 40 | + let log_mean = timings.iter().map(|x| x.ln()).sum::<f64>() / iterations as f64; |
| 41 | + let log_variance = timings |
| 42 | + .iter() |
| 43 | + .map(|x| (x.ln() - log_mean).powi(2)) |
| 44 | + .sum::<f64>() |
| 45 | + / iterations as f64; |
| 46 | + let stddev = log_variance.sqrt().exp(); |
| 47 | + |
| 48 | + BenchmarkStats { |
| 49 | + mean, |
| 50 | + stddev, |
| 51 | + p50, |
| 52 | + p95, |
| 53 | + p99, |
| 54 | + } |
| 55 | + } |
| 56 | + |
| 57 | + fn print(&self, label: &str) { |
| 58 | + println!(" {}:", label); |
| 59 | + println!(" distribution: log-normal"); |
| 60 | + println!(" mean (geom): {:.2} µs", self.mean / 1000.0); |
| 61 | + println!(" std dev (geom): {:.2}x", self.stddev); |
| 62 | + println!(" p50 (median): {:.2} µs", self.p50 / 1000.0); |
| 63 | + println!(" p95: {:.2} µs", self.p95 / 1000.0); |
| 64 | + println!(" p99: {:.2} µs", self.p99 / 1000.0); |
| 65 | + } |
| 66 | +} |
| 67 | + |
| 68 | +struct BenchmarkResult { |
| 69 | + label: String, |
| 70 | + stats_old: BenchmarkStats, |
| 71 | + stats_optimized: BenchmarkStats, |
| 72 | +} |
| 73 | + |
| 74 | +impl BenchmarkResult { |
| 75 | + fn speedup_mean(&self) -> f64 { |
| 76 | + self.stats_old.mean / self.stats_optimized.mean |
| 77 | + } |
| 78 | +} |
| 79 | + |
| 80 | +/// Run a benchmark comparing get_attributes_old vs get_attributes |
| 81 | +fn benchmark_attributes( |
| 82 | + session: &Session, |
| 83 | + object: ObjectHandle, |
| 84 | + attributes: &[AttributeType], |
| 85 | + iterations: usize, |
| 86 | + label: &str, |
| 87 | +) -> Result<BenchmarkResult, Box<dyn std::error::Error>> { |
| 88 | + println!("\n=== {} ===", label); |
| 89 | + |
| 90 | + // Benchmark get_attributes_old (original implementation) |
| 91 | + println!( |
| 92 | + "Benchmarking get_attributes_old() - {} iterations...", |
| 93 | + iterations |
| 94 | + ); |
| 95 | + let mut timings_old = Vec::with_capacity(iterations); |
| 96 | + for _ in 0..iterations { |
| 97 | + let start = Instant::now(); |
| 98 | + let _attrs = session.get_attributes_old(object, attributes)?; |
| 99 | + timings_old.push(start.elapsed().as_nanos() as f64); |
| 100 | + } |
| 101 | + |
| 102 | + // Benchmark get_attributes (optimized implementation) |
| 103 | + println!( |
| 104 | + "Benchmarking get_attributes() - {} iterations...", |
| 105 | + iterations |
| 106 | + ); |
| 107 | + let mut timings_optimized = Vec::with_capacity(iterations); |
| 108 | + for _ in 0..iterations { |
| 109 | + let start = Instant::now(); |
| 110 | + let _attrs = session.get_attributes(object, attributes)?; |
| 111 | + timings_optimized.push(start.elapsed().as_nanos() as f64); |
| 112 | + } |
| 113 | + |
| 114 | + let stats_old = BenchmarkStats::from_timings(timings_old); |
| 115 | + let stats_optimized = BenchmarkStats::from_timings(timings_optimized); |
| 116 | + |
| 117 | + println!("\nResults:"); |
| 118 | + stats_old.print("Original implementation"); |
| 119 | + stats_optimized.print("Optimized implementation"); |
| 120 | + |
| 121 | + let speedup_mean = stats_old.mean / stats_optimized.mean; |
| 122 | + let speedup_p95 = stats_old.p95 / stats_optimized.p95; |
| 123 | + println!("\nSpeedup:"); |
| 124 | + println!(" Based on mean (geom): {:.2}x", speedup_mean); |
| 125 | + println!(" Based on p95: {:.2}x", speedup_p95); |
| 126 | + |
| 127 | + // Verify both methods return the same results |
| 128 | + let attrs_old = session.get_attributes_old(object, attributes)?; |
| 129 | + let attrs_optimized = session.get_attributes(object, attributes)?; |
| 130 | + |
| 131 | + println!("\nVerifying correctness..."); |
| 132 | + println!( |
| 133 | + " Original implementation returned {} attributes", |
| 134 | + attrs_old.len() |
| 135 | + ); |
| 136 | + println!( |
| 137 | + " Optimized implementation returned {} attributes", |
| 138 | + attrs_optimized.len() |
| 139 | + ); |
| 140 | + |
| 141 | + if attrs_old.len() != attrs_optimized.len() { |
| 142 | + println!(" ✗ Implementations returned different number of attributes!"); |
| 143 | + } else { |
| 144 | + println!(" ✓ Both implementations returned the same number of attributes"); |
| 145 | + |
| 146 | + // Verify the order is the same |
| 147 | + let mut order_matches = true; |
| 148 | + for (i, (old_attr, opt_attr)) in attrs_old.iter().zip(attrs_optimized.iter()).enumerate() { |
| 149 | + if std::mem::discriminant(old_attr) != std::mem::discriminant(opt_attr) { |
| 150 | + println!( |
| 151 | + " ✗ Attribute at position {} differs: {:?} vs {:?}", |
| 152 | + i, old_attr, opt_attr |
| 153 | + ); |
| 154 | + order_matches = false; |
| 155 | + } |
| 156 | + } |
| 157 | + |
| 158 | + if order_matches { |
| 159 | + println!(" ✓ Attributes are in the same order"); |
| 160 | + } |
| 161 | + } |
| 162 | + |
| 163 | + Ok(BenchmarkResult { |
| 164 | + label: label.to_string(), |
| 165 | + stats_old, |
| 166 | + stats_optimized, |
| 167 | + }) |
| 168 | +} |
| 169 | + |
| 170 | +fn print_summary_table(results: &[BenchmarkResult]) { |
| 171 | + println!("\n"); |
| 172 | + println!("╔═════════════════════════════════════════════════════════════════════════════════════════════════╗"); |
| 173 | + println!("║ BENCHMARK SUMMARY TABLE ║"); |
| 174 | + println!("╠═══════════════════╦═════════════╦═════════════╦═════════════╦═════════════╦═════════════╦═══════╣"); |
| 175 | + println!( |
| 176 | + "║ {:^17} ║ {:>11} ║ {:>11} ║ {:>11} ║ {:>11} ║ {:>11} ║ {:^5} ║", |
| 177 | + "Test Case", "Orig Mean", "Orig p95", "Opt Mean", "Opt p95", "Speedup", "Unit" |
| 178 | + ); |
| 179 | + println!("╠═══════════════════╬═════════════╬═════════════╬═════════════╬═════════════╬═════════════╬═══════╣"); |
| 180 | + |
| 181 | + // Each row is a test case |
| 182 | + for result in results { |
| 183 | + println!( |
| 184 | + "║ {:17} ║ {:11.2} ║ {:11.2} ║ {:11.2} ║ {:11.2} ║ {:11.2} ║ {:>5} ║", |
| 185 | + result.label, |
| 186 | + result.stats_old.mean / 1000.0, |
| 187 | + result.stats_old.p95 / 1000.0, |
| 188 | + result.stats_optimized.mean / 1000.0, |
| 189 | + result.stats_optimized.p95 / 1000.0, |
| 190 | + result.speedup_mean(), |
| 191 | + "µs/x" |
| 192 | + ); |
| 193 | + } |
| 194 | + |
| 195 | + println!("╚═══════════════════╩═════════════╩═════════════╩═════════════╩═════════════╩═════════════╩═══════╝"); |
| 196 | +} |
| 197 | + |
| 198 | +fn main() -> Result<(), Box<dyn std::error::Error>> { |
| 199 | + // how many iterations to run, default to 1000 |
| 200 | + let iterations = env::var("TEST_BENCHMARK_ITERATIONS") |
| 201 | + .unwrap_or_else(|_| "1000".to_string()) |
| 202 | + .parse::<usize>()?; |
| 203 | + |
| 204 | + let pkcs11 = Pkcs11::new( |
| 205 | + env::var("TEST_PKCS11_MODULE") |
| 206 | + .unwrap_or_else(|_| "/usr/lib/softhsm/libsofthsm2.so".to_string()), |
| 207 | + )?; |
| 208 | + |
| 209 | + pkcs11.initialize(CInitializeArgs::OsThreads)?; |
| 210 | + |
| 211 | + let slot = pkcs11 |
| 212 | + .get_slots_with_token()? |
| 213 | + .into_iter() |
| 214 | + .next() |
| 215 | + .ok_or("No slot available")?; |
| 216 | + |
| 217 | + let session = pkcs11.open_rw_session(slot)?; |
| 218 | + |
| 219 | + session.login(UserType::User, Some(&AuthPin::new("fedcba123456".into())))?; |
| 220 | + |
| 221 | + // Generate a test RSA key pair |
| 222 | + let mechanism = Mechanism::RsaPkcsKeyPairGen; |
| 223 | + let public_exponent: Vec<u8> = vec![0x01, 0x00, 0x01]; |
| 224 | + let modulus_bits = 2048; |
| 225 | + |
| 226 | + let pub_key_template = vec![ |
| 227 | + Attribute::Token(false), // Don't persist |
| 228 | + Attribute::Private(false), |
| 229 | + Attribute::PublicExponent(public_exponent), |
| 230 | + Attribute::ModulusBits(modulus_bits.into()), |
| 231 | + Attribute::Verify(true), |
| 232 | + Attribute::Label("Benchmark Key".into()), |
| 233 | + Attribute::Id(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]), |
| 234 | + ]; |
| 235 | + |
| 236 | + let priv_key_template = vec![Attribute::Token(false), Attribute::Sign(true)]; |
| 237 | + |
| 238 | + println!("Generating RSA key pair for benchmarking..."); |
| 239 | + let (public, _private) = |
| 240 | + session.generate_key_pair(&mechanism, &pub_key_template, &priv_key_template)?; |
| 241 | + |
| 242 | + let mut results = Vec::new(); |
| 243 | + |
| 244 | + // Test 1: Multiple attributes (mix of fixed and variable length) |
| 245 | + let multiple_attributes = vec![ |
| 246 | + AttributeType::Class, // CK_ULONG (fixed, 8 bytes) |
| 247 | + AttributeType::Label, // Variable length |
| 248 | + AttributeType::Id, // Variable length |
| 249 | + AttributeType::KeyType, // CK_ULONG (fixed, 8 bytes) |
| 250 | + AttributeType::Token, // CK_BBOOL as CK_ULONG (fixed, 8 bytes) |
| 251 | + AttributeType::Private, // CK_BBOOL as CK_ULONG (fixed, 8 bytes) |
| 252 | + AttributeType::Modulus, // Large variable (256 bytes for 2048-bit key) |
| 253 | + AttributeType::PublicExponent, // Variable, typically 3 bytes |
| 254 | + AttributeType::Verify, // CK_BBOOL as CK_ULONG (fixed, 8 bytes) |
| 255 | + AttributeType::Encrypt, // CK_BBOOL as CK_ULONG (fixed, 8 bytes) |
| 256 | + AttributeType::ModulusBits, // CK_ULONG (fixed, 8 bytes) |
| 257 | + ]; |
| 258 | + |
| 259 | + results.push(benchmark_attributes( |
| 260 | + &session, |
| 261 | + public, |
| 262 | + &multiple_attributes, |
| 263 | + iterations, |
| 264 | + "Multiple", |
| 265 | + )?); |
| 266 | + |
| 267 | + // Test 2: Single fixed-length attribute (CK_ULONG) |
| 268 | + let single_fixed = vec![AttributeType::KeyType]; |
| 269 | + |
| 270 | + results.push(benchmark_attributes( |
| 271 | + &session, |
| 272 | + public, |
| 273 | + &single_fixed, |
| 274 | + iterations, |
| 275 | + "Single-fixed", |
| 276 | + )?); |
| 277 | + |
| 278 | + // Test 3: Single variable-length attribute (large) |
| 279 | + let single_variable_large = vec![AttributeType::Modulus]; |
| 280 | + |
| 281 | + results.push(benchmark_attributes( |
| 282 | + &session, |
| 283 | + public, |
| 284 | + &single_variable_large, |
| 285 | + iterations, |
| 286 | + "Single-variable", |
| 287 | + )?); |
| 288 | + |
| 289 | + // Test 4: Single attribute that doesn't exist (EC point for RSA key) |
| 290 | + let single_nonexistent = vec![AttributeType::EcPoint]; |
| 291 | + |
| 292 | + results.push(benchmark_attributes( |
| 293 | + &session, |
| 294 | + public, |
| 295 | + &single_nonexistent, |
| 296 | + iterations, |
| 297 | + "Single-nonexist", |
| 298 | + )?); |
| 299 | + |
| 300 | + // Print summary table |
| 301 | + print_summary_table(&results); |
| 302 | + |
| 303 | + // Clean up |
| 304 | + session.destroy_object(public)?; |
| 305 | + |
| 306 | + Ok(()) |
| 307 | +} |
0 commit comments