From e63b2fab056da1f667e7f4a0de368aebea0b9c7b Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sat, 22 Mar 2025 05:03:03 +0000 Subject: [PATCH] Clean up icount benchmarks * Delete some memcpy tests that were a bit excessive * Always use the same offset of 65 * Add a memmove test with aligned source and destination * Improve printing output and add more comments * Use a constant for 1 MiB so it shows up in the benchmark logs --- testcrate/benches/mem_icount.rs | 199 ++++++++++++++++++-------------- 1 file changed, 112 insertions(+), 87 deletions(-) diff --git a/testcrate/benches/mem_icount.rs b/testcrate/benches/mem_icount.rs index be3d13dff..63045f6e1 100644 --- a/testcrate/benches/mem_icount.rs +++ b/testcrate/benches/mem_icount.rs @@ -7,7 +7,9 @@ use std::{ops, slice}; use compiler_builtins::mem::{memcmp, memcpy, memmove, memset}; use iai_callgrind::{library_benchmark, library_benchmark_group, main}; -const PAGE_SIZE: usize = 0x1000; +const PAGE_SIZE: usize = 0x1000; // 4 kiB +const MAX_ALIGN: usize = 512; // assume we may use avx512 operations one day +const MEG1: usize = 1 << 20; // 1 MiB #[derive(Clone)] #[repr(C, align(0x1000))] @@ -62,7 +64,7 @@ mod mcpy { fn setup(cfg: Cfg) -> (usize, AlignedSlice, AlignedSlice) { let Cfg { len, s_off, d_off } = cfg; - println!("{len} bytes, {s_off} src offset, {d_off} dst offset"); + println!("bytes: {len} bytes, src offset: {s_off}, dst offset: {d_off}"); let mut src = AlignedSlice::new_zeroed(len, s_off); let dst = AlignedSlice::new_zeroed(len, d_off); src.fill(1); @@ -71,57 +73,38 @@ mod mcpy { #[library_benchmark] #[benches::aligned( + // Both aligned args = [ Cfg { len: 16, s_off: 0, d_off: 0 }, - Cfg { len: 16, s_off: 0, d_off: 0 }, - Cfg { len: 28, s_off: 0, d_off: 0 }, Cfg { len: 32, s_off: 0, d_off: 0 }, - Cfg { len: 36, s_off: 0, d_off: 0 }, - Cfg { len: 60, s_off: 0, d_off: 0 }, Cfg { len: 64, s_off: 0, d_off: 0 }, - Cfg { len: 68, s_off: 0, d_off: 0 }, - Cfg { len: 128, s_off: 0, d_off: 0 }, - Cfg { len: 256, s_off: 0, d_off: 0 }, Cfg { len: 512, s_off: 0, d_off: 0 }, - Cfg { len: 1024, s_off: 0, d_off: 0 }, Cfg { len: 4096, s_off: 0, d_off: 0 }, - Cfg { len: 1048576, s_off: 0, d_off: 0 }, + Cfg { len: MEG1, s_off: 0, d_off: 0 }, ], setup = setup, )] #[benches::offset( + // Both at the same offset args = [ Cfg { len: 16, s_off: 65, d_off: 65 }, - Cfg { len: 28, s_off: 65, d_off: 65 }, Cfg { len: 32, s_off: 65, d_off: 65 }, - Cfg { len: 36, s_off: 65, d_off: 65 }, - Cfg { len: 60, s_off: 65, d_off: 65 }, Cfg { len: 64, s_off: 65, d_off: 65 }, - Cfg { len: 68, s_off: 65, d_off: 65 }, - Cfg { len: 128, s_off: 65, d_off: 65 }, - Cfg { len: 256, s_off: 65, d_off: 65 }, Cfg { len: 512, s_off: 65, d_off: 65 }, - Cfg { len: 1024, s_off: 65, d_off: 65 }, Cfg { len: 4096, s_off: 65, d_off: 65 }, - Cfg { len: 1048576, s_off: 65, d_off: 65 }, + Cfg { len: MEG1, s_off: 65, d_off: 65 }, ], setup = setup, )] #[benches::misaligned( + // `src` and `dst` both misaligned by different amounts args = [ Cfg { len: 16, s_off: 65, d_off: 66 }, - Cfg { len: 28, s_off: 65, d_off: 66 }, Cfg { len: 32, s_off: 65, d_off: 66 }, - Cfg { len: 36, s_off: 65, d_off: 66 }, - Cfg { len: 60, s_off: 65, d_off: 66 }, Cfg { len: 64, s_off: 65, d_off: 66 }, - Cfg { len: 68, s_off: 65, d_off: 66 }, - Cfg { len: 128, s_off: 65, d_off: 66 }, - Cfg { len: 256, s_off: 65, d_off: 66 }, Cfg { len: 512, s_off: 65, d_off: 66 }, - Cfg { len: 1024, s_off: 65, d_off: 66 }, Cfg { len: 4096, s_off: 65, d_off: 66 }, - Cfg { len: 1048576, s_off: 65, d_off: 66 }, + Cfg { len: MEG1, s_off: 65, d_off: 66 }, ], setup = setup, )] @@ -147,7 +130,7 @@ mod mset { } fn setup(Cfg { len, offset }: Cfg) -> (usize, AlignedSlice) { - println!("{len} bytes, {offset} offset"); + println!("bytes: {len}, offset: {offset}"); (len, AlignedSlice::new_zeroed(len, offset)) } @@ -159,7 +142,7 @@ mod mset { Cfg { len: 64, offset: 0 }, Cfg { len: 512, offset: 0 }, Cfg { len: 4096, offset: 0 }, - Cfg { len: 1048576, offset: 0 }, + Cfg { len: MEG1, offset: 0 }, ], setup = setup, )] @@ -170,7 +153,7 @@ mod mset { Cfg { len: 64, offset: 65 }, Cfg { len: 512, offset: 65 }, Cfg { len: 4096, offset: 65 }, - Cfg { len: 1048576, offset: 65 }, + Cfg { len: MEG1, offset: 65 }, ], setup = setup, )] @@ -198,7 +181,7 @@ mod mcmp { fn setup(cfg: Cfg) -> (usize, AlignedSlice, AlignedSlice) { let Cfg { len, s_off, d_off } = cfg; - println!("{len} bytes, {s_off} src offset, {d_off} dst offset"); + println!("bytes: {len}, src offset: {s_off}, dst offset: {d_off}"); let b1 = AlignedSlice::new_zeroed(len, s_off); let mut b2 = AlignedSlice::new_zeroed(len, d_off); b2[len - 1] = 1; @@ -207,35 +190,38 @@ mod mcmp { #[library_benchmark] #[benches::aligned( + // Both aligned args = [ Cfg { len: 16, s_off: 0, d_off: 0 }, Cfg { len: 32, s_off: 0, d_off: 0 }, Cfg { len: 64, s_off: 0, d_off: 0 }, Cfg { len: 512, s_off: 0, d_off: 0 }, Cfg { len: 4096, s_off: 0, d_off: 0 }, - Cfg { len: 1048576, s_off: 0, d_off: 0 }, + Cfg { len: MEG1, s_off: 0, d_off: 0 }, ], setup = setup )] #[benches::offset( + // Both at the same offset args = [ Cfg { len: 16, s_off: 65, d_off: 65 }, Cfg { len: 32, s_off: 65, d_off: 65 }, Cfg { len: 64, s_off: 65, d_off: 65 }, Cfg { len: 512, s_off: 65, d_off: 65 }, Cfg { len: 4096, s_off: 65, d_off: 65 }, - Cfg { len: 1048576, s_off: 65, d_off: 65 }, + Cfg { len: MEG1, s_off: 65, d_off: 65 }, ], setup = setup )] #[benches::misaligned( + // `src` and `dst` both misaligned by different amounts args = [ Cfg { len: 16, s_off: 65, d_off: 66 }, Cfg { len: 32, s_off: 65, d_off: 66 }, Cfg { len: 64, s_off: 65, d_off: 66 }, Cfg { len: 512, s_off: 65, d_off: 66 }, Cfg { len: 4096, s_off: 65, d_off: 66 }, - Cfg { len: 1048576, s_off: 65, d_off: 66 }, + Cfg { len: MEG1, s_off: 65, d_off: 66 }, ], setup = setup )] @@ -254,7 +240,7 @@ mod mcmp { mod mmove { use super::*; - use Spread::{Large, Medium, Small}; + use Spread::{Aligned, Large, Medium, Small}; struct Cfg { len: usize, @@ -263,6 +249,8 @@ mod mmove { } enum Spread { + /// `src` and `dst` are close and have the same alignment (or offset). + Aligned, /// `src` and `dst` are close. Small, /// `src` and `dst` are halfway offset in the buffer. @@ -271,10 +259,16 @@ mod mmove { Large, } + // Note that small and large are fn calculate_spread(len: usize, spread: Spread) -> usize { match spread { + // Note that this test doesn't make sense for lengths less than len=128 + Aligned => { + assert!(len > MAX_ALIGN, "aligned memset would have no overlap"); + MAX_ALIGN + } Small => 1, - Medium => len / 2, + Medium => (len / 2) + 1, // add 1 so all are misaligned Large => len - 1, } } @@ -282,8 +276,8 @@ mod mmove { fn setup_forward(cfg: Cfg) -> (usize, usize, AlignedSlice) { let Cfg { len, spread, off } = cfg; let spread = calculate_spread(len, spread); - println!("{len} bytes, {spread} spread, {off} offset"); - assert!(spread < len, "otherwise this just tests memcpy"); + println!("bytes: {len}, spread: {spread}, offset: {off}, forward"); + assert!(spread < len, "memmove tests should have some overlap"); let mut buf = AlignedSlice::new_zeroed(len + spread, off); let mut fill: usize = 0; buf[..len].fill_with(|| { @@ -296,8 +290,8 @@ mod mmove { fn setup_backward(cfg: Cfg) -> (usize, usize, AlignedSlice) { let Cfg { len, spread, off } = cfg; let spread = calculate_spread(len, spread); - println!("{len} bytes, {spread} spread, {off} offset"); - assert!(spread < len, "otherwise this just tests memcpy"); + println!("bytes: {len}, spread: {spread}, offset: {off}, backward"); + assert!(spread < len, "memmove tests should have some overlap"); let mut buf = AlignedSlice::new_zeroed(len + spread, off); let mut fill: usize = 0; buf[spread..].fill_with(|| { @@ -308,6 +302,14 @@ mod mmove { } #[library_benchmark] + #[benches::aligned( + args = [ + // Don't test small spreads since there is no overlap + Cfg { len: 4096, spread: Aligned, off: 0 }, + Cfg { len: MEG1, spread: Aligned, off: 0 }, + ], + setup = setup_forward + )] #[benches::small_spread( args = [ Cfg { len: 16, spread: Small, off: 0 }, @@ -315,7 +317,7 @@ mod mmove { Cfg { len: 64, spread: Small, off: 0 }, Cfg { len: 512, spread: Small, off: 0 }, Cfg { len: 4096, spread: Small, off: 0 }, - Cfg { len: 1048576, spread: Small, off: 0 }, + Cfg { len: MEG1, spread: Small, off: 0 }, ], setup = setup_forward )] @@ -326,7 +328,7 @@ mod mmove { Cfg { len: 64, spread: Medium, off: 0 }, Cfg { len: 512, spread: Medium, off: 0 }, Cfg { len: 4096, spread: Medium, off: 0 }, - Cfg { len: 1048576, spread: Medium, off: 0 }, + Cfg { len: MEG1, spread: Medium, off: 0 }, ], setup = setup_forward )] @@ -337,40 +339,47 @@ mod mmove { Cfg { len: 64, spread: Large, off: 0 }, Cfg { len: 512, spread: Large, off: 0 }, Cfg { len: 4096, spread: Large, off: 0 }, - Cfg { len: 1048576, spread: Large, off: 0 }, + Cfg { len: MEG1, spread: Large, off: 0 }, ], setup = setup_forward )] - #[benches::small_spread_offset( + #[benches::aligned_off( + args = [ + Cfg { len: 4096, spread: Aligned, off: 65 }, + Cfg { len: MEG1, spread: Aligned, off: 65 }, + ], + setup = setup_forward + )] + #[benches::small_spread_off( args = [ - Cfg { len: 16, spread: Small, off: 63 }, - Cfg { len: 32, spread: Small, off: 63 }, - Cfg { len: 64, spread: Small, off: 63 }, - Cfg { len: 512, spread: Small, off: 63 }, - Cfg { len: 4096, spread: Small, off: 63 }, - Cfg { len: 1048576, spread: Small, off: 63 }, + Cfg { len: 16, spread: Small, off: 65 }, + Cfg { len: 32, spread: Small, off: 65 }, + Cfg { len: 64, spread: Small, off: 65 }, + Cfg { len: 512, spread: Small, off: 65 }, + Cfg { len: 4096, spread: Small, off: 65 }, + Cfg { len: MEG1, spread: Small, off: 65 }, ], setup = setup_forward )] - #[benches::medium_spread_offset( + #[benches::medium_spread_off( args = [ - Cfg { len: 16, spread: Medium, off: 63 }, - Cfg { len: 32, spread: Medium, off: 63 }, - Cfg { len: 64, spread: Medium, off: 63 }, - Cfg { len: 512, spread: Medium, off: 63 }, - Cfg { len: 4096, spread: Medium, off: 63 }, - Cfg { len: 1048576, spread: Medium, off: 63 }, + Cfg { len: 16, spread: Medium, off: 65 }, + Cfg { len: 32, spread: Medium, off: 65 }, + Cfg { len: 64, spread: Medium, off: 65 }, + Cfg { len: 512, spread: Medium, off: 65 }, + Cfg { len: 4096, spread: Medium, off: 65 }, + Cfg { len: MEG1, spread: Medium, off: 65 }, ], setup = setup_forward )] - #[benches::large_spread_offset( + #[benches::large_spread_off( args = [ - Cfg { len: 16, spread: Large, off: 63 }, - Cfg { len: 32, spread: Large, off: 63 }, - Cfg { len: 64, spread: Large, off: 63 }, - Cfg { len: 512, spread: Large, off: 63 }, - Cfg { len: 4096, spread: Large, off: 63 }, - Cfg { len: 1048576, spread: Large, off: 63 }, + Cfg { len: 16, spread: Large, off: 65 }, + Cfg { len: 32, spread: Large, off: 65 }, + Cfg { len: 64, spread: Large, off: 65 }, + Cfg { len: 512, spread: Large, off: 65 }, + Cfg { len: 4096, spread: Large, off: 65 }, + Cfg { len: MEG1, spread: Large, off: 65 }, ], setup = setup_forward )] @@ -386,6 +395,14 @@ mod mmove { } #[library_benchmark] + #[benches::aligned( + args = [ + // Don't test small spreads since there is no overlap + Cfg { len: 4096, spread: Aligned, off: 0 }, + Cfg { len: MEG1, spread: Aligned, off: 0 }, + ], + setup = setup_backward + )] #[benches::small_spread( args = [ Cfg { len: 16, spread: Small, off: 0 }, @@ -393,18 +410,18 @@ mod mmove { Cfg { len: 64, spread: Small, off: 0 }, Cfg { len: 512, spread: Small, off: 0 }, Cfg { len: 4096, spread: Small, off: 0 }, - Cfg { len: 1048576, spread: Small, off: 0 }, + Cfg { len: MEG1, spread: Small, off: 0 }, ], setup = setup_backward )] - #[benches::middle( + #[benches::medium_spread( args = [ Cfg { len: 16, spread: Medium, off: 0 }, Cfg { len: 32, spread: Medium, off: 0 }, Cfg { len: 64, spread: Medium, off: 0 }, Cfg { len: 512, spread: Medium, off: 0 }, Cfg { len: 4096, spread: Medium, off: 0 }, - Cfg { len: 1048576, spread: Medium, off: 0 }, + Cfg { len: MEG1, spread: Medium, off: 0 }, ], setup = setup_backward )] @@ -415,40 +432,48 @@ mod mmove { Cfg { len: 64, spread: Large, off: 0 }, Cfg { len: 512, spread: Large, off: 0 }, Cfg { len: 4096, spread: Large, off: 0 }, - Cfg { len: 1048576, spread: Large, off: 0 }, + Cfg { len: MEG1, spread: Large, off: 0 }, + ], + setup = setup_backward + )] + #[benches::aligned_off( + args = [ + // Don't test small spreads since there is no overlap + Cfg { len: 4096, spread: Aligned, off: 65 }, + Cfg { len: MEG1, spread: Aligned, off: 65 }, ], setup = setup_backward )] #[benches::small_spread_off( args = [ - Cfg { len: 16, spread: Small, off: 63 }, - Cfg { len: 32, spread: Small, off: 63 }, - Cfg { len: 64, spread: Small, off: 63 }, - Cfg { len: 512, spread: Small, off: 63 }, - Cfg { len: 4096, spread: Small, off: 63 }, - Cfg { len: 1048576, spread: Small, off: 63 }, + Cfg { len: 16, spread: Small, off: 65 }, + Cfg { len: 32, spread: Small, off: 65 }, + Cfg { len: 64, spread: Small, off: 65 }, + Cfg { len: 512, spread: Small, off: 65 }, + Cfg { len: 4096, spread: Small, off: 65 }, + Cfg { len: MEG1, spread: Small, off: 65 }, ], setup = setup_backward )] - #[benches::middle_off( + #[benches::medium_spread_off( args = [ - Cfg { len: 16, spread: Medium, off: 63 }, - Cfg { len: 32, spread: Medium, off: 63 }, - Cfg { len: 64, spread: Medium, off: 63 }, - Cfg { len: 512, spread: Medium, off: 63 }, - Cfg { len: 4096, spread: Medium, off: 63 }, - Cfg { len: 1048576, spread: Medium, off: 63 }, + Cfg { len: 16, spread: Medium, off: 65 }, + Cfg { len: 32, spread: Medium, off: 65 }, + Cfg { len: 64, spread: Medium, off: 65 }, + Cfg { len: 512, spread: Medium, off: 65 }, + Cfg { len: 4096, spread: Medium, off: 65 }, + Cfg { len: MEG1, spread: Medium, off: 65 }, ], setup = setup_backward )] #[benches::large_spread_off( args = [ - Cfg { len: 16, spread: Large, off: 63 }, - Cfg { len: 32, spread: Large, off: 63 }, - Cfg { len: 64, spread: Large, off: 63 }, - Cfg { len: 512, spread: Large, off: 63 }, - Cfg { len: 4096, spread: Large, off: 63 }, - Cfg { len: 1048576, spread: Large, off: 63 }, + Cfg { len: 16, spread: Large, off: 65 }, + Cfg { len: 32, spread: Large, off: 65 }, + Cfg { len: 64, spread: Large, off: 65 }, + Cfg { len: 512, spread: Large, off: 65 }, + Cfg { len: 4096, spread: Large, off: 65 }, + Cfg { len: MEG1, spread: Large, off: 65 }, ], setup = setup_backward )]