From 2fab3e35097a359f086362410a481576d2e0e099 Mon Sep 17 00:00:00 2001 From: jantti Date: Wed, 6 Sep 2023 12:42:31 +0900 Subject: [PATCH] Add edge bias feature to bias for testing edge cases. * Works by randomly changing some generated values to be one of the random edge cases. The first tests are done with 100% edge cases, and then the chance goes down over the number of test cases until it's 0%. * The average amount of edge case tests can be controlled by edge_bias configuration option. It defaults to 0.25. * In failure the edge bias is persisted to the persistence file along with the seed, so it should be reproducible. * Also fix time::Duration test because the edge bias system found a problem in it. (Might need more modifications.) * Fix float_samplers test subsequent_splits_always_match_bounds. --- proptest/src/arbitrary/_std/time.rs | 7 +- proptest/src/num.rs | 238 ++++++++++++++++++ proptest/src/num/float_samplers.rs | 37 +-- proptest/src/test_runner/config.rs | 71 ++++++ .../test_runner/failure_persistence/file.rs | 106 +++++--- .../test_runner/failure_persistence/map.rs | 19 +- .../test_runner/failure_persistence/mod.rs | 47 +++- .../test_runner/failure_persistence/noop.rs | 7 +- proptest/src/test_runner/rng.rs | 10 +- proptest/src/test_runner/runner.rs | 63 ++++- 10 files changed, 522 insertions(+), 83 deletions(-) diff --git a/proptest/src/arbitrary/_std/time.rs b/proptest/src/arbitrary/_std/time.rs index 7d940439..b6944d0a 100644 --- a/proptest/src/arbitrary/_std/time.rs +++ b/proptest/src/arbitrary/_std/time.rs @@ -17,8 +17,11 @@ use crate::num; use crate::strategy::statics::{self, static_map}; arbitrary!(Duration, SMapped<(u64, u32), Self>; - static_map(any::<(u64, u32)>(), |(a, b)| Duration::new(a, b)) -); + static_map(any::<(u64, u32)>(), |(a, b)| + // Duration::new panics if nanoseconds are over one billion (one second) + // and overflowing into the seconds would overflow the second counter. + Duration::new(a, b % 1_000_000_000) +)); // Instant::now() "never" returns the same Instant, so no shrinking may occur! arbitrary!(Instant; Self::now()); diff --git a/proptest/src/num.rs b/proptest/src/num.rs index 675cb59f..2b0d0271 100644 --- a/proptest/src/num.rs +++ b/proptest/src/num.rs @@ -38,6 +38,193 @@ pub fn sample_uniform_incl( Uniform::new_inclusive(start, end).sample(run.rng()) } +trait SampleEdgeCase { + fn sample_edge_case(runner: &mut TestRunner, epsilon: T) -> Self; +} + +macro_rules! impl_sample_edge_case_signed_impl { + ($typ: ty) => { + impl SampleEdgeCase<$typ> for $typ { + fn sample_edge_case( + runner: &mut TestRunner, + epsilon: $typ, + ) -> $typ { + match sample_uniform(runner, 0, 7) { + 0 => 0, + 1 => epsilon, + 2 => -epsilon, + 3 => <$typ>::MIN, + 4 => <$typ>::MAX, + 5 => <$typ>::MIN + epsilon, + 6 => <$typ>::MAX - epsilon, + _ => unreachable!(), + } + } + } + }; +} + +macro_rules! impl_sample_edge_case_signed { + ($($typ: ty),*) => { + $(impl_sample_edge_case_signed_impl!($typ);)* + }; +} + +macro_rules! impl_sample_edge_case_unsigned_impl { + ($typ: ty) => { + impl SampleEdgeCase<$typ> for $typ { + fn sample_edge_case( + runner: &mut TestRunner, + epsilon: $typ, + ) -> $typ { + match sample_uniform(runner, 0, 4) { + 0 => 0, + 1 => epsilon, + 2 => <$typ>::MAX, + 3 => <$typ>::MAX - epsilon, + _ => unreachable!(), + } + } + } + }; +} + +macro_rules! impl_sample_edge_case_unsigned { + ($($typ: ty),*) => { + $(impl_sample_edge_case_unsigned_impl!($typ);)* + }; +} + +impl_sample_edge_case_signed!(i8, i16, i32, i64, i128, isize); +impl_sample_edge_case_unsigned!(u8, u16, u32, u64, u128, usize); + +trait SampleEdgeCaseRangeExclusive { + fn sample_edge_case_range_exclusive( + runner: &mut TestRunner, + start: T, + end: T, + epsilon: T, + ) -> Self; +} + +macro_rules! impl_sample_edge_case_range_exclusive_impl { + ($typ: ty) => { + impl SampleEdgeCaseRangeExclusive<$typ> for $typ { + fn sample_edge_case_range_exclusive( + runner: &mut TestRunner, + start: $typ, + end: $typ, + epsilon: $typ, + ) -> $typ { + match sample_uniform(runner, 0, 4) { + 0 => start, + 1 => { + num_traits::clamp(start + epsilon, start, end - epsilon) + } + 2 => { + if start < end - epsilon { + end - epsilon * 2 as $typ + } else { + start + } + } + 3 => end - epsilon, + _ => unreachable!(), + } + } + } + }; +} + +macro_rules! impl_sample_edge_case_range_exclusive { + ($($typ: ty),*) => { + $(impl_sample_edge_case_range_exclusive_impl!($typ);)* + }; +} + +impl_sample_edge_case_range_exclusive!( + i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize, f32, f64 +); + +trait SampleEdgeCaseRangeInclusive { + fn sample_edge_case_range_inclusive( + runner: &mut TestRunner, + start: T, + end: T, + epsilon: T, + ) -> Self; +} + +macro_rules! impl_sample_edge_case_range_inclusive_impl { + ($typ: ty) => { + impl SampleEdgeCaseRangeInclusive<$typ> for $typ { + fn sample_edge_case_range_inclusive( + runner: &mut TestRunner, + start: $typ, + end: $typ, + epsilon: $typ, + ) -> $typ { + match sample_uniform(runner, 0, 4) { + 0 => start, + 1 => num_traits::clamp(start + epsilon, start, end), + 2 => num_traits::clamp(end - epsilon, start, end), + 3 => end, + _ => unreachable!(), + } + } + } + }; +} + +macro_rules! impl_sample_edge_case_range_inclusive { + ($($typ: ty),*) => { + $(impl_sample_edge_case_range_inclusive_impl!($typ);)* + }; +} + +impl_sample_edge_case_range_inclusive!( + i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize, f32, f64 +); + +trait SampleEdgeCaseFloat { + fn sample_edge_case_float(runner: &mut TestRunner) -> Self; +} + +macro_rules! impl_sample_edge_case_float_impl { + ($typ: ty) => { + impl SampleEdgeCaseFloat<$typ> for $typ { + fn sample_edge_case_float(runner: &mut TestRunner) -> $typ { + match sample_uniform(runner, 0, 11) { + 0 => 0.0, + 1 => -0.0, + 2 => 1.0, + 3 => -1.0, + 4 => <$typ>::MIN, + 5 => <$typ>::MAX, + 6 => <$typ>::MIN_POSITIVE, + 7 => -<$typ>::MIN_POSITIVE, + 8 => <$typ>::EPSILON, + // One ULP from MIN and MAX + 9 => <$typ>::from_bits(<$typ>::MIN.to_bits() - 1), + 10 => <$typ>::from_bits(<$typ>::MAX.to_bits() - 1), + // 11 => <$typ>::NAN, + // 12 => <$typ>::NEG_INFINITY, + // 13 => <$typ>::INFINITY, + _ => unreachable!(), + } + } + } + }; +} + +macro_rules! impl_sample_edge_case_float { + ($($typ: ty),*) => { + $(impl_sample_edge_case_float_impl!($typ);)* + }; +} + +impl_sample_edge_case_float!(f32, f64); + macro_rules! int_any { ($typ: ident) => { /// Type of the `ANY` constant. @@ -53,6 +240,14 @@ macro_rules! int_any { type Value = $typ; fn new_tree(&self, runner: &mut TestRunner) -> NewTree { + if runner.eval_edge_bias() { + return Ok(BinarySearch::new( + $crate::num::SampleEdgeCase::sample_edge_case( + runner, 1, + ), + )); + } + Ok(BinarySearch::new(runner.rng().gen())) } } @@ -76,6 +271,14 @@ macro_rules! numeric_api { ); } + if runner.eval_edge_bias() { + return Ok(BinarySearch::new( + $crate::num::SampleEdgeCaseRangeExclusive::sample_edge_case_range_exclusive( + runner, self.start, self.end, $epsilon, + ), + )); + } + Ok(BinarySearch::new_clamped( self.start, $crate::num::sample_uniform::<$sample_typ>( @@ -102,6 +305,14 @@ macro_rules! numeric_api { ); } + if runner.eval_edge_bias() { + return Ok(BinarySearch::new( + $crate::num::SampleEdgeCaseRangeInclusive::sample_edge_case_range_inclusive( + runner, (*self.start()), (*self.end()), $epsilon, + ), + )); + } + Ok(BinarySearch::new_clamped( *self.start(), $crate::num::sample_uniform_incl::<$sample_typ>( @@ -120,6 +331,13 @@ macro_rules! numeric_api { type Value = $typ; fn new_tree(&self, runner: &mut TestRunner) -> NewTree { + if runner.eval_edge_bias() { + return Ok(BinarySearch::new( + $crate::num::SampleEdgeCaseRangeInclusive::sample_edge_case_range_inclusive( + runner, self.start, ::core::$typ::MAX, $epsilon, + ), + )); + } Ok(BinarySearch::new_clamped( self.start, $crate::num::sample_uniform_incl::<$sample_typ>( @@ -138,6 +356,13 @@ macro_rules! numeric_api { type Value = $typ; fn new_tree(&self, runner: &mut TestRunner) -> NewTree { + if runner.eval_edge_bias() { + return Ok(BinarySearch::new( + $crate::num::SampleEdgeCaseRangeExclusive::sample_edge_case_range_exclusive( + runner, ::core::$typ::MIN, self.end, $epsilon, + ), + )); + } Ok(BinarySearch::new_clamped( ::core::$typ::MIN, $crate::num::sample_uniform::<$sample_typ>( @@ -156,6 +381,13 @@ macro_rules! numeric_api { type Value = $typ; fn new_tree(&self, runner: &mut TestRunner) -> NewTree { + if runner.eval_edge_bias() { + return Ok(BinarySearch::new( + $crate::num::SampleEdgeCaseRangeInclusive::sample_edge_case_range_inclusive( + runner, ::core::$typ::MIN, self.end, $epsilon, + ), + )); + } Ok(BinarySearch::new_clamped( ::core::$typ::MIN, $crate::num::sample_uniform_incl::<$sample_typ>( @@ -627,6 +859,12 @@ macro_rules! float_any { fn new_tree(&self, runner: &mut TestRunner) -> NewTree { let flags = self.0.normalise(); + + if runner.eval_edge_bias() { + return Ok(BinarySearch::new_with_types( + $crate::num::SampleEdgeCaseFloat::sample_edge_case_float(runner), flags)) + } + let sign_mask = if flags.contains(FloatTypes::NEGATIVE) { $typ::SIGN_MASK } else { diff --git a/proptest/src/num/float_samplers.rs b/proptest/src/num/float_samplers.rs index 55b6cd03..4a232f62 100644 --- a/proptest/src/num/float_samplers.rs +++ b/proptest/src/num/float_samplers.rs @@ -438,20 +438,29 @@ macro_rules! float_sampler { let intervals = split_interval([low, high]); let size = (intervals.count - 1) as usize; - let interval = intervals.get(index.index(size) as $int_typ); - let small_intervals = split_interval(interval); - - let start = small_intervals.get(0); - let end = small_intervals.get(small_intervals.count - 1); - let (low_interval, high_interval) = if start[0] < end[0] { - (start, end) - } else { - (end, start) - }; - - prop_assert!( - interval[0] == low_interval[0] && - interval[1] == high_interval[1]); + if size <= 0 + { + prop_assert!((intervals.start == high && intervals.step < 0.0) || + (intervals.start == low && intervals.step > 0.0)); + prop_assert!(intervals.count == 1); + } + else + { + let interval = intervals.get(index.index(size) as $int_typ); + let small_intervals = split_interval(interval); + + let start = small_intervals.get(0); + let end = small_intervals.get(small_intervals.count - 1); + let (low_interval, high_interval) = if start[0] < end[0] { + (start, end) + } else { + (end, start) + }; + + prop_assert!( + interval[0] == low_interval[0] && + interval[1] == high_interval[1]); + } } } } diff --git a/proptest/src/test_runner/config.rs b/proptest/src/test_runner/config.rs index 0a5dc1c8..537b7646 100644 --- a/proptest/src/test_runner/config.rs +++ b/proptest/src/test_runner/config.rs @@ -28,6 +28,8 @@ use crate::test_runner::FileFailurePersistence; #[cfg(feature = "std")] const CASES: &str = "PROPTEST_CASES"; #[cfg(feature = "std")] +const EDGE_BIAS: &str = "PROPTEST_EDGE_BIAS"; +#[cfg(feature = "std")] const MAX_LOCAL_REJECTS: &str = "PROPTEST_MAX_LOCAL_REJECTS"; #[cfg(feature = "std")] const MAX_GLOBAL_REJECTS: &str = "PROPTEST_MAX_GLOBAL_REJECTS"; @@ -85,6 +87,9 @@ pub fn contextualize_config(mut result: Config) -> Config { { match var.as_str() { CASES => parse_or_warn(&value, &mut result.cases, "u32", CASES), + EDGE_BIAS => { + parse_or_warn(&value, &mut result.edge_bias, "f32", EDGE_BIAS) + } MAX_LOCAL_REJECTS => parse_or_warn( &value, &mut result.max_local_rejects, @@ -158,6 +163,7 @@ pub fn contextualize_config(result: Config) -> Config { fn default_default_config() -> Config { Config { cases: 256, + edge_bias: 0.25f32, max_local_rejects: 65_536, max_global_rejects: 1024, max_flat_map_regens: 1_000_000, @@ -204,6 +210,30 @@ pub struct Config { /// when the `std` feature is enabled, which it is by default.) pub cases: u32, + /// Bias towards edge cases of the input domain, meaning things like + /// MIN, MIN + 1, -1, 0, 1, MAX - 1, MAX. And in case of floats, NaN's and + /// infinity. + /// + /// This only applies to integers, floats and ranges of those. + /// + /// The number indicates the fraction of the cases on average that will be + /// edge cases. + /// + /// 0.0 would mean no bias towards edge cases at any point. + /// + /// 1.0 means only edge cases. + /// + /// 0.5 means that chance of edge cases reaches 50% at half-way through. + /// + /// Any non-zero value means edge cases start at 100% chance and then + /// smoothly decreases to 0% chance at the end. It decreases in a two-part + /// linear function whose area is edge_bias, so there is always a + /// combination of edge cases with other edge cases, edge cases with random + /// values, and random values with random values. + /// + /// The default is 0.25. + pub edge_bias: f32, + /// The maximum number of individual inputs that may be rejected before the /// test as a whole aborts. /// @@ -428,6 +458,47 @@ impl Config { } } + /// Constructs a `Config` only differing from the `default()` in the + /// edge_bias. + /// + /// This is simply a more concise alternative to using field-record update + /// syntax: + /// + /// ``` + /// # use proptest::test_runner::Config; + /// assert_eq!( + /// Config::with_edge_bias(0.5f32), + /// Config { edge_bias: 0.5f32, .. Config::default() } + /// ); + /// ``` + pub fn with_edge_bias(edge_bias: f32) -> Self { + Self { + edge_bias, + ..Config::default() + } + } + + /// Constructs a `Config` only differing from the `default()` in the + /// number of test cases and edge_bias. + /// + /// This is simply a more concise alternative to using field-record update + /// syntax: + /// + /// ``` + /// # use proptest::test_runner::Config; + /// assert_eq!( + /// Config::with_cases_and_edge_bias(1000, 0.5), + /// Config { cases: 1000, edge_bias: 0.5, .. Config::default() } + /// ); + /// ``` + pub fn with_cases_and_edge_bias(cases: u32, edge_bias: f32) -> Self { + Self { + cases, + edge_bias, + ..Config::default() + } + } + /// Constructs a `Config` only differing from the `default()` in the /// source_file of the present test. /// diff --git a/proptest/src/test_runner/failure_persistence/file.rs b/proptest/src/test_runner/failure_persistence/file.rs index 21799adb..aaf0e23e 100644 --- a/proptest/src/test_runner/failure_persistence/file.rs +++ b/proptest/src/test_runner/failure_persistence/file.rs @@ -21,7 +21,8 @@ use std::vec::Vec; use self::FileFailurePersistence::*; use crate::test_runner::failure_persistence::{ - FailurePersistence, PersistedSeed, + from_base16, to_base16, FailurePersistence, PersistedEdgeBias, + PersistedSeed, }; /// Describes how failing test cases are persisted. @@ -84,7 +85,7 @@ impl FailurePersistence for FileFailurePersistence { fn load_persisted_failures2( &self, source_file: Option<&'static str>, - ) -> Vec { + ) -> Vec<(PersistedSeed, PersistedEdgeBias)> { let p = self.resolve( source_file .and_then(|s| absolutize_source_file(Path::new(s))) @@ -93,21 +94,25 @@ impl FailurePersistence for FileFailurePersistence { ); let path: Option<&PathBuf> = p.as_ref(); - let result: io::Result> = path.map_or_else( - || Ok(vec![]), - |path| { - // .ok() instead of .unwrap() so we don't propagate panics here - let _lock = PERSISTENCE_LOCK.read().ok(); - io::BufReader::new(fs::File::open(path)?) - .lines() - .enumerate() - .filter_map(|(lineno, line)| match line { - Err(err) => Some(Err(err)), - Ok(line) => parse_seed_line(line, path, lineno).map(Ok), - }) - .collect() - }, - ); + let result: io::Result> = path + .map_or_else( + || Ok(vec![]), + |path| { + // .ok() instead of .unwrap() so we don't propagate panics here + let _lock = PERSISTENCE_LOCK.read().ok(); + io::BufReader::new(fs::File::open(path)?) + .lines() + .enumerate() + .filter_map(|(lineno, line)| match line { + Err(err) => Some(Err(err)), + Ok(line) => parse_seed_line(line, path, lineno) + .map({ + |(seed, edge_bias)| Ok((seed, edge_bias)) + }), + }) + .collect() + }, + ); unwrap_or!(result, err => { if io::ErrorKind::NotFound != err.kind() { @@ -127,6 +132,7 @@ impl FailurePersistence for FileFailurePersistence { &mut self, source_file: Option<&'static str>, seed: PersistedSeed, + current_edge_bias: PersistedEdgeBias, shrunken_value: &dyn Debug, ) { let path = self.resolve(source_file.map(Path::new)); @@ -141,8 +147,13 @@ impl FailurePersistence for FileFailurePersistence { .expect("proptest: couldn't write header."); } - write_seed_line(&mut to_write, &seed, shrunken_value) - .expect("proptest: couldn't write seed line."); + write_seed_line( + &mut to_write, + &seed, + current_edge_bias, + shrunken_value, + ) + .expect("proptest: couldn't write seed line."); if let Err(e) = write_seed_data_to_file(&path, &to_write) { eprintln!( @@ -151,14 +162,17 @@ impl FailurePersistence for FileFailurePersistence { e ); } else { + let mut edge_string = String::new(); + to_base16(&mut edge_string, ¤t_edge_bias); + eprintln!( "proptest: Saving this and future failures in {}\n\ proptest: If this test was run on a CI system, you may \ wish to add the following line to your copy of the file.{}\n\ - {}", + {},{}", path.display(), if is_new { " (You may need to create it.)" } else { "" }, - seed); + seed, edge_string); } } } @@ -253,36 +267,60 @@ fn parse_seed_line( mut line: String, path: &Path, lineno: usize, -) -> Option { +) -> Option<(PersistedSeed, PersistedEdgeBias)> { // Remove anything after and including '#': if let Some(comment_start) = line.find('#') { line.truncate(comment_start); } if line.len() > 0 { - let ret = line.parse::().ok(); - if !ret.is_some() { - eprintln!( - "proptest: {}:{}: unparsable line, ignoring", - path.display(), - lineno + 1 - ); + let seed; + let mut edge_bias: PersistedEdgeBias = 0f32.to_le_bytes(); + if let Some(comma_position) = line.find(',') { + seed = line[0..comma_position].parse::().ok(); + // The result is safe to ignore to not spam the log in case old + // cases exist, in which case edge_bias will be 0 to match old + // behaviour. + // We read 8 characters for 4 bytes. + if line.len() >= comma_position + 9 { + from_base16( + &mut edge_bias, + &line[comma_position + 1..comma_position + 9], + ); + } + } else { + seed = line.parse::().ok(); + edge_bias = 0f32.to_le_bytes(); + } + match seed { + Some(seed) => { + return Some((seed, edge_bias)); + } + None => { + eprintln!( + "proptest: {}:{}: unparsable line, ignoring", + path.display(), + lineno + 1 + ); + } } - return ret; } - None } fn write_seed_line( buf: &mut Vec, seed: &PersistedSeed, + current_edge_bias: PersistedEdgeBias, shrunken_value: &dyn Debug, ) -> io::Result<()> { - // Write the seed itself - write!(buf, "{}", seed.to_string())?; + // Write the seed and edge bias. Note: f32 should be roundtrip-safe: + // https://github.com/rust-lang/rust/pull/27307 + let mut edge_string = String::new(); + to_base16(&mut edge_string, ¤t_edge_bias); + write!(buf, "{},{}", seed.to_string(), edge_string)?; - // Write out comment: + // Write out comment let debug_start = buf.len(); write!(buf, " # shrinks to {:?}", shrunken_value)?; diff --git a/proptest/src/test_runner/failure_persistence/map.rs b/proptest/src/test_runner/failure_persistence/map.rs index 322e554c..8cde2dda 100644 --- a/proptest/src/test_runner/failure_persistence/map.rs +++ b/proptest/src/test_runner/failure_persistence/map.rs @@ -11,6 +11,7 @@ use crate::std_facade::{fmt, BTreeMap, BTreeSet, Box, Vec}; use core::any::Any; use crate::test_runner::failure_persistence::FailurePersistence; +use crate::test_runner::failure_persistence::PersistedEdgeBias; use crate::test_runner::failure_persistence::PersistedSeed; /// Failure persistence option that loads and saves seeds in memory @@ -20,14 +21,14 @@ use crate::test_runner::failure_persistence::PersistedSeed; #[derive(Clone, Debug, Default, PartialEq)] pub struct MapFailurePersistence { /// Backing map, keyed by source_file. - pub map: BTreeMap<&'static str, BTreeSet>, + pub map: BTreeMap<&'static str, BTreeSet<(PersistedSeed, [u8; 4])>>, } impl FailurePersistence for MapFailurePersistence { fn load_persisted_failures2( &self, source_file: Option<&'static str>, - ) -> Vec { + ) -> Vec<(PersistedSeed, PersistedEdgeBias)> { source_file .and_then(|source| self.map.get(source)) .map(|seeds| seeds.iter().cloned().collect::>()) @@ -38,14 +39,16 @@ impl FailurePersistence for MapFailurePersistence { &mut self, source_file: Option<&'static str>, seed: PersistedSeed, + current_edge_bias: PersistedEdgeBias, _shrunken_value: &dyn fmt::Debug, ) { let s = match source_file { Some(sf) => sf, None => return, }; - let set = self.map.entry(s).or_insert_with(BTreeSet::new); - set.insert(seed); + let set: &mut BTreeSet<(PersistedSeed, PersistedEdgeBias)> = + self.map.entry(s).or_insert_with(BTreeSet::new); + set.insert((seed, current_edge_bias)); } fn box_clone(&self) -> Box { @@ -79,10 +82,10 @@ mod tests { #[test] fn seeds_recoverable() { let mut p = MapFailurePersistence::default(); - p.save_persisted_failure2(HI_PATH, INC_SEED, &""); + p.save_persisted_failure2(HI_PATH, INC_SEED, INC_EDGE_BIAS, &""); let restored = p.load_persisted_failures2(HI_PATH); assert_eq!(1, restored.len()); - assert_eq!(INC_SEED, *restored.first().unwrap()); + assert_eq!((INC_SEED, INC_EDGE_BIAS), *restored.first().unwrap()); assert!(p.load_persisted_failures2(None).is_empty()); assert!(p.load_persisted_failures2(UNREL_PATH).is_empty()); @@ -91,8 +94,8 @@ mod tests { #[test] fn seeds_deduplicated() { let mut p = MapFailurePersistence::default(); - p.save_persisted_failure2(HI_PATH, INC_SEED, &""); - p.save_persisted_failure2(HI_PATH, INC_SEED, &""); + p.save_persisted_failure2(HI_PATH, INC_SEED, INC_EDGE_BIAS, &""); + p.save_persisted_failure2(HI_PATH, INC_SEED, INC_EDGE_BIAS, &""); let restored = p.load_persisted_failures2(HI_PATH); assert_eq!(1, restored.len()); } diff --git a/proptest/src/test_runner/failure_persistence/mod.rs b/proptest/src/test_runner/failure_persistence/mod.rs index f48d571b..b45d529c 100644 --- a/proptest/src/test_runner/failure_persistence/mod.rs +++ b/proptest/src/test_runner/failure_persistence/mod.rs @@ -7,7 +7,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -use crate::std_facade::{fmt, Box, Vec}; +use crate::std_facade::{fmt, Box, String, Vec}; use core::any::Any; use core::fmt::Display; use core::result::Result; @@ -32,6 +32,9 @@ use crate::test_runner::Seed; #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct PersistedSeed(pub(crate) Seed); +/// Floating point edge bias as bytes to be put inside BTreeMap/set. +pub type PersistedEdgeBias = [u8; 4]; + impl Display for PersistedSeed { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}", self.0.to_persistence()) @@ -64,10 +67,12 @@ pub trait FailurePersistence: Send + Sync + fmt::Debug { fn load_persisted_failures2( &self, source_file: Option<&'static str>, - ) -> Vec { + ) -> Vec<(PersistedSeed, PersistedEdgeBias)> { self.load_persisted_failures(source_file) .into_iter() - .map(|seed| PersistedSeed(Seed::XorShift(seed))) + .map(|(seed, edge_bias)| { + (PersistedSeed(Seed::XorShift(seed)), edge_bias) + }) .collect() } @@ -80,7 +85,7 @@ pub trait FailurePersistence: Send + Sync + fmt::Debug { fn load_persisted_failures( &self, source_file: Option<&'static str>, - ) -> Vec<[u8; 16]> { + ) -> Vec<([u8; 16], PersistedEdgeBias)> { panic!("load_persisted_failures2 not implemented"); } @@ -93,12 +98,16 @@ pub trait FailurePersistence: Send + Sync + fmt::Debug { &mut self, source_file: Option<&'static str>, seed: PersistedSeed, + current_edge_bias: PersistedEdgeBias, shrunken_value: &dyn fmt::Debug, ) { match seed.0 { - Seed::XorShift(seed) => { - self.save_persisted_failure(source_file, seed, shrunken_value) - } + Seed::XorShift(seed) => self.save_persisted_failure( + source_file, + seed, + current_edge_bias, + shrunken_value, + ), _ => (), } } @@ -113,6 +122,7 @@ pub trait FailurePersistence: Send + Sync + fmt::Debug { &mut self, source_file: Option<&'static str>, seed: [u8; 16], + current_edge_bias: PersistedEdgeBias, shrunken_value: &dyn fmt::Debug, ) { panic!("save_persisted_failure2 not implemented"); @@ -142,15 +152,36 @@ impl Clone for Box { } } +pub(crate) fn to_base16(dst: &mut String, src: &[u8]) { + for byte in src { + dst.push_str(&format!("{:02x}", byte)); + } +} + +pub(crate) fn from_base16(dst: &mut [u8], src: &str) -> Option<()> { + if dst.len() * 2 != src.len() { + return None; + } + + for (dst_byte, src_pair) in dst.into_iter().zip(src.as_bytes().chunks(2)) { + *dst_byte = + u8::from_str_radix(std::str::from_utf8(src_pair).ok()?, 16).ok()?; + } + + Some(()) +} + #[cfg(test)] mod tests { - use super::PersistedSeed; + use super::{PersistedEdgeBias, PersistedSeed}; use crate::test_runner::rng::Seed; pub const INC_SEED: PersistedSeed = PersistedSeed(Seed::XorShift([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, ])); + pub const INC_EDGE_BIAS: PersistedEdgeBias = [0, 0, 0, 0]; + pub const HI_PATH: Option<&str> = Some("hi"); pub const UNREL_PATH: Option<&str> = Some("unrelated"); } diff --git a/proptest/src/test_runner/failure_persistence/noop.rs b/proptest/src/test_runner/failure_persistence/noop.rs index 59538392..28e63fd6 100644 --- a/proptest/src/test_runner/failure_persistence/noop.rs +++ b/proptest/src/test_runner/failure_persistence/noop.rs @@ -11,7 +11,7 @@ use crate::std_facade::{fmt, Box, Vec}; use core::any::Any; use crate::test_runner::failure_persistence::{ - FailurePersistence, PersistedSeed, + FailurePersistence, PersistedEdgeBias, PersistedSeed, }; /// Failure persistence option that loads and saves nothing at all. @@ -22,7 +22,7 @@ impl FailurePersistence for NoopFailurePersistence { fn load_persisted_failures2( &self, _source_file: Option<&'static str>, - ) -> Vec { + ) -> Vec<(PersistedSeed, PersistedEdgeBias)> { Vec::new() } @@ -30,6 +30,7 @@ impl FailurePersistence for NoopFailurePersistence { &mut self, _source_file: Option<&'static str>, _seed: PersistedSeed, + _current_edge_bias: PersistedEdgeBias, _shrunken_value: &dyn fmt::Debug, ) { } @@ -68,7 +69,7 @@ mod tests { #[test] fn seeds_not_recoverable() { let mut p = NoopFailurePersistence::default(); - p.save_persisted_failure2(HI_PATH, INC_SEED, &""); + p.save_persisted_failure2(HI_PATH, INC_SEED, INC_EDGE_BIAS, &""); assert!(p.load_persisted_failures2(HI_PATH).is_empty()); assert!(p.load_persisted_failures2(None).is_empty()); assert!(p.load_persisted_failures2(UNREL_PATH).is_empty()); diff --git a/proptest/src/test_runner/rng.rs b/proptest/src/test_runner/rng.rs index 31dd3a35..839483a3 100644 --- a/proptest/src/test_runner/rng.rs +++ b/proptest/src/test_runner/rng.rs @@ -9,12 +9,14 @@ use crate::std_facade::{Arc, String, ToOwned, Vec}; use core::result::Result; -use core::{fmt, str, u8, convert::TryInto}; +use core::{convert::TryInto, fmt, str, u8}; use rand::{self, Rng, RngCore, SeedableRng}; use rand_chacha::ChaChaRng; use rand_xorshift::XorShiftRng; +use crate::test_runner::failure_persistence::to_base16; + /// Identifies a particular RNG algorithm supported by proptest. /// /// Proptest supports dynamic configuration of algorithms to allow it to @@ -347,12 +349,6 @@ impl Seed { } pub(crate) fn to_persistence(&self) -> String { - fn to_base16(dst: &mut String, src: &[u8]) { - for byte in src { - dst.push_str(&format!("{:02x}", byte)); - } - } - match *self { Seed::XorShift(ref seed) => { let dwords = [ diff --git a/proptest/src/test_runner/runner.rs b/proptest/src/test_runner/runner.rs index d209dff6..aa575f8f 100644 --- a/proptest/src/test_runner/runner.rs +++ b/proptest/src/test_runner/runner.rs @@ -14,6 +14,7 @@ use core::{fmt, iter}; #[cfg(feature = "std")] use std::panic::{self, AssertUnwindSafe}; +use rand::Rng; #[cfg(feature = "fork")] use rusty_fork; #[cfg(feature = "fork")] @@ -35,6 +36,8 @@ use crate::test_runner::replay; use crate::test_runner::result_cache::*; use crate::test_runner::rng::TestRng; +use super::PersistedEdgeBias; + #[cfg(feature = "fork")] const ENV_FORK_FILE: &'static str = "_PROPTEST_FORKFILE"; @@ -71,6 +74,7 @@ type RejectionDetail = BTreeMap; pub struct TestRunner { config: Config, successes: u32, + current_edge_bias: f32, local_rejects: u32, global_rejects: u32, rng: TestRng, @@ -85,6 +89,7 @@ impl fmt::Debug for TestRunner { f.debug_struct("TestRunner") .field("config", &self.config) .field("successes", &self.successes) + .field("current_edge_bias", &self.current_edge_bias) .field("local_rejects", &self.local_rejects) .field("global_rejects", &self.global_rejects) .field("rng", &"") @@ -340,6 +345,7 @@ impl TestRunner { TestRunner { config: config, successes: 0, + current_edge_bias: 0f32, local_rejects: 0, global_rejects: 0, rng: rng, @@ -356,6 +362,7 @@ impl TestRunner { TestRunner { config: self.config.clone(), successes: 0, + current_edge_bias: 0f32, local_rejects: 0, global_rejects: 0, rng: self.new_rng(), @@ -370,6 +377,19 @@ impl TestRunner { &mut self.rng } + /// Returns the current edge bias for this test run. + pub fn current_edge_bias(&self) -> f32 { + self.current_edge_bias + } + + /// Evaluates if we should generate edge bias case. + pub fn eval_edge_bias(&mut self) -> bool { + // We do not want to use up rng if current edge bias is 0 so we can + // stay backwards compatible in case current_edge_bias is not set. + self.current_edge_bias > 0f32 + && self.rng.gen_range(0f32..1f32) <= self.current_edge_bias + } + /// Create a new, independent but deterministic RNG from the RNG in this /// runner. pub fn new_rng(&mut self) -> TestRng { @@ -584,19 +604,20 @@ impl TestRunner { ) -> TestRunResult { let old_rng = self.rng.clone(); - let persisted_failure_seeds: Vec = self - .config - .failure_persistence - .as_ref() - .map(|f| f.load_persisted_failures2(self.config.source_file)) - .unwrap_or_default(); + let persisted_failure_seeds: Vec<(PersistedSeed, PersistedEdgeBias)> = + self.config + .failure_persistence + .as_ref() + .map(|f| f.load_persisted_failures2(self.config.source_file)) + .unwrap_or_default(); let mut result_cache = self.new_cache(); - for PersistedSeed(persisted_seed) in + for (PersistedSeed(persisted_seed), edge_bias_bytes) in persisted_failure_seeds.into_iter().rev() { self.rng.set_seed(persisted_seed); + self.current_edge_bias = f32::from_le_bytes(edge_bias_bytes); self.gen_and_run_case( strategy, &test, @@ -611,6 +632,33 @@ impl TestRunner { while self.successes < self.config.cases { // Generate a new seed and make an RNG from that so that we know // what seed to persist if this case fails. + + if self.config.edge_bias <= 0f32 { + self.current_edge_bias = 0f32; + } else if self.config.edge_bias >= 1f32 { + self.current_edge_bias = 1f32; + } else { + // Value from 0.0 to 1.0 representing how far we are through the + // tests. Infinity is OK here in case of cases == 1. + let x = self.successes as f32 / (self.config.cases - 1) as f32; + if x.is_infinite() { + self.current_edge_bias = self.config.edge_bias; + } else { + // Separate the curve into two linear parts whose total area + // is p. Cases of p <= 0 and p >= 1 are handled above. + let p = self.config.edge_bias; + if x < p { + self.current_edge_bias = 1f32 + x * (p - 1f32) / p; + } else { + self.current_edge_bias = (x - 1f32) * p / (p - 1f32); + } + } + } + + if self.current_edge_bias.is_nan() { + // Might happen if cases is 1 or edge_bias is 0. + self.current_edge_bias = self.config.edge_bias; + } let seed = self.rng.gen_get_seed(); let result = self.gen_and_run_case( strategy, @@ -633,6 +681,7 @@ impl TestRunner { failure_persistence.save_persisted_failure2( *source_file, PersistedSeed(seed), + self.current_edge_bias.to_le_bytes(), value, ); }