From 373f0652720a76c5bf61997b6b934e073f138406 Mon Sep 17 00:00:00 2001 From: Andreas Longva Date: Fri, 5 Jul 2024 16:47:44 +0200 Subject: [PATCH] [proptest] Implement is_minimal_case --- proptest/src/is_minimal_case.rs | 61 ++++++++++++++++++++++++++++++ proptest/src/lib.rs | 3 ++ proptest/src/test_runner/runner.rs | 26 ++++++++++++- 3 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 proptest/src/is_minimal_case.rs diff --git a/proptest/src/is_minimal_case.rs b/proptest/src/is_minimal_case.rs new file mode 100644 index 00000000..98cea22a --- /dev/null +++ b/proptest/src/is_minimal_case.rs @@ -0,0 +1,61 @@ +use core::cell::Cell; + +thread_local! { + static IS_MINIMAL_CASE: Cell = Cell::new(false); +} + +/// When run inside a property test, indicates whether the current case being tested +/// is the minimal test case. +/// +/// `proptest` typically runs a large number of test cases for each +/// property test. If it finds a failing test case, it tries to shrink it +/// in the hopes of finding a simpler test case. When debugging a failing +/// property test, we are often only interested in the actual minimal +/// failing case. After the minimal test case has been identified, +/// the test is rerun with the minimal input, and this function +/// returns `true` when called inside the test. +/// +/// The results are undefined if property tests are nested, meaning that a property test +/// is run inside another property test. +/// +/// # Example +/// +/// ```rust +/// use proptest::{proptest, prop_assert, is_minimal_case}; +/// # fn export_to_file_for_analysis() {} +/// +/// proptest! { +/// #[test] +/// fn test_is_not_five(num in 0 .. 10) { +/// if is_minimal_case() { +/// eprintln!("Minimal test case is {num:?}"); +/// export_to_file_for_analysis(num); +/// } +/// +/// prop_assert!(num != 5); +/// } +/// } +/// ``` +pub fn is_minimal_case() -> bool { + IS_MINIMAL_CASE.with(|cell| cell.get()) +} + +/// Helper struct that helps to ensure panic safety when entering a minimal case. +/// +/// Specifically, if the test case panics, we must ensure that we still +/// correctly reset the thread-local variable. +#[non_exhaustive] +pub(crate) struct MinimalCaseGuard; + +impl MinimalCaseGuard { + pub(crate) fn begin_minimal_case() -> Self { + IS_MINIMAL_CASE.with(|cell| cell.replace(true)); + Self + } +} + +impl Drop for MinimalCaseGuard { + fn drop(&mut self) { + IS_MINIMAL_CASE.with(|cell| cell.replace(false)); + } +} \ No newline at end of file diff --git a/proptest/src/lib.rs b/proptest/src/lib.rs index 980b8a6e..d4bafb87 100644 --- a/proptest/src/lib.rs +++ b/proptest/src/lib.rs @@ -45,6 +45,9 @@ extern crate alloc; #[macro_use] mod product_tuple; +mod is_minimal_case; +pub use is_minimal_case::is_minimal_case; + #[macro_use] extern crate bitflags; #[cfg(feature = "bit-set")] diff --git a/proptest/src/test_runner/runner.rs b/proptest/src/test_runner/runner.rs index d209dff6..e9bbe245 100644 --- a/proptest/src/test_runner/runner.rs +++ b/proptest/src/test_runner/runner.rs @@ -24,7 +24,7 @@ use std::env; use std::fs; #[cfg(feature = "fork")] use tempfile; - +use crate::is_minimal_case::MinimalCaseGuard; use crate::strategy::*; use crate::test_runner::config::*; use crate::test_runner::errors::*; @@ -737,13 +737,35 @@ impl TestRunner { let why = self .shrink( &mut case, - test, + &test, replay_from_fork, result_cache, fork_output, is_from_persisted_seed, ) .unwrap_or(why); + + // Run minimal test again + let _guard = MinimalCaseGuard::begin_minimal_case(); + let minimal_result = call_test( + self, + case.current(), + &test, + &mut iter::empty(), + &mut *noop_result_cache(), + // TODO: What should fork_output be? + fork_output, + is_from_persisted_seed, + ); + + if !matches!(minimal_result, Err(TestCaseError::Fail(_))) { + // TODO: Is it appropriate to use eprintln! here? + // It seems appropriate to atleast notify the user somehow + // that the minimal test case does not consistently fail + eprintln!("unexpected behavior: minimal case did not result in test \ + failure on second test run"); + } + Err(TestError::Fail(why, case.current())) } Err(TestCaseError::Reject(whence)) => {