Merge pull request #332 from Pat-Lafon/main

Continue on fail when max_fail is gt 0

Author: jmid

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ## NEXT RELEASE
 
-- ...
+- Restore `Test.make`'s `max_fail` parameter which was accidentally broken in 0.18
 
 ## 0.24
 

src/core/QCheck2.ml

@@ -884,7 +884,7 @@ module Gen = struct
     make_primitive
       ~gen:(fun st -> gen st |> Tree.root)
       ~shrink
-  
+
   let no_shrink (gen: 'a t) : 'a t = set_shrink (fun _ -> Seq.empty) gen
 
   let (let+) = (>|=)
@@ -1868,6 +1868,7 @@ module Test = struct
     state.step state.test.name state.test input Failure;
     state.cur_max_fail <- state.cur_max_fail - 1;
     R.fail state.res ~steps ~msg_l input;
+    if state.cur_max_fail > 0 then CR_continue else
     CR_yield state.res
 
   (* [check_state state] applies [state.test] repeatedly ([iter] times)

test/core/QCheck2_expect_test.expected.ocaml4.32

@@ -107,6 +107,24 @@ Test should_fail_sort_id failed (5 shrink steps):
 
 [1; 0]
 
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (3 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (8 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (8 shrink steps):
+
+[0; 1]
+
 === Error ======================================================================
 
 Test should_error_raise_exn errored on (1 shrink steps):
@@ -1664,7 +1682,7 @@ stats dist:
     6.. 10:                                                                   0
 ================================================================================
 1 warning(s)
-failure (74 tests failed, 3 tests errored, ran 166 tests)
+failure (75 tests failed, 3 tests errored, ran 167 tests)
 random seed: 153870556
 
 +++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

test/core/QCheck2_expect_test.expected.ocaml4.64

@@ -169,6 +169,24 @@ Test should_fail_sort_id failed (5 shrink steps):
 
 [1; 0]
 
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (3 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (8 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (8 shrink steps):
+
+[0; 1]
+
 === Error ======================================================================
 
 Test should_error_raise_exn errored on (1 shrink steps):
@@ -1726,7 +1744,7 @@ stats dist:
     6.. 10:                                                                   0
 ================================================================================
 1 warning(s)
-failure (74 tests failed, 3 tests errored, ran 166 tests)
+failure (75 tests failed, 3 tests errored, ran 167 tests)
 random seed: 153870556
 
 +++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

test/core/QCheck2_expect_test.expected.ocaml5.32

@@ -90,6 +90,24 @@ Test should_fail_sort_id failed (8 shrink steps):
 
 [1; 0]
 
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (4 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (8 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (2 shrink steps):
+
+[0; 1]
+
 === Error ======================================================================
 
 Test should_error_raise_exn errored on (1 shrink steps):
@@ -1648,7 +1666,7 @@ stats dist:
    -3..  0: #######################################################        1631
 ================================================================================
 1 warning(s)
-failure (74 tests failed, 3 tests errored, ran 166 tests)
+failure (75 tests failed, 3 tests errored, ran 167 tests)
 random seed: 153870556
 
 +++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

test/core/QCheck2_expect_test.expected.ocaml5.64

@@ -152,6 +152,24 @@ Test should_fail_sort_id failed (8 shrink steps):
 
 [1; 0]
 
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (4 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (8 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (2 shrink steps):
+
+[0; 1]
+
 === Error ======================================================================
 
 Test should_error_raise_exn errored on (1 shrink steps):
@@ -1710,7 +1728,7 @@ stats dist:
    -3..  0: #######################################################        1631
 ================================================================================
 1 warning(s)
-failure (74 tests failed, 3 tests errored, ran 166 tests)
+failure (75 tests failed, 3 tests errored, ran 167 tests)
 random seed: 153870556
 
 +++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

test/core/QCheck2_tests.ml

@@ -51,6 +51,10 @@ module Overall = struct
     Test.make ~name:"should_fail_sort_id" ~count:10 ~print:Print.(list int)
       Gen.(small_list small_int) (fun l -> l = List.sort compare l)
 
+  let max_fail =
+    Test.make ~name:"max_fail" ~count:1000 ~max_fail:3 ~print:Print.(list int)
+      Gen.(list small_nat) (fun l -> l = List.rev l)
+
   exception Error
 
   let error =
@@ -142,6 +146,7 @@ module Overall = struct
   let tests = [
     passing;
     failing;
+    max_fail;
     error;
     collect;
     stats;

test/core/QCheck_expect_test.expected.ocaml4.32

@@ -83,6 +83,24 @@ Test should_fail_sort_id failed (13 shrink steps):
 
 [1; 0]
 
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (16 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (19 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (11 shrink steps):
+
+[1; 0]
+
 === Error ======================================================================
 
 Test should_error_raise_exn errored on (31 shrink steps):
@@ -1641,7 +1659,7 @@ stats dist:
     6.. 10:                                                                   0
 ================================================================================
 1 warning(s)
-failure (74 tests failed, 3 tests errored, ran 174 tests)
+failure (75 tests failed, 3 tests errored, ran 175 tests)
 random seed: 153870556
 
 +++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

test/core/QCheck_expect_test.expected.ocaml4.64

@@ -115,6 +115,24 @@ Test should_fail_sort_id failed (13 shrink steps):
 
 [1; 0]
 
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (16 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (19 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (11 shrink steps):
+
+[1; 0]
+
 === Error ======================================================================
 
 Test should_error_raise_exn errored on (63 shrink steps):
@@ -1673,7 +1691,7 @@ stats dist:
     6.. 10:                                                                   0
 ================================================================================
 1 warning(s)
-failure (74 tests failed, 3 tests errored, ran 174 tests)
+failure (75 tests failed, 3 tests errored, ran 175 tests)
 random seed: 153870556
 
 +++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++

test/core/QCheck_expect_test.expected.ocaml5.32

@@ -93,6 +93,24 @@ Test should_fail_sort_id failed (10 shrink steps):
 
 [1; 0]
 
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (20 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (14 shrink steps):
+
+[0; 1]
+
+--- Failure --------------------------------------------------------------------
+
+Test max_fail failed (16 shrink steps):
+
+[1; 0]
+
 === Error ======================================================================
 
 Test should_error_raise_exn errored on (30 shrink steps):
@@ -1652,7 +1670,7 @@ stats dist:
    -3..  0: #######################################################        1631
 ================================================================================
 1 warning(s)
-failure (74 tests failed, 3 tests errored, ran 174 tests)
+failure (75 tests failed, 3 tests errored, ran 175 tests)
 random seed: 153870556
 
 +++ Stats for int_dist_empty_bucket ++++++++++++++++++++++++++++++++++++++++++++++++++++++++