feat: update timed out test counting logic

Since timed out tests can now be treated as either successes or
failures, update the logic for handling them. This commit also adds
tests for exercising edge cases between `on-timeout="pass"` and
retries/flaky tests

Author: eduardorittner

fixture-data/src/nextest_tests.rs

@@ -47,6 +47,10 @@ pub static EXPECTED_TEST_SUITES: LazyLock<IdOrdMap<TestSuiteFixture>> = LazyLock
                     "test_slow_timeout_subprocess",
                     TestCaseFixtureStatus::IgnoredPass,
                 ),
+                TestCaseFixture::new(
+                    "test_slow_timeout_flaky_mod_4",
+                    TestCaseFixtureStatus::IgnoredFail
+                ),
                 TestCaseFixture::new("test_stdin_closed", TestCaseFixtureStatus::Pass),
                 TestCaseFixture::new("test_subprocess_doesnt_exit", TestCaseFixtureStatus::Leak),
                 TestCaseFixture::new("test_subprocess_doesnt_exit_fail", TestCaseFixtureStatus::FailLeak),

fixtures/nextest-tests/.config/nextest.toml

@@ -88,6 +88,23 @@ test-group = '@global'
 [profile.with-timeout-success]
 slow-timeout = { period = "10ms", terminate-after = 2, on-timeout = "pass" }
 
+[[profile.with-timeout-success.overrides]]
+filter = 'test(=test_slow_timeout_2)'
+slow-timeout = { period = "500ms", terminate-after = 2, on-timeout = "fail" }
+test-group = '@global'
+
+[[profile.with-timeout-success.overrides]]
+filter = 'test(=test_slow_timeout_subprocess)'
+slow-timeout = { period = "10ms", terminate-after = 2, on-timeout = "fail" }
+
+[profile.with-timeout-retries-success]
+slow-timeout = { period = "10ms", terminate-after = 2, on-timeout = "pass" }
+retries = 2
+
+[profile.with-timeout-success-flaky]
+slow-timeout = { period = "10ms", terminate-after = 2, on-timeout = "pass" }
+retries = 3
+
 [profile.with-junit]
 retries = 2
 

fixtures/nextest-tests/tests/basic.rs

@@ -319,6 +319,16 @@ fn test_slow_timeout_subprocess() {
     cmd.output().unwrap();
 }
 
+#[test]
+#[ignore]
+fn test_slow_timeout_flaky_mod_4() {
+    let nextest_attempt = nextest_attempt();
+    if nextest_attempt % 4 != 0 {
+        panic!("Failed because attempt {} % 4 != 0", nextest_attempt)
+    }
+    std::thread::sleep(std::time::Duration::from_secs(4));
+}
+
 #[test]
 fn test_result_failure() -> Result<(), std::io::Error> {
     Err(std::io::Error::new(

integration-tests/tests/integration/snapshots/integration__list_with_default_set_bound_all.snap

@@ -1,7 +1,6 @@
 ---
 source: integration-tests/tests/integration/main.rs
 expression: bound_all_output.stdout_as_str()
-snapshot_kind: text
 ---
 cdylib-example:
     tests::test_multiply_two_cdylib

integration-tests/tests/integration/snapshots/integration__show_config_test_groups-2.snap

@@ -1,7 +1,6 @@
 ---
 source: integration-tests/tests/integration/main.rs
 expression: default_profile_all_output.stdout_as_str()
-snapshot_kind: text
 ---
 group: flaky (max threads = 4)
     (no matches)

integration-tests/tests/integration/snapshots/integration__show_config_test_groups-3.snap

@@ -1,7 +1,6 @@
 ---
 source: integration-tests/tests/integration/main.rs
 expression: with_retries_output.stdout_as_str()
-snapshot_kind: text
 ---
 group: flaky (max threads = 4)
   * override for with-retries profile with filter 'test(test_flaky_mod) | test(~test_flaky_mod_4)':

integration-tests/tests/integration/snapshots/integration__show_config_test_groups-4.snap

@@ -1,7 +1,6 @@
 ---
 source: integration-tests/tests/integration/main.rs
 expression: with_retries_all_output.stdout_as_str()
-snapshot_kind: text
 ---
 group: flaky (max threads = 4)
   * override for with-retries profile with filter 'test(test_flaky_mod) | test(~test_flaky_mod_4)':

nextest-runner/src/config/elements/slow_timeout.rs

@@ -205,6 +205,19 @@ mod tests {
 
         ; "timeout result failure"
     )]
+    #[test_case(
+        indoc! {r#"
+            [profile.default]
+            slow-timeout = { period = "60s", on-timeout = "pass" }
+
+            [profile.ci]
+            slow-timeout = { period = "30s", on-timeout = "fail" }
+        "#},
+        Ok(SlowTimeout { period: Duration::from_secs(60), terminate_after: None, grace_period: Duration::from_secs(10), on_timeout: SlowTimeoutResult::Pass }),
+        Some(SlowTimeout { period: Duration::from_secs(30), terminate_after: None, grace_period: Duration::from_secs(10), on_timeout: SlowTimeoutResult::Fail })
+
+        ; "override on-timeout option"
+    )]
     #[test_case(
         indoc! {r#"
             [profile.default]

nextest-runner/src/reporter/events.rs

@@ -534,7 +534,7 @@ pub struct RunStats {
     /// The number of tests that passed on retry.
     pub flaky: usize,
 
-    /// The number of tests that failed.
+    /// The number of tests that failed. Includes `leaky_failed`.
     pub failed: usize,
 
     /// The number of failed tests that were slow.
@@ -701,7 +701,20 @@ impl RunStats {
                     self.failed_slow += 1;
                 }
             }
-            ExecutionResult::Timeout { .. } => self.failed_timed_out += 1,
+            ExecutionResult::Timeout {
+                result: SlowTimeoutResult::Pass,
+            } => {
+                self.passed += 1;
+                self.passed_timed_out += 1;
+                if run_statuses.len() > 1 {
+                    self.flaky += 1;
+                }
+            }
+            ExecutionResult::Timeout {
+                result: SlowTimeoutResult::Fail,
+            } => {
+                self.failed_timed_out += 1;
+            }
             ExecutionResult::ExecFail => self.exec_failed += 1,
         }
     }
@@ -1597,7 +1610,18 @@ mod tests {
                 initial_run_count: 42,
                 not_run: 0
             }),
-            "timed out => failure"
+            "timed out => failure {:?} {:?}",
+            RunStats {
+                initial_run_count: 42,
+                finished_count: 42,
+                failed_timed_out: 1,
+                ..RunStats::default()
+            }
+            .summarize_final(),
+            FinalRunStats::Failed(RunStatsFailureKind::Test {
+                initial_run_count: 42,
+                not_run: 0
+            }),
         );
         assert_eq!(
             RunStats {