Add global-timeout option

The global-timeout option gives an upper bound to how long a test run
can last. When this limit is reached, the run is cancelled (similar to
sending a signal to the process) and the results we got so far are
displayed.

Author: roblabla

nextest-runner/default-config.toml

@@ -97,6 +97,11 @@ slow-timeout = { period = "60s" }
 # See <https://nexte.st/docs/features/leaky-tests> for more information.
 leak-timeout = "100ms"
 
+# Stop all tests after the configured global timeout.
+# Defaults to 30 year, which is a large enough value to feel "infinite" without
+# running into overflows on various platforms.
+global-timeout = "30y"
+
 # `nextest archive` automatically includes any build output required by a standard build.
 # However sometimes extra non-standard files are required.
 # To address this, "archive.include" specifies additional paths that will be included in the archive.

nextest-runner/src/config/config_impl.rs

@@ -4,9 +4,9 @@
 use super::{
     ArchiveConfig, CompiledByProfile, CompiledData, CompiledDefaultFilter, ConfigExperimental,
     CustomTestGroup, DefaultJunitImpl, DeserializedOverride, DeserializedProfileScriptConfig,
-    JunitConfig, JunitImpl, MaxFail, NextestVersionDeserialize, RetryPolicy, ScriptConfig,
-    ScriptId, SettingSource, SetupScripts, SlowTimeout, TestGroup, TestGroupConfig, TestSettings,
-    TestThreads, ThreadsRequired, ToolConfigFile, leak_timeout::LeakTimeout,
+    GlobalTimeout, JunitConfig, JunitImpl, MaxFail, NextestVersionDeserialize, RetryPolicy,
+    ScriptConfig, ScriptId, SettingSource, SetupScripts, SlowTimeout, TestGroup, TestGroupConfig,
+    TestSettings, TestThreads, ThreadsRequired, ToolConfigFile, leak_timeout::LeakTimeout,
 };
 use crate::{
     config::{ListSettings, ProfileScriptType, ScriptInfo, SetupScriptConfig},
@@ -1009,6 +1009,13 @@ impl<'cfg> EvaluatableProfile<'cfg> {
             .unwrap_or(self.default_profile.slow_timeout)
     }
 
+    /// Returns the time after which we should stop running tests.
+    pub fn global_timeout(&self) -> GlobalTimeout {
+        self.custom_profile
+            .and_then(|profile| profile.global_timeout)
+            .unwrap_or(self.default_profile.global_timeout)
+    }
+
     /// Returns the time after which a child process that hasn't closed its handles is marked as
     /// leaky.
     pub fn leak_timeout(&self) -> LeakTimeout {
@@ -1210,6 +1217,7 @@ pub(super) struct DefaultProfileImpl {
     success_output: TestOutputDisplay,
     max_fail: MaxFail,
     slow_timeout: SlowTimeout,
+    global_timeout: GlobalTimeout,
     leak_timeout: LeakTimeout,
     overrides: Vec<DeserializedOverride>,
     scripts: Vec<DeserializedProfileScriptConfig>,
@@ -1249,6 +1257,9 @@ impl DefaultProfileImpl {
             slow_timeout: p
                 .slow_timeout
                 .expect("slow-timeout present in default profile"),
+            global_timeout: p
+                .global_timeout
+                .expect("global-timeout present in default profile"),
             leak_timeout: p
                 .leak_timeout
                 .expect("leak-timeout present in default profile"),
@@ -1302,6 +1313,8 @@ pub(super) struct CustomProfileImpl {
     max_fail: Option<MaxFail>,
     #[serde(default, deserialize_with = "super::deserialize_slow_timeout")]
     slow_timeout: Option<SlowTimeout>,
+    #[serde(default)]
+    global_timeout: Option<GlobalTimeout>,
     #[serde(default, deserialize_with = "super::deserialize_leak_timeout")]
     leak_timeout: Option<LeakTimeout>,
     #[serde(default)]

nextest-runner/src/config/global_timeout.rs

@@ -0,0 +1,120 @@
+// Copyright (c) The nextest Contributors
+// SPDX-License-Identifier: MIT OR Apache-2.0
+
+use serde::{Deserialize, Deserializer};
+use std::time::Duration;
+
+/// Type for the global-timeout config key.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct GlobalTimeout {
+    pub(crate) period: Duration,
+}
+
+impl<'de> Deserialize<'de> for GlobalTimeout {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        Ok(GlobalTimeout {
+            period: humantime_serde::deserialize(deserializer)?,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::{
+        NextestConfig,
+        test_helpers::{build_platforms, temp_workspace},
+    };
+    use camino_tempfile::tempdir;
+    use indoc::indoc;
+    use nextest_filtering::ParseContext;
+    use test_case::test_case;
+
+    #[test_case(
+        "",
+        Ok(GlobalTimeout { period: Duration::from_secs(946728000) }),
+        None
+
+        ; "empty config is expected to use the hardcoded values"
+    )]
+    #[test_case(
+        indoc! {r#"
+            [profile.default]
+            global-timeout = "30s"
+        "#},
+        Ok(GlobalTimeout { period: Duration::from_secs(30) }),
+        None
+
+        ; "overrides the default profile"
+    )]
+    #[test_case(
+        indoc! {r#"
+            [profile.default]
+            global-timeout = "30s"
+
+            [profile.ci]
+            global-timeout = "60s"
+        "#},
+        Ok(GlobalTimeout { period: Duration::from_secs(30) }),
+        Some(GlobalTimeout { period: Duration::from_secs(60) })
+
+        ; "adds a custom profile 'ci'"
+    )]
+    fn globaltimeout_adheres_to_hierarchy(
+        config_contents: &str,
+        expected_default: Result<GlobalTimeout, &str>,
+        maybe_expected_ci: Option<GlobalTimeout>,
+    ) {
+        let workspace_dir = tempdir().unwrap();
+
+        let graph = temp_workspace(&workspace_dir, config_contents);
+
+        let pcx = ParseContext::new(&graph);
+
+        let nextest_config_result = NextestConfig::from_sources(
+            graph.workspace().root(),
+            &pcx,
+            None,
+            &[][..],
+            &Default::default(),
+        );
+
+        match expected_default {
+            Ok(expected_default) => {
+                let nextest_config = nextest_config_result.expect("config file should parse");
+
+                assert_eq!(
+                    nextest_config
+                        .profile("default")
+                        .expect("default profile should exist")
+                        .apply_build_platforms(&build_platforms())
+                        .global_timeout(),
+                    expected_default,
+                );
+
+                if let Some(expected_ci) = maybe_expected_ci {
+                    assert_eq!(
+                        nextest_config
+                            .profile("ci")
+                            .expect("ci profile should exist")
+                            .apply_build_platforms(&build_platforms())
+                            .global_timeout(),
+                        expected_ci,
+                    );
+                }
+            }
+
+            Err(expected_err_str) => {
+                let err_str = format!("{:?}", nextest_config_result.unwrap_err());
+
+                assert!(
+                    err_str.contains(expected_err_str),
+                    "expected error string not found: {err_str}",
+                )
+            }
+        }
+    }
+}

nextest-runner/src/config/mod.rs

@@ -21,6 +21,7 @@
 
 mod archive;
 mod config_impl;
+mod global_timeout;
 mod helpers;
 mod identifier;
 mod junit;
@@ -40,6 +41,7 @@ mod track_default;
 
 pub use archive::*;
 pub use config_impl::*;
+pub use global_timeout::*;
 pub use identifier::*;
 pub use junit::*;
 pub use leak_timeout::*;

nextest-runner/src/config/slow_timeout.rs

@@ -1,6 +1,7 @@
 // Copyright (c) The nextest Contributors
 // SPDX-License-Identifier: MIT OR Apache-2.0
 
+use crate::time::far_future_duration;
 use serde::{Deserialize, de::IntoDeserializer};
 use std::{fmt, num::NonZeroUsize, time::Duration};
 
@@ -19,8 +20,7 @@ pub struct SlowTimeout {
 impl SlowTimeout {
     /// A reasonable value for "maximum slow timeout".
     pub(crate) const VERY_LARGE: Self = Self {
-        // See far_future() in pausable_sleep.rs for why this is roughly 30 years.
-        period: Duration::from_secs(86400 * 365 * 30),
+        period: far_future_duration(),
         terminate_after: None,
         grace_period: Duration::from_secs(10),
     };

nextest-runner/src/reporter/displayer/progress.rs

@@ -482,6 +482,7 @@ fn progress_bar_cancel_prefix(reason: CancelReason, styles: &Styles) -> String {
         CancelReason::SetupScriptFailure
         | CancelReason::TestFailure
         | CancelReason::ReportError
+        | CancelReason::GlobalTimeout
         | CancelReason::Signal
         | CancelReason::Interrupt => "Cancelling",
         CancelReason::SecondSignal => "Killing",

nextest-runner/src/reporter/events.rs

@@ -904,6 +904,9 @@ pub enum CancelReason {
     /// An error occurred while reporting results.
     ReportError,
 
+    /// The global timeout was exceeded.
+    GlobalTimeout,
+
     /// A termination signal (on Unix, SIGTERM or SIGHUP) was received.
     Signal,
 
@@ -922,6 +925,7 @@ impl CancelReason {
             CancelReason::ReportError => "reporting error",
             CancelReason::Signal => "signal",
             CancelReason::Interrupt => "interrupt",
+            CancelReason::GlobalTimeout => "timeout",
             CancelReason::SecondSignal => "second signal",
         }
     }

nextest-runner/src/runner/dispatcher.rs

@@ -44,6 +44,7 @@ pub(super) struct DispatcherContext<'a, F> {
     stopwatch: StopwatchStart,
     run_stats: RunStats,
     max_fail: MaxFail,
+    global_timeout: Duration,
     running_setup_script: Option<ContextSetupScript<'a>>,
     running_tests: BTreeMap<TestInstanceId<'a>, ContextTestInstance<'a>>,
     cancel_state: Option<CancelReason>,
@@ -63,6 +64,7 @@ where
         cli_args: Vec<String>,
         initial_run_count: usize,
         max_fail: MaxFail,
+        global_timeout: Duration,
     ) -> Self {
         Self {
             callback: DebugIgnore(callback),
@@ -75,6 +77,7 @@ where
                 ..RunStats::default()
             },
             max_fail,
+            global_timeout,
             running_setup_script: None,
             running_tests: BTreeMap::new(),
             cancel_state: None,
@@ -103,9 +106,14 @@ where
         let mut signals_done = false;
         let mut inputs_done = false;
         let mut report_cancel_rx_done = false;
+        let mut global_timeout_sleep =
+            std::pin::pin!(crate::time::pausable_sleep(self.global_timeout));
 
         loop {
             let internal_event = tokio::select! {
+                _ = &mut global_timeout_sleep => {
+                    InternalEvent::GlobalTimeout
+                },
                 internal_event = executor_rx.recv() => {
                     match internal_event {
                         Some(event) => InternalEvent::Executor(event),
@@ -201,13 +209,20 @@ where
                     // Restore the terminal state.
                     input_handler.suspend();
 
+                    // Pause the global timeout while suspended.
+                    global_timeout_sleep.as_mut().pause();
+
                     // Now stop nextest itself.
                     super::os::raise_stop();
                 }
                 #[cfg(unix)]
                 HandleEventResponse::JobControl(JobControlEvent::Continue) => {
                     // Nextest has been resumed. Resume the input handler, as well as all the tests.
                     input_handler.resume();
+
+                    // Resume the global timeout.
+                    global_timeout_sleep.as_mut().resume();
+
                     self.broadcast_request(RunUnitRequest::Signal(SignalRequest::Continue));
                 }
                 #[cfg(not(unix))]
@@ -285,6 +300,10 @@ where
                             // A test failure has caused cancellation to begin.
                             self.broadcast_request(RunUnitRequest::OtherCancel);
                         }
+                        CancelEvent::GlobalTimeout => {
+                            // The global timeout has expired, causing cancellation to begin.
+                            self.broadcast_request(RunUnitRequest::OtherCancel);
+                        }
                         CancelEvent::Signal(req) => {
                             // A signal has caused cancellation to begin. Let all the child
                             // processes know about the signal, and continue to handle
@@ -534,6 +553,9 @@ where
                 })
             }
             InternalEvent::Signal(event) => self.handle_signal_event(event),
+            InternalEvent::GlobalTimeout => {
+                self.begin_cancel(CancelReason::GlobalTimeout, CancelEvent::GlobalTimeout)
+            }
             InternalEvent::Input(InputEvent::Info) => {
                 // Print current statistics.
                 HandleEventResponse::Info(InfoEvent::Input)
@@ -850,6 +872,7 @@ enum InternalEvent<'a> {
     Signal(SignalEvent),
     Input(InputEvent),
     ReportCancel,
+    GlobalTimeout,
 }
 
 /// The return result of `handle_event`.
@@ -883,6 +906,7 @@ enum InfoEvent {
 enum CancelEvent {
     Report,
     TestFailure,
+    GlobalTimeout,
     Signal(ShutdownRequest),
 }
 
@@ -919,7 +943,9 @@ mod tests {
             vec![],
             0,
             MaxFail::All,
+            crate::time::far_future_duration(),
         );
+
         cx.disable_signal_3_times_panic = true;
 
         // Begin cancellation with a report error.

nextest-runner/src/runner/imp.rs

@@ -255,6 +255,7 @@ impl<'a> TestRunnerInner<'a> {
             self.cli_args.clone(),
             self.test_list.run_count(),
             self.max_fail,
+            self.profile.global_timeout().period,
         );
 
         let executor_cx = ExecutorContext::new(

nextest-runner/src/time/pausable_sleep.rs

@@ -126,11 +126,15 @@ enum SleepPauseState {
 
 // Cribbed from tokio.
 fn far_future() -> Instant {
+    Instant::now() + far_future_duration()
+}
+
+pub(crate) const fn far_future_duration() -> Duration {
     // Roughly 30 years from now.
     // API does not provide a way to obtain max `Instant`
     // or convert specific date in the future to instant.
     // 1000 years overflows on macOS, 100 years overflows on FreeBSD.
-    Instant::now() + Duration::from_secs(86400 * 365 * 30)
+    Duration::from_secs(86400 * 365 * 30)
 }
 
 #[cfg(test)]