Add --no-fail-fast to compiletest (#2145)

Compiletest behavior has changed to always failing fast after #2045. This PR introduces a --no-fail-fast flag to compiletest which will execute the entire suite regardless of failure.

The regression script uses --no-fail-fast so all failures in a suite are printed as part of the CI.

Author: celinval

scripts/kani-regression.sh

@@ -74,7 +74,8 @@ for testp in "${TESTS[@]}"; do
   suite=${testl[0]}
   mode=${testl[1]}
   echo "Check compiletest suite=$suite mode=$mode"
-  cargo run -p compiletest --quiet -- --suite $suite --mode $mode --quiet
+  cargo run -p compiletest --quiet -- --suite $suite --mode $mode \
+      --quiet --no-fail-fast
 done
 
 # Check codegen for the standard library

tools/compiletest/src/common.rs

@@ -134,6 +134,10 @@ pub struct Config {
     /// Timeout duration for each test.
     pub timeout: Option<Duration>,
 
+    /// Whether we will abort execution when a failure occurs.
+    /// When set to false, this will execute the entire test suite regardless of any failure.
+    pub fail_fast: bool,
+
     /// Whether we will run the tests or not.
     pub dry_run: bool,
 }

tools/compiletest/src/main.rs

@@ -86,6 +86,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         .optflag("h", "help", "show this message")
         .optopt("", "edition", "default Rust edition", "EDITION")
         .optopt("", "timeout", "the timeout for each test in seconds", "TIMEOUT")
+        .optflag("", "no-fail-fast", "run all tests regardless of failure")
         .optflag("", "dry-run", "don't actually run the tests")
     ;
 
@@ -157,6 +158,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         color,
         edition: matches.opt_str("edition"),
         force_rerun: matches.opt_present("force-rerun"),
+        fail_fast: !matches.opt_present("no-fail-fast"),
         dry_run: matches.opt_present("dry-run"),
         timeout,
     }
@@ -176,6 +178,8 @@ pub fn log_config(config: &Config) {
     logv(c, format!("verbose: {}", config.verbose));
     logv(c, format!("quiet: {}", config.quiet));
     logv(c, format!("timeout: {:?}", config.timeout));
+    logv(c, format!("fail-fast: {:?}", config.fail_fast));
+    logv(c, format!("dry-run: {:?}", config.dry_run));
     logv(
         c,
         format!(
@@ -285,7 +289,7 @@ pub fn test_opts(config: &Config) -> test::TestOpts {
         list: false,
         options: test::Options::new(),
         time_options: None,
-        fail_fast: true,
+        fail_fast: config.fail_fast,
         force_run_in_process: false,
     }
 }